diff --git a/app/Livewire/Admin/Vpn.php b/app/Livewire/Admin/Vpn.php index ee37b99..1c75128 100644 --- a/app/Livewire/Admin/Vpn.php +++ b/app/Livewire/Admin/Vpn.php @@ -352,6 +352,12 @@ class Vpn extends Component '[Interface]', 'PrivateKey = '.$keypair->privateKey, 'Address = '.$ip.'/32', + // Only present when the console is actually reachable inside the + // tunnel. Setting it otherwise would hand the device a resolver + // that does not exist and take its name resolution with it. + ...(config('admin_access.vpn_ready') + ? ['DNS = '.config('provisioning.wireguard.hub_address', '10.66.0.1')] + : []), '', '[Peer]', 'PublicKey = '.$hub->publicKey(), diff --git a/config/admin_access.php b/config/admin_access.php index bb22304..f6e2baf 100644 --- a/config/admin_access.php +++ b/config/admin_access.php @@ -53,6 +53,34 @@ return [ */ 'vpn_config_key' => env('VPN_CONFIG_KEY', ''), + /* + | The hostname the console answers on INSIDE the tunnel. + | + | Set, the VPN gateway and resolver are started (compose profile `vpn`) and + | issued client configs point their DNS at the hub. Empty, none of that + | exists and the VPN is only a management network — which is what an + | installation without the extra services should be. + | + | Deliberately the SAME name the console already uses publicly: the + | certificate is bound to the name, not to the address serving it, so + | nothing new has to be issued and nothing about the internal network ends + | up in a public DNS zone. + */ + 'vpn_internal_host' => strtolower(trim((string) env('VPN_INTERNAL_HOST', ''))), + + /* + | Whether the tunnel-side gateway is actually READY, not merely wanted. + | + | The hostname alone is not enough: the deployment refuses to start the + | gateway until a certificate for it has been found, and a client config + | that names a resolver nobody started takes the device's whole name + | resolution down for as long as the tunnel is up. The certificate path is + | written by the same step that starts the services, so it is the honest + | signal. + */ + 'vpn_ready' => strtolower(trim((string) env('VPN_INTERNAL_HOST', ''))) !== '' + && filter_var(env('VPN_READY', false), FILTER_VALIDATE_BOOLEAN), + /* | Key for credentials stored in the console (32 bytes, base64). | diff --git a/config/provisioning.php b/config/provisioning.php index cc99b57..87d23d4 100644 --- a/config/provisioning.php +++ b/config/provisioning.php @@ -154,6 +154,9 @@ return [ // CluPilot VM acts as the WireGuard hub; hosts join it during onboarding. 'wireguard' => [ + // The hub's own address inside the tunnel — what a client uses as its + // resolver when the console is reachable there. + 'hub_address' => env('CLUPILOT_WG_HUB_ADDRESS', '10.66.0.1'), 'subnet' => $wgSubnet, 'hub_ip' => env('CLUPILOT_WG_HUB_IP', $wgHubDefault), // handshake target 'endpoint' => env('CLUPILOT_WG_ENDPOINT', ''), // host:port reachable by peers diff --git a/deploy/install-agent.sh b/deploy/install-agent.sh index 54b4740..c286e57 100755 --- a/deploy/install-agent.sh +++ b/deploy/install-agent.sh @@ -139,6 +139,135 @@ EOF visudo -cf /etc/sudoers.d/clupilot-caddy-reload >/dev/null || rm -f /etc/sudoers.d/clupilot-caddy-reload fi +# ── The console inside the tunnel ──────────────────────────────────────────── +# The gateway serves the console on the WireGuard hub address using the +# certificate the public Caddy already obtains for that same hostname — a +# certificate is bound to the NAME, not to the address serving it. Finding it +# has to happen here, as root: the storage is Caddy's and the paths contain the +# ACME directory, which is not something to ask an operator to type. +VPN_HOST="$(sed -n 's/^VPN_INTERNAL_HOST=//p' "$ROOT/.env" 2>/dev/null | tail -1)" + +if [[ -z "$VPN_HOST" ]]; then + # Cleared. Reconciled here too, because the updater exits early when the + # checkout has not moved — so without this there may be no path at all that + # stops the services, and clients would keep being handed a resolver that + # is no longer meant to exist. + runuser -u "$APP_USER" -- sh -c " + cd '$ROOT' + if grep -qE '^COMPOSE_PROFILES=.*vpn' .env 2>/dev/null || grep -qE '^VPN_READY=' .env 2>/dev/null; then + docker compose --profile vpn stop vpn-dns vpn-gateway >/dev/null 2>&1 || true + docker compose --profile vpn rm -f vpn-dns vpn-gateway >/dev/null 2>&1 || true + sed -i '/^VPN_CERT_PATH=/d;/^VPN_KEY_PATH=/d;/^VPN_READY=/d' .env + profiles=\"\$(sed -n 's/^COMPOSE_PROFILES=//p' .env | tail -1)\" + profiles=\"\$(printf '%s' \"\$profiles\" | tr ',' '\n' | grep -vx 'vpn' | grep -v '^\$' | paste -sd, - || true)\" + sed -i '/^COMPOSE_PROFILES=/d' .env + [ -n \"\$profiles\" ] && printf 'COMPOSE_PROFILES=%s\n' \"\$profiles\" >> .env + docker compose exec -T app php artisan config:clear >/dev/null 2>&1 || true + echo ' Tunnel gateway disabled — VPN_INTERNAL_HOST is empty.' + fi + " || true +fi + +if [[ -n "$VPN_HOST" ]]; then + # From the project's .env, because that is the file Compose reads for the + # mount. Looking only at the default would report "no certificate" on an + # installation whose storage is somewhere else. + CADDY_DATA="$(sed -n 's/^CADDY_DATA_DIR=//p' "$ROOT/.env" 2>/dev/null | tail -1)" + CADDY_DATA="${CADDY_DATA:-${CADDY_DATA_DIR:-/var/lib/caddy/.local/share/caddy}}" + # -print -quit, not | head -1: under `set -o pipefail` a find that is still + # producing output when head exits returns SIGPIPE, and the whole agent + # installation aborts — on the perfectly ordinary case of one certificate + # existing under two issuer directories. + cert="$(find "$CADDY_DATA/certificates" -name "${VPN_HOST}.crt" -print -quit 2>/dev/null)" + key="${cert%.crt}.key" + + if [[ -n "$cert" && -f "$key" ]]; then + # Paths as the gateway container sees them: the storage is mounted at /certs. + rel="${cert#$CADDY_DATA/}" + runuser -u "$APP_USER" -- sh -c " + cd '$ROOT' + sed -i '/^VPN_CERT_PATH=/d;/^VPN_KEY_PATH=/d' .env + printf 'VPN_CERT_PATH=/certs/%s\n' '$rel' >> .env + printf 'VPN_KEY_PATH=/certs/%s\n' '${rel%.crt}.key' >> .env + " + # No chmod. The gateway container runs as root and reads these through + # the read-only mount already; making the private key world-readable + # would let every local account copy the console's TLS identity. + # Enabled AND started here. The updater exits early when the checkout + # has not moved, so leaving this to "the next deploy" leaves the + # gateway stopped and the success message a lie. + runuser -u "$APP_USER" -- sh -c " + cd '$ROOT' + grep -qE '^COMPOSE_PROFILES=.*vpn' .env \ + || { grep -qE '^COMPOSE_PROFILES=' .env \ + && sed -i 's/^COMPOSE_PROFILES=\\(.*\\)\$/COMPOSE_PROFILES=\\1,vpn/' .env \ + || printf 'COMPOSE_PROFILES=vpn\n' >> .env; } + docker compose --profile vpn up -d vpn-dns vpn-gateway + " >/dev/null 2>&1 || true + + # `up -d` succeeds once the containers are CREATED. One that starts and + # exits — Caddy unable to read the certificate, or to bind the hub + # address — still counts as success there, so the state is checked. + sleep 3 + vpn_started=0 + if runuser -u "$APP_USER" -- sh -c "cd '$ROOT' && docker compose --profile vpn ps --status running --services" 2>/dev/null \ + | grep -q '^vpn-gateway$' \ + && runuser -u "$APP_USER" -- sh -c "cd '$ROOT' && docker compose --profile vpn ps --status running --services" 2>/dev/null \ + | grep -q '^vpn-dns$'; then + vpn_started=1 + fi + + # The marker, and only on success. Certificate paths alone would report + # the tunnel ready after a failed start, and the application would hand + # out configs naming a resolver that never came up. + runuser -u "$APP_USER" -- sh -c " + cd '$ROOT' + sed -i '/^VPN_READY=/d' .env + printf 'VPN_READY=%s\n' '$([[ $vpn_started -eq 1 ]] && echo true || echo false)' >> .env + " + + if [[ $vpn_started -ne 1 ]]; then + echo " ! Could not start the tunnel gateway — check: docker compose --profile vpn up -d" + fi + + # The application caches its configuration. Without rebuilding it here, + # vpn_ready stays false and client configs keep omitting the tunnel + # resolver — the gateway running but nothing pointed at it. + runuser -u "$APP_USER" -- sh -c " + cd '$ROOT' + docker compose exec -T app php artisan config:clear >/dev/null 2>&1 + docker compose exec -T app php artisan config:cache >/dev/null 2>&1 + " || true + + # Only when it is true. Announcing success while the gateway failed to + # start is worse than silence: it sends the operator looking somewhere + # else entirely. + if [[ $vpn_started -eq 1 ]]; then + echo " Console reachable inside the tunnel at https://$VPN_HOST" + fi + else + # Nothing usable for the CURRENT hostname. Leaving the old paths, the + # profile and the readiness marker in place would have the application + # advertise the renamed endpoint as ready while the gateway still + # served the previous name. + runuser -u "$APP_USER" -- sh -c " + cd '$ROOT' + sed -i '/^VPN_CERT_PATH=/d;/^VPN_KEY_PATH=/d;/^VPN_READY=/d' .env + docker compose --profile vpn stop vpn-dns vpn-gateway >/dev/null 2>&1 || true + # And the profile with it, or the next ordinary `up -d` starts a + # Caddy with empty tls paths that can only crash. + profiles=\"\$(sed -n 's/^COMPOSE_PROFILES=//p' .env | tail -1)\" + profiles=\"\$(printf '%s' \"\$profiles\" | tr ',' '\n' | grep -vx 'vpn' | grep -v '^\$' | paste -sd, - || true)\" + sed -i '/^COMPOSE_PROFILES=/d' .env + [ -n \"\$profiles\" ] && printf 'COMPOSE_PROFILES=%s\n' \"\$profiles\" >> .env + docker compose exec -T app php artisan config:clear >/dev/null 2>&1 || true + " || true + + echo " ! No certificate for $VPN_HOST in $CADDY_DATA yet." + echo " Reach https://$VPN_HOST publicly once so Caddy obtains it, then run this again." + fi +fi + systemctl daemon-reload systemctl enable --now clupilot-update-agent.timer >/dev/null diff --git a/deploy/update-agent.sh b/deploy/update-agent.sh index d871867..bb1bc57 100755 --- a/deploy/update-agent.sh +++ b/deploy/update-agent.sh @@ -80,7 +80,35 @@ sync_console_allowlist() { fi } +# The tunnel gateway serves a certificate the PUBLIC Caddy owns and renews. Its +# tls directive loads the file once at startup, so after a renewal it would go +# on presenting the old one until something restarted it — and eventually serve +# an expired certificate to the only people who can reach the console. +sync_vpn_certificate() { + local stamp seen="$STATE_DIR/.vpn-cert-stamp" path + + path="$(sed -n 's/^VPN_CERT_PATH=//p' "$ROOT/.env" 2>/dev/null | tail -1)" + [[ -n "$path" ]] || return 0 + + # Read from INSIDE the gateway, not from the host. Caddy's storage belongs + # to its own service account and is deliberately left that way — this agent + # runs unprivileged and cannot even traverse it, so a host-side check would + # silently never fire and the gateway would serve an expired certificate. + stamp="$(docker compose exec -T vpn-gateway stat -c %Y "$path" 2>/dev/null | tr -d '\r\n')" + [[ -n "$stamp" ]] || return 0 + + if [[ ! -f "$seen" ]] || [[ "$(cat "$seen" 2>/dev/null)" != "$stamp" ]]; then + # Stamped only on success. Recording a restart that did not happen means + # the next run considers this certificate handled and never retries — + # leaving the gateway serving the old one until the NEXT renewal. + if docker compose --profile vpn restart vpn-gateway >/dev/null 2>&1; then + printf '%s' "$stamp" > "$seen" + fi + fi +} + sync_console_allowlist +sync_vpn_certificate MODE="$(release_mode)" BRANCH="$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo main)" diff --git a/deploy/update.sh b/deploy/update.sh index b9ec9e3..6f9fcb4 100755 --- a/deploy/update.sh +++ b/deploy/update.sh @@ -101,6 +101,114 @@ fi # instead of declaring "already up to date". deployed="$(cat "$STATE_FILE" 2>/dev/null || echo '')" +# Readiness is re-established after every start, never carried over. `up -d` +# succeeds once a container is CREATED, so one that starts and exits — an +# unreadable certificate, an address it cannot bind — would leave a previous +# VPN_READY=true in place and the application still advertising a resolver that +# is no longer running. +# +# A function because there are two ways out of this script: a full deploy, and +# the shortcut taken when the checkout has not moved. Both start services; both +# have to record what actually came up. +reconcile_vpn_readiness() { + grep -qE '^COMPOSE_PROFILES=.*vpn' .env 2>/dev/null || return 0 + + sleep 3 + local running vpn_ready + running="$(docker compose --profile vpn ps --status running --services 2>/dev/null || true)" + + if grep -q '^vpn-gateway$' <<<"$running" && grep -q '^vpn-dns$' <<<"$running"; then + vpn_ready=true + else + vpn_ready=false + printf '\033[1;33m !\033[0m %s\n' "The tunnel gateway is not running — clients will not be given its resolver." + fi + + if [[ "$(sed -n 's/^VPN_READY=//p' .env | tail -1)" != "$vpn_ready" ]]; then + sed -i '/^VPN_READY=/d' .env + printf 'VPN_READY=%s\n' "$vpn_ready" >> .env + in_app php artisan config:clear >/dev/null 2>&1 || true + in_app php artisan config:cache >/dev/null 2>&1 || true + fi +} + +# The VPN gateway and resolver live behind a compose profile, and behind a +# certificate. Enabling the profile on the hostname alone would start a Caddy +# with an empty tls directive — it fails to load, the internal console never +# comes up, and the application is meanwhile handing out client configs that +# point at a resolver nobody started. So: both, or neither, and BEFORE the +# services are brought up rather than after. +if ! grep -qE '^VPN_INTERNAL_HOST=..' .env 2>/dev/null; then + # Cleared. Leaving the profile in place keeps both services running against + # a placeholder hostname and stale certificate paths, so "empty disables it" + # would not be true. + if grep -qE '^COMPOSE_PROFILES=.*vpn' .env 2>/dev/null; then + # Rebuilt from the remaining entries rather than cut out in place: + # deleting "vpn" from "ci,vpn" by substitution leaves "ci," and Compose + # reads the empty entry as a profile. + profiles="$(sed -n 's/^COMPOSE_PROFILES=//p' .env | tail -1)" + # `|| true`: with only "vpn" in the list, grep finds nothing and exits + # 1 — which under `set -e` aborts the update mid-flight, in maintenance + # mode, with the services it was about to stop still running. + profiles="$(printf '%s' "$profiles" | tr ',' '\n' | grep -vx 'vpn' | grep -v '^$' | paste -sd, - || true)" + sed -i '/^COMPOSE_PROFILES=/d' .env + [[ -n "$profiles" ]] && printf 'COMPOSE_PROFILES=%s\n' "$profiles" >> .env + docker compose --profile vpn stop vpn-dns vpn-gateway >/dev/null 2>&1 || true + docker compose --profile vpn rm -f vpn-dns vpn-gateway >/dev/null 2>&1 || true + # The application caches its configuration, so clearing .env alone + # would leave it issuing configs that name a resolver just stopped. + in_app php artisan config:clear >/dev/null 2>&1 || true + log "Disabled the vpn compose profile — VPN_INTERNAL_HOST is empty" + fi +elif grep -qE '^VPN_INTERNAL_HOST=..' .env 2>/dev/null; then + vpn_cert="$(sed -n 's/^VPN_CERT_PATH=//p' .env | tail -1)" + vpn_key="$(sed -n 's/^VPN_KEY_PATH=//p' .env | tail -1)" + + vpn_host="$(sed -n 's/^VPN_INTERNAL_HOST=//p' .env | tail -1)" + + # The certificate has to belong to the CURRENT hostname. After a rename the + # old paths are still set and still exist, and the gateway would start + # serving the previous name's certificate to clients resolving the new one. + if [[ -n "$vpn_cert" && "$(basename "$vpn_cert")" != "${vpn_host}.crt" ]]; then + sed -i '/^VPN_CERT_PATH=/d;/^VPN_KEY_PATH=/d;/^VPN_READY=/d' .env + vpn_cert=''; vpn_key='' + in_app php artisan config:clear >/dev/null 2>&1 || true + printf '\033[1;33m !\033[0m %s\n' "The tunnel certificate does not match VPN_INTERNAL_HOST — cleared." + fi + + if [[ -z "$vpn_cert" || -z "$vpn_key" ]]; then + # Both or neither: leaving the profile on with empty tls paths starts a + # Caddy that cannot load its configuration and crashes forever. + if grep -qE '^COMPOSE_PROFILES=.*vpn' .env 2>/dev/null; then + docker compose --profile vpn stop vpn-dns vpn-gateway >/dev/null 2>&1 || true + profiles="$(sed -n 's/^COMPOSE_PROFILES=//p' .env | tail -1)" + profiles="$(printf '%s' "$profiles" | tr ',' '\n' | grep -vx 'vpn' | grep -v '^$' | paste -sd, - || true)" + sed -i '/^COMPOSE_PROFILES=/d' .env + [[ -n "$profiles" ]] && printf 'COMPOSE_PROFILES=%s\n' "$profiles" >> .env + sed -i '/^VPN_READY=/d' .env + fi + fi + + if [[ -n "$vpn_cert" && -n "$vpn_key" ]]; then + if ! grep -qE '^COMPOSE_PROFILES=.*vpn' .env; then + if grep -qE '^COMPOSE_PROFILES=' .env; then + sed -i 's/^COMPOSE_PROFILES=\(.*\)$/COMPOSE_PROFILES=\1,vpn/' .env + else + printf 'COMPOSE_PROFILES=vpn\n' >> .env + fi + log "Enabled the vpn compose profile" + # Started here, not left to the deploy below: the update may exit + # early as already deployed, and the profile would then be marked + # on with nothing running behind it. + docker compose --profile vpn up -d vpn-dns vpn-gateway >/dev/null 2>&1 || true + reconcile_vpn_readiness + fi + else + printf '\033[1;33m !\033[0m %s\n' "VPN_INTERNAL_HOST is set but the certificate paths are not." + printf ' %s\n' "The console is not reachable inside the tunnel. Run: sudo bash $(pwd)/deploy/install-agent.sh" + fi +fi + if [[ "$before" == "$target" && "$deployed" == "$target" ]]; then # Nothing to deploy, but possibly something to decide: pinning a server to # a release that happens to be the commit it is already on is a real and @@ -208,6 +316,8 @@ docker compose up -d # keep running the code from before the update. docker compose restart queue queue-provisioning scheduler reverb +reconcile_vpn_readiness + log "Leaving maintenance mode" in_app php artisan up >/dev/null down=0 diff --git a/docker-compose.yml b/docker-compose.yml index b096ae9..d28e791 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -82,6 +82,58 @@ services: - redis - mariadb + # ── The console, reachable from inside the tunnel ─────────────────────────── + # + # Both of these share the provisioning container's network namespace, which is + # where wg0 (10.66.0.1) lives. That is the whole point: they answer ON the + # tunnel address, so a VPN client needs no route beyond the management subnet + # it already has, the host's docker bridge is never exposed to the tunnel, and + # the console sees the client's real 10.66.0.x address rather than a proxy's. + # + # Nothing here is published to the host. If the tunnel is down, neither exists + # as far as the outside world is concerned. + # + # Both are optional: an installation without VPN_INTERNAL_HOST configured + # simply does not start them (profile `vpn`). + vpn-dns: + image: 4km3/dnsmasq:2.90-r0 + restart: unless-stopped + profiles: ["vpn"] + network_mode: "service:queue-provisioning" + cap_add: + - NET_ADMIN + command: >- + --keep-in-foreground --log-facility=- + --listen-address=${CLUPILOT_WG_HUB_ADDRESS:-10.66.0.1} --bind-interfaces + --no-resolv --server=1.1.1.1 --server=9.9.9.9 + --address=/${VPN_INTERNAL_HOST:-admin.invalid}/${CLUPILOT_WG_HUB_ADDRESS:-10.66.0.1} + depends_on: + - queue-provisioning + + vpn-gateway: + image: caddy:2-alpine + restart: unless-stopped + profiles: ["vpn"] + network_mode: "service:queue-provisioning" + volumes: + - ./docker/caddy/vpn.Caddyfile:/etc/caddy/Caddyfile:ro + # The certificate the public Caddy already obtains and renews. Read-only, + # and shared rather than duplicated: a second ACME client would be a + # second thing to renew, and the private address it would have to answer + # on cannot satisfy an HTTP challenge anyway. + - ${CADDY_DATA_DIR:-/var/lib/caddy/.local/share/caddy}:/certs:ro + environment: + VPN_INTERNAL_HOST: ${VPN_INTERNAL_HOST:-admin.invalid} + # The same address the client is told to use as its resolver. Hard-coding + # it here while the application reads a configurable one would produce a + # config pointing at an address nothing listens on. + VPN_HUB_ADDRESS: ${CLUPILOT_WG_HUB_ADDRESS:-10.66.0.1} + VPN_CERT_PATH: ${VPN_CERT_PATH:-} + VPN_KEY_PATH: ${VPN_KEY_PATH:-} + depends_on: + - queue-provisioning + - app + scheduler: image: clupilot-app:dev restart: unless-stopped diff --git a/docker/caddy/vpn.Caddyfile b/docker/caddy/vpn.Caddyfile new file mode 100644 index 0000000..458ffc5 --- /dev/null +++ b/docker/caddy/vpn.Caddyfile @@ -0,0 +1,30 @@ +# The console, served on the WireGuard hub address. +# +# Runs inside the provisioning container's network namespace, so it listens on +# 10.66.0.1 — the tunnel address itself. A VPN client therefore needs no route +# beyond the management subnet it already has, and the request arrives at the +# application with its real 10.66.0.x source, which is what the console's own +# IP allowlist checks. +# +# The certificate is the one the public Caddy already obtains and renews for +# this hostname. A certificate is bound to the NAME, not to the address that +# serves it, so the same file is valid here. Shared read-only rather than +# duplicated: a second ACME client would be a second thing to renew, and a +# private address cannot satisfy an HTTP challenge in any case. +{ + admin off + auto_https off +} + +https://{$VPN_INTERNAL_HOST}:443 { + bind {$VPN_HUB_ADDRESS} + tls {$VPN_CERT_PATH} {$VPN_KEY_PATH} + + # To the application directly. Resolvable because this container shares the + # provisioning container's namespace, which is on the compose network. + reverse_proxy app:80 { + header_up X-Forwarded-For {remote_host} + header_up X-Forwarded-Proto https + header_up Host {host} + } +} diff --git a/tests/Feature/Admin/VpnTest.php b/tests/Feature/Admin/VpnTest.php index 2807661..69656de 100644 --- a/tests/Feature/Admin/VpnTest.php +++ b/tests/Feature/Admin/VpnTest.php @@ -895,3 +895,23 @@ it('does not route the WireGuard endpoint through its own tunnel', function () { expect($method->invoke(null, $endpoint))->toBe('10.66.0.0/24', $endpoint); } }); + +it('points a client at the tunnel resolver only when the console is reachable there', function () { + // Setting DNS otherwise hands the device a resolver that does not exist and + // takes its name resolution down with it for as long as the tunnel is up. + config()->set('provisioning.wireguard.hub_address', '10.66.0.1'); + + $build = new ReflectionMethod(App\Livewire\Admin\Vpn::class, 'clientConfig'); + $build->setAccessible(true); + $page = new App\Livewire\Admin\Vpn; + + $keypair = new App\Services\Wireguard\Keypair('priv', 'pub'); + + // Not ready: no resolver is running, so naming one would take the device's + // whole name resolution down for as long as the tunnel is up. + config()->set('admin_access.vpn_ready', false); + expect($build->invoke($page, $keypair, '10.66.0.9'))->not->toContain('DNS ='); + + config()->set('admin_access.vpn_ready', true); + expect($build->invoke($page, $keypair, '10.66.0.9'))->toContain('DNS = 10.66.0.1'); +});