init

scripts/deploy.sh

@@ -0,0 +1,69 @@
+#!/bin/sh
+# Deploy a service to the local VMs.
+#
+# Usage:
+#   scripts/deploy.sh <service>                        # deploy service
+#   scripts/deploy.sh <service> --skip docker,apache   # skip dependencies
+#   scripts/deploy.sh <service> --restore              # restore "provisioned" first
+#   scripts/deploy.sh <service> --restore initialized  # restore specific snapshot first
+#
+# Examples:
+#   scripts/deploy.sh bugzilla
+#   scripts/deploy.sh bugzilla --skip docker,apache
+#   scripts/deploy.sh prosody --restore
+#   scripts/deploy.sh authentik --restore initialized
+set -eu
+. "$(dirname "$0")/env.sh"
+
+service=""
+skip_tags=""
+restore_snapshot=""
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --skip)
+            skip_tags="$2"
+            shift 2
+            ;;
+        --restore)
+            restore_snapshot="${2:-provisioned}"
+            if [ $# -ge 2 ] && case "$2" in -*) false;; *) true;; esac; then
+                shift 2
+            else
+                shift
+            fi
+            ;;
+        -*)
+            echo "Unknown option: $1" >&2
+            exit 1
+            ;;
+        *)
+            if [ -z "$service" ]; then
+                service="$1"
+            else
+                echo "Unexpected argument: $1" >&2
+                exit 1
+            fi
+            shift
+            ;;
+    esac
+done
+
+if [ -z "$service" ]; then
+    echo "Usage: deploy.sh <service> [--skip tags] [--restore [snapshot]]" >&2
+    exit 1
+fi
+
+if [ -n "$restore_snapshot" ]; then
+    echo "==> Restoring to '$restore_snapshot'"
+    "$SCRIPT_DIR/vm/restore.sh" "$restore_snapshot"
+fi
+
+echo "==> Deploying: $service"
+if [ -n "$skip_tags" ]; then
+    ansible-playbook -i "$INVENTORY" "$PLAYBOOK" --vault-password-file "$VAULT_PASS" \
+        --tags "$service" --skip-tags "$skip_tags"
+else
+    ansible-playbook -i "$INVENTORY" "$PLAYBOOK" --vault-password-file "$VAULT_PASS" \
+        --tags "$service"
+fi

scripts/env.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+# Shared environment for top-level workflow scripts.
+# Source this — don't execute directly.
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+INVENTORY=ansible/inventories/debug/hosts.ini
+PLAYBOOK=ansible/playbooks/setup.yml
+VAULT_PASS=.vault-pass-debug
+
+export QEMU_VM="$(command -v qemu-vm)"

scripts/local/setup-hosts.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+# Add /etc/hosts entries for VM service domains.  Requires root.
+set -eu
+
+# Domains on vm-proxy (WG: 10.0.0.1)
+PROXY_DOMAINS="
+tiararodney.com
+chat.tiararodney.com
+comments.tiararodney.com
+bugs.code.tiararodney.com
+dockerhub.oci.code.tiararodney.com
+ghcr.oci.code.tiararodney.com
+crates.code.tiararodney.com
+pypi.code.tiararodney.com
+"
+
+# Domains on vm-idp (WG: 10.0.0.2)
+IDP_DOMAINS="
+accounts.tiararodney.com
+"
+
+for domain in $PROXY_DOMAINS; do
+    grep -q "$domain" /etc/hosts 2>/dev/null && continue
+    echo "10.0.0.1 $domain" >> /etc/hosts
+    echo "    added $domain -> 10.0.0.1"
+done
+
+for domain in $IDP_DOMAINS; do
+    grep -q "$domain" /etc/hosts 2>/dev/null && continue
+    echo "10.0.0.2 $domain" >> /etc/hosts
+    echo "    added $domain -> 10.0.0.2"
+done

scripts/local/setup-wireguard.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+# Configure local WireGuard interface to peer with the VMs.  Requires root.
+#
+# Expects /etc/wireguard/private.key and /etc/wireguard/public.key to exist.
+# Generate with:
+#   wg genkey | tee /etc/wireguard/private.key | wg pubkey > /etc/wireguard/public.key
+set -eu
+
+SSH_USER=debian
+SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR"
+PROXY_HOST=10.10.0.2
+IDP_HOST=10.10.0.3
+
+WG_IFACE=wg-dev
+LOCAL_WG_IP=10.0.0.3/24
+WG_PORT=51820
+
+# Get the VM public keys
+PROXY_PUBKEY=$(ssh $SSH_OPTS "${SSH_USER}@${PROXY_HOST}" 'sudo cat /etc/wireguard/public.key' 2>/dev/null)
+IDP_PUBKEY=$(ssh $SSH_OPTS "${SSH_USER}@${IDP_HOST}" 'sudo cat /etc/wireguard/public.key' 2>/dev/null)
+
+if [ -z "$PROXY_PUBKEY" ] || [ -z "$IDP_PUBKEY" ]; then
+    echo "Failed to retrieve VM WireGuard public keys." >&2
+    echo "Has the wireguard role been deployed?" >&2
+    exit 1
+fi
+
+cat > /etc/wireguard/${WG_IFACE}.conf <<EOF
+[Interface]
+Address = ${LOCAL_WG_IP}
+PrivateKey = $(cat /etc/wireguard/private.key)
+
+[Peer]
+# vm-proxy
+PublicKey = ${PROXY_PUBKEY}
+Endpoint = ${PROXY_HOST}:${WG_PORT}
+AllowedIPs = 10.0.0.1/32
+
+[Peer]
+# vm-idp
+PublicKey = ${IDP_PUBKEY}
+Endpoint = ${IDP_HOST}:${WG_PORT}
+AllowedIPs = 10.0.0.2/32
+EOF
+
+wg-quick down ${WG_IFACE} 2>/dev/null || true
+wg-quick up ${WG_IFACE}
+
+echo "==> WireGuard ${WG_IFACE} up (${LOCAL_WG_IP})"

scripts/local/teardown-hosts.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+# Remove VM service domain entries from /etc/hosts.  Requires root.
+set -eu
+
+DOMAINS="
+tiararodney.com
+chat.tiararodney.com
+comments.tiararodney.com
+bugs.code.tiararodney.com
+dockerhub.oci.code.tiararodney.com
+ghcr.oci.code.tiararodney.com
+crates.code.tiararodney.com
+pypi.code.tiararodney.com
+accounts.tiararodney.com
+"
+
+for domain in $DOMAINS; do
+    sed -i "/[[:space:]]${domain}$/d" /etc/hosts
+done
+
+echo "==> /etc/hosts entries removed"

scripts/local/teardown-wireguard.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+# Remove local WireGuard interface.  Requires root.
+set -eu
+
+WG_IFACE=wg-dev
+
+wg-quick down ${WG_IFACE} 2>/dev/null || true
+rm -f /etc/wireguard/${WG_IFACE}.conf
+
+echo "==> WireGuard ${WG_IFACE} removed"

scripts/prod-deploy.sh

@@ -0,0 +1,45 @@
+#!/bin/sh
+# Deploy to production.
+#
+# Usage:
+#   scripts/prod-deploy.sh                  # full deployment
+#   scripts/prod-deploy.sh <service>        # targeted deployment
+#   scripts/prod-deploy.sh <service> --skip docker,apache
+set -eu
+
+INVENTORY=ansible/inventories/prod/hosts.ini
+PLAYBOOK=ansible/playbooks/setup.yml
+
+service=""
+skip_tags=""
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --skip)
+            skip_tags="$2"
+            shift 2
+            ;;
+        -*)
+            echo "Unknown option: $1" >&2
+            exit 1
+            ;;
+        *)
+            service="$1"
+            shift
+            ;;
+    esac
+done
+
+if [ -n "$service" ]; then
+    echo "==> Deploying to production: $service"
+    if [ -n "$skip_tags" ]; then
+        exec ansible-playbook -i "$INVENTORY" "$PLAYBOOK" \
+            --tags "$service" --skip-tags "$skip_tags"
+    else
+        exec ansible-playbook -i "$INVENTORY" "$PLAYBOOK" \
+            --tags "$service"
+    fi
+else
+    echo "==> Full production deployment"
+    exec ansible-playbook -i "$INVENTORY" "$PLAYBOOK"
+fi

scripts/provision.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+# Full first-time provisioning: set up networking, create VMs, run the
+# complete playbook, then snapshot as "provisioned" for fast iteration.
+set -eu
+. "$(dirname "$0")/env.sh"
+
+echo "==> Setting up bridge network"
+mkdir -p .local/qemu
+sudo --preserve-env=QEMU_VM "$SCRIPT_DIR/vm/setup-network.sh"
+
+echo "==> Creating VMs"
+"$SCRIPT_DIR/vm/create.sh"
+
+echo "==> Starting VMs"
+"$SCRIPT_DIR/vm/start.sh"
+
+echo "==> Running full playbook"
+ansible-playbook -i "$INVENTORY" "$PLAYBOOK" --vault-password-file "$VAULT_PASS"
+
+echo "==> Setting up local WireGuard"
+sudo "$SCRIPT_DIR/local/setup-wireguard.sh"
+
+echo "==> Setting up local /etc/hosts"
+sudo "$SCRIPT_DIR/local/setup-hosts.sh"
+
+echo "==> Snapshotting 'provisioned'"
+"$SCRIPT_DIR/vm/snapshot.sh" provisioned
+
+echo "==> Done. Iterate with: scripts/deploy.sh <service>"

scripts/reproduce-containerd-referrers.sh

@@ -0,0 +1,98 @@
+#!/bin/sh
+# Reproduce: containerd-snapshotter pull-through mirror referrers failure
+#
+# Prerequisites: Docker Engine installed, root access
+# This script:
+#   1. Starts a pull-through registry mirror for Docker Hub
+#   2. Enables containerd-snapshotter
+#   3. Configures Docker to use the mirror via hosts.toml
+#   4. Pulls an image — fails with "failed to decode referrers index"
+#
+# Run as root.
+set -eu
+
+MIRROR_PORT=5555
+MIRROR_NAME=registry-mirror-test
+
+cleanup() {
+    echo "==> Cleaning up"
+    docker rm -f "$MIRROR_NAME" 2>/dev/null || true
+    rm -rf /etc/docker/certs.d/docker.io
+    rm -f /tmp/mirror-config.yml
+
+    # Restore daemon.json
+    if [ -f /etc/docker/daemon.json.bak ]; then
+        mv /etc/docker/daemon.json.bak /etc/docker/daemon.json
+    else
+        rm -f /etc/docker/daemon.json
+    fi
+    systemctl restart docker 2>/dev/null || true
+}
+trap cleanup EXIT
+
+echo "==> Step 1: Start pull-through registry mirror"
+cat > /tmp/mirror-config.yml <<EOF
+version: 0.1
+log:
+  fields:
+    service: registry
+storage:
+  filesystem:
+    rootdirectory: /var/lib/registry
+  delete:
+    enabled: true
+http:
+  addr: :5000
+proxy:
+  remoteurl: https://registry-1.docker.io
+EOF
+
+docker rm -f "$MIRROR_NAME" 2>/dev/null || true
+docker run -d \
+    --name "$MIRROR_NAME" \
+    -p "${MIRROR_PORT}:5000" \
+    -v /tmp/mirror-config.yml:/etc/docker/registry/config.yml:ro \
+    registry:2
+
+echo "    Waiting for mirror to start..."
+sleep 3
+
+echo "==> Step 2: Enable containerd-snapshotter"
+if [ -f /etc/docker/daemon.json ]; then
+    cp /etc/docker/daemon.json /etc/docker/daemon.json.bak
+fi
+cat > /etc/docker/daemon.json <<EOF
+{
+  "features": {
+    "containerd-snapshotter": true
+  }
+}
+EOF
+
+echo "==> Step 3: Configure mirror via hosts.toml"
+mkdir -p /etc/docker/certs.d/docker.io
+cat > /etc/docker/certs.d/docker.io/hosts.toml <<EOF
+server = "https://docker.io"
+
+[host."http://127.0.0.1:${MIRROR_PORT}"]
+  capabilities = ["pull", "resolve"]
+EOF
+
+echo "==> Step 4: Restart Docker"
+systemctl restart docker
+sleep 3
+
+echo "==> Step 5: Remove cached image and pull through mirror"
+docker image rm alpine:latest 2>/dev/null || true
+echo "    Pulling alpine:latest..."
+if docker pull alpine:latest; then
+    echo "    PASS: Pull succeeded (issue may be fixed)"
+else
+    echo "    FAIL: Pull failed with referrers error (issue reproduced)"
+fi
+
+echo
+echo "==> Docker version:"
+docker version --format '{{.Server.Version}}'
+echo "==> Storage driver:"
+docker info --format '{{.Driver}}'

scripts/reset.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+# Restore VMs to a snapshot.
+#
+# Usage:
+#   scripts/reset.sh                # restore to "initialized"
+#   scripts/reset.sh provisioned    # restore to "provisioned"
+set -eu
+. "$(dirname "$0")/env.sh"
+
+label="${1:-initialized}"
+
+exec "$SCRIPT_DIR/vm/restore.sh" "$label"

scripts/resume.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+# Resume VMs — start them back up.
+set -eu
+. "$(dirname "$0")/env.sh"
+
+exec "$SCRIPT_DIR/vm/start.sh"

scripts/suspend.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+# Suspend VMs — stop them gracefully.
+set -eu
+. "$(dirname "$0")/env.sh"
+
+exec "$SCRIPT_DIR/vm/stop.sh"

scripts/teardown.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+# Full teardown: destroy VMs and clean up local machine config.
+set -eu
+. "$(dirname "$0")/env.sh"
+
+echo "==> Destroying VMs"
+"$SCRIPT_DIR/vm/destroy.sh"
+
+echo "==> Tearing down local WireGuard"
+sudo "$SCRIPT_DIR/local/teardown-wireguard.sh"
+
+echo "==> Removing local /etc/hosts entries"
+sudo "$SCRIPT_DIR/local/teardown-hosts.sh"
+
+echo "==> Done. Run scripts/provision.sh to start over."

scripts/vm/create.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+# First-time setup: download image, create volumes, provision VMs,
+# wait for cloud-init, then snapshot as "initialized".
+set -eu
+. "$(dirname "$0")/env.sh"
+
+echo "==> Downloading cloud image"
+"$QEMU_VM" image download
+
+echo "==> Creating volumes"
+"$QEMU_VM" volume create "$PROXY_VM" --size 20G --backing debian-12
+"$QEMU_VM" volume create "$IDP_VM" --size 20G --backing debian-12
+
+echo "==> Preparing TAP interfaces"
+sudo --preserve-env=QEMU_VM "$(dirname "$0")/setup-taps.sh"
+
+echo "==> Clearing stale SSH host keys"
+clear_host_keys
+
+echo "==> Creating instances (in screen sessions)"
+screen -dmS "$PROXY_VM" \
+    "$QEMU_VM" instance create "$PROXY_VM" 0 \
+        --network "$NETWORK" --ip "$PROXY_IP" --headless
+
+screen -dmS "$IDP_VM" \
+    "$QEMU_VM" instance create "$IDP_VM" 0 \
+        --network "$NETWORK" --ip "$IDP_IP" --headless
+
+echo "==> Waiting for cloud-init"
+wait_ssh "$PROXY_HOST"
+# shellcheck disable=SC2086
+ssh $SSH_OPTS "${SSH_USER}@${PROXY_HOST}" 'cloud-init status --wait'
+echo "    $PROXY_VM ready"
+
+wait_ssh "$IDP_HOST"
+# shellcheck disable=SC2086
+ssh $SSH_OPTS "${SSH_USER}@${IDP_HOST}" 'cloud-init status --wait'
+echo "    $IDP_VM ready"
+
+echo "==> Stopping for initial snapshot"
+"$QEMU_VM" instance stop "$PROXY_VM"
+"$QEMU_VM" instance stop "$IDP_VM"
+
+echo "==> Snapshotting 'initialized'"
+"$QEMU_VM" volume snapshot "$PROXY_VM" initialized
+"$QEMU_VM" volume snapshot "$IDP_VM" initialized
+
+echo "==> Done. Run: scripts/vm/start.sh"

scripts/vm/destroy.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+# Destroy everything: stop VMs, delete volumes.
+# Use scripts/vm/create.sh to start over.
+set -eu
+. "$(dirname "$0")/env.sh"
+
+"$(dirname "$0")/stop.sh"
+
+echo "==> Deleting volumes"
+"$QEMU_VM" volume delete "$PROXY_VM" 2>/dev/null || true
+"$QEMU_VM" volume delete "$IDP_VM" 2>/dev/null || true
+
+echo "==> Destroyed. Run scripts/vm/create.sh to recreate."

scripts/vm/env.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+# Shared environment for all VM scripts.
+# Source this — don't execute directly.
+
+# Caller must set QEMU_VM, or we fall back to PATH lookup.
+QEMU_VM="${QEMU_VM:-qemu-vm}"
+
+BRIDGE=qemu-br0
+GATEWAY=10.10.0.1/24
+SUBNET=10.10.0.0/24
+NETWORK=lab
+
+PROXY_VM=vm-proxy
+PROXY_IP=10.10.0.2/24
+
+IDP_VM=vm-idp
+IDP_IP=10.10.0.3/24
+
+PROXY_HOST="${PROXY_IP%%/*}"
+IDP_HOST="${IDP_IP%%/*}"
+
+SSH_USER=debian
+SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR"
+
+clear_host_keys() {
+    ssh-keygen -R "$PROXY_HOST" 2>/dev/null || true
+    ssh-keygen -R "$IDP_HOST" 2>/dev/null || true
+}
+
+learn_host_keys() {
+    ssh-keyscan -H "$PROXY_HOST" >> ~/.ssh/known_hosts 2>/dev/null
+    ssh-keyscan -H "$IDP_HOST" >> ~/.ssh/known_hosts 2>/dev/null
+}
+
+wait_ssh() {
+    _host=$1 _i=0
+    while [ "$_i" -lt 60 ]; do
+        # shellcheck disable=SC2086
+        if ssh $SSH_OPTS "${SSH_USER}@${_host}" true 2>/dev/null; then
+            return 0
+        fi
+        _i=$((_i + 1))
+        sleep 1
+    done
+    echo "SSH timeout: ${_host}" >&2
+    return 1
+}

scripts/vm/restore.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+# Restore both VM volumes to a snapshot and start them.
+#
+# Usage: scripts/vm/restore.sh <label>
+#   e.g. scripts/vm/restore.sh initialized
+set -eu
+. "$(dirname "$0")/env.sh"
+
+label="${1:?Usage: restore.sh <label>}"
+
+"$(dirname "$0")/stop.sh"
+
+echo "==> Restoring '$label'"
+"$QEMU_VM" volume restore "$PROXY_VM" "$label"
+"$QEMU_VM" volume restore "$IDP_VM" "$label"
+
+echo "==> Clearing stale SSH host keys"
+clear_host_keys
+
+exec "$(dirname "$0")/start.sh"

scripts/vm/setup-network.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+# Create and activate the bridge network.  Requires root.
+set -eu
+. "$(dirname "$0")/env.sh"
+
+"$QEMU_VM" network create "$NETWORK" \
+    --type bridge \
+    --bridge "$BRIDGE" \
+    --gateway "$GATEWAY" \
+    --subnet "$SUBNET" 2>/dev/null || true
+
+exec "$QEMU_VM" network setup "$NETWORK"

scripts/vm/setup-taps.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+# Pre-create TAP interfaces for both VMs.  Requires root.
+set -eu
+. "$(dirname "$0")/env.sh"
+
+"$QEMU_VM" network attach "$NETWORK" "$PROXY_VM"
+"$QEMU_VM" network attach "$NETWORK" "$IDP_VM"

scripts/vm/snapshot.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+# Snapshot both VM volumes.  Stops VMs first.
+#
+# Usage: scripts/vm/snapshot.sh <label>
+set -eu
+. "$(dirname "$0")/env.sh"
+
+label="${1:?Usage: snapshot.sh <label>}"
+
+"$(dirname "$0")/stop.sh"
+
+echo "==> Snapshotting '$label'"
+"$QEMU_VM" volume snapshot "$PROXY_VM" "$label"
+"$QEMU_VM" volume snapshot "$IDP_VM" "$label"
+
+echo "==> Done. Restart with: scripts/vm/start.sh"

scripts/vm/start.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+# Start VMs in detached GNU screen sessions.
+# Attach with: screen -r vm-proxy / screen -r vm-idp
+set -eu
+. "$(dirname "$0")/env.sh"
+
+echo "==> Ensuring bridge network"
+sudo --preserve-env=QEMU_VM "$(dirname "$0")/setup-network.sh"
+
+echo "==> Preparing TAP interfaces"
+sudo --preserve-env=QEMU_VM "$(dirname "$0")/setup-taps.sh"
+
+echo "==> Starting $PROXY_VM"
+screen -dmS "$PROXY_VM" "$QEMU_VM" instance start "$PROXY_VM"
+
+echo "==> Starting $IDP_VM"
+screen -dmS "$IDP_VM" "$QEMU_VM" instance start "$IDP_VM"
+
+echo "==> Waiting for SSH"
+wait_ssh "$PROXY_HOST" && echo "    $PROXY_VM ready"
+wait_ssh "$IDP_HOST" && echo "    $IDP_VM ready"
+
+echo "==> Learning host keys"
+learn_host_keys
+
+echo "==> Attach with: screen -r vm-proxy / screen -r vm-idp"

scripts/vm/status.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+# Show VM and volume status, plus screen sessions.
+set -eu
+. "$(dirname "$0")/env.sh"
+
+echo "==> Instances"
+"$QEMU_VM" instance list
+echo
+echo "==> Volumes"
+"$QEMU_VM" volume list
+echo
+echo "==> Screen sessions"
+screen -ls 2>/dev/null | grep -E 'vm-(proxy|idp)' || echo "    (none)"

scripts/vm/stop.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+# Stop both VMs gracefully.
+set -eu
+. "$(dirname "$0")/env.sh"
+
+"$QEMU_VM" instance stop "$PROXY_VM" || true
+"$QEMU_VM" instance stop "$IDP_VM" || true