fix builder stuff
This commit is contained in:
parent
90cff4f16a
commit
43998b73d7
5 changed files with 184 additions and 96 deletions
|
|
@ -10,17 +10,32 @@
|
|||
substituters = [
|
||||
"https://cache.nixos.org"
|
||||
"https://cache.axiomania.org/main"
|
||||
"https://axium.cachix.org"
|
||||
];
|
||||
trusted-public-keys = [
|
||||
"main:Uz5F0MbXItVx2XCmBbEAMmQ0T6+DZDgLaXWalh1k++o="
|
||||
"axium.cachix.org-1:BfzPfRTbbCYmaQrVLSWchgsR4ScA9ZCZ389FyWspUH8="
|
||||
];
|
||||
};
|
||||
|
||||
programs.tmux = {
|
||||
enable = true;
|
||||
extraConfig = "set -g mouse on";
|
||||
};
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
git
|
||||
tmux
|
||||
attic-client
|
||||
cachix
|
||||
vim
|
||||
htop
|
||||
curl
|
||||
jq
|
||||
file
|
||||
binutils # readelf, objdump, nm, strings, size
|
||||
patchelf
|
||||
zip
|
||||
unzip
|
||||
];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,34 +3,28 @@
|
|||
let
|
||||
sshKeyPath = "/home/lebowski/.ssh/id_ed25519";
|
||||
sshKeyName = "V3";
|
||||
serverName = "nix-builder";
|
||||
serverName = "builder";
|
||||
serverType = "ccx43";
|
||||
snapshotType = "ccx13";
|
||||
snapshotDesc = "nixos-builder";
|
||||
location = "nbg1";
|
||||
flakeDir = "/home/lebowski/NixOS";
|
||||
|
||||
sshCmd = "ssh -o StrictHostKeyChecking=no -o BatchMode=yes -i ${sshKeyPath}";
|
||||
|
||||
# Spin up a builder from the NixOS snapshot
|
||||
# Boot a builder from the NixOS snapshot
|
||||
builder-up = pkgs.writeShellApplication {
|
||||
name = "builder-up";
|
||||
runtimeInputs = with pkgs; [ hcloud openssh ];
|
||||
text = ''
|
||||
SERVER_NAME="${serverName}"
|
||||
SSH_KEY_PATH="${sshKeyPath}"
|
||||
|
||||
# ── Check if server already exists ──
|
||||
if hcloud server describe "$SERVER_NAME" &>/dev/null; then
|
||||
IP=$(hcloud server ip "$SERVER_NAME")
|
||||
echo "Server '$SERVER_NAME' already running at $IP"
|
||||
if hcloud --context builder server describe "${serverName}" &>/dev/null; then
|
||||
IP=$(hcloud --context builder server ip "${serverName}")
|
||||
echo "Builder already running at $IP"
|
||||
echo " builder-ssh"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Find snapshot ──
|
||||
SNAPSHOT_ID=$(hcloud image list --type snapshot --selector description="${snapshotDesc}" -o noheader -o columns=id 2>/dev/null | head -1)
|
||||
if [ -z "$SNAPSHOT_ID" ]; then
|
||||
# Fallback: search by description
|
||||
SNAPSHOT_ID=$(hcloud image list --type snapshot -o noheader -o columns=id,description | grep "${snapshotDesc}" | awk '{print $1}' | head -1)
|
||||
fi
|
||||
SNAPSHOT_ID=$(hcloud --context builder image list --type snapshot -o noheader -o columns=id,description | grep "${snapshotDesc}" | awk '{print $1}' | head -1)
|
||||
|
||||
if [ -z "$SNAPSHOT_ID" ]; then
|
||||
echo "ERROR: No snapshot '${snapshotDesc}' found."
|
||||
|
|
@ -38,19 +32,16 @@ let
|
|||
exit 1
|
||||
fi
|
||||
|
||||
# ── Create server from snapshot ──
|
||||
echo "==> Creating ${serverType} in ${location} from snapshot $SNAPSHOT_ID..."
|
||||
hcloud server create \
|
||||
--name "$SERVER_NAME" \
|
||||
echo "==> Creating ${serverType} in ${location}..."
|
||||
hcloud --context builder server create \
|
||||
--name "${serverName}" \
|
||||
--type "${serverType}" \
|
||||
--image "$SNAPSHOT_ID" \
|
||||
--location "${location}" \
|
||||
--label role=nix-builder \
|
||||
--ssh-key "${sshKeyName}"
|
||||
|
||||
IP=$(hcloud server ip "$SERVER_NAME")
|
||||
IP=$(hcloud --context builder server ip "${serverName}")
|
||||
|
||||
# ── Wait for SSH ──
|
||||
echo "==> Waiting for SSH..."
|
||||
for i in $(seq 1 30); do
|
||||
if ${sshCmd} "root@''${IP}" true 2>/dev/null; then
|
||||
|
|
@ -63,13 +54,13 @@ let
|
|||
sleep 5
|
||||
done
|
||||
|
||||
# ── Update known_hosts ──
|
||||
ssh-keygen -R "$IP" 2>/dev/null || true
|
||||
ssh-keyscan -H "$IP" >> ~/.ssh/known_hosts 2>/dev/null
|
||||
|
||||
echo ""
|
||||
echo "=== Builder ready ==="
|
||||
echo " builder-ssh"
|
||||
IP=$(hcloud --context builder server ip "${serverName}")
|
||||
ssh -i "${sshKeyPath}" -t "root@$IP" "tmux new-session -A -s build"
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
@ -78,16 +69,13 @@ let
|
|||
name = "builder-ssh";
|
||||
runtimeInputs = with pkgs; [ hcloud openssh ];
|
||||
text = ''
|
||||
SERVER_NAME="${serverName}"
|
||||
SSH_KEY_PATH="${sshKeyPath}"
|
||||
|
||||
if ! hcloud server describe "$SERVER_NAME" &>/dev/null; then
|
||||
echo "No server '$SERVER_NAME' found. Run builder-up first."
|
||||
if ! hcloud --context builder server describe "${serverName}" &>/dev/null; then
|
||||
echo "No builder found. Run builder-up first."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
IP=$(hcloud server ip "$SERVER_NAME")
|
||||
ssh -i "$SSH_KEY_PATH" -t "root@$IP" "tmux new-session -A -s build"
|
||||
IP=$(hcloud --context builder server ip "${serverName}")
|
||||
ssh -i "${sshKeyPath}" -t "root@$IP" "tmux new-session -A -s build"
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
@ -96,26 +84,135 @@ let
|
|||
name = "builder-down";
|
||||
runtimeInputs = with pkgs; [ hcloud openssh ];
|
||||
text = ''
|
||||
SERVER_NAME="${serverName}"
|
||||
|
||||
if hcloud server describe "$SERVER_NAME" &>/dev/null; then
|
||||
IP=$(hcloud server ip "$SERVER_NAME")
|
||||
echo "==> Deleting '$SERVER_NAME' ($IP)..."
|
||||
hcloud server delete "$SERVER_NAME"
|
||||
if hcloud --context builder server describe "${serverName}" &>/dev/null; then
|
||||
IP=$(hcloud --context builder server ip "${serverName}")
|
||||
echo "==> Deleting '${serverName}' ($IP)..."
|
||||
hcloud --context builder server delete "${serverName}"
|
||||
ssh-keygen -R "$IP" 2>/dev/null || true
|
||||
echo " Done."
|
||||
else
|
||||
echo "No server '$SERVER_NAME' found."
|
||||
echo "No builder found."
|
||||
fi
|
||||
'';
|
||||
};
|
||||
|
||||
# Create or update the NixOS builder snapshot
|
||||
# 1. Spins up a cheap server
|
||||
# 2. Installs NixOS via nixos-anywhere
|
||||
# 3. Rebuilds with latest config
|
||||
# 4. Sets up credentials (attic, git)
|
||||
# 5. Cleans up, snapshots, deletes server
|
||||
builder-snapshot = pkgs.writeShellApplication {
|
||||
name = "builder-snapshot";
|
||||
runtimeInputs = with pkgs; [ hcloud openssh nixos-anywhere ];
|
||||
text = ''
|
||||
TMP_SERVER="snapshot-tmp"
|
||||
|
||||
MISSING=""
|
||||
[ -z "''${ATTIC_TOKEN:-}" ] && MISSING="$MISSING ATTIC_TOKEN"
|
||||
[ -z "''${FORGEJO_TOKEN:-}" ] && MISSING="$MISSING FORGEJO_TOKEN"
|
||||
[ -z "''${CACHIX_TOKEN:-}" ] && MISSING="$MISSING CACHIX_TOKEN"
|
||||
|
||||
if [ -n "$MISSING" ]; then
|
||||
echo "ERROR: Missing environment variables:$MISSING"
|
||||
echo ""
|
||||
echo " export ATTIC_TOKEN=<attic-jwt-token>"
|
||||
echo " export FORGEJO_TOKEN=<forgejo-access-token>"
|
||||
echo " export CACHIX_TOKEN=<cachix-auth-token>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "==> Creating ${snapshotType} for snapshot..."
|
||||
hcloud --context builder server create \
|
||||
--name "$TMP_SERVER" \
|
||||
--type "${snapshotType}" \
|
||||
--image ubuntu-24.04 \
|
||||
--location "${location}" \
|
||||
--ssh-key "${sshKeyName}"
|
||||
|
||||
IP=$(hcloud --context builder server ip "$TMP_SERVER")
|
||||
|
||||
echo "==> Waiting for SSH..."
|
||||
for i in $(seq 1 30); do
|
||||
if ${sshCmd} "root@''${IP}" true 2>/dev/null; then
|
||||
break
|
||||
fi
|
||||
if [ "$i" -eq 30 ]; then
|
||||
echo "ERROR: SSH did not become available"
|
||||
hcloud --context builder server delete "$TMP_SERVER"
|
||||
exit 1
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
|
||||
ssh-keygen -R "$IP" 2>/dev/null || true
|
||||
ssh-keyscan -H "$IP" >> ~/.ssh/known_hosts 2>/dev/null
|
||||
|
||||
echo "==> Installing NixOS via nixos-anywhere..."
|
||||
nixos-anywhere --flake "${flakeDir}#builder" "root@$IP"
|
||||
|
||||
echo "==> Waiting for NixOS to boot..."
|
||||
sleep 15
|
||||
ssh-keygen -R "$IP" 2>/dev/null || true
|
||||
for i in $(seq 1 30); do
|
||||
if ${sshCmd} "root@''${IP}" true 2>/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
ssh-keyscan -H "$IP" >> ~/.ssh/known_hosts 2>/dev/null
|
||||
|
||||
echo "==> Rebuilding with latest config..."
|
||||
nixos-rebuild switch --flake "${flakeDir}#builder" --target-host "root@$IP"
|
||||
|
||||
echo "==> Setting up credentials..."
|
||||
${sshCmd} "root@''${IP}" "attic login axiomania https://cache.axiomania.org ''${ATTIC_TOKEN}"
|
||||
${sshCmd} "root@''${IP}" bash -s -- "''${FORGEJO_TOKEN}" <<'SETUP'
|
||||
TOKEN="$1"
|
||||
git config --global credential.https://git.axiomania.org.helper store
|
||||
echo "https://lebowski:$TOKEN@git.axiomania.org" > /root/.git-credentials
|
||||
chmod 600 /root/.git-credentials
|
||||
SETUP
|
||||
${sshCmd} "root@''${IP}" bash -s -- "''${CACHIX_TOKEN}" <<'CACHIX'
|
||||
echo "$1" | cachix authtoken --stdin
|
||||
CACHIX
|
||||
|
||||
echo "==> Garbage collecting..."
|
||||
${sshCmd} "root@''${IP}" "nix-collect-garbage -d && nix store optimise"
|
||||
|
||||
# ── Delete old snapshot(s) ──
|
||||
OLD=$(hcloud --context builder image list --type snapshot -o noheader -o columns=id,description | grep "${snapshotDesc}" | awk '{print $1}')
|
||||
if [ -n "$OLD" ]; then
|
||||
echo "==> Deleting old snapshot(s)..."
|
||||
echo "$OLD" | while read -r sid; do
|
||||
hcloud --context builder image delete "$sid"
|
||||
done
|
||||
fi
|
||||
|
||||
echo "==> Shutting down for clean snapshot..."
|
||||
hcloud --context builder server shutdown "$TMP_SERVER"
|
||||
sleep 10
|
||||
|
||||
echo "==> Creating snapshot..."
|
||||
hcloud --context builder server create-image --type snapshot --description "${snapshotDesc}" "$TMP_SERVER"
|
||||
|
||||
echo "==> Deleting temporary server..."
|
||||
hcloud --context builder server delete "$TMP_SERVER"
|
||||
ssh-keygen -R "$IP" 2>/dev/null || true
|
||||
|
||||
echo ""
|
||||
echo "=== Snapshot '${snapshotDesc}' created ==="
|
||||
echo " Use 'builder-up' to spin up a builder."
|
||||
'';
|
||||
};
|
||||
|
||||
in
|
||||
{
|
||||
environment.systemPackages = [
|
||||
builder-up
|
||||
builder-ssh
|
||||
builder-down
|
||||
builder-snapshot
|
||||
pkgs.hcloud
|
||||
];
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue