fix builder stuff

Hetzner/builder.nix

@@ -10,17 +10,32 @@
     substituters = [
       "https://cache.nixos.org"
       "https://cache.axiomania.org/main"
+      "https://axium.cachix.org"
     ];
     trusted-public-keys = [
       "main:Uz5F0MbXItVx2XCmBbEAMmQ0T6+DZDgLaXWalh1k++o="
+      "axium.cachix.org-1:BfzPfRTbbCYmaQrVLSWchgsR4ScA9ZCZ389FyWspUH8="
     ];
   };
 
+  programs.tmux = {
+    enable = true;
+    extraConfig = "set -g mouse on";
+  };
+
   environment.systemPackages = with pkgs; [
     git
     tmux
     attic-client
+    cachix
     vim
     htop
+    curl
+    jq
+    file
+    binutils  # readelf, objdump, nm, strings, size
+    patchelf
+    zip
+    unzip
   ];
 }

Hetzner/setup.nix

@@ -3,34 +3,28 @@
 let
   sshKeyPath = "/home/lebowski/.ssh/id_ed25519";
   sshKeyName = "V3";
-  serverName = "nix-builder";
+  serverName = "builder";
   serverType = "ccx43";
+  snapshotType = "ccx13";
+  snapshotDesc = "nixos-builder";
   location = "nbg1";
+  flakeDir = "/home/lebowski/NixOS";
 
   sshCmd = "ssh -o StrictHostKeyChecking=no -o BatchMode=yes -i ${sshKeyPath}";
 
-  # Spin up a builder from the NixOS snapshot
+  # Boot a builder from the NixOS snapshot
   builder-up = pkgs.writeShellApplication {
     name = "builder-up";
     runtimeInputs = with pkgs; [ hcloud openssh ];
     text = ''
-      SERVER_NAME="${serverName}"
+      if hcloud --context builder server describe "${serverName}" &>/dev/null; then
-      SSH_KEY_PATH="${sshKeyPath}"
+        IP=$(hcloud --context builder server ip "${serverName}")
-
+        echo "Builder already running at $IP"
-      # ── Check if server already exists ──
-      if hcloud server describe "$SERVER_NAME" &>/dev/null; then
-        IP=$(hcloud server ip "$SERVER_NAME")
-        echo "Server '$SERVER_NAME' already running at $IP"
         echo "  builder-ssh"
         exit 0
       fi
 
-      # ── Find snapshot ──
+      SNAPSHOT_ID=$(hcloud --context builder image list --type snapshot -o noheader -o columns=id,description | grep "${snapshotDesc}" | awk '{print $1}' | head -1)
-      SNAPSHOT_ID=$(hcloud image list --type snapshot --selector description="${snapshotDesc}" -o noheader -o columns=id 2>/dev/null | head -1)
-      if [ -z "$SNAPSHOT_ID" ]; then
-        # Fallback: search by description
-        SNAPSHOT_ID=$(hcloud image list --type snapshot -o noheader -o columns=id,description | grep "${snapshotDesc}" | awk '{print $1}' | head -1)
-      fi
 
       if [ -z "$SNAPSHOT_ID" ]; then
         echo "ERROR: No snapshot '${snapshotDesc}' found."
@@ -38,19 +32,16 @@ let
         exit 1
       fi
 
-      # ── Create server from snapshot ──
+      echo "==> Creating ${serverType} in ${location}..."
-      echo "==> Creating ${serverType} in ${location} from snapshot $SNAPSHOT_ID..."
+      hcloud --context builder server create \
-      hcloud server create \
+        --name "${serverName}" \
-        --name "$SERVER_NAME" \
         --type "${serverType}" \
         --image "$SNAPSHOT_ID" \
         --location "${location}" \
-        --label role=nix-builder \
         --ssh-key "${sshKeyName}"
 
-      IP=$(hcloud server ip "$SERVER_NAME")
+      IP=$(hcloud --context builder server ip "${serverName}")
 
-      # ── Wait for SSH ──
       echo "==> Waiting for SSH..."
       for i in $(seq 1 30); do
         if ${sshCmd} "root@''${IP}" true 2>/dev/null; then
@@ -63,13 +54,13 @@ let
         sleep 5
       done
 
-      # ── Update known_hosts ──
       ssh-keygen -R "$IP" 2>/dev/null || true
       ssh-keyscan -H "$IP" >> ~/.ssh/known_hosts 2>/dev/null
 
       echo ""
       echo "=== Builder ready ==="
-      echo "  builder-ssh"
+      IP=$(hcloud --context builder server ip "${serverName}")
+      ssh -i "${sshKeyPath}" -t "root@$IP" "tmux new-session -A -s build"
     '';
   };
 
@@ -78,16 +69,13 @@ let
     name = "builder-ssh";
     runtimeInputs = with pkgs; [ hcloud openssh ];
     text = ''
-      SERVER_NAME="${serverName}"
+      if ! hcloud --context builder server describe "${serverName}" &>/dev/null; then
-      SSH_KEY_PATH="${sshKeyPath}"
+        echo "No builder found. Run builder-up first."
-
-      if ! hcloud server describe "$SERVER_NAME" &>/dev/null; then
-        echo "No server '$SERVER_NAME' found. Run builder-up first."
         exit 1
       fi
 
-      IP=$(hcloud server ip "$SERVER_NAME")
+      IP=$(hcloud --context builder server ip "${serverName}")
-      ssh -i "$SSH_KEY_PATH" -t "root@$IP" "tmux new-session -A -s build"
+      ssh -i "${sshKeyPath}" -t "root@$IP" "tmux new-session -A -s build"
     '';
   };
 
@@ -96,26 +84,135 @@ let
     name = "builder-down";
     runtimeInputs = with pkgs; [ hcloud openssh ];
     text = ''
-      SERVER_NAME="${serverName}"
+      if hcloud --context builder server describe "${serverName}" &>/dev/null; then
-
+        IP=$(hcloud --context builder server ip "${serverName}")
-      if hcloud server describe "$SERVER_NAME" &>/dev/null; then
+        echo "==> Deleting '${serverName}' ($IP)..."
-        IP=$(hcloud server ip "$SERVER_NAME")
+        hcloud --context builder server delete "${serverName}"
-        echo "==> Deleting '$SERVER_NAME' ($IP)..."
-        hcloud server delete "$SERVER_NAME"
         ssh-keygen -R "$IP" 2>/dev/null || true
         echo "    Done."
       else
-        echo "No server '$SERVER_NAME' found."
+        echo "No builder found."
       fi
     '';
   };
 
+  # Create or update the NixOS builder snapshot
+  # 1. Spins up a cheap server
+  # 2. Installs NixOS via nixos-anywhere
+  # 3. Rebuilds with latest config
+  # 4. Sets up credentials (attic, git)
+  # 5. Cleans up, snapshots, deletes server
+  builder-snapshot = pkgs.writeShellApplication {
+    name = "builder-snapshot";
+    runtimeInputs = with pkgs; [ hcloud openssh nixos-anywhere ];
+    text = ''
+      TMP_SERVER="snapshot-tmp"
+
+      MISSING=""
+      [ -z "''${ATTIC_TOKEN:-}" ] && MISSING="$MISSING ATTIC_TOKEN"
+      [ -z "''${FORGEJO_TOKEN:-}" ] && MISSING="$MISSING FORGEJO_TOKEN"
+      [ -z "''${CACHIX_TOKEN:-}" ] && MISSING="$MISSING CACHIX_TOKEN"
+
+      if [ -n "$MISSING" ]; then
+        echo "ERROR: Missing environment variables:$MISSING"
+        echo ""
+        echo "  export ATTIC_TOKEN=<attic-jwt-token>"
+        echo "  export FORGEJO_TOKEN=<forgejo-access-token>"
+        echo "  export CACHIX_TOKEN=<cachix-auth-token>"
+        exit 1
+      fi
+
+      echo "==> Creating ${snapshotType} for snapshot..."
+      hcloud --context builder server create \
+        --name "$TMP_SERVER" \
+        --type "${snapshotType}" \
+        --image ubuntu-24.04 \
+        --location "${location}" \
+        --ssh-key "${sshKeyName}"
+
+      IP=$(hcloud --context builder server ip "$TMP_SERVER")
+
+      echo "==> Waiting for SSH..."
+      for i in $(seq 1 30); do
+        if ${sshCmd} "root@''${IP}" true 2>/dev/null; then
+          break
+        fi
+        if [ "$i" -eq 30 ]; then
+          echo "ERROR: SSH did not become available"
+          hcloud --context builder server delete "$TMP_SERVER"
+          exit 1
+        fi
+        sleep 5
+      done
+
+      ssh-keygen -R "$IP" 2>/dev/null || true
+      ssh-keyscan -H "$IP" >> ~/.ssh/known_hosts 2>/dev/null
+
+      echo "==> Installing NixOS via nixos-anywhere..."
+      nixos-anywhere --flake "${flakeDir}#builder" "root@$IP"
+
+      echo "==> Waiting for NixOS to boot..."
+      sleep 15
+      ssh-keygen -R "$IP" 2>/dev/null || true
+      for i in $(seq 1 30); do
+        if ${sshCmd} "root@''${IP}" true 2>/dev/null; then
+          break
+        fi
+        sleep 5
+      done
+      ssh-keyscan -H "$IP" >> ~/.ssh/known_hosts 2>/dev/null
+
+      echo "==> Rebuilding with latest config..."
+      nixos-rebuild switch --flake "${flakeDir}#builder" --target-host "root@$IP"
+
+      echo "==> Setting up credentials..."
+      ${sshCmd} "root@''${IP}" "attic login axiomania https://cache.axiomania.org ''${ATTIC_TOKEN}"
+      ${sshCmd} "root@''${IP}" bash -s -- "''${FORGEJO_TOKEN}" <<'SETUP'
+        TOKEN="$1"
+        git config --global credential.https://git.axiomania.org.helper store
+        echo "https://lebowski:$TOKEN@git.axiomania.org" > /root/.git-credentials
+        chmod 600 /root/.git-credentials
+SETUP
+      ${sshCmd} "root@''${IP}" bash -s -- "''${CACHIX_TOKEN}" <<'CACHIX'
+        echo "$1" | cachix authtoken --stdin
+CACHIX
+
+      echo "==> Garbage collecting..."
+      ${sshCmd} "root@''${IP}" "nix-collect-garbage -d && nix store optimise"
+
+      # ── Delete old snapshot(s) ──
+      OLD=$(hcloud --context builder image list --type snapshot -o noheader -o columns=id,description | grep "${snapshotDesc}" | awk '{print $1}')
+      if [ -n "$OLD" ]; then
+        echo "==> Deleting old snapshot(s)..."
+        echo "$OLD" | while read -r sid; do
+          hcloud --context builder image delete "$sid"
+        done
+      fi
+
+      echo "==> Shutting down for clean snapshot..."
+      hcloud --context builder server shutdown "$TMP_SERVER"
+      sleep 10
+
+      echo "==> Creating snapshot..."
+      hcloud --context builder server create-image --type snapshot --description "${snapshotDesc}" "$TMP_SERVER"
+
+      echo "==> Deleting temporary server..."
+      hcloud --context builder server delete "$TMP_SERVER"
+      ssh-keygen -R "$IP" 2>/dev/null || true
+
+      echo ""
+      echo "=== Snapshot '${snapshotDesc}' created ==="
+      echo "  Use 'builder-up' to spin up a builder."
+    '';
+  };
+
 in
 {
   environment.systemPackages = [
     builder-up
     builder-ssh
     builder-down
+    builder-snapshot
     pkgs.hcloud
   ];
 }

configuration.nix

@@ -11,7 +11,7 @@
     ./Suckless/suckless.nix
     ./ryzenadj.nix
     # ./virtualisation.nix
-    # ./Hetzner/setup.nix
+    ./Hetzner/setup.nix
   ];
 
   nix.settings = {
@@ -51,6 +51,19 @@
 
   networking.hostName = "v3";
 
+  # Resolve VPN-only services to the WireGuard IP so traffic
+  # goes through the tunnel and nginx sees a 10.100.0.x source
+  networking.hosts."10.100.0.1" = [
+    "vault.axiomania.org"
+    "music.axiomania.org"
+    "sync.axiomania.org"
+    "books.axiomania.org"
+    "audio.axiomania.org"
+    "docs.axiomania.org"
+    "search.axiomania.org"
+    "home.axiomania.org"
+  ];
+
   networking.networkmanager.enable = true;
   hardware.bluetooth.enable = true;
 
@@ -147,13 +160,13 @@
 
   # Profile 1: Internal services only (split tunnel)
   networking.wg-quick.interfaces.wg-services = {
-    autostart = false;
+    autostart = true;
     address = [ "10.100.0.2/24" ];
     privateKeyFile = "/etc/wireguard/private.key";
     peers = [{
       publicKey = "F2hvz4vx9VrM6IZ2zMUG2FMPMCMwmfxGH9qocbe4q3U=";
       endpoint = "178.104.15.221:51820";
-      allowedIPs = [ "10.100.0.0/24" "178.104.15.221/32" ];
+      allowedIPs = [ "10.100.0.0/24" ];
       persistentKeepalive = 25;
     }];
   };

git.nix

@@ -14,35 +14,9 @@
       init.defaultBranch = "main";
       pull.rebase = true;
       push.autoSetupRemote = true;
-      # Use SSH for GitHub
       url."ssh://git@github.com/".insteadOf = "https://github.com/";
+      url."ssh://forgejo@git.axiomania.org:2222/".insteadOf = "https://git.axiomania.org/";
     };
-
-    # Delta - better diff viewer
-    # delta = {
-    #   enable = true;
-    #   options = {
-    #     navigate = true;
-    #     light = false;
-    #     line-numbers = true;
-    #   };
-    # };
-
-    # Aliases
-    # aliases = {
-    #   co = "checkout";
-    #   br = "branch";
-    #   ci = "commit";
-    #   st = "status";
-    #   lg = "log --oneline --graph --decorate";
-    # };
-
-    # SSH signing (for verified commits)
-    # signing = {
-    #   key = "~/.ssh/id_ed25519.pub";
-    #   signByDefault = true;
-    # };
-    # extraConfig.gpg.format = "ssh";
   };
 
   # ============================================================================
@@ -50,19 +24,29 @@
   # ============================================================================
   programs.ssh = {
     enable = true;
+    package = pkgs-stable.openssh;
     enableDefaultConfig = false;
     matchBlocks = {
+      "*" = {
+        addKeysToAgent = "yes";
+        serverAliveInterval = 60;
+        controlMaster = "auto";
+        controlPersist = "10m";
+      };
       "github.com" = {
-        host = "github.com";
         identityFile = "~/.ssh/id_ed25519";
         identitiesOnly = true;
       };
-      # Add more hosts as needed
+      "git.axiomania.org" = {
-      # "gitlab.com" = {
+        port = 2222;
-      #   host = "gitlab.com";
+        identityFile = "~/.ssh/id_ed25519";
-      #   identityFile = "~/.ssh/id_ed25519";
+        identitiesOnly = true;
-      #   identitiesOnly = true;
+      };
-      # };
+      "vps" = {
+        hostname = "178.104.15.221";
+        user = "root";
+        identityFile = "~/.ssh/id_ed25519";
+      };
     };
   };
 }

home.nix

@@ -109,28 +109,7 @@ in
     package = pkgs-stable.foliate;
   };
 
-  # --- SSH ---
-  programs.ssh = {
-    enable = true;
-    package = pkgs-stable.openssh;
 
-    matchBlocks = {
-      "*" = {
-        addKeysToAgent = "yes";
-        serverAliveInterval = 60;
-        controlMaster = "auto";
-        controlPersist = "10m";
-      };
-      "git.axiomania.org" = {
-        port = 2222;
-      };
-      "vps" = {
-        hostname = "178.104.15.221";
-        user = "root";
-        identityFile = "~/.ssh/id_ed25519";
-      };
-    };
-  };
 
   # ============================================================================
   # Disable version check - we intentionally use unstable home-manager with stable nixpkgs