From 5dd992c05d9950fbe8f3fac98eea76c92047d8cf Mon Sep 17 00:00:00 2001
From: Martin Weinelt <hexa@darmstadt.ccc.de>
Date: Mon, 23 Dec 2024 00:00:58 +0100
Subject: [PATCH] builders: init profile based builder configuration

---
 build/hydra.nix                           |  6 ++
 build/pluto/prometheus/exporters/node.nix |  8 +++
 builders/boot/efi-grub.nix                | 20 +++++++
 builders/common/hardening.nix             | 11 ++++
 builders/common/network.nix               | 13 +++++
 builders/common/nix.nix                   | 39 +++++++++++++
 builders/common/node-exporter.nix         | 15 +++++
 builders/common/ssh.nix                   | 11 ++++
 builders/common/system.nix                | 20 +++++++
 builders/common/tools.nix                 | 17 ++++++
 builders/common/update.nix                |  8 +++
 builders/common/users.nix                 | 37 ++++++++++++
 builders/disk-layouts/efi-zfs-raid0.nix   | 69 +++++++++++++++++++++++
 builders/flake-module.nix                 | 37 ++++++++++++
 builders/instances/elated-minsky.nix      | 40 +++++++++++++
 builders/instances/goofy-hopcroft.nix     | 40 +++++++++++++
 builders/instances/sleepy-brown.nix       | 41 ++++++++++++++
 builders/network/autoconfig.nix           | 19 +++++++
 builders/profiles/hetzner-ax101r.nix      | 20 +++++++
 builders/profiles/hetzner-rx220.nix       | 20 +++++++
 flake.nix                                 |  1 +
 terraform/dns.tf                          | 32 +++++++++++
 22 files changed, 524 insertions(+)
 create mode 100644 builders/boot/efi-grub.nix
 create mode 100644 builders/common/hardening.nix
 create mode 100644 builders/common/network.nix
 create mode 100644 builders/common/nix.nix
 create mode 100644 builders/common/node-exporter.nix
 create mode 100644 builders/common/ssh.nix
 create mode 100644 builders/common/system.nix
 create mode 100644 builders/common/tools.nix
 create mode 100644 builders/common/update.nix
 create mode 100644 builders/common/users.nix
 create mode 100644 builders/disk-layouts/efi-zfs-raid0.nix
 create mode 100644 builders/flake-module.nix
 create mode 100644 builders/instances/elated-minsky.nix
 create mode 100644 builders/instances/goofy-hopcroft.nix
 create mode 100644 builders/instances/sleepy-brown.nix
 create mode 100644 builders/network/autoconfig.nix
 create mode 100644 builders/profiles/hetzner-ax101r.nix
 create mode 100644 builders/profiles/hetzner-rx220.nix

diff --git a/build/hydra.nix b/build/hydra.nix
index 5db141e7..281bea83 100644
--- a/build/hydra.nix
+++ b/build/hydra.nix
@@ -148,6 +148,12 @@ in
 
   # These IPs and SSH public keys are specifically provisioned for Hydra
   services.openssh.knownHosts = {
+    # x86_64-linux at Hetzner
+    "elated-minsky.builder.nixos.org".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIvrJpd3aynfPVGGG/s7MtRFz/S6M4dtqvqKI3Da7O7+";
+    "sleepy-brown.builder.nixos.org".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOh4/3m7o6H3J5QG711aJdlSUVvlC8yW6KoqAES3Fy6I";
+    # aarch64-linux at Hetzner
+    "goofy-hopcroft.builder.nixos.org".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICTJEi+nQNd7hzNYN3cLBK/0JCkmwmyC1I+b5nMI7+dd";
+
     # M1 Macs in North America
     "*.foundation.detsys.dev" = {
       certAuthority = true;
diff --git a/build/pluto/prometheus/exporters/node.nix b/build/pluto/prometheus/exporters/node.nix
index 2c66358f..4bb0717b 100644
--- a/build/pluto/prometheus/exporters/node.nix
+++ b/build/pluto/prometheus/exporters/node.nix
@@ -42,6 +42,14 @@
               "eager-heisenberg.mac.nixos.org:9100"
             ];
           }
+          {
+            labels.role = "builders";
+            targets = [
+              "elated-minsky.builder.nixos.org:9100"
+              "sleepy-brown.builder.nixos.org:9100"
+              "goofy-hopcroft.builder.nixos.org:9100"
+            ];
+          }
         ];
       }
     ];
diff --git a/builders/boot/efi-grub.nix b/builders/boot/efi-grub.nix
new file mode 100644
index 00000000..b7f830bb
--- /dev/null
+++ b/builders/boot/efi-grub.nix
@@ -0,0 +1,20 @@
+{
+  boot.loader = {
+    efi.canTouchEfiVariables = false;
+    grub = {
+      enable = true;
+      efiSupport = true;
+      efiInstallAsRemovable = true;
+      mirroredBoots = [
+        {
+          devices = [ "nodev" ];
+          path = "/efi/a";
+        }
+        {
+          devices = [ "nodev" ];
+          path = "/efi/b";
+        }
+      ];
+    };
+  };
+}
diff --git a/builders/common/hardening.nix b/builders/common/hardening.nix
new file mode 100644
index 00000000..30e7e753
--- /dev/null
+++ b/builders/common/hardening.nix
@@ -0,0 +1,11 @@
+{
+  # no priviledge escalation through sudo or polkit
+  security.sudo.execWheelOnly = true;
+  security.polkit.enable = false;
+
+  # no password authentication
+  services.openssh.settings = {
+    KbdInteractiveAuthentication = false;
+    PasswordAuthentication = false;
+  };
+}
diff --git a/builders/common/network.nix b/builders/common/network.nix
new file mode 100644
index 00000000..b6141c26
--- /dev/null
+++ b/builders/common/network.nix
@@ -0,0 +1,13 @@
+{
+  networking = {
+    domain = "builders.nixos.org";
+
+    firewall = {
+      # too spammy, rotates dmesg too quickly
+      logRefusedConnections = false;
+    };
+
+    # we use networkd instead
+    useDHCP = false;
+  };
+}
diff --git a/builders/common/nix.nix b/builders/common/nix.nix
new file mode 100644
index 00000000..5b5cc649
--- /dev/null
+++ b/builders/common/nix.nix
@@ -0,0 +1,39 @@
+{
+  config,
+  ...
+}:
+
+{
+  nix = {
+    nrBuildUsers = config.nix.settings.max-jobs + 32;
+
+    gc =
+      let
+        maxFreed = 100; # GB
+      in
+      {
+        automatic = true;
+        dates = "*:0/30"; # every 30 minutes
+        options = "--max-freed \"$((${toString maxFreed} * 1024**3 - 1024 * $(df --output=avail /nix/store | tail -n 1)))\"";
+      };
+
+    settings = {
+      builders-use-substitutes = true;
+      extra-experimental-features = [
+        "cgroups"
+        "nix-command"
+        "no-url-literals"
+        "flakes"
+      ];
+      system-features = [
+        "kvm"
+        "nixos-test"
+      ];
+      trusted-users = [
+        "build"
+        "root"
+      ];
+      use-cgroups = true;
+    };
+  };
+}
diff --git a/builders/common/node-exporter.nix b/builders/common/node-exporter.nix
new file mode 100644
index 00000000..5a02de6b
--- /dev/null
+++ b/builders/common/node-exporter.nix
@@ -0,0 +1,15 @@
+{
+  config,
+  ...
+}:
+
+{
+  networking.firewall.allowedTCPPorts = [
+    config.services.prometheus.exporters.node.port
+  ];
+
+  services.prometheus.exporters.node = {
+    enable = true;
+    enabledCollectors = [ "systemd" ];
+  };
+}
diff --git a/builders/common/ssh.nix b/builders/common/ssh.nix
new file mode 100644
index 00000000..669c6209
--- /dev/null
+++ b/builders/common/ssh.nix
@@ -0,0 +1,11 @@
+{
+  lib,
+  ...
+}:
+
+{
+  services.openssh = {
+    enable = true;
+    authorizedKeysFiles = lib.mkForce [ "/etc/ssh/authorized_keys.d/%u" ];
+  };
+}
diff --git a/builders/common/system.nix b/builders/common/system.nix
new file mode 100644
index 00000000..e112e9ae
--- /dev/null
+++ b/builders/common/system.nix
@@ -0,0 +1,20 @@
+{
+  pkgs,
+  ...
+}:
+
+{
+  # apply microcode to fix functional and security issues
+  hardware.enableRedistributableFirmware = true;
+  hardware.cpu.amd.updateMicrocode = pkgs.stdenv.isx86_64;
+  hardware.cpu.intel.updateMicrocode = pkgs.stdenv.isx86_64;
+
+  # enable kernel same-page merging for improved vm test performance
+  hardware.ksm.enable = true;
+
+  # discard blocks weekly
+  services.fstrim.enable = true;
+
+  # use memory more efficiently at the cost of some compute
+  zramSwap.enable = true;
+}
diff --git a/builders/common/tools.nix b/builders/common/tools.nix
new file mode 100644
index 00000000..0738c931
--- /dev/null
+++ b/builders/common/tools.nix
@@ -0,0 +1,17 @@
+{
+  pkgs,
+  ...
+}:
+
+{
+  environment.systemPackages = with pkgs; [
+    atop
+    ethtool
+    htop
+    lm_sensors
+    nvme-cli
+    pciutils
+    smartmontools
+    usbutils
+  ];
+}
diff --git a/builders/common/update.nix b/builders/common/update.nix
new file mode 100644
index 00000000..86963a46
--- /dev/null
+++ b/builders/common/update.nix
@@ -0,0 +1,8 @@
+{
+  system.autoUpgrade = {
+    enable = true;
+    dates = "daily";
+    flake = "git+https://github.com/nixos/infra.git?ref=master";
+    allowReboot = true;
+  };
+}
diff --git a/builders/common/users.nix b/builders/common/users.nix
new file mode 100644
index 00000000..45b4c1db
--- /dev/null
+++ b/builders/common/users.nix
@@ -0,0 +1,37 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  sshKeys = {
+    hydra-queue-runner-rhea = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOdxl6gDS7h3oeBBja2RSBxeS51Kp44av8OAJPPJwuU/ hydra-queue-runner@rhea";
+  };
+
+  authorizedNixStoreKey =
+    key:
+    let
+      environment = lib.concatStringsSep " " [
+        "NIX_SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
+      ];
+    in
+    "command=\"${environment} ${config.nix.package}/bin/nix-store --serve --write\" ${key}";
+in
+
+{
+  users = {
+    mutableUsers = false;
+    users = {
+      build = {
+        isNormalUser = true;
+        uid = 2000;
+        openssh.authorizedKeys.keys = [
+          (authorizedNixStoreKey sshKeys.hydra-queue-runner-rhea)
+        ];
+      };
+
+      root.openssh.authorizedKeys.keys = (import ../../ssh-keys.nix).infra-core;
+    };
+  };
+}
diff --git a/builders/disk-layouts/efi-zfs-raid0.nix b/builders/disk-layouts/efi-zfs-raid0.nix
new file mode 100644
index 00000000..6bf931d1
--- /dev/null
+++ b/builders/disk-layouts/efi-zfs-raid0.nix
@@ -0,0 +1,69 @@
+{
+  disk1 ? "/dev/nvme0n1",
+  disk2 ? "/dev/nvme1n1",
+}:
+let
+  mkDiskLayout = id: {
+    type = "gpt";
+    partitions = {
+      esp = {
+        type = "EF00";
+        size = "512M";
+        content = {
+          type = "filesystem";
+          format = "vfat";
+          mountpoint = "/efi/${id}";
+        };
+      };
+      zdev = {
+        size = "100%";
+        content = {
+          type = "zfs";
+          pool = "zroot";
+        };
+      };
+    };
+  };
+in
+{
+  disk = {
+    a = {
+      type = "disk";
+      device = disk1;
+      content = mkDiskLayout "a";
+    };
+
+    b = {
+      type = "disk";
+      device = disk2;
+      content = mkDiskLayout "b";
+    };
+  };
+
+  zpool.zroot = {
+    mode = ""; # RAID 0
+    options.ashift = "12"; # 4k blocks
+
+    rootFsOptions = {
+      acltype = "posixacl";
+      atime = "off";
+      compression = "on";
+      mountpoint = "none";
+      xattr = "sa";
+    };
+
+    datasets = {
+      root = {
+        type = "zfs_fs";
+        mountpoint = "/";
+      };
+      reserved = {
+        type = "zfs_fs";
+        options = {
+          canmount = "off";
+          refreservation = "16G"; # roughly one system closure
+        };
+      };
+    };
+  };
+}
diff --git a/builders/flake-module.nix b/builders/flake-module.nix
new file mode 100644
index 00000000..b887897e
--- /dev/null
+++ b/builders/flake-module.nix
@@ -0,0 +1,37 @@
+{ inputs, ... }:
+{
+  flake.nixosConfigurations =
+    let
+      mkNixOS =
+        system: config:
+        inputs.nixpkgs.lib.nixosSystem {
+          inherit system;
+
+          modules = [
+            inputs.disko.nixosModules.disko
+
+            ./common/hardening.nix
+            ./common/network.nix
+            ./common/nix.nix
+            ./common/node-exporter.nix
+            ./common/system.nix
+            ./common/tools.nix
+            ./common/update.nix
+            ./common/users.nix
+            ./common/ssh.nix
+
+            ../modules/rasdaemon.nix
+
+            config
+          ];
+        };
+    in
+    {
+      # Epyc 9454P (48C/96T), 256 GB DDR4 RAM, 2x 1.92TB PCIe4 NVME
+      elated-minsky = mkNixOS "x86_64-linux" ./instances/elated-minsky.nix;
+      sleepy-brown= mkNixOS "x86_64-linux" ./instances/sleepy-brown.nix;
+
+      # Ampere Q80-30 (80C), 256 GB DDR4 RAM, 2x3.84TB PCIe4 NVME
+      goofy-hopcroft = mkNixOS "aarch64-linux" ./instances/goofy-hopcroft.nix;
+    };
+}
diff --git a/builders/instances/elated-minsky.nix b/builders/instances/elated-minsky.nix
new file mode 100644
index 00000000..a4bb0828
--- /dev/null
+++ b/builders/instances/elated-minsky.nix
@@ -0,0 +1,40 @@
+{
+  imports = [
+    ../profiles/hetzner-ax101r.nix
+  ];
+
+  nix.settings = {
+    cores = 2;
+    max-jobs = 48;
+  };
+
+  networking = {
+    hostName = "elated-minsky";
+    domain = "builders.nixos.org";
+    useDHCP = false;
+  };
+
+  systemd.network = {
+    enable = true;
+    networks = {
+      "30-enp193s0f0np0" = {
+        matchConfig = {
+          MACAddress = "9c:6b:00:4e:1a:6a";
+          Type = "ether";
+        };
+        linkConfig.RequiredForOnline = true;
+        networkConfig.Description = "WAN";
+        address = [
+          "167.235.95.99/26"
+          "2a01:4f8:2220:1b03::1/64"
+        ];
+        routes = [
+          { Gateway = "167.235.95.65"; }
+          { Gateway = "fe80::1"; }
+        ];
+      };
+    };
+  };
+
+  system.stateVersion = "24.11";
+}
diff --git a/builders/instances/goofy-hopcroft.nix b/builders/instances/goofy-hopcroft.nix
new file mode 100644
index 00000000..d2b57953
--- /dev/null
+++ b/builders/instances/goofy-hopcroft.nix
@@ -0,0 +1,40 @@
+{
+  imports = [
+    ../profiles/hetzner-rx220.nix
+  ];
+
+  nix.settings = {
+    cores = 2;
+    max-jobs = 40;
+  };
+
+  networking = {
+    hostName = "goofy-hopcroft";
+    domain = "builders.nixos.org";
+    useDHCP = false;
+  };
+
+  systemd.network = {
+    enable = true;
+    networks = {
+      "30-enP3p2s0f0" = {
+        matchConfig = {
+          MACAddress = "74:56:3c:8c:01:a9";
+          Type = "ether";
+        };
+        linkConfig.RequiredForOnline = true;
+        networkConfig.Description = "WAN";
+        address = [
+          "135.181.225.104/26"
+          "2a01:4f9:3071:2d8b::1/64"
+        ];
+        routes = [
+          { Gateway = "135.181.225.65"; }
+          { Gateway = "fe80::1"; }
+        ];
+      };
+    };
+  };
+
+  system.stateVersion = "24.11";
+}
diff --git a/builders/instances/sleepy-brown.nix b/builders/instances/sleepy-brown.nix
new file mode 100644
index 00000000..d4255b94
--- /dev/null
+++ b/builders/instances/sleepy-brown.nix
@@ -0,0 +1,41 @@
+{
+  imports = [
+    ../profiles/hetzner-ax101r.nix
+  ];
+
+  nix.settings = {
+    cores = 24;
+    max-jobs = 4;
+    system-features = [ "big-parallel" ];
+  };
+
+  networking = {
+    hostName = "sleepy-brown";
+    domain = "builders.nixos.org";
+    useDHCP = false;
+  };
+
+  systemd.network = {
+    enable = true;
+    networks = {
+      "30-enp193s0f0np0" = {
+        matchConfig = {
+          MACAddress = "9c:6b:00:4e:fd:2d";
+          Type = "ether";
+        };
+        linkConfig.RequiredForOnline = true;
+        networkConfig.Description = "WAN";
+        address = [
+          "162.55.130.51/26"
+          "2a01:4f8:271:5c14::1/64"
+        ];
+        routes = [
+          { Gateway = "162.55.130.1"; }
+          { Gateway = "fe80::1"; }
+        ];
+      };
+    };
+  };
+
+  system.stateVersion = "24.11";
+}
diff --git a/builders/network/autoconfig.nix b/builders/network/autoconfig.nix
new file mode 100644
index 00000000..57920420
--- /dev/null
+++ b/builders/network/autoconfig.nix
@@ -0,0 +1,19 @@
+{
+  networking.useDHCP = false;
+
+  systemd.network = {
+    enable = true;
+    networks = {
+      "99-autoconfig" = {
+        matchConfig = {
+          Kind = "!*";
+          Type = "ether";
+        };
+        networkConfig = {
+          DHCP = "yes";
+          IPv6AcceptRA = true;
+        };
+      };
+    };
+  };
+}
diff --git a/builders/profiles/hetzner-ax101r.nix b/builders/profiles/hetzner-ax101r.nix
new file mode 100644
index 00000000..19610a67
--- /dev/null
+++ b/builders/profiles/hetzner-ax101r.nix
@@ -0,0 +1,20 @@
+{
+  imports = [
+    ../boot/efi-grub.nix
+  ];
+
+  disko.devices = import ../disk-layouts/efi-zfs-raid0.nix { };
+  boot.supportedFilesystems.zfs = true;
+  networking.hostId = "91312b0a";
+
+  # 96G for build roots, 160G for working memory
+  boot.tmp = {
+    useTmpfs = true;
+    tmpfsSize = "96G";
+  };
+
+  boot.initrd.availableKernelModules = [
+    "nvme"
+    "usbhid"
+  ];
+}
diff --git a/builders/profiles/hetzner-rx220.nix b/builders/profiles/hetzner-rx220.nix
new file mode 100644
index 00000000..19610a67
--- /dev/null
+++ b/builders/profiles/hetzner-rx220.nix
@@ -0,0 +1,20 @@
+{
+  imports = [
+    ../boot/efi-grub.nix
+  ];
+
+  disko.devices = import ../disk-layouts/efi-zfs-raid0.nix { };
+  boot.supportedFilesystems.zfs = true;
+  networking.hostId = "91312b0a";
+
+  # 96G for build roots, 160G for working memory
+  boot.tmp = {
+    useTmpfs = true;
+    tmpfsSize = "96G";
+  };
+
+  boot.initrd.availableKernelModules = [
+    "nvme"
+    "usbhid"
+  ];
+}
diff --git a/flake.nix b/flake.nix
index 77e69b11..18d3b00e 100644
--- a/flake.nix
+++ b/flake.nix
@@ -54,6 +54,7 @@
         "aarch64-darwin"
       ];
       imports = [
+        ./builders/flake-module.nix
         ./formatter/flake-module.nix
         ./checks/flake-module.nix
         ./terraform/flake-module.nix
diff --git a/terraform/dns.tf b/terraform/dns.tf
index e7563cfd..514494ac 100644
--- a/terraform/dns.tf
+++ b/terraform/dns.tf
@@ -206,6 +206,38 @@ locals {
       type     = "TXT"
       value    = "v=spf1 include:spf.improvmx.com include:_mailcust.gandi.net ~all"
     },
+    {
+      # hetzner ax162-r 2548595
+      hostname = "elated-minsky.builder.nixos.org"
+      type     = "A"
+      value    = "167.235.95.99"
+    },
+    {
+      hostname = "elated-minsky.builder.nixos.org"
+      type     = "AAAA"
+      value    = "2a01:4f8:2220:1b03::1"
+    },
+    {
+      # hetzner ax162-r 2566166
+      hostname = "sleepy-brown.builder.nixos.org"
+      type     = "A"
+      value    = "162.55.130.51"
+    },
+    {
+      hostname = "sleepy-brown.builder.nixos.org"
+      type     = "AAAA"
+      value    = "2a01:4f8:271:5c14::1"
+    },
+    {
+      hostname = "goofy-hopcroft.builder.nixos.org"
+      type     = "A"
+      value    = "135.181.225.104"
+    },
+    {
+      hostname = "goofy-hopcroft.builder.nixos.org"
+      type     = "AAAA"
+      value    = "2a01:4f9:3071:2d8b::1"
+    },
     {
       # hetzner m1 1638981
       hostname = "intense-heron.mac.nixos.org"