builders: init profile based builder configuration

build/hydra.nix

@@ -148,6 +148,12 @@ in
 
   # These IPs and SSH public keys are specifically provisioned for Hydra
   services.openssh.knownHosts = {
+    # x86_64-linux at Hetzner
+    "elated-minsky.builder.nixos.org".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIvrJpd3aynfPVGGG/s7MtRFz/S6M4dtqvqKI3Da7O7+";
+    "sleepy-brown.builder.nixos.org".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOh4/3m7o6H3J5QG711aJdlSUVvlC8yW6KoqAES3Fy6I";
+    # aarch64-linux at Hetzner
+    "goofy-hopcroft.builder.nixos.org".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICTJEi+nQNd7hzNYN3cLBK/0JCkmwmyC1I+b5nMI7+dd";
+
     # M1 Macs in North America
     "*.foundation.detsys.dev" = {
       certAuthority = true;

build/pluto/prometheus/exporters/node.nix

@@ -42,6 +42,14 @@
               "eager-heisenberg.mac.nixos.org:9100"
             ];
           }
+          {
+            labels.role = "builders";
+            targets = [
+              "elated-minsky.builder.nixos.org:9100"
+              "sleepy-brown.builder.nixos.org:9100"
+              "goofy-hopcroft.builder.nixos.org:9100"
+            ];
+          }
         ];
       }
     ];

builders/boot/efi-grub.nix

@@ -0,0 +1,20 @@
+{
+  boot.loader = {
+    efi.canTouchEfiVariables = false;
+    grub = {
+      enable = true;
+      efiSupport = true;
+      efiInstallAsRemovable = true;
+      mirroredBoots = [
+        {
+          devices = [ "nodev" ];
+          path = "/efi/a";
+        }
+        {
+          devices = [ "nodev" ];
+          path = "/efi/b";
+        }
+      ];
+    };
+  };
+}

builders/common/hardening.nix

@@ -0,0 +1,11 @@
+{
+  # no priviledge escalation through sudo or polkit
+  security.sudo.execWheelOnly = true;
+  security.polkit.enable = false;
+
+  # no password authentication
+  services.openssh.settings = {
+    KbdInteractiveAuthentication = false;
+    PasswordAuthentication = false;
+  };
+}

builders/common/network.nix

@@ -0,0 +1,13 @@
+{
+  networking = {
+    domain = "builders.nixos.org";
+
+    firewall = {
+      # too spammy, rotates dmesg too quickly
+      logRefusedConnections = false;
+    };
+
+    # we use networkd instead
+    useDHCP = false;
+  };
+}

builders/common/nix.nix

@@ -0,0 +1,39 @@
+{
+  config,
+  ...
+}:
+
+{
+  nix = {
+    nrBuildUsers = config.nix.settings.max-jobs + 32;
+
+    gc =
+      let
+        maxFreed = 100; # GB
+      in
+      {
+        automatic = true;
+        dates = "*:0/30"; # every 30 minutes
+        options = "--max-freed \"$((${toString maxFreed} * 1024**3 - 1024 * $(df --output=avail /nix/store | tail -n 1)))\"";
+      };
+
+    settings = {
+      builders-use-substitutes = true;
+      extra-experimental-features = [
+        "cgroups"
+        "nix-command"
+        "no-url-literals"
+        "flakes"
+      ];
+      system-features = [
+        "kvm"
+        "nixos-test"
+      ];
+      trusted-users = [
+        "build"
+        "root"
+      ];
+      use-cgroups = true;
+    };
+  };
+}

builders/common/node-exporter.nix

@@ -0,0 +1,15 @@
+{
+  config,
+  ...
+}:
+
+{
+  networking.firewall.allowedTCPPorts = [
+    config.services.prometheus.exporters.node.port
+  ];
+
+  services.prometheus.exporters.node = {
+    enable = true;
+    enabledCollectors = [ "systemd" ];
+  };
+}

builders/common/ssh.nix

@@ -0,0 +1,11 @@
+{
+  lib,
+  ...
+}:
+
+{
+  services.openssh = {
+    enable = true;
+    authorizedKeysFiles = lib.mkForce [ "/etc/ssh/authorized_keys.d/%u" ];
+  };
+}

builders/common/system.nix

@@ -0,0 +1,20 @@
+{
+  pkgs,
+  ...
+}:
+
+{
+  # apply microcode to fix functional and security issues
+  hardware.enableRedistributableFirmware = true;
+  hardware.cpu.amd.updateMicrocode = pkgs.stdenv.isx86_64;
+  hardware.cpu.intel.updateMicrocode = pkgs.stdenv.isx86_64;
+
+  # enable kernel same-page merging for improved vm test performance
+  hardware.ksm.enable = true;
+
+  # discard blocks weekly
+  services.fstrim.enable = true;
+
+  # use memory more efficiently at the cost of some compute
+  zramSwap.enable = true;
+}

builders/common/tools.nix

@@ -0,0 +1,17 @@
+{
+  pkgs,
+  ...
+}:
+
+{
+  environment.systemPackages = with pkgs; [
+    atop
+    ethtool
+    htop
+    lm_sensors
+    nvme-cli
+    pciutils
+    smartmontools
+    usbutils
+  ];
+}

builders/common/update.nix

@@ -0,0 +1,8 @@
+{
+  system.autoUpgrade = {
+    enable = true;
+    dates = "daily";
+    flake = "git+https://github.com/nixos/infra.git?ref=master";
+    allowReboot = true;
+  };
+}

builders/common/users.nix

@@ -0,0 +1,37 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  sshKeys = {
+    hydra-queue-runner-rhea = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOdxl6gDS7h3oeBBja2RSBxeS51Kp44av8OAJPPJwuU/ hydra-queue-runner@rhea";
+  };
+
+  authorizedNixStoreKey =
+    key:
+    let
+      environment = lib.concatStringsSep " " [
+        "NIX_SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
+      ];
+    in
+    "command=\"${environment} ${config.nix.package}/bin/nix-store --serve --write\" ${key}";
+in
+
+{
+  users = {
+    mutableUsers = false;
+    users = {
+      build = {
+        isNormalUser = true;
+        uid = 2000;
+        openssh.authorizedKeys.keys = [
+          (authorizedNixStoreKey sshKeys.hydra-queue-runner-rhea)
+        ];
+      };
+
+      root.openssh.authorizedKeys.keys = (import ../../ssh-keys.nix).infra-core;
+    };
+  };
+}

builders/disk-layouts/efi-zfs-raid0.nix

@@ -0,0 +1,69 @@
+{
+  disk1 ? "/dev/nvme0n1",
+  disk2 ? "/dev/nvme1n1",
+}:
+let
+  mkDiskLayout = id: {
+    type = "gpt";
+    partitions = {
+      esp = {
+        type = "EF00";
+        size = "512M";
+        content = {
+          type = "filesystem";
+          format = "vfat";
+          mountpoint = "/efi/${id}";
+        };
+      };
+      zdev = {
+        size = "100%";
+        content = {
+          type = "zfs";
+          pool = "zroot";
+        };
+      };
+    };
+  };
+in
+{
+  disk = {
+    a = {
+      type = "disk";
+      device = disk1;
+      content = mkDiskLayout "a";
+    };
+
+    b = {
+      type = "disk";
+      device = disk2;
+      content = mkDiskLayout "b";
+    };
+  };
+
+  zpool.zroot = {
+    mode = ""; # RAID 0
+    options.ashift = "12"; # 4k blocks
+
+    rootFsOptions = {
+      acltype = "posixacl";
+      atime = "off";
+      compression = "on";
+      mountpoint = "none";
+      xattr = "sa";
+    };
+
+    datasets = {
+      root = {
+        type = "zfs_fs";
+        mountpoint = "/";
+      };
+      reserved = {
+        type = "zfs_fs";
+        options = {
+          canmount = "off";
+          refreservation = "16G"; # roughly one system closure
+        };
+      };
+    };
+  };
+}

builders/flake-module.nix

@@ -0,0 +1,37 @@
+{ inputs, ... }:
+{
+  flake.nixosConfigurations =
+    let
+      mkNixOS =
+        system: config:
+        inputs.nixpkgs.lib.nixosSystem {
+          inherit system;
+
+          modules = [
+            inputs.disko.nixosModules.disko
+
+            ./common/hardening.nix
+            ./common/network.nix
+            ./common/nix.nix
+            ./common/node-exporter.nix
+            ./common/system.nix
+            ./common/tools.nix
+            ./common/update.nix
+            ./common/users.nix
+            ./common/ssh.nix
+
+            ../modules/rasdaemon.nix
+
+            config
+          ];
+        };
+    in
+    {
+      # Epyc 9454P (48C/96T), 256 GB DDR4 RAM, 2x 1.92TB PCIe4 NVME
+      elated-minsky = mkNixOS "x86_64-linux" ./instances/elated-minsky.nix;
+      sleepy-brown= mkNixOS "x86_64-linux" ./instances/sleepy-brown.nix;
+
+      # Ampere Q80-30 (80C), 256 GB DDR4 RAM, 2x3.84TB PCIe4 NVME
+      goofy-hopcroft = mkNixOS "aarch64-linux" ./instances/goofy-hopcroft.nix;
+    };
+}

builders/instances/elated-minsky.nix

@@ -0,0 +1,40 @@
+{
+  imports = [
+    ../profiles/hetzner-ax101r.nix
+  ];
+
+  nix.settings = {
+    cores = 2;
+    max-jobs = 48;
+  };
+
+  networking = {
+    hostName = "elated-minsky";
+    domain = "builders.nixos.org";
+    useDHCP = false;
+  };
+
+  systemd.network = {
+    enable = true;
+    networks = {
+      "30-enp193s0f0np0" = {
+        matchConfig = {
+          MACAddress = "9c:6b:00:4e:1a:6a";
+          Type = "ether";
+        };
+        linkConfig.RequiredForOnline = true;
+        networkConfig.Description = "WAN";
+        address = [
+          "167.235.95.99/26"
+          "2a01:4f8:2220:1b03::1/64"
+        ];
+        routes = [
+          { Gateway = "167.235.95.65"; }
+          { Gateway = "fe80::1"; }
+        ];
+      };
+    };
+  };
+
+  system.stateVersion = "24.11";
+}