Move to AL2023

README.md

@@ -34,6 +34,9 @@ In addition, the following tools are recommended:
 
 ### AMI
 
+The Terraform modules in this repository require the use of AL2023 for the
+Linux version for all AMIs.
+
 The Terraform modules in this repository rely on the AMIs used for worker
 nodes to be optimized for XRd. The easiest way to achieve this is to use the
 [XRd Packer](https://github.com/ios-xr/xrd-packer) templates to generate

examples/ha/infra/data.tf

@@ -8,7 +8,7 @@ data "aws_ami" "eks_optimized" {
 
   filter {
     name   = "name"
-    values = ["amazon-eks-node-${data.aws_eks_cluster.this.version}-*"]
+    values = ["amazon-eks-node-al2023-x86_64-standard-${data.aws_eks_cluster.this.version}-*"]
   }
 
   filter {

examples/ha/workload/main.tf

@@ -95,7 +95,7 @@ module "peer" {
   source = "../../../modules/aws/linux-pod-with-net-attach"
 
   name       = "peer"
-  device     = "eth1"
+  device     = "ens6"
   ip_address = "10.0.10.12/24"
   gateway    = "10.0.10.20"
   routes     = ["10.0.11.0/24", "10.0.13.0/24"]
@@ -108,7 +108,7 @@ module "cnf_vrid1" {
   source = "../../../modules/aws/linux-pod-with-net-attach"
 
   name       = "cnf-vrid1"
-  device     = "eth2"
+  device     = "ens7"
   ip_address = "10.0.11.12/24"
   gateway    = "10.0.11.20"
   routes     = ["10.0.10.0/24"]
@@ -121,7 +121,7 @@ module "cnf_vrid2" {
   source = "../../../modules/aws/linux-pod-with-net-attach"
 
   name       = "cnf-vrid2"
-  device     = "eth3"
+  device     = "ens8"
   ip_address = "10.0.13.12/24"
   gateway    = "10.0.13.1"
   routes     = ["10.0.10.0/24"]

examples/overlay/infra/data.tf

@@ -4,7 +4,7 @@ data "aws_ami" "eks_optimized" {
 
   filter {
     name   = "name"
-    values = ["amazon-eks-node-${data.aws_eks_cluster.this.version}-*"]
+    values = ["amazon-eks-node-al2023-x86_64-standard-${data.aws_eks_cluster.this.version}-*"]
   }
 
   filter {

examples/overlay/workload/main.tf

@@ -76,7 +76,7 @@ module "cnf" {
   source = "../../../modules/aws/linux-pod-with-net-attach"
 
   name       = "cnf"
-  device     = "eth1"
+  device     = "ens6"
   ip_address = "10.0.10.10/24"
   gateway    = "10.0.10.11"
   routes     = ["10.0.13.0/24"]
@@ -89,7 +89,7 @@ module "peer" {
   source = "../../../modules/aws/linux-pod-with-net-attach"
 
   name       = "peer"
-  device     = "eth2"
+  device     = "ens7"
   ip_address = "10.0.13.10/24"
   gateway    = "10.0.13.12"
   routes     = ["10.0.10.0/24"]

modules/aws/node/data.tf

@@ -8,3 +8,7 @@ data "aws_ami" "this" {
 data "aws_ec2_instance_type" "this" {
   instance_type = var.instance_type
 }
+
+data "aws_eks_cluster" "this" {
+  name = var.cluster_name
+}

modules/aws/node/main.tf

@@ -156,20 +156,19 @@ resource "aws_instance" "this" {
   user_data = templatefile(
     "${path.module}/templates/user-data.tftpl",
     {
-      xrd_bootstrap  = local.is_xrd_ami
-      hugepages_gb   = local.hugepages_gb
-      isolated_cores = local.isolated_cores
-      cluster_name   = var.cluster_name
-      kubelet_extra_args = format(
-        "%s%s",
-        (
-          local.kubelet_node_labels_arg != null ?
-          "--node-labels ${local.kubelet_node_labels_arg}" :
-          ""
-        ),
-        var.kubelet_extra_args != null ? " ${var.kubelet_extra_args}" : "",
+      name                  = data.aws_eks_cluster.this.name
+      api_endpoint          = data.aws_eks_cluster.this.endpoint
+      certificate_authority = data.aws_eks_cluster.this.certificate_authority[0].data
+      cidr                  = data.aws_eks_cluster.this.kubernetes_network_config[0].service_ipv4_cidr
+      kubelet_flags = concat(
+        ["--node-labels=${local.kubelet_node_labels_arg}"],
+        var.kubelet_extra_args
       )
+      hugepages_gb         = local.hugepages_gb
+      isolated_cores       = local.isolated_cores
       additional_user_data = var.user_data
+      xrd_bootstrap        = local.is_xrd_ami
+      private_ip           = var.private_ip_address
     }
   )
 
@@ -194,6 +193,11 @@ resource "aws_instance" "this" {
 }
 
 resource "aws_network_interface" "this" {
+  # Wait for kubelet to start before attaching the network interfaces.
+  depends_on = [
+    kubernetes_job.wait
+  ]
+
   for_each = {
     for i, ni in var.network_interfaces :
     i => ni

modules/aws/node/templates/user-data.tftpl

@@ -1,7 +1,32 @@
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="BOUNDARY"
+
+--BOUNDARY
+Content-Type: application/node.eks.aws
+
+---
+apiVersion: node.eks.aws/v1alpha1
+kind: NodeConfig
+spec:
+  cluster:
+    name: ${name}
+    apiServerEndpoint: ${api_endpoint}
+    certificateAuthority: ${certificate_authority}
+    cidr: ${cidr}
+  kubelet:
+    flags:
+    %{~ for flag in kubelet_flags ~}
+    - ${flag}
+    %{~ endfor ~}
+
+--BOUNDARY
+Content-Type: text/x-shellscript; charset="us-ascii"
+
 #!/bin/bash
 %{~ if xrd_bootstrap }
 HUGEPAGES_GB=${hugepages_gb} ISOLATED_CORES=${isolated_cores} /etc/xrd/bootstrap.sh
 %{~ endif }
-/etc/eks/bootstrap.sh ${cluster_name} --kubelet-extra-args '${kubelet_extra_args}'
 ${additional_user_data}
 reboot
+
+--BOUNDARY--

modules/aws/node/variables.tf

@@ -95,11 +95,11 @@ variable "isolated_cores" {
 
 variable "kubelet_extra_args" {
   description = <<-EOT
-  Extra arguments to pass to kubelet when booting the node.
+  List of extra command-line kubelet arguments to be appended to the defaults.
   Note that node labels must be specified via the 'labels' variable.
   EOT
-  type        = string
-  default     = null
+  type        = list(string)
+  default     = []
 }
 
 variable "labels" {

tests/ut/terraform/node/variables.tf

@@ -101,11 +101,11 @@ variable "isolated_cores" {
 
 variable "kubelet_extra_args" {
   description = <<-EOT
-  Extra arguments to pass to kubelet when booting the node.
+  List of extra command-line kubelet arguments to be appended to the defaults.
   Note that node labels must be specified via the 'labels' variable.
   EOT
-  type        = string
-  default     = null
+  type        = list(string)
+  default     = []
 }
 
 variable "labels" {

tests/ut/test_node.py

@@ -1,6 +1,7 @@
 import base64
 import json
 import subprocess
+import textwrap
 import uuid
 from pathlib import Path
 from typing import Any
@@ -15,6 +16,7 @@
     Subnet,
     Vpc,
 )
+from mypy_boto3_eks import EKSClient
 from mypy_boto3_iam import IAMServiceResource
 from mypy_boto3_iam.service_resource import InstanceProfile
 
@@ -66,6 +68,14 @@ def subnet(vpc: Vpc) -> Subnet:
     )
 
 
+@pytest.fixture
+def other_subnet(vpc: Vpc) -> Subnet:
+    return vpc.create_subnet(
+        AvailabilityZone="eu-west-1a",
+        CidrBlock="10.0.1.0/24",
+    )
+
+
 @pytest.fixture
 def key_pair(ec2: EC2ServiceResource) -> KeyPair:
     return ec2.create_key_pair(KeyName=str(uuid.uuid4()))
@@ -92,6 +102,7 @@ def iam_instance_profile(iam: IAMServiceResource) -> InstanceProfile:
         "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy",
         "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly",
         "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy",
+        "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy",
     ):
         role.attach_policy(PolicyArn=policy_arn)
 
@@ -116,19 +127,42 @@ def security_group(ec2: EC2ServiceResource, vpc: Vpc) -> SecurityGroup:
     return sg
 
 
+@pytest.fixture()
+def eks_cluster(
+    eks_client: EKSClient,
+    iam_instance_profile: InstanceProfile,
+    subnet: Subnet,
+    other_subnet: Subnet,
+    security_group: SecurityGroup,
+) -> dict[str, Any]:
+    name = str(uuid.uuid4())
+    return eks_client.create_cluster(
+        name=name,
+        roleArn=iam_instance_profile.roles_attribute[0]["Arn"],
+        resourcesVpcConfig={
+            "subnetIds": [subnet.id, other_subnet.id],
+            "securityGroupIds": [security_group.id],
+            "endpointPublicAccess": False,
+            "endpointPrivateAccess": False,
+            "publicAccessCidrs": [],
+        },
+    )
+
+
 @pytest.fixture
 def base_vars(
     subnet: Subnet,
     key_pair: KeyPair,
     iam_instance_profile: InstanceProfile,
+    eks_cluster: dict[str, Any],
 ) -> dict[str, Any]:
     # This AMI should exist in the Moto server.
     # Refer to https://github.com/getmoto/moto/blob/master/moto/ec2/resources/amis.json.
     ami = "ami-03cf127a"
 
     return {
         "ami": ami,
-        "cluster_name": str(uuid.uuid4()),
+        "cluster_name": eks_cluster["cluster"]["name"],
         "name": str(uuid.uuid4()),
         "private_ip_address": "10.0.0.10",
         "security_groups": [],
@@ -162,7 +196,12 @@ def _assert_tag(instance: Instance, tag_key: str, tag_value: str) -> None:
         raise AssertionError(f"tag '{tag_key}' does not exist")
 
 
-def test_defaults(ec2, tf: Terraform, base_vars: dict[str, Any]):
+def test_defaults(
+    ec2,
+    tf: Terraform,
+    base_vars: dict[str, Any],
+    eks_cluster: dict[str, Any],
+):
     tf.apply(vars=base_vars)
     outputs = Outputs.from_terraform(tf)
     instance = ec2.Instance(outputs.id)
@@ -182,7 +221,23 @@ def test_defaults(ec2, tf: Terraform, base_vars: dict[str, Any]):
         "Value"
     ]
     user_data = base64.b64decode(user_data).decode()
-    assert f"/etc/eks/bootstrap.sh {base_vars['cluster_name']}" in user_data
+    expected_node_config = textwrap.dedent(
+        f"""\
+        ---
+        apiVersion: node.eks.aws/v1alpha1
+        kind: NodeConfig
+        spec:
+          cluster:
+            name: {base_vars['cluster_name']}
+            apiServerEndpoint: {eks_cluster['cluster']['endpoint']}
+            certificateAuthority: {eks_cluster['cluster']['certificateAuthority']['data']}
+            cidr: {eks_cluster['cluster']['kubernetesNetworkConfig']['serviceIpv4Cidr']}
+          kubelet:
+            flags:
+            - --node-labels=ios-xr.cisco.com/name={base_vars['name']}""",
+    )
+
+    assert expected_node_config in user_data
 
     # There should be no public IP address assigned.
     assert not instance.public_ip_address
@@ -250,7 +305,7 @@ def test_unknown_instance_type(ec2, tf: Terraform, base_vars: dict[str, Any]):
 
 
 def test_kubelet_extra_args(ec2, tf: Terraform, base_vars: dict[str, Any]):
-    vars = base_vars | {"kubelet_extra_args": "foo bar baz"}
+    vars = base_vars | {"kubelet_extra_args": ["foo", "bar"]}
     tf.apply(vars=vars)
     outputs = Outputs.from_terraform(tf)
     instance = ec2.Instance(outputs.id)
@@ -260,8 +315,10 @@ def test_kubelet_extra_args(ec2, tf: Terraform, base_vars: dict[str, Any]):
     ]
     user_data = base64.b64decode(user_data).decode()
     assert (
-        f"""--kubelet-extra-args '--node-labels ios-xr.cisco.com/name={base_vars["name"]} foo bar baz'"""
-        in user_data
+        "    flags:\n"
+        f"    - --node-labels=ios-xr.cisco.com/name={vars['name']}\n"
+        "    - foo\n"
+        "    - bar\n" in user_data
     )