From 58b16742630f31c28ad01c7c8a2a2f51c96c2636 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Tue, 3 Dec 2024 04:44:04 -0800
Subject: [PATCH 01/37] .

---
 examples/ha/infra/data.tf                  |  2 +-
 modules/aws/node/data.tf                   |  4 ++++
 modules/aws/node/main.tf                   | 18 ++++--------------
 modules/aws/node/templates/user-data.tftpl | 16 +++++++++-------
 modules/aws/node/variables.tf              |  1 +
 5 files changed, 19 insertions(+), 22 deletions(-)

diff --git a/examples/ha/infra/data.tf b/examples/ha/infra/data.tf
index c4386d6..5943a00 100644
--- a/examples/ha/infra/data.tf
+++ b/examples/ha/infra/data.tf
@@ -8,7 +8,7 @@ data "aws_ami" "eks_optimized" {
 
   filter {
     name   = "name"
-    values = ["amazon-eks-node-${data.aws_eks_cluster.this.version}-*"]
+    values = ["amazon-eks-node-al2023-x86_64-standard-${data.aws_eks_cluster.this.version}-*"]
   }
 
   filter {
diff --git a/modules/aws/node/data.tf b/modules/aws/node/data.tf
index c250064..9f656b3 100644
--- a/modules/aws/node/data.tf
+++ b/modules/aws/node/data.tf
@@ -8,3 +8,7 @@ data "aws_ami" "this" {
 data "aws_ec2_instance_type" "this" {
   instance_type = var.instance_type
 }
+
+data "aws_eks_cluster" "this" {
+  name = var.cluster_name
+}
\ No newline at end of file
diff --git a/modules/aws/node/main.tf b/modules/aws/node/main.tf
index 9d00b79..75593df 100644
--- a/modules/aws/node/main.tf
+++ b/modules/aws/node/main.tf
@@ -156,20 +156,10 @@ resource "aws_instance" "this" {
   user_data = templatefile(
     "${path.module}/templates/user-data.tftpl",
     {
-      xrd_bootstrap  = local.is_xrd_ami
-      hugepages_gb   = local.hugepages_gb
-      isolated_cores = local.isolated_cores
-      cluster_name   = var.cluster_name
-      kubelet_extra_args = format(
-        "%s%s",
-        (
-          local.kubelet_node_labels_arg != null ?
-          "--node-labels ${local.kubelet_node_labels_arg}" :
-          ""
-        ),
-        var.kubelet_extra_args != null ? " ${var.kubelet_extra_args}" : "",
-      )
-      additional_user_data = var.user_data
+      name = data.aws_eks_cluster.this.name
+      api_endpoint = data.aws_eks_cluster.this.endpoint
+      certificate_authority = data.aws_eks_cluster.this.certificate_authority[0].data
+      cidr = data.aws_eks_cluster.this.vpc_config[0].public_access_cidr
     }
   )
 
diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index 102168d..bed1bc0 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -1,7 +1,9 @@
-#!/bin/bash
-%{~ if xrd_bootstrap }
-HUGEPAGES_GB=${hugepages_gb} ISOLATED_CORES=${isolated_cores} /etc/xrd/bootstrap.sh
-%{~ endif }
-/etc/eks/bootstrap.sh ${cluster_name} --kubelet-extra-args '${kubelet_extra_args}'
-${additional_user_data}
-reboot
+---
+apiVersion: node.eks.aws/v1alpha1
+kind: NodeConfig
+spec:
+  cluster:
+    name: ${name}
+    apiServerEndpoint: ${api_endpoint}
+    certificateAuthority: ${certificate_authority}
+    cidr: ${cidr}
\ No newline at end of file
diff --git a/modules/aws/node/variables.tf b/modules/aws/node/variables.tf
index 01c1149..4ba4992 100644
--- a/modules/aws/node/variables.tf
+++ b/modules/aws/node/variables.tf
@@ -164,3 +164,4 @@ variable "xrd_vr_cp_num_cpus" {
   type        = number
   default     = null
 }
+=
\ No newline at end of file

From e7fc25166199d3b651271774513172106e4035ca Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Tue, 3 Dec 2024 04:45:52 -0800
Subject: [PATCH 02/37] .

---
 modules/aws/node/variables.tf | 1 -
 1 file changed, 1 deletion(-)

diff --git a/modules/aws/node/variables.tf b/modules/aws/node/variables.tf
index 4ba4992..01c1149 100644
--- a/modules/aws/node/variables.tf
+++ b/modules/aws/node/variables.tf
@@ -164,4 +164,3 @@ variable "xrd_vr_cp_num_cpus" {
   type        = number
   default     = null
 }
-=
\ No newline at end of file

From d2fbbc9485e77126985e6a80584aafd7f570ead4 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Tue, 3 Dec 2024 04:48:12 -0800
Subject: [PATCH 03/37] .

---
 modules/aws/node/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/aws/node/main.tf b/modules/aws/node/main.tf
index 75593df..d7702db 100644
--- a/modules/aws/node/main.tf
+++ b/modules/aws/node/main.tf
@@ -159,7 +159,7 @@ resource "aws_instance" "this" {
       name = data.aws_eks_cluster.this.name
       api_endpoint = data.aws_eks_cluster.this.endpoint
       certificate_authority = data.aws_eks_cluster.this.certificate_authority[0].data
-      cidr = data.aws_eks_cluster.this.vpc_config[0].public_access_cidr
+      cidr = data.aws_eks_cluster.this.vpc_config[0].public_access_cidrs[0]
     }
   )
 

From 4715e191810b82d3cf265ff7879dd5cd374cc467 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Tue, 3 Dec 2024 04:49:38 -0800
Subject: [PATCH 04/37] .

---
 modules/aws/node/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/aws/node/main.tf b/modules/aws/node/main.tf
index d7702db..c0cf2e7 100644
--- a/modules/aws/node/main.tf
+++ b/modules/aws/node/main.tf
@@ -159,7 +159,7 @@ resource "aws_instance" "this" {
       name = data.aws_eks_cluster.this.name
       api_endpoint = data.aws_eks_cluster.this.endpoint
       certificate_authority = data.aws_eks_cluster.this.certificate_authority[0].data
-      cidr = data.aws_eks_cluster.this.vpc_config[0].public_access_cidrs[0]
+      cidr = data.aws_eks_cluster.this.kubernetes_network_config[0].service_ipv4_cidr
     }
   )
 

From a169e73c3b768fb4b7af31fa33b2497f33fc3c1e Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Tue, 3 Dec 2024 07:43:46 -0800
Subject: [PATCH 05/37] .

---
 modules/aws/node/main.tf                   | 3 +++
 modules/aws/node/templates/user-data.tftpl | 9 ++++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/modules/aws/node/main.tf b/modules/aws/node/main.tf
index c0cf2e7..cc92d9a 100644
--- a/modules/aws/node/main.tf
+++ b/modules/aws/node/main.tf
@@ -160,6 +160,9 @@ resource "aws_instance" "this" {
       api_endpoint = data.aws_eks_cluster.this.endpoint
       certificate_authority = data.aws_eks_cluster.this.certificate_authority[0].data
       cidr = data.aws_eks_cluster.this.kubernetes_network_config[0].service_ipv4_cidr
+      kubelet_extra_args = [
+        "--node-labels=${local.kubelet_node_labels_arg}"
+      ]
     }
   )
 
diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index bed1bc0..cf73161 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -6,4 +6,11 @@ spec:
     name: ${name}
     apiServerEndpoint: ${api_endpoint}
     certificateAuthority: ${certificate_authority}
-    cidr: ${cidr}
\ No newline at end of file
+    cidr: ${cidr}
+  %{~ if len(kubelet_flags) > 0 ~}
+  kubelet:
+    flags:
+    %{~ for flag in kubelet_flags }
+    - ${flag}
+    %{~ endfor }
+  %{~ end if }

From d25a1655379ff3e2c288350f708a71ce6a1c9418 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Tue, 3 Dec 2024 07:44:44 -0800
Subject: [PATCH 06/37] .

---
 modules/aws/node/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/aws/node/main.tf b/modules/aws/node/main.tf
index cc92d9a..ee78916 100644
--- a/modules/aws/node/main.tf
+++ b/modules/aws/node/main.tf
@@ -160,7 +160,7 @@ resource "aws_instance" "this" {
       api_endpoint = data.aws_eks_cluster.this.endpoint
       certificate_authority = data.aws_eks_cluster.this.certificate_authority[0].data
       cidr = data.aws_eks_cluster.this.kubernetes_network_config[0].service_ipv4_cidr
-      kubelet_extra_args = [
+      kubelet_flags = [
         "--node-labels=${local.kubelet_node_labels_arg}"
       ]
     }

From 2a7ad5fbee605940aff758dc785865a7019ed0f6 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Tue, 3 Dec 2024 07:45:34 -0800
Subject: [PATCH 07/37] .

---
 modules/aws/node/templates/user-data.tftpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index cf73161..c721360 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -13,4 +13,4 @@ spec:
     %{~ for flag in kubelet_flags }
     - ${flag}
     %{~ endfor }
-  %{~ end if }
+  %{~ endif }

From a3ff84e6fec0b3748449e7765bccb2575ef1b3d8 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Tue, 3 Dec 2024 07:47:45 -0800
Subject: [PATCH 08/37] .

---
 modules/aws/node/templates/user-data.tftpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index c721360..013659a 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -7,7 +7,7 @@ spec:
     apiServerEndpoint: ${api_endpoint}
     certificateAuthority: ${certificate_authority}
     cidr: ${cidr}
-  %{~ if len(kubelet_flags) > 0 ~}
+  %{~ if length(kubelet_flags) > 0 ~}
   kubelet:
     flags:
     %{~ for flag in kubelet_flags }

From c8b7096906183fc5dcd978d501a61cee8fdab895 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Tue, 3 Dec 2024 07:57:17 -0800
Subject: [PATCH 09/37] .

---
 modules/aws/node/main.tf                   |  3 +++
 modules/aws/node/templates/user-data.tftpl | 23 ++++++++++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/modules/aws/node/main.tf b/modules/aws/node/main.tf
index ee78916..8cfb7ef 100644
--- a/modules/aws/node/main.tf
+++ b/modules/aws/node/main.tf
@@ -163,6 +163,9 @@ resource "aws_instance" "this" {
       kubelet_flags = [
         "--node-labels=${local.kubelet_node_labels_arg}"
       ]
+      hugepages_gb   = local.hugepages_gb
+      isolated_cores = local.isolated_cores
+      additional_user_data = var.user_data
     }
   )
 
diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index 013659a..c88fd80 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -1,3 +1,17 @@
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="BOUNDARY"
+
+--BOUNDARY
+Content-Type: text/x-shellscript; charset="us-ascii"
+
+#!/bin/bash
+%{~ if xrd_bootstrap }
+HUGEPAGES_GB=${hugepages_gb} ISOLATED_CORES=${isolated_cores} /etc/xrd/bootstrap.sh
+%{~ endif }
+
+--BOUNDARY
+Content-Type: application/node.eks.aws
+
 ---
 apiVersion: node.eks.aws/v1alpha1
 kind: NodeConfig
@@ -14,3 +28,12 @@ spec:
     - ${flag}
     %{~ endfor }
   %{~ endif }
+
+--BOUNDARY
+Content-Type: text/x-shellscript; charset="us-ascii"
+
+#!/bin/bash
+${additional_user_data}
+reboot
+
+--BOUNDARY--
\ No newline at end of file

From a1f2eb77d32e41a3a645cc719a2e9cf154f6dca2 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Tue, 3 Dec 2024 08:02:10 -0800
Subject: [PATCH 10/37] .

---
 modules/aws/node/main.tf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/modules/aws/node/main.tf b/modules/aws/node/main.tf
index 8cfb7ef..60fa955 100644
--- a/modules/aws/node/main.tf
+++ b/modules/aws/node/main.tf
@@ -166,6 +166,7 @@ resource "aws_instance" "this" {
       hugepages_gb   = local.hugepages_gb
       isolated_cores = local.isolated_cores
       additional_user_data = var.user_data
+      xrd_bootstrap  = local.is_xrd_ami
     }
   )
 

From ce53075d0b69593288773d24c2885d533a3153b9 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Thu, 5 Dec 2024 01:39:38 -0800
Subject: [PATCH 11/37] .

---
 modules/aws/node/templates/user-data.tftpl | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index c88fd80..2001af7 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -8,6 +8,10 @@ Content-Type: text/x-shellscript; charset="us-ascii"
 %{~ if xrd_bootstrap }
 HUGEPAGES_GB=${hugepages_gb} ISOLATED_CORES=${isolated_cores} /etc/xrd/bootstrap.sh
 %{~ endif }
+${additional_user_data}
+%{~ if xrd_bootstrap || length(additional_user_data) > 0 }
+reboot
+%{~ endif }
 
 --BOUNDARY
 Content-Type: application/node.eks.aws
@@ -29,11 +33,4 @@ spec:
     %{~ endfor }
   %{~ endif }
 
---BOUNDARY
-Content-Type: text/x-shellscript; charset="us-ascii"
-
-#!/bin/bash
-${additional_user_data}
-reboot
-
 --BOUNDARY--
\ No newline at end of file

From 28c189d403b2494c9d97133f521975f978876dfc Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Thu, 5 Dec 2024 01:58:20 -0800
Subject: [PATCH 12/37] .

---
 modules/aws/node/templates/user-data.tftpl | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index 2001af7..25f1818 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -9,9 +9,6 @@ Content-Type: text/x-shellscript; charset="us-ascii"
 HUGEPAGES_GB=${hugepages_gb} ISOLATED_CORES=${isolated_cores} /etc/xrd/bootstrap.sh
 %{~ endif }
 ${additional_user_data}
-%{~ if xrd_bootstrap || length(additional_user_data) > 0 }
-reboot
-%{~ endif }
 
 --BOUNDARY
 Content-Type: application/node.eks.aws

From f13881ef07806b2ad0ae31c227df4ec365235cac Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Thu, 5 Dec 2024 02:32:57 -0800
Subject: [PATCH 13/37] .

---
 modules/aws/node/templates/user-data.tftpl | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index 25f1818..803503c 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -30,4 +30,11 @@ spec:
     %{~ endfor }
   %{~ endif }
 
+--BOUNDARY
+  Content-Type: text/x-shellscript; charset="us-ascii"
+
+  #!/bin/bash
+  ${additional_user_data}
+  reboot
+
 --BOUNDARY--
\ No newline at end of file

From b0bc560e421a8f661dd718575a6a0d6717c755bb Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Thu, 5 Dec 2024 02:43:57 -0800
Subject: [PATCH 14/37] .

---
 modules/aws/node/templates/user-data.tftpl | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index 803503c..f6ab3b2 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -8,7 +8,6 @@ Content-Type: text/x-shellscript; charset="us-ascii"
 %{~ if xrd_bootstrap }
 HUGEPAGES_GB=${hugepages_gb} ISOLATED_CORES=${isolated_cores} /etc/xrd/bootstrap.sh
 %{~ endif }
-${additional_user_data}
 
 --BOUNDARY
 Content-Type: application/node.eks.aws
@@ -31,10 +30,9 @@ spec:
   %{~ endif }
 
 --BOUNDARY
-  Content-Type: text/x-shellscript; charset="us-ascii"
+Content-Type: text/x-shellscript; charset="us-ascii"
 
-  #!/bin/bash
-  ${additional_user_data}
-  reboot
+${additional_user_data}
+reboot
 
 --BOUNDARY--
\ No newline at end of file

From 76fe89f26a5fc2fcb5fb83345764a85d7db88284 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Thu, 5 Dec 2024 03:00:05 -0800
Subject: [PATCH 15/37] .

---
 modules/aws/node/templates/user-data.tftpl | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index f6ab3b2..39b1eda 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -8,6 +8,10 @@ Content-Type: text/x-shellscript; charset="us-ascii"
 %{~ if xrd_bootstrap }
 HUGEPAGES_GB=${hugepages_gb} ISOLATED_CORES=${isolated_cores} /etc/xrd/bootstrap.sh
 %{~ endif }
+${additional_user_data}
+%{~ if xrd_bootstrap || lengtht(additional_user_data) > 0 }
+reboot
+%{~ endif }
 
 --BOUNDARY
 Content-Type: application/node.eks.aws
@@ -29,10 +33,4 @@ spec:
     %{~ endfor }
   %{~ endif }
 
---BOUNDARY
-Content-Type: text/x-shellscript; charset="us-ascii"
-
-${additional_user_data}
-reboot
-
 --BOUNDARY--
\ No newline at end of file

From 9073409180144672147f0577507e8ad5c21a3a9b Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Thu, 5 Dec 2024 03:03:49 -0800
Subject: [PATCH 16/37] .

---
 modules/aws/node/templates/user-data.tftpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index 39b1eda..2001af7 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -9,7 +9,7 @@ Content-Type: text/x-shellscript; charset="us-ascii"
 HUGEPAGES_GB=${hugepages_gb} ISOLATED_CORES=${isolated_cores} /etc/xrd/bootstrap.sh
 %{~ endif }
 ${additional_user_data}
-%{~ if xrd_bootstrap || lengtht(additional_user_data) > 0 }
+%{~ if xrd_bootstrap || length(additional_user_data) > 0 }
 reboot
 %{~ endif }
 

From 03a711b735c29eb352ee405d173df22f95635fd9 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Thu, 5 Dec 2024 03:26:50 -0800
Subject: [PATCH 17/37] .

---
 modules/aws/node/templates/user-data.tftpl | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index 2001af7..25f1818 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -9,9 +9,6 @@ Content-Type: text/x-shellscript; charset="us-ascii"
 HUGEPAGES_GB=${hugepages_gb} ISOLATED_CORES=${isolated_cores} /etc/xrd/bootstrap.sh
 %{~ endif }
 ${additional_user_data}
-%{~ if xrd_bootstrap || length(additional_user_data) > 0 }
-reboot
-%{~ endif }
 
 --BOUNDARY
 Content-Type: application/node.eks.aws

From 6bdb8cf341eb8517c476a2268fc2d843f7d1890d Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Thu, 5 Dec 2024 03:34:34 -0800
Subject: [PATCH 18/37] .

---
 modules/aws/node/templates/user-data.tftpl | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index 25f1818..b20a852 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -1,15 +1,6 @@
 MIME-Version: 1.0
 Content-Type: multipart/mixed; boundary="BOUNDARY"
 
---BOUNDARY
-Content-Type: text/x-shellscript; charset="us-ascii"
-
-#!/bin/bash
-%{~ if xrd_bootstrap }
-HUGEPAGES_GB=${hugepages_gb} ISOLATED_CORES=${isolated_cores} /etc/xrd/bootstrap.sh
-%{~ endif }
-${additional_user_data}
-
 --BOUNDARY
 Content-Type: application/node.eks.aws
 

From 83d85bad5f01f39edb52787f33813525ed341713 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Thu, 5 Dec 2024 03:42:08 -0800
Subject: [PATCH 19/37] .

---
 modules/aws/node/templates/user-data.tftpl | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index b20a852..9cbb508 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -13,12 +13,5 @@ spec:
     apiServerEndpoint: ${api_endpoint}
     certificateAuthority: ${certificate_authority}
     cidr: ${cidr}
-  %{~ if length(kubelet_flags) > 0 ~}
-  kubelet:
-    flags:
-    %{~ for flag in kubelet_flags }
-    - ${flag}
-    %{~ endfor }
-  %{~ endif }
 
 --BOUNDARY--
\ No newline at end of file

From 2c55e58693c501a5fd20de7ecef14d7a2bbeea87 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Thu, 5 Dec 2024 05:53:52 -0800
Subject: [PATCH 20/37] .

---
 modules/aws/node/templates/user-data.tftpl | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index 9cbb508..1af98de 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -1,9 +1,3 @@
-MIME-Version: 1.0
-Content-Type: multipart/mixed; boundary="BOUNDARY"
-
---BOUNDARY
-Content-Type: application/node.eks.aws
-
 ---
 apiVersion: node.eks.aws/v1alpha1
 kind: NodeConfig
@@ -13,5 +7,10 @@ spec:
     apiServerEndpoint: ${api_endpoint}
     certificateAuthority: ${certificate_authority}
     cidr: ${cidr}
-
---BOUNDARY--
\ No newline at end of file
+  %{~ if length(kubelet_flags) > 0 ~}
+  kubelet:
+    flags:
+    %{~ for flag in kubelet_flags }
+    - ${flag}
+    %{~ endfor }
+  %{~ endif }
\ No newline at end of file

From a21578af04c257e7b8571c9d67969435582d8ea3 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Thu, 5 Dec 2024 06:18:21 -0800
Subject: [PATCH 21/37] .

---
 modules/aws/node/templates/user-data.tftpl | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index 1af98de..25f1818 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -1,3 +1,18 @@
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="BOUNDARY"
+
+--BOUNDARY
+Content-Type: text/x-shellscript; charset="us-ascii"
+
+#!/bin/bash
+%{~ if xrd_bootstrap }
+HUGEPAGES_GB=${hugepages_gb} ISOLATED_CORES=${isolated_cores} /etc/xrd/bootstrap.sh
+%{~ endif }
+${additional_user_data}
+
+--BOUNDARY
+Content-Type: application/node.eks.aws
+
 ---
 apiVersion: node.eks.aws/v1alpha1
 kind: NodeConfig
@@ -13,4 +28,6 @@ spec:
     %{~ for flag in kubelet_flags }
     - ${flag}
     %{~ endfor }
-  %{~ endif }
\ No newline at end of file
+  %{~ endif }
+
+--BOUNDARY--
\ No newline at end of file

From fe6a063efbb0300221824ed569fa961b1fb38507 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Thu, 5 Dec 2024 06:45:31 -0800
Subject: [PATCH 22/37] .

---
 modules/aws/node/templates/user-data.tftpl | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index 25f1818..4e7a00e 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -5,10 +5,7 @@ Content-Type: multipart/mixed; boundary="BOUNDARY"
 Content-Type: text/x-shellscript; charset="us-ascii"
 
 #!/bin/bash
-%{~ if xrd_bootstrap }
 HUGEPAGES_GB=${hugepages_gb} ISOLATED_CORES=${isolated_cores} /etc/xrd/bootstrap.sh
-%{~ endif }
-${additional_user_data}
 
 --BOUNDARY
 Content-Type: application/node.eks.aws
@@ -22,12 +19,10 @@ spec:
     apiServerEndpoint: ${api_endpoint}
     certificateAuthority: ${certificate_authority}
     cidr: ${cidr}
-  %{~ if length(kubelet_flags) > 0 ~}
   kubelet:
     flags:
     %{~ for flag in kubelet_flags }
     - ${flag}
     %{~ endfor }
-  %{~ endif }
 
 --BOUNDARY--
\ No newline at end of file

From df79400716b44256278de1e664f8f6e35f914bf5 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Thu, 5 Dec 2024 06:47:15 -0800
Subject: [PATCH 23/37] .

---
 modules/aws/node/templates/user-data.tftpl | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index 4e7a00e..6cd6ea5 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -1,12 +1,6 @@
 MIME-Version: 1.0
 Content-Type: multipart/mixed; boundary="BOUNDARY"
 
---BOUNDARY
-Content-Type: text/x-shellscript; charset="us-ascii"
-
-#!/bin/bash
-HUGEPAGES_GB=${hugepages_gb} ISOLATED_CORES=${isolated_cores} /etc/xrd/bootstrap.sh
-
 --BOUNDARY
 Content-Type: application/node.eks.aws
 

From 27a2a8c1f0fdcee843e1a26891f0c803cf730303 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Fri, 6 Dec 2024 03:03:49 -0800
Subject: [PATCH 24/37] .

---
 modules/aws/node/templates/user-data.tftpl | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index 6cd6ea5..35f3cb7 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -19,4 +19,14 @@ spec:
     - ${flag}
     %{~ endfor }
 
+--BOUNDARY
+Content-Type: text/x-shellscript; charset="us-ascii"
+
+#!/bin/bash
+%{~ if xrd_bootstrap }
+HUGEPAGES_GB=${hugepages_gb} ISOLATED_CORES=${isolated_cores} /etc/xrd/bootstrap.sh
+%{~ endif }
+${additional_user_data}
+reboot
+
 --BOUNDARY--
\ No newline at end of file

From 6f864695dec57fafc653623149ea0a5981a70a67 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Fri, 6 Dec 2024 03:57:15 -0800
Subject: [PATCH 25/37] .

---
 modules/aws/node/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/aws/node/main.tf b/modules/aws/node/main.tf
index 60fa955..e2b9eda 100644
--- a/modules/aws/node/main.tf
+++ b/modules/aws/node/main.tf
@@ -159,7 +159,7 @@ resource "aws_instance" "this" {
       name = data.aws_eks_cluster.this.name
       api_endpoint = data.aws_eks_cluster.this.endpoint
       certificate_authority = data.aws_eks_cluster.this.certificate_authority[0].data
-      cidr = data.aws_eks_cluster.this.kubernetes_network_config[0].service_ipv4_cidr
+      cidr = var.private_ip_address
       kubelet_flags = [
         "--node-labels=${local.kubelet_node_labels_arg}"
       ]

From a34ec7fe7f2a8f1db39bd387cd1e58716c397842 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Fri, 6 Dec 2024 04:09:52 -0800
Subject: [PATCH 26/37] .

---
 modules/aws/node/templates/user-data.tftpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index 35f3cb7..97e98ba 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -12,7 +12,7 @@ spec:
     name: ${name}
     apiServerEndpoint: ${api_endpoint}
     certificateAuthority: ${certificate_authority}
-    cidr: ${cidr}
+    cidr: 10.0.100.0/24
   kubelet:
     flags:
     %{~ for flag in kubelet_flags }

From 7899e270b4df632b6317d2d988a89f01cf7abbb4 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Fri, 6 Dec 2024 04:34:53 -0800
Subject: [PATCH 27/37] .

---
 modules/aws/node/main.tf                   |  2 +-
 modules/aws/node/templates/user-data.tftpl | 15 +++++++++++++--
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/modules/aws/node/main.tf b/modules/aws/node/main.tf
index e2b9eda..60fa955 100644
--- a/modules/aws/node/main.tf
+++ b/modules/aws/node/main.tf
@@ -159,7 +159,7 @@ resource "aws_instance" "this" {
       name = data.aws_eks_cluster.this.name
       api_endpoint = data.aws_eks_cluster.this.endpoint
       certificate_authority = data.aws_eks_cluster.this.certificate_authority[0].data
-      cidr = var.private_ip_address
+      cidr = data.aws_eks_cluster.this.kubernetes_network_config[0].service_ipv4_cidr
       kubelet_flags = [
         "--node-labels=${local.kubelet_node_labels_arg}"
       ]
diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index 97e98ba..a4ff546 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -1,18 +1,29 @@
 MIME-Version: 1.0
 Content-Type: multipart/mixed; boundary="BOUNDARY"
 
+--BOUNDARY
+Content-Type: text/x-shellscript; charset="us-ascii"
+
+#!/bin/bash
+%{~ if xrd_bootstrap }
+HUGEPAGES_GB=${hugepages_gb} ISOLATED_CORES=${isolated_cores} /etc/xrd/bootstrap.sh
+%{~ endif }
+${additional_user_data}
+reboot
+
+
 --BOUNDARY
 Content-Type: application/node.eks.aws
 
 ---
 apiVersion: node.eks.aws/v1alpha1
-kind: NodeConfig
+kind: NodeCnfig
 spec:
   cluster:
     name: ${name}
     apiServerEndpoint: ${api_endpoint}
     certificateAuthority: ${certificate_authority}
-    cidr: 10.0.100.0/24
+    cidr: ${cidr}
   kubelet:
     flags:
     %{~ for flag in kubelet_flags }

From f0077cb9856d314be73318e15b3ef9d03c5defcd Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Fri, 6 Dec 2024 04:46:03 -0800
Subject: [PATCH 28/37] .

---
 modules/aws/node/main.tf                   | 1 +
 modules/aws/node/templates/user-data.tftpl | 9 ++-------
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/modules/aws/node/main.tf b/modules/aws/node/main.tf
index 60fa955..34eab15 100644
--- a/modules/aws/node/main.tf
+++ b/modules/aws/node/main.tf
@@ -167,6 +167,7 @@ resource "aws_instance" "this" {
       isolated_cores = local.isolated_cores
       additional_user_data = var.user_data
       xrd_bootstrap  = local.is_xrd_ami
+      private_ip     = var.private_ip_address
     }
   )
 
diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index a4ff546..1252fc8 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -5,19 +5,14 @@ Content-Type: multipart/mixed; boundary="BOUNDARY"
 Content-Type: text/x-shellscript; charset="us-ascii"
 
 #!/bin/bash
-%{~ if xrd_bootstrap }
-HUGEPAGES_GB=${hugepages_gb} ISOLATED_CORES=${isolated_cores} /etc/xrd/bootstrap.sh
-%{~ endif }
-${additional_user_data}
-reboot
-
+/etc/xrd/delete_default_routes_except.sh private_ip
 
 --BOUNDARY
 Content-Type: application/node.eks.aws
 
 ---
 apiVersion: node.eks.aws/v1alpha1
-kind: NodeCnfig
+kind: NodeConfig
 spec:
   cluster:
     name: ${name}

From 901588237efcf3044bf8c20abd0d68ee705bd6ff Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Fri, 6 Dec 2024 06:10:33 -0800
Subject: [PATCH 29/37] .

---
 modules/aws/node/main.tf                   | 5 +++++
 modules/aws/node/templates/user-data.tftpl | 6 ------
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/modules/aws/node/main.tf b/modules/aws/node/main.tf
index 34eab15..229b49b 100644
--- a/modules/aws/node/main.tf
+++ b/modules/aws/node/main.tf
@@ -192,6 +192,11 @@ resource "aws_instance" "this" {
 }
 
 resource "aws_network_interface" "this" {
+  # Wait for kubelet to start before attaching the network interfaces.
+  depends_on = [
+    kubernetes_job.wait
+  ]
+
   for_each = {
     for i, ni in var.network_interfaces :
     i => ni
diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index 1252fc8..35f3cb7 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -1,12 +1,6 @@
 MIME-Version: 1.0
 Content-Type: multipart/mixed; boundary="BOUNDARY"
 
---BOUNDARY
-Content-Type: text/x-shellscript; charset="us-ascii"
-
-#!/bin/bash
-/etc/xrd/delete_default_routes_except.sh private_ip
-
 --BOUNDARY
 Content-Type: application/node.eks.aws
 

From 0622383f8e80a739fdee0bb7d8a69f086faba28e Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Tue, 10 Dec 2024 02:47:35 -0800
Subject: [PATCH 30/37] .

---
 aws-quickstart                 |  1 +
 examples/overlay/infra/data.tf |  2 +-
 examples/overlay/infra/main.tf | 13 +++++++++++++
 modules/aws/eks-config/main.tf |  6 ------
 modules/aws/node/main.tf       |  4 +---
 5 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/aws-quickstart b/aws-quickstart
index 19fd321..22fd544 100755
--- a/aws-quickstart
+++ b/aws-quickstart
@@ -141,6 +141,7 @@ ensure_ami_exists () {
     --filters \
       "Name=tag:Generated_By,Values=xrd-packer" \
       "Name=tag:Kubernetes_Version,Values=${KUBERNETES_VERSION}" \
+      "Name=tag:Amazon_Linux_Version,Values=AL2023" \
     --query "Images[*].[ImageId]" \
     --output text \
     | head -n 1)
diff --git a/examples/overlay/infra/data.tf b/examples/overlay/infra/data.tf
index 938fa1c..feff999 100644
--- a/examples/overlay/infra/data.tf
+++ b/examples/overlay/infra/data.tf
@@ -4,7 +4,7 @@ data "aws_ami" "eks_optimized" {
 
   filter {
     name   = "name"
-    values = ["amazon-eks-node-${data.aws_eks_cluster.this.version}-*"]
+    values = ["amazon-eks-node-al2023-x86_64-standard-${data.aws_eks_cluster.this.version}-*"]
   }
 
   filter {
diff --git a/examples/overlay/infra/main.tf b/examples/overlay/infra/main.tf
index d5afa9f..6f9a2f7 100644
--- a/examples/overlay/infra/main.tf
+++ b/examples/overlay/infra/main.tf
@@ -57,6 +57,19 @@ resource "aws_security_group" "data" {
   }
 }
 
+data "http" "multus_yaml" {
+  url = "https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/master/config/multus/v4.0.2-eksbuild.1/multus-daemonset-thick.yml"
+}
+
+
+resource "kubernetes_manifest" "multus" {
+  for_each = toset(compact(split("---", data.http.multus_yaml.response_body)))
+  depends_on = [module.node]
+
+  manifest = yamldecode(each.key)
+}
+
+
 module "eks_config" {
   source = "../../../modules/aws/eks-config"
 
diff --git a/modules/aws/eks-config/main.tf b/modules/aws/eks-config/main.tf
index f46ee82..c5a2b6f 100644
--- a/modules/aws/eks-config/main.tf
+++ b/modules/aws/eks-config/main.tf
@@ -53,9 +53,3 @@ resource "helm_release" "ebs_csi" {
   }
   wait = false
 }
-
-resource "kubernetes_manifest" "multus" {
-  for_each = toset(compact(split("---", data.http.multus_yaml.response_body)))
-
-  manifest = yamldecode(each.key)
-}
diff --git a/modules/aws/node/main.tf b/modules/aws/node/main.tf
index 229b49b..89affba 100644
--- a/modules/aws/node/main.tf
+++ b/modules/aws/node/main.tf
@@ -160,9 +160,7 @@ resource "aws_instance" "this" {
       api_endpoint = data.aws_eks_cluster.this.endpoint
       certificate_authority = data.aws_eks_cluster.this.certificate_authority[0].data
       cidr = data.aws_eks_cluster.this.kubernetes_network_config[0].service_ipv4_cidr
-      kubelet_flags = [
-        "--node-labels=${local.kubelet_node_labels_arg}"
-      ]
+      kubelet_flags = ["--node-labels=${local.kubelet_node_labels_arg}"]
       hugepages_gb   = local.hugepages_gb
       isolated_cores = local.isolated_cores
       additional_user_data = var.user_data

From 04c7c6cb663b935c8706c54d3524bd0dbe33c7db Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Wed, 11 Dec 2024 02:25:36 -0800
Subject: [PATCH 31/37] .

---
 examples/ha/workload/main.tf      | 6 +++---
 examples/overlay/workload/main.tf | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/examples/ha/workload/main.tf b/examples/ha/workload/main.tf
index d13e674..0f3f535 100644
--- a/examples/ha/workload/main.tf
+++ b/examples/ha/workload/main.tf
@@ -95,7 +95,7 @@ module "peer" {
   source = "../../../modules/aws/linux-pod-with-net-attach"
 
   name       = "peer"
-  device     = "eth1"
+  device     = "ens6"
   ip_address = "10.0.10.12/24"
   gateway    = "10.0.10.20"
   routes     = ["10.0.11.0/24", "10.0.13.0/24"]
@@ -108,7 +108,7 @@ module "cnf_vrid1" {
   source = "../../../modules/aws/linux-pod-with-net-attach"
 
   name       = "cnf-vrid1"
-  device     = "eth2"
+  device     = "ens7"
   ip_address = "10.0.11.12/24"
   gateway    = "10.0.11.20"
   routes     = ["10.0.10.0/24"]
@@ -121,7 +121,7 @@ module "cnf_vrid2" {
   source = "../../../modules/aws/linux-pod-with-net-attach"
 
   name       = "cnf-vrid2"
-  device     = "eth3"
+  device     = "ens8"
   ip_address = "10.0.13.12/24"
   gateway    = "10.0.13.1"
   routes     = ["10.0.10.0/24"]
diff --git a/examples/overlay/workload/main.tf b/examples/overlay/workload/main.tf
index 45914d9..66974d0 100644
--- a/examples/overlay/workload/main.tf
+++ b/examples/overlay/workload/main.tf
@@ -76,7 +76,7 @@ module "cnf" {
   source = "../../../modules/aws/linux-pod-with-net-attach"
 
   name       = "cnf"
-  device     = "eth1"
+  device     = "ens6"
   ip_address = "10.0.10.10/24"
   gateway    = "10.0.10.11"
   routes     = ["10.0.13.0/24"]
@@ -89,7 +89,7 @@ module "peer" {
   source = "../../../modules/aws/linux-pod-with-net-attach"
 
   name       = "peer"
-  device     = "eth2"
+  device     = "ens7"
   ip_address = "10.0.13.10/24"
   gateway    = "10.0.13.12"
   routes     = ["10.0.10.0/24"]

From 6cbfca7bb2b33043070b163b31157a58d419bb43 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Wed, 11 Dec 2024 03:31:58 -0800
Subject: [PATCH 32/37] .

---
 examples/ha/infra/main.tf      | 7 +++++++
 modules/aws/eks-config/main.tf | 6 ++++++
 2 files changed, 13 insertions(+)

diff --git a/examples/ha/infra/main.tf b/examples/ha/infra/main.tf
index 6057c73..e7dd6cb 100644
--- a/examples/ha/infra/main.tf
+++ b/examples/ha/infra/main.tf
@@ -84,6 +84,13 @@ module "xrd_ami" {
   count  = var.node_ami == null ? 1 : 0
 
   cluster_version = data.aws_eks_cluster.this.version
+
+  filters = [
+    {
+      name   = "tag:Amazon_Linux_Version"
+      values = ["AL2023"]
+    }
+  ]
 }
 
 locals {
diff --git a/modules/aws/eks-config/main.tf b/modules/aws/eks-config/main.tf
index c5a2b6f..f46ee82 100644
--- a/modules/aws/eks-config/main.tf
+++ b/modules/aws/eks-config/main.tf
@@ -53,3 +53,9 @@ resource "helm_release" "ebs_csi" {
   }
   wait = false
 }
+
+resource "kubernetes_manifest" "multus" {
+  for_each = toset(compact(split("---", data.http.multus_yaml.response_body)))
+
+  manifest = yamldecode(each.key)
+}

From c5b5e849d06c39903d316e092a2d91899a38833a Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Wed, 11 Dec 2024 06:38:33 -0800
Subject: [PATCH 33/37] savepoint

---
 README.md                      |  3 +++
 aws-quickstart                 |  1 -
 examples/ha/infra/main.tf      |  7 -------
 examples/overlay/infra/main.tf | 13 -------------
 4 files changed, 3 insertions(+), 21 deletions(-)

diff --git a/README.md b/README.md
index 7496696..d3d9c99 100644
--- a/README.md
+++ b/README.md
@@ -34,6 +34,9 @@ In addition, the following tools are recommended:
 
 ### AMI
 
+The Terraform modules in this repository require the use of AL2023 for the
+Linux version for all AMIs.
+
 The Terraform modules in this repository rely on the AMIs used for worker
 nodes to be optimized for XRd. The easiest way to achieve this is to use the
 [XRd Packer](https://github.com/ios-xr/xrd-packer) templates to generate
diff --git a/aws-quickstart b/aws-quickstart
index 22fd544..19fd321 100755
--- a/aws-quickstart
+++ b/aws-quickstart
@@ -141,7 +141,6 @@ ensure_ami_exists () {
     --filters \
       "Name=tag:Generated_By,Values=xrd-packer" \
       "Name=tag:Kubernetes_Version,Values=${KUBERNETES_VERSION}" \
-      "Name=tag:Amazon_Linux_Version,Values=AL2023" \
     --query "Images[*].[ImageId]" \
     --output text \
     | head -n 1)
diff --git a/examples/ha/infra/main.tf b/examples/ha/infra/main.tf
index e7dd6cb..6057c73 100644
--- a/examples/ha/infra/main.tf
+++ b/examples/ha/infra/main.tf
@@ -84,13 +84,6 @@ module "xrd_ami" {
   count  = var.node_ami == null ? 1 : 0
 
   cluster_version = data.aws_eks_cluster.this.version
-
-  filters = [
-    {
-      name   = "tag:Amazon_Linux_Version"
-      values = ["AL2023"]
-    }
-  ]
 }
 
 locals {
diff --git a/examples/overlay/infra/main.tf b/examples/overlay/infra/main.tf
index 6f9a2f7..d5afa9f 100644
--- a/examples/overlay/infra/main.tf
+++ b/examples/overlay/infra/main.tf
@@ -57,19 +57,6 @@ resource "aws_security_group" "data" {
   }
 }
 
-data "http" "multus_yaml" {
-  url = "https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/master/config/multus/v4.0.2-eksbuild.1/multus-daemonset-thick.yml"
-}
-
-
-resource "kubernetes_manifest" "multus" {
-  for_each = toset(compact(split("---", data.http.multus_yaml.response_body)))
-  depends_on = [module.node]
-
-  manifest = yamldecode(each.key)
-}
-
-
 module "eks_config" {
   source = "../../../modules/aws/eks-config"
 

From fcbc5ead4bb6febb986af668153a6d33ff67fab4 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Wed, 11 Dec 2024 07:43:48 -0800
Subject: [PATCH 34/37] kubelet extra args

---
 modules/aws/node/main.tf      | 5 ++++-
 modules/aws/node/variables.tf | 6 +++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/modules/aws/node/main.tf b/modules/aws/node/main.tf
index 89affba..3229db5 100644
--- a/modules/aws/node/main.tf
+++ b/modules/aws/node/main.tf
@@ -160,7 +160,10 @@ resource "aws_instance" "this" {
       api_endpoint = data.aws_eks_cluster.this.endpoint
       certificate_authority = data.aws_eks_cluster.this.certificate_authority[0].data
       cidr = data.aws_eks_cluster.this.kubernetes_network_config[0].service_ipv4_cidr
-      kubelet_flags = ["--node-labels=${local.kubelet_node_labels_arg}"]
+      kubelet_flags = concat(
+        ["--node-labels=${local.kubelet_node_labels_arg}"],
+        var.kubelet_extra_args
+      )
       hugepages_gb   = local.hugepages_gb
       isolated_cores = local.isolated_cores
       additional_user_data = var.user_data
diff --git a/modules/aws/node/variables.tf b/modules/aws/node/variables.tf
index 01c1149..bdca23b 100644
--- a/modules/aws/node/variables.tf
+++ b/modules/aws/node/variables.tf
@@ -95,11 +95,11 @@ variable "isolated_cores" {
 
 variable "kubelet_extra_args" {
   description = <<-EOT
-  Extra arguments to pass to kubelet when booting the node.
+  List of extra command-line kubelet arguments to be appended to the defaults.
   Note that node labels must be specified via the 'labels' variable.
   EOT
-  type        = string
-  default     = null
+  type        = list(string)
+  default     = []
 }
 
 variable "labels" {

From 6ef71603c15e3d87681393d19af2031ce18919c3 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Thu, 12 Dec 2024 01:21:37 -0800
Subject: [PATCH 35/37] UT fixed

---
 modules/aws/node/templates/user-data.tftpl |  6 +-
 tests/ut/terraform/node/variables.tf       |  6 +-
 tests/ut/test_node.py                      | 68 ++++++++++++++++++++--
 3 files changed, 69 insertions(+), 11 deletions(-)

diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl
index 35f3cb7..6f7b956 100644
--- a/modules/aws/node/templates/user-data.tftpl
+++ b/modules/aws/node/templates/user-data.tftpl
@@ -15,9 +15,9 @@ spec:
     cidr: ${cidr}
   kubelet:
     flags:
-    %{~ for flag in kubelet_flags }
+    %{~ for flag in kubelet_flags ~}
     - ${flag}
-    %{~ endfor }
+    %{~ endfor ~}
 
 --BOUNDARY
 Content-Type: text/x-shellscript; charset="us-ascii"
@@ -29,4 +29,4 @@ HUGEPAGES_GB=${hugepages_gb} ISOLATED_CORES=${isolated_cores} /etc/xrd/bootstrap
 ${additional_user_data}
 reboot
 
---BOUNDARY--
\ No newline at end of file
+--BOUNDARY--
diff --git a/tests/ut/terraform/node/variables.tf b/tests/ut/terraform/node/variables.tf
index 570bdd2..b22f87c 100644
--- a/tests/ut/terraform/node/variables.tf
+++ b/tests/ut/terraform/node/variables.tf
@@ -101,11 +101,11 @@ variable "isolated_cores" {
 
 variable "kubelet_extra_args" {
   description = <<-EOT
-  Extra arguments to pass to kubelet when booting the node.
+  List of extra command-line kubelet arguments to be appended to the defaults.
   Note that node labels must be specified via the 'labels' variable.
   EOT
-  type        = string
-  default     = null
+  type        = list(string)
+  default     = []
 }
 
 variable "labels" {
diff --git a/tests/ut/test_node.py b/tests/ut/test_node.py
index a3d350c..36ffe79 100644
--- a/tests/ut/test_node.py
+++ b/tests/ut/test_node.py
@@ -3,6 +3,7 @@
 import subprocess
 import uuid
 from pathlib import Path
+import textwrap
 from typing import Any
 
 import pytest
@@ -15,6 +16,7 @@
     Subnet,
     Vpc,
 )
+from mypy_boto3_eks import EKSClient
 from mypy_boto3_iam import IAMServiceResource
 from mypy_boto3_iam.service_resource import InstanceProfile
 
@@ -66,6 +68,14 @@ def subnet(vpc: Vpc) -> Subnet:
     )
 
 
+@pytest.fixture
+def other_subnet(vpc: Vpc) -> Subnet:
+    return vpc.create_subnet(
+        AvailabilityZone="eu-west-1a",
+        CidrBlock="10.0.1.0/24",
+    )
+
+
 @pytest.fixture
 def key_pair(ec2: EC2ServiceResource) -> KeyPair:
     return ec2.create_key_pair(KeyName=str(uuid.uuid4()))
@@ -92,6 +102,7 @@ def iam_instance_profile(iam: IAMServiceResource) -> InstanceProfile:
         "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy",
         "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly",
         "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy",
+        "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy",
     ):
         role.attach_policy(PolicyArn=policy_arn)
 
@@ -116,11 +127,34 @@ def security_group(ec2: EC2ServiceResource, vpc: Vpc) -> SecurityGroup:
     return sg
 
 
+@pytest.fixture()
+def eks_cluster(
+    eks_client: EKSClient,
+    iam_instance_profile: InstanceProfile,
+    subnet: Subnet,
+    other_subnet: Subnet,
+    security_group: SecurityGroup,
+) -> dict[str, Any]:
+    name = str(uuid.uuid4())
+    return eks_client.create_cluster(
+        name=name,
+        roleArn=iam_instance_profile.roles_attribute[0]["Arn"],
+        resourcesVpcConfig={
+            "subnetIds": [subnet.id, other_subnet.id],
+            "securityGroupIds": [security_group.id],
+            "endpointPublicAccess": False,
+            "endpointPrivateAccess": False,
+            "publicAccessCidrs": [],
+        },
+    )
+
+
 @pytest.fixture
 def base_vars(
     subnet: Subnet,
     key_pair: KeyPair,
     iam_instance_profile: InstanceProfile,
+    eks_cluster: dict[str, Any],
 ) -> dict[str, Any]:
     # This AMI should exist in the Moto server.
     # Refer to https://github.com/getmoto/moto/blob/master/moto/ec2/resources/amis.json.
@@ -128,7 +162,7 @@ def base_vars(
 
     return {
         "ami": ami,
-        "cluster_name": str(uuid.uuid4()),
+        "cluster_name": eks_cluster["cluster"]["name"],
         "name": str(uuid.uuid4()),
         "private_ip_address": "10.0.0.10",
         "security_groups": [],
@@ -162,7 +196,12 @@ def _assert_tag(instance: Instance, tag_key: str, tag_value: str) -> None:
         raise AssertionError(f"tag '{tag_key}' does not exist")
 
 
-def test_defaults(ec2, tf: Terraform, base_vars: dict[str, Any]):
+def test_defaults(
+    ec2,
+    tf: Terraform,
+    base_vars: dict[str, Any],
+    eks_cluster: dict[str, Any],
+):
     tf.apply(vars=base_vars)
     outputs = Outputs.from_terraform(tf)
     instance = ec2.Instance(outputs.id)
@@ -182,7 +221,23 @@ def test_defaults(ec2, tf: Terraform, base_vars: dict[str, Any]):
         "Value"
     ]
     user_data = base64.b64decode(user_data).decode()
-    assert f"/etc/eks/bootstrap.sh {base_vars['cluster_name']}" in user_data
+    expected_node_config = textwrap.dedent(
+        f"""\
+        ---
+        apiVersion: node.eks.aws/v1alpha1
+        kind: NodeConfig
+        spec:
+          cluster:
+            name: {base_vars['cluster_name']}
+            apiServerEndpoint: {eks_cluster['cluster']['endpoint']}
+            certificateAuthority: {eks_cluster['cluster']['certificateAuthority']['data']}
+            cidr: {eks_cluster['cluster']['kubernetesNetworkConfig']['serviceIpv4Cidr']}
+          kubelet:
+            flags:
+            - --node-labels=ios-xr.cisco.com/name={base_vars['name']}""",
+    )
+
+    assert expected_node_config in user_data
 
     # There should be no public IP address assigned.
     assert not instance.public_ip_address
@@ -250,7 +305,7 @@ def test_unknown_instance_type(ec2, tf: Terraform, base_vars: dict[str, Any]):
 
 
 def test_kubelet_extra_args(ec2, tf: Terraform, base_vars: dict[str, Any]):
-    vars = base_vars | {"kubelet_extra_args": "foo bar baz"}
+    vars = base_vars | {"kubelet_extra_args": ["foo", "bar"]}
     tf.apply(vars=vars)
     outputs = Outputs.from_terraform(tf)
     instance = ec2.Instance(outputs.id)
@@ -260,7 +315,10 @@ def test_kubelet_extra_args(ec2, tf: Terraform, base_vars: dict[str, Any]):
     ]
     user_data = base64.b64decode(user_data).decode()
     assert (
-        f"""--kubelet-extra-args '--node-labels ios-xr.cisco.com/name={base_vars["name"]} foo bar baz'"""
+        "    flags:\n"
+        f"    - --node-labels=ios-xr.cisco.com/name={vars['name']}\n"
+        "    - foo\n"
+        "    - bar\n"
         in user_data
     )
 

From 2373d79cb1bdb96158551dffeb3b0a10a82f3754 Mon Sep 17 00:00:00 2001
From: Ed Beaty <ebeaty@cisco.com>
Date: Thu, 12 Dec 2024 01:37:49 -0800
Subject: [PATCH 36/37] linted

---
 modules/aws/node/data.tf |  2 +-
 modules/aws/node/main.tf | 14 +++++++-------
 tests/ut/test_node.py    |  5 ++---
 3 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/modules/aws/node/data.tf b/modules/aws/node/data.tf
index 9f656b3..268b75e 100644
--- a/modules/aws/node/data.tf
+++ b/modules/aws/node/data.tf
@@ -11,4 +11,4 @@ data "aws_ec2_instance_type" "this" {
 
 data "aws_eks_cluster" "this" {
   name = var.cluster_name
-}
\ No newline at end of file
+}
diff --git a/modules/aws/node/main.tf b/modules/aws/node/main.tf
index 3229db5..158578e 100644
--- a/modules/aws/node/main.tf
+++ b/modules/aws/node/main.tf
@@ -156,19 +156,19 @@ resource "aws_instance" "this" {
   user_data = templatefile(
     "${path.module}/templates/user-data.tftpl",
     {
-      name = data.aws_eks_cluster.this.name
-      api_endpoint = data.aws_eks_cluster.this.endpoint
+      name                  = data.aws_eks_cluster.this.name
+      api_endpoint          = data.aws_eks_cluster.this.endpoint
       certificate_authority = data.aws_eks_cluster.this.certificate_authority[0].data
-      cidr = data.aws_eks_cluster.this.kubernetes_network_config[0].service_ipv4_cidr
+      cidr                  = data.aws_eks_cluster.this.kubernetes_network_config[0].service_ipv4_cidr
       kubelet_flags = concat(
         ["--node-labels=${local.kubelet_node_labels_arg}"],
         var.kubelet_extra_args
       )
-      hugepages_gb   = local.hugepages_gb
-      isolated_cores = local.isolated_cores
+      hugepages_gb         = local.hugepages_gb
+      isolated_cores       = local.isolated_cores
       additional_user_data = var.user_data
-      xrd_bootstrap  = local.is_xrd_ami
-      private_ip     = var.private_ip_address
+      xrd_bootstrap        = local.is_xrd_ami
+      private_ip           = var.private_ip_address
     }
   )
 
diff --git a/tests/ut/test_node.py b/tests/ut/test_node.py
index 36ffe79..904df50 100644
--- a/tests/ut/test_node.py
+++ b/tests/ut/test_node.py
@@ -1,9 +1,9 @@
 import base64
 import json
 import subprocess
+import textwrap
 import uuid
 from pathlib import Path
-import textwrap
 from typing import Any
 
 import pytest
@@ -318,8 +318,7 @@ def test_kubelet_extra_args(ec2, tf: Terraform, base_vars: dict[str, Any]):
         "    flags:\n"
         f"    - --node-labels=ios-xr.cisco.com/name={vars['name']}\n"
         "    - foo\n"
-        "    - bar\n"
-        in user_data
+        "    - bar\n" in user_data
     )
 
 

From 88a615f1caa998462fcd04ebb65acabf4c96c95b Mon Sep 17 00:00:00 2001
From: ebeaty-cisco <150805304+ebeaty-cisco@users.noreply.github.com>
Date: Wed, 15 Jan 2025 14:04:33 +0000
Subject: [PATCH 37/37] updated m6in.16xlarge, m5n.24xlarge and m5.25xlarge to
 use 17 cpu cores (#31)

---
 modules/aws/node-props/main.tf | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/modules/aws/node-props/main.tf b/modules/aws/node-props/main.tf
index d6a3eb7..0db5640 100644
--- a/modules/aws/node-props/main.tf
+++ b/modules/aws/node-props/main.tf
@@ -12,12 +12,17 @@ locals {
       }
 
       "m5.24xlarge" = {
-        cpuset       = "12-23"
+        cpuset       = "7-23"
         hugepages_gb = 6
       }
 
       "m5n.24xlarge" = {
-        cpuset       = "12-23"
+        cpuset       = "7-23"
+        hugepages_gb = 6
+      }
+
+      "m6in.16xlarge" = {
+        cpuset       = "15-31"
         hugepages_gb = 6
       }
     }