Merge pull request #29 from junior/5G-NF-Infra-w-calico-example

quick fix 0.8.12

VERSION

@@ -1 +1 @@
-0.8.11
+0.8.12

examples/5G-NF-Infra/oke.tf

@@ -79,5 +79,6 @@ EOF
   # Cluster Tools
   # ingress_nginx_enabled = true
   # cert_manager_enabled  = true
-  prometheus_enabled = true
+  prometheus_enabled     = true
+  metrics_server_enabled = true
 }

main.tf

@@ -119,6 +119,7 @@ module "oke_node_pools" {
   node_pool_node_shape_config_ocpus         = each.value.node_pool_node_shape_config_ocpus
   node_pool_node_shape_config_memory_in_gbs = each.value.node_pool_node_shape_config_memory_in_gbs
   existent_oke_nodepool_id_for_autoscaler   = each.value.existent_oke_nodepool_id_for_autoscaler
+  node_pool_autoscaler_enabled              = try(each.value.node_pool_autoscaler_enabled, true)
   node_pool_oke_init_params                 = each.value.node_pool_oke_init_params
   node_pool_cloud_init_parts                = each.value.node_pool_cloud_init_parts
   public_ssh_key                            = local.workers_public_ssh_key
@@ -179,8 +180,8 @@ module "oke_cluster_autoscaler" {
   region = var.region
 
   ## Enable Cluster Autoscaler
-  cluster_autoscaler_enabled = var.cluster_autoscaler_enabled
-  oke_node_pools             = values(module.oke_node_pools)
+  # cluster_autoscaler_enabled = var.cluster_autoscaler_enabled
+  oke_node_pools = [for node_pool in values(module.oke_node_pools) : node_pool if node_pool.node_pool_autoscaler_enabled]
 
   depends_on = [module.oke, module.oke_node_pools]
 }
@@ -617,6 +618,15 @@ locals {
           udp_options      = { max = -1, min = -1, source_port_range = null }
           icmp_options     = null
           }, {
+          description      = "Allow Pods to communicate with Worker Nodes"
+          destination      = lookup(local.network_cidrs, "NODES-REGIONAL-SUBNET-CIDR")
+          destination_type = "SERVICE_CIDR_BLOCK"
+          protocol         = local.security_list_ports.tcp_protocol_number
+          stateless        = false
+          tcp_options      = { max = -1, min = -1, source_port_range = null }
+          udp_options      = { max = -1, min = -1, source_port_range = null }
+          icmp_options     = null
+          }, {
           description      = "Pod to Kubernetes API endpoint communication (when using VCN-native pod networking)"
           destination      = lookup(local.network_cidrs, "ENDPOINT-REGIONAL-SUBNET-CIDR")
           destination_type = "CIDR_BLOCK"

modules/oke-cluster-autoscaler/main.tf

@@ -3,24 +3,25 @@
 # 
 
 locals {
-  cluster_autoscaler_supported_k8s_versions           = { "1.21" = "1.21.1-3", "1.22" = "1.22.2-4", "1.23" = "1.23.0-4", "1.24" = "1.23.0-4" } # There's no API to get that list. Need to be updated manually
+  cluster_autoscaler_supported_k8s_versions           = var.cluster_autoscaler_supported_k8s_versions # There's no API to get that list. Need to be updated manually
   cluster_autoscaler_image_version                    = lookup(local.cluster_autoscaler_supported_k8s_versions, local.k8s_major_minor_version, reverse(values(local.cluster_autoscaler_supported_k8s_versions))[0])
   cluster_autoscaler_default_region                   = "us-ashburn-1"
   cluster_autoscaler_image_regions                    = ["us-ashburn-1", "us-phoenix-1", "uk-london-1", "eu-frankfurt-1"]
   cluster_autoscaler_image_region                     = contains(local.cluster_autoscaler_image_regions, var.region) ? var.region : local.cluster_autoscaler_default_region
-  cluster_autoscaler_image                            = "${local.cluster_autoscaler_image_region}.ocir.io/oracle/oci-cluster-autoscaler:${local.cluster_autoscaler_image_version}"
-  cluster_autoscaler_log_level_verbosity              = 4
+  cluster_autoscaler_image                            = var.custom_cluster_autoscaler_image != "" ? var.custom_cluster_autoscaler_image : "${local.cluster_autoscaler_image_region}.ocir.io/oracle/oci-cluster-autoscaler:${local.cluster_autoscaler_image_version}"
+  cluster_autoscaler_log_level_verbosity              = var.cluster_autoscaler_log_level_verbosity
   cluster_autoscaler_node_pools                       = [for map in var.oke_node_pools[*] : "--nodes=${map.node_pool_min_nodes}:${map.node_pool_max_nodes}:${map.node_pool_id}"]
-  cluster_autoscaler_max_node_provision_time          = "25m"
-  cluster_autoscaler_scale_down_delay_after_add       = "10m"
-  cluster_autoscaler_scale_down_unneeded_time         = "10m"
-  cluster_autoscaler_unremovable_node_recheck_timeout = "5m"
+  cluster_autoscaler_max_node_provision_time          = var.cluster_autoscaler_max_node_provision_time
+  cluster_autoscaler_scale_down_delay_after_add       = var.cluster_autoscaler_scale_down_delay_after_add
+  cluster_autoscaler_scale_down_unneeded_time         = var.cluster_autoscaler_scale_down_unneeded_time
+  cluster_autoscaler_unremovable_node_recheck_timeout = var.cluster_autoscaler_unremovable_node_recheck_timeout
   cluster_autoscaler_enabled                          = alltrue([contains(keys(local.cluster_autoscaler_supported_k8s_versions), local.k8s_major_minor_version)]) ? var.cluster_autoscaler_enabled : false
+  cluster_autoscaler_cloud_provider                   = local.k8s_major_minor_version < "1.24" ? "oci" : "oci-oke"
   k8s_major_minor_version                             = regex("\\d+(?:\\.(?:\\d+|x)(?:))", var.oke_node_pools.0.node_k8s_version)
 }
 
 # NOTE: Service Account Terraform resource is not supported with Kubernetes 1.24.
-resource "kubernetes_service_account" "cluster_autoscaler_sa" {
+resource "kubernetes_service_account_v1" "cluster_autoscaler_sa" {
   metadata {
     name      = "cluster-autoscaler"
     namespace = "kube-system"
@@ -29,25 +30,25 @@ resource "kubernetes_service_account" "cluster_autoscaler_sa" {
       k8s-app   = "cluster-autoscaler"
     }
   }
-  automount_service_account_token = false
+  automount_service_account_token = true # false
 
   count = local.cluster_autoscaler_enabled ? 1 : 0
 }
-resource "kubernetes_secret" "cluster_autoscaler_sa_secret" {
-  metadata {
-    name      = "cluster-autoscaler-token-secret"
-    namespace = "kube-system"
-    annotations = {
-      "kubernetes.io/service-account.name"      = "cluster-autoscaler"
-      "kubernetes.io/service-account.namespace" = "kube-system"
-    }
-  }
-  type = "kubernetes.io/service-account-token"
+# resource "kubernetes_secret" "cluster_autoscaler_sa_secret" {
+#   metadata {
+#     name      = "cluster-autoscaler-token-secret"
+#     namespace = "kube-system"
+#     annotations = {
+#       "kubernetes.io/service-account.name"      = "cluster-autoscaler"
+#       "kubernetes.io/service-account.namespace" = "kube-system"
+#     }
+#   }
+#   type = "kubernetes.io/service-account-token"
 
-  depends_on = [kubernetes_service_account.cluster_autoscaler_sa]
+#   depends_on = [kubernetes_service_account.cluster_autoscaler_sa]
 
-  count = local.cluster_autoscaler_enabled ? 1 : 0
-}
+#   count = local.cluster_autoscaler_enabled ? 1 : 0
+# }
 resource "kubernetes_cluster_role" "cluster_autoscaler_cr" {
   metadata {
     name = "cluster-autoscaler"
@@ -206,7 +207,7 @@ resource "kubernetes_deployment" "cluster_autoscaler_deployment" {
   }
 
   spec {
-    replicas = 3
+    replicas = var.cluster_autoscaler_num_of_replicas
 
     selector {
       match_labels = {
@@ -246,7 +247,7 @@ resource "kubernetes_deployment" "cluster_autoscaler_deployment" {
             "./cluster-autoscaler",
             "--v=${local.cluster_autoscaler_log_level_verbosity}",
             "--stderrthreshold=info",
-            "--cloud-provider=oci",
+            "--cloud-provider=${local.cluster_autoscaler_cloud_provider}",
             "--max-node-provision-time=${local.cluster_autoscaler_max_node_provision_time}",
             "--scale-down-delay-after-add=${local.cluster_autoscaler_scale_down_delay_after_add}",
             "--scale-down-unneeded-time=${local.cluster_autoscaler_scale_down_unneeded_time}",
@@ -257,7 +258,8 @@ resource "kubernetes_deployment" "cluster_autoscaler_deployment" {
             "--balancing-ignore-label=internal_addr",
             "--balancing-ignore-label=oci.oraclecloud.com/fault-domain"
             ],
-          local.cluster_autoscaler_node_pools)
+            local.cluster_autoscaler_node_pools,
+          var.cluster_autoscaler_extra_args)
           image_pull_policy = "Always"
           env {
             name  = "OKE_USE_INSTANCE_PRINCIPAL"

modules/oke-cluster-autoscaler/variables.tf

@@ -8,13 +8,51 @@ variable "cluster_autoscaler_enabled" {
   default     = true
   description = "Enables OKE cluster autoscaler. Node pools will auto scale based on the resources usage"
 }
-variable "cluster_autoscaler_min_nodes" {
+# variable "cluster_autoscaler_min_nodes" {
+#   default     = 3
+#   description = "Minimum number of nodes on the node pool to be scheduled by the Kubernetes"
+# }
+# variable "cluster_autoscaler_max_nodes" {
+#   default     = 10
+#   description = "Maximum number of nodes on the node pool to be scheduled by the Kubernetes"
+# }
+variable "cluster_autoscaler_supported_k8s_versions" {
+  type = map(string)
+
+  default     = { "1.22" = "1.22.2-4", "1.23" = "1.23.0-4", "1.24" = "1.24.0-5", "1.25" = "1.25.0-6" } # There's no API to get that list. Need to be updated manually
+  description = "Supported Kubernetes versions for OKE cluster autoscaler"
+}
+variable "custom_cluster_autoscaler_image" {
+  default     = ""
+  description = "Custom Image for OKE cluster autoscaler"
+}
+variable "cluster_autoscaler_log_level_verbosity" {
+  default     = 4
+  description = "Log level verbosity for OKE cluster autoscaler"
+}
+variable "cluster_autoscaler_max_node_provision_time" {
+  default     = "25m"
+  description = "Maximum time in minutes for a node to be provisioned. If the node is not ready after this time, it will be deleted and recreated"
+}
+variable "cluster_autoscaler_scale_down_delay_after_add" {
+  default     = "10m"
+  description = "Time to wait after scale up before attempting to scale down"
+}
+variable "cluster_autoscaler_scale_down_unneeded_time" {
+  default     = "10m"
+  description = "Time after which a node should be deleted after it has been unneeded for this long"
+}
+variable "cluster_autoscaler_unremovable_node_recheck_timeout" {
+  default     = "5m"
+  description = "Time after which a node which failed to be removed is retried"
+}
+variable "cluster_autoscaler_num_of_replicas" {
   default     = 3
-  description = "Minimum number of nodes on the node pool to be scheduled by the Kubernetes"
+  description = "Number of replicas for OKE cluster autoscaler"
 }
-variable "cluster_autoscaler_max_nodes" {
-  default     = 10
-  description = "Maximum number of nodes on the node pool to be scheduled by the Kubernetes"
+variable "cluster_autoscaler_extra_args" {
+  default     = []
+  description = "Extra arguments to pass to OKE cluster autoscaler"
 }
 
 ## OKE Node Pool Details

modules/oke-node-pool/main.tf

@@ -56,13 +56,6 @@ resource "oci_containerengine_node_pool" "oke_node_pool" {
   node_metadata = {
     user_data = anytrue([var.node_pool_oke_init_params != "", var.node_pool_cloud_init_parts != []]) ? data.cloudinit_config.nodes.rendered : null
   }
-  # dynamic "node_metadata" {
-  #   for_each = alltrue([var.node_pool_oke_init_params != "", var.node_pool_cloud_init_parts != []]) ? [1] : []
-
-  #   content {
-  #     user_data = data.cloudinit_config.nodes.rendered
-  #   }
-  # }
 
   initial_node_labels {
     key   = "name"
@@ -78,6 +71,12 @@ resource "oci_containerengine_node_pool" "oke_node_pool" {
     }
   }
 
+  lifecycle {
+    ignore_changes = [
+      node_config_details.0.size
+    ]
+  }
+
   count = var.create_new_node_pool ? 1 : 0
 }
 

modules/oke-node-pool/outputs.tf

@@ -17,3 +17,6 @@ output "node_pool_id" {
 output "node_k8s_version" {
   value = local.node_k8s_version
 }
+output "node_pool_autoscaler_enabled" {
+  value = var.node_pool_autoscaler_enabled
+}

modules/oke-node-pool/variables.tf

@@ -73,6 +73,10 @@ variable "existent_oke_nodepool_id_for_autoscaler" {
   default     = ""
   description = "Nodepool Id of the existent OKE to use with Cluster Autoscaler"
 }
+variable "node_pool_autoscaler_enabled" {
+  default     = true
+  description = "Enable Cluster Autoscaler for the node pool"
+}
 variable "image_operating_system" {
   default     = "Oracle Linux"
   description = "The OS/image installed on all nodes in the node pool."

modules/oke/main.tf

@@ -57,8 +57,9 @@ resource "oci_containerengine_cluster" "oke_cluster" {
 
 # Local kubeconfig for when using Terraform locally. Not used by Oracle Resource Manager
 resource "local_file" "oke_kubeconfig" {
-  content  = data.oci_containerengine_cluster_kube_config.oke.content
-  filename = "${path.root}/generated/kubeconfig"
+  content         = data.oci_containerengine_cluster_kube_config.oke.content
+  filename        = "${path.root}/generated/kubeconfig"
+  file_permission = "0644"
 }
 
 # Get OKE options

providers.tf

@@ -71,6 +71,7 @@ provider "oci" {
 provider "kubernetes" {
   host                   = local.cluster_endpoint
   cluster_ca_certificate = local.cluster_ca_certificate
+  insecure               = local.external_private_endpoint
   exec {
     api_version = "client.authentication.k8s.io/v1beta1"
     args        = ["ce", "cluster", "generate-token", "--cluster-id", local.cluster_id, "--region", local.cluster_region]
@@ -83,6 +84,7 @@ provider "helm" {
   kubernetes {
     host                   = local.cluster_endpoint
     cluster_ca_certificate = local.cluster_ca_certificate
+    insecure               = local.external_private_endpoint
     exec {
       api_version = "client.authentication.k8s.io/v1beta1"
       args        = ["ce", "cluster", "generate-token", "--cluster-id", local.cluster_id, "--region", local.cluster_region]
@@ -95,7 +97,8 @@ locals {
   cluster_endpoint = (var.cluster_endpoint_visibility == "Private") ? (
     "https://${module.oke.orm_private_endpoint_oke_api_ip_address}:6443") : (
   yamldecode(module.oke.kubeconfig)["clusters"][0]["cluster"]["server"])
-  cluster_ca_certificate = base64decode(yamldecode(module.oke.kubeconfig)["clusters"][0]["cluster"]["certificate-authority-data"])
-  cluster_id             = yamldecode(module.oke.kubeconfig)["users"][0]["user"]["exec"]["args"][4]
-  cluster_region         = yamldecode(module.oke.kubeconfig)["users"][0]["user"]["exec"]["args"][6]
+  external_private_endpoint = (var.cluster_endpoint_visibility == "Private") ? true : false
+  cluster_ca_certificate    = base64decode(yamldecode(module.oke.kubeconfig)["clusters"][0]["cluster"]["certificate-authority-data"])
+  cluster_id                = yamldecode(module.oke.kubeconfig)["users"][0]["user"]["exec"]["args"][4]
+  cluster_region            = yamldecode(module.oke.kubeconfig)["users"][0]["user"]["exec"]["args"][6]
 }