Merge pull request #503 from robinmordasiewicz/dev99

adding init

manifests/infrastructure/flannel/HelmRelease.yaml

@@ -0,0 +1,20 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: flannel
+  namespace: cluster-config
+spec:
+  targetNamespace: kube-flannel
+  releaseName: flannel
+  chart:
+    spec:
+      chart: flannel
+      sourceRef:
+        kind: HelmRepository
+        name: flannel-repo
+        namespace: cluster-config
+  interval: 10m0s
+  install:
+    remediation:
+      retries: 3

manifests/infrastructure/flannel/HelmRepository.yaml

@@ -0,0 +1,9 @@
+  ---
+  apiVersion: source.toolkit.fluxcd.io/v1beta2
+  kind: HelmRepository
+metadata:
+  name: flannel-repo
+  namespace: cluster-config
+spec:
+  interval: 30m
+  url: https://flannel-io.github.io/flannel/

manifests/infrastructure/flannel/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - HelmRepository.yaml
+  - HelmRelease.yaml
+  - namespace.yaml

manifests/infrastructure/flannel/namespace.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kube-flannel
+  labels:
+    name: kube-flannel
+    pod-security.kubernetes.io/enforce: privileged

manifests/infrastructure/kustomization.yaml

@@ -3,4 +3,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - rbac.yaml
-  - fortiweb-ingress
+  - ./flannel
+  - ./fortiweb-ingress

terraform/spoke-k8s_cluster.tf

@@ -14,6 +14,21 @@ resource "azurerm_log_analytics_workspace" "log_analytics" {
   sku                 = "PerGB2018"
 }
 
+resource "azurerm_user_assigned_identity" "my_identity" {
+  name                = "myUserAssignedIdentity"
+  resource_group_name = azurerm_resource_group.azure_resource_group.name
+  location            = azurerm_resource_group.azure_resource_group.location
+}
+resource "azurerm_role_assignment" "kubernetes_contributor" {
+  principal_id   = azurerm_user_assigned_identity.my_identity.principal_id
+  role_definition_name = "Contributor"
+  scope          = azurerm_resource_group.azure_resource_group.id
+}
+resource "azurerm_role_assignment" "route_table_network_contributor" {
+  principal_id   = azurerm_user_assigned_identity.my_identity.principal_id
+  role_definition_name = "Network Contributor"
+  scope          = azurerm_resource_group.azure_resource_group.id
+}
 resource "azurerm_kubernetes_cluster" "kubernetes_cluster" {
   depends_on                        = [azurerm_virtual_network_peering.spoke-to-hub_virtual_network_peering, azurerm_linux_virtual_machine.hub-nva_virtual_machine]
   name                              = "spoke_kubernetes_cluster"
@@ -42,18 +57,24 @@ resource "azurerm_kubernetes_cluster" "kubernetes_cluster" {
     node_count                  = 1
     vm_size                     = local.vm-image["aks"].size
     os_sku                      = "AzureLinux"
+    max_pods                    = "50"
+    vnet_subnet_id              = azurerm_subnet.spoke_subnet.id
+    #vnet_subnet_id              = azurerm_subnet.spoke_subnet.id
     upgrade_settings {
       max_surge = "10%"
     }
   }
   network_profile {
-    network_plugin    = "azure"
-    network_policy    = "azure"
-    load_balancer_sku = "standard"
+    network_plugin    = "none"
+    #network_policy    = "azure"
+    #load_balancer_sku = "standard"
+    #service_cidr      = var.spoke-k8s_service_cidr
+    #dns_service_ip    = var.spoke-ks8_dns_service_ip
   }
 
   identity {
-    type = "SystemAssigned"
+    type         = "UserAssigned"
+    identity_ids = [azurerm_user_assigned_identity.my_identity.id]
   }
 }
 
@@ -72,6 +93,7 @@ resource "azurerm_kubernetes_cluster_node_pool" "node-pool" {
   os_disk_size_gb       = "256"
   max_pods              = "50"
   zones                 = ["1"]
+  #vnet_subnet_id        = azurerm_subnet.spoke_subnet.id
 }
 
 resource "azurerm_kubernetes_cluster_extension" "flux_extension" {
@@ -92,9 +114,33 @@ resource "null_resource" "kube_config" {
   triggers = {
     always_run = timestamp()
   }
-  depends_on = [azurerm_kubernetes_flux_configuration.flux_configuration]
+  depends_on = [azurerm_kubernetes_cluster.kubernetes_cluster]
+  provisioner "local-exec" {
+    command = "echo \"${azurerm_kubernetes_cluster.kubernetes_cluster.kube_config_raw}\" > ~/.kube/config && chmod 600 ~/.kube/config"
+  }
+}
+
+resource "null_resource" "secret" {
+  triggers = {
+    always_run = timestamp()
+  }
+  depends_on = [azurerm_kubernetes_flux_configuration.flux_configuration, null_resource.kube_config]
+
   provisioner "local-exec" {
-    command = "echo \"${azurerm_kubernetes_cluster.kubernetes_cluster.kube_config_raw}\" > ~/.kube/config && chmod 600 ~/.kube/config && kubectl create secret generic fortiweb-login --from-literal=username=${random_pet.admin_username.id} --from-literal=password=${random_password.admin_password.result} --namespace=fortiweb-ingress"
+    interpreter = ["bash", "-c"]
+    command = <<-EOF
+      kubectl apply -f - <<EOF2
+      apiVersion: v1
+      kind: Secret
+      metadata:
+        name: fortiweb-login
+        namespace: fortiweb-ingress
+      type: Opaque
+      data:
+        username: $(echo -n "${random_pet.admin_username.id}" | base64)
+        password: $(echo -n "${random_password.admin_password.result}" | base64)
+      EOF2
+    EOF
   }
 }
 
@@ -142,4 +188,4 @@ output "kube_config" {
   description = "kube config"
   value       = azurerm_kubernetes_cluster.kubernetes_cluster.kube_config_raw
   sensitive   = true
-}
+}

terraform/spoke-network.tf

@@ -22,6 +22,13 @@ resource "azurerm_subnet" "spoke_subnet" {
   virtual_network_name = azurerm_virtual_network.spoke_virtual_network.name
 }
 
+#resource "azurerm_subnet" "spoke_aks_subnet" {
+#  address_prefixes     = [var.spoke-aks-subnet_prefix]
+#  name                 = var.spoke-aks-subnet_name
+#  resource_group_name  = azurerm_resource_group.azure_resource_group.name
+#  virtual_network_name = azurerm_virtual_network.spoke_virtual_network.name
+#}
+
 resource "azurerm_route_table" "spoke_route_table" {
   name                = "spoke_route_table"
   location            = azurerm_resource_group.azure_resource_group.location

terraform/terraform.auto.tfvars

@@ -11,10 +11,14 @@ hub-nva-management-action            = "Allow"
 hub-nva-management-ip                = "10.0.0.4"
 hub-nva-vip                          = "10.0.0.5"
 hub-nva-gateway                      = "10.0.0.37"
-spoke-virtual-network_address_prefix = "10.1.1.0/24"
+spoke-virtual-network_address_prefix = "10.1.0.0/24"
 spoke-subnet_name                    = "spoke_subnet"
-spoke-subnet_prefix                  = "10.1.1.0/24"
+spoke-subnet_prefix                  = "10.1.0.0/24"
+spoke-aks-subnet_name                = "spoke-aks-subnet_name"
+spoke-aks-subnet_prefix              = "10.1.0.0/24"
+spoke-k8s_service_cidr               = "10.2.0.0/16"
+spoke-ks8_dns_service_ip             = "10.2.0.10"
 spoke-check-internet-up-ip           = "8.8.8.8"
-spoke-linux-server-ip                = "10.1.1.5"
+spoke-linux-server-ip                = "10.1.0.5"
 spoke-linux-server-image-gpu         = false
 spoke-k8s-node-pool-gpu              = false

terraform/variables.tf

@@ -269,6 +269,45 @@ variable "spoke-subnet_prefix" {
     error_message = "The subnet must be in the format of 'xxx.xxx.xxx.xxx/xx', where xxx is between 0 and 255, and xx is between 0 and 32."
   }
 }
+variable "spoke-aks-subnet_name" {
+  default     = "spoke-aks-subnet_name"
+  description = "Spoke aks Subnet Name."
+  type        = string
+  validation {
+    condition     = can(regex("^[a-zA-Z0-9_-]*$", var.spoke-aks-subnet_name))
+    error_message = "The value must consist of alphanumeric characters, underscores, or dashes only."
+  }
+}
+
+variable "spoke-aks-subnet_prefix" {
+  default     = "10.1.2.0/24"
+  description = "Spoke Pod Subnet Prefix."
+  type        = string
+  validation {
+    condition     = can(regex("^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\\.){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])/(3[0-2]|[12]?[0-9])$", var.spoke-aks-subnet_prefix))
+    error_message = "The subnet must be in the format of 'xxx.xxx.xxx.xxx/xx', where xxx is between 0 and 255, and xx is between 0 and 32."
+  }
+}
+
+variable "spoke-k8s_service_cidr" {
+  default     = "10.1.2.0/24"
+  description = "Spoke k8s service cidr."
+  type        = string
+  validation {
+    condition     = can(regex("^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\\.){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])/(3[0-2]|[12]?[0-9])$", var.spoke-k8s_service_cidr))
+    error_message = "The subnet must be in the format of 'xxx.xxx.xxx.xxx/xx', where xxx is between 0 and 255, and xx is between 0 and 32."
+  } 
+}
+
+variable "spoke-ks8_dns_service_ip" {
+  default     = "10.2.0.10"
+  description = "Spoke k8s dns service ip"
+  type        = string
+  validation {
+    condition     = can(regex("^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$", var.spoke-ks8_dns_service_ip))
+    error_message = "The IP address must be a valid IPv4 format (e.g., 10.2.0.10)."
+  }
+}
 
 variable "spoke-linux-server-ip" {
   default     = "10.1.1.5"