diff --git a/bin/main/application.yaml b/bin/main/application.yaml index a147c18..03fb30b 100644 --- a/bin/main/application.yaml +++ b/bin/main/application.yaml @@ -66,7 +66,7 @@ aes: key: fb1755281b0ca6184a0ee644e6477ee7 cbtumblebug: - url: ${TUMBLEBUG_URL:210.217.178.130} + url: ${TUMBLEBUG_URL:13.125.215.182} port: ${TUMBLEBUG_PORT:1323} id: ${TUMBLEBUG_ID:default} pass: ${TUMBLEBUG_PASSWORD:default} diff --git a/kubeconfig b/kubeconfig deleted file mode 100644 index 8136d4a..0000000 --- a/kubeconfig +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM3akNDQWRhZ0F3SUJBZ0lRVUpEaHRmSXFTb2VMcGU2blc2RUVSVEFOQmdrcWhraUc5dzBCQVFzRkFEQWYKTVIwd0d3WURWUVFEREJSamMzWTRaM1ppWldKa05YTTNNems0YlhOeU1EQWVGdzB5TkRFeE1qQXdNVEV3TXpWYQpGdzB5T1RFeE1qQXdNVEV3TXpWYU1COHhIVEFiQmdOVkJBTU1GR056ZGpobmRtSmxZbVExY3pjek9UaHRjM0l3Ck1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBMXVtV0lqTVBHSTF2VnJhbmV1REEKakxDTGxMcEtxa3ZmY3NiT3N5SlpTVFAwMlJ2MTRDRHNwam5ibWU0QWVCb3hzUlBzc0l3emJBVnpUSVFWTW5adgoreDlQVlpZV0EzdEdaVGQ5aURPMjFlUmllWUpVdjA3eW9NeUNaTDN5Qy9YYW1reFQzT3hQR2JVaC8ySkU4ZytnCi85OURnNVA2UUFSQmc3TWFNdE9GYTlqYnJEVzQ0elVOQ1VDb2VxL2NkUlVDQVl0d1dkeHdRUlphNGF5QVlGdXEKZnRwM3c1WEdzWlNhYlA4K0YyZE0wU0dEOEFVVG1sNjVlUVpZWGdmRGVydWEwQW9Iclk1RUdrUzl2OHc4L3V2MApLaGErOEJjMlBjbmo2YTV4c1FUejdpVEdOL1BMc3ZwRzBTbm15UGFQam4vYWxzVWZ2OVV0QUdlUEIvOWtkTTVpCjNRSURBUUFCb3lZd0pEQVNCZ05WSFJNQkFmOEVDREFHQVFIL0FnRUFNQTRHQTFVZER3RUIvd1FFQXdJQ0JEQU4KQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBSUVlMzJDQWtFSlozMnQvdXBhckYyNi9QZWwzc05UZjJobENEVHZGRQpyZHVVb2tqdk4rbHlqSU1SYzE4ZWpvM29rZXR6K1dLYll2ZjNWMDZFckVnY043d0E3L2dlWEJ4TGJXeVpjVmVhCmkzSm1uNUd0ZWlYZ3BsczIwTmtSQlV3QTFpVG9ldDdjRmY0UVp2amQzNzJDcnVwekVSR3NSdkpYVDBDa1R5bTkKSHE4c0xzNXVsNnpXc3RHdlp5cDIxekQrU2ZNZ1loRUhudWxXbDg1b0c1U0FjVnFDQStWVkt6NytzaTA1clV3Rgo5YlUyN3JEV0xPeGlYWFpSY3BYQ0ZXRmxhNTlqQVg3YzFRWlBrWHc4SWVJb1oxa2JBYUlpUFhVcnJ3VHhqU1g2Clh0VC9QWXJEZFBHL3RENjlEbDdUQVJ0c1Z2aTE0cmJUNVgrWG9LbUtsNEIvOFE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://81b5ace4-nks-kr1.container.nhncloud.com:6443 - name: "toast-csv8gvbebd5s7398msr0" -contexts: -- context: - cluster: "toast-csv8gvbebd5s7398msr0" - user: admin - name: default -current-context: default -kind: Config -preferences: {} -users: -- name: admin - user: - client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMwVENDQWJtZ0F3SUJBZ0lSQU42MzBKYkYwMGw4aW92QVBKUkxjSXN3RFFZSktvWklodmNOQVFFTEJRQXcKSHpFZE1Cc0dBMVVFQXd3VVkzTjJPR2QyWW1WaVpEVnpOek01T0cxemNqQXdIaGNOTWpReE1USTBNRGd6TXpJMQpXaGNOTWpreE1USTBNRGd6TXpJMVdqQXBNUTR3REFZRFZRUUREQVZoWkcxcGJqRVhNQlVHQTFVRUNnd09jM2x6CmRHVnRPbTFoYzNSbGNuTXdnZ0VpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElCRHdBd2dnRUtBb0lCQVFEU2JQQUkKcHZZRjc2bjJzV1d2WUNnR1ZXaGRrMjJPUHpaUXFLYzRlaDRsMjhDV2tjNlNBUy96RDhYOWRCWm9tNm0yRDF6YQpoUEtNVVVyYjNobDUzV2hqR3ZRb25tVVc3eFZVVWd5K0UyUG9zNXFrcHBXTXVmMFg5Z0MydzY4ZzdpZVVWcFRhCkhaS2IrQkxnTVVsVFVKS0hkUlFYeTlXOWp4WG9xWDJ2KzNIa1BnNDhKSHU4UytQa2tqSFBLVy8yZnJZTHNJTHcKUlh2YlptVmREYWRDREozTHp2UlZzak00LzhtbFVnbUtBNXBmd2ZOS3hYMlgvdUE4T0V5bXdFcEZGZ3pNL3lNdApTT0ZFNTRDMFZLNU51UXB6N2RNVksvTjYvL24yZ0s3cnZSdENrY1kycCtld24rUFB6cmxsUERub21wRmhoOVVNCjgxbmNSK3RDL2FOcE41Vy9BZ01CQUFFd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFDckgxbVpBS1pXNzltUFYKcWh1V291Z0FPVnh3TkIvSStqYkl3bjdaeDhrUXpReWgzV0JLSENNUjFKYXhaVTlKZUxqNUNLZVV1SU51b2ljawpGeEF3cUJhMGlDbDlIRmNEWHN1WEJoNDJNRnY0TlRMSFdWdXAvcDZPenFYR1N4YXRzdXZtampVVWVWbDJ5VUVxClh3VU9JZ2g4Uy91aXlsTitXUlJQMExGVDVJbkxxK2Zkay9JbWpyMnRiQ1A2djZZdVdIeTNsbVZrbStsMzZpVjYKUnRxZ2RNdUhBQUk1UGxvNHQ2Q3pOZHVaNXhXdVlEeHlxRHZ6QzZTL08xOGZYTHFQbEhkak9KWXdVbXc5QWJzdQphWDNnOHF6K2VoUmZJZGI4dmxwN2dCQUNFRXNkR09iQnhjRERlSDV5WDZJYzJGTTM0dzMzTWw1Zk5WSkxiWWpwCmpSaFRwUGM9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0= - client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBMG16d0NLYjJCZStwOXJGbHIyQW9CbFZvWFpOdGpqODJVS2luT0hvZUpkdkFscEhPCmtnRXY4dy9GL1hRV2FKdXB0ZzljMm9UeWpGRksyOTRaZWQxb1l4cjBLSjVsRnU4VlZGSU12aE5qNkxPYXBLYVYKakxuOUYvWUF0c092SU80bmxGYVUyaDJTbS9nUzRERkpVMUNTaDNVVUY4dlZ2WThWNktsOXIvdHg1RDRPUENSNwp2RXZqNUpJeHp5bHY5bjYyQzdDQzhFVjcyMlpsWFEyblFneWR5ODcwVmJJek9QL0pwVklKaWdPYVg4SHpTc1Y5CmwvN2dQRGhNcHNCS1JSWU16UDhqTFVqaFJPZUF0RlN1VGJrS2MrM1RGU3Z6ZXYvNTlvQ3U2NzBiUXBIR05xZm4Kc0ovano4NjVaVHc1NkpxUllZZlZEUE5aM0VmclF2MmphVGVWdndJREFRQUJBb0lCQUhiK3Zkcld1QWZ4V1JjMQppSUJkaXlUUnhad3RVK1grSHpRTHU4eTNXamNraDB0ZWx3ZC8vbUtnaCtGbGRZcWdoSjk5SHBCQVlIMHpOM2NRCnFTMStKSnJ1MGlYcU5OWjhTMzZNaThJdGF2RmNKOVoya2Rtc0Q5RXQvNGRiTDcwYWtHMVZiUVhQZFdvWUpteDcKMHJCM3Z2VTFJekMxeDJCVCt4ckFSbUJ2ZkIrS1piY2JPeHI2UUpWRHVuN1RCWTJxS0M3dFJWT3FrMmwwMVRoeAo2VjFPaW1vUFJkc1psNXBWRXNoMUQyci9zeFVNdzVxK1JHZWRFaFNEWnJqemxub1RyRHAveTlwRVFGYWFqN3RBCnhwZm1UaUd4Z3BaVTROeVNNNE01WllUQlhPSVp3TUkrUGNIdm9GL3RxNW9QYmduZ3JEd3I3WUFtUG1uc2d1anIKOVlxMWlYa0NnWUVBNktOOFRWbEI4NXR5Q2t0OUFYYW90Z09FdW94eEJCOHJUeU11RkxXZ0tudGdyNWpvWmFXOApBQUp1UWFqZGR3QUZOdmJvUGY1MGgvdVB3NDFSVFB1cVFLcVQzMFdvQWdSWjFjVGxXSTdITDBjbi8wM015UUxFClRCTFZaYzNIeTFnUGNPQ2NxTTIvcnFtMjgrTHkrUHo1RzZTSXFnYVJDN1dEUmhnZTZCU2xMSXNDZ1lFQTU0NXAKMko4Y2RDNXV5TkN3WDR2MTdZTWl3THlpUTJQSSs4bjl1MmV5YXpOYUtVakxQNjVvSHRaR3BpMWZiYjMvbkI4Rgp4Rmw2Q0FEMTdzdm11bjVJaTlFMndqanQ2RmZ0M2RoVzZ2cDVmRHpnYnVHdnEvMWF5SVdXMGdRdUgwaXBTUytQCjVCWGtaZmxyNmxkOG8xWTVDYm50dmgwRmtWcUlvSFhmZDdqRDNoMENnWUF2dGovckdUQkdodzMzZ0kzdHUrbFUKRXhxb3E5OE42bHFXaTIrSy9ja1FqV2RCL0VsRmQ1MDhIME4vb2VOeDRZWXdSS2xTOVpudXZiQTVlZ1UvbGFuUQpRMnU2Y2UzSUpCNU1rbW83SkdWRGxBTm5BQXpNK1ExTFVOMkZXSlB1WjJ3YlNXVzMxU0xIN3JUQUJqNnd4Q2xPCit3cXdYRkpwSlZ2OEJ6eG5lcng1aXdLQmdRRFNNNWpvZWNBVnc5VHozV1pPeGpwUkN6c3dKYzFUU2JFNHlTdloKeFEvVmV2OTgvTVR0SVVFSnhWRExKK002TjBGa0UxRmo1aW4rUEFrRlp2b2tEK0dCYzBHM1hJQ3o2NlpXUGo3ZwovdEhyczBPMXhKQjVtL3Vlam83VVJkRXN1d1JhR0tBUjJNSEd4SDV2ZEU0RDVKRFF3SVVPa1QvdHJIYmMyMFFxClpZTUJEUUtCZ0NIbUxXb1h3MUs3cWhOK3pOM1ZSNnNOeU5UUmNqWjVDOHN6THJ6RGVodmVvSjRiSHM3WXZxc0YKZnpVTHUxbmIzVTMxd3ljaWw3MHQ4ODVCSVhncWlyRk82S0ljVlV0U0prbk1Tb3l0SnVKR0t3THNaRmg4MHVZZwpLNU5vekJjbEN1RCtzOG1xZVdWT2FjeDdHa3dLVU5kcHdjaXdicHJwR0FTWmVRV0poVEFiCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== \ No newline at end of file diff --git a/src/main/java/kr/co/mcmp/ape/cbtumblebug/dto/K8sClusterDto.java b/src/main/java/kr/co/mcmp/ape/cbtumblebug/dto/K8sClusterDto.java index 6039208..b463e73 100644 --- a/src/main/java/kr/co/mcmp/ape/cbtumblebug/dto/K8sClusterDto.java +++ b/src/main/java/kr/co/mcmp/ape/cbtumblebug/dto/K8sClusterDto.java @@ -1,16 +1,19 @@ package kr.co.mcmp.ape.cbtumblebug.dto; +import java.util.List; +import java.util.Map; + import com.fasterxml.jackson.annotation.JsonProperty; + import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; - -import java.util.List; -import java.util.Map; +import lombok.ToString; @Data @NoArgsConstructor +@ToString @AllArgsConstructor @ApiModel(description = "K8s Cluster 정보") public class K8sClusterDto { @@ -69,6 +72,7 @@ public class K8sClusterDto { @Data @NoArgsConstructor + @ToString @AllArgsConstructor public static class ConnectionConfig { @JsonProperty("configName") @@ -105,6 +109,7 @@ public static class ConnectionConfig { @Data @NoArgsConstructor @AllArgsConstructor + @ToString public static class RegionZoneInfo { @JsonProperty("assignedRegion") private String assignedRegion; @@ -116,6 +121,7 @@ public static class RegionZoneInfo { @Data @NoArgsConstructor @AllArgsConstructor + @ToString public static class RegionDetail { @JsonProperty("regionId") private String regionId; @@ -136,6 +142,7 @@ public static class RegionDetail { @Data @NoArgsConstructor @AllArgsConstructor + @ToString public static class Location { @JsonProperty("display") private String display; @@ -150,6 +157,7 @@ public static class Location { @Data @NoArgsConstructor @AllArgsConstructor + @ToString public static class CspViewK8sClusterDetail { @JsonProperty("IId") private IID iid; @@ -180,6 +188,7 @@ public static class CspViewK8sClusterDetail { } @Data + @ToString @NoArgsConstructor @AllArgsConstructor public static class IID { @@ -191,6 +200,7 @@ public static class IID { } @Data + @ToString @NoArgsConstructor @AllArgsConstructor public static class Network { @@ -210,6 +220,7 @@ public static class Network { @Data @NoArgsConstructor @AllArgsConstructor + @ToString public static class NodeGroup { @JsonProperty("IId") private IID iid; @@ -254,6 +265,7 @@ public static class NodeGroup { @Data @NoArgsConstructor @AllArgsConstructor + @ToString public static class AccessInfo { @JsonProperty("Endpoint") private String endpoint; @@ -265,6 +277,7 @@ public static class AccessInfo { @Data @NoArgsConstructor @AllArgsConstructor + @ToString public static class Addons { @JsonProperty("KeyValueList") private List keyValueList; @@ -273,6 +286,7 @@ public static class Addons { @Data @NoArgsConstructor @AllArgsConstructor + @ToString public static class KeyValue { @JsonProperty("key") private String key; diff --git a/src/main/java/kr/co/mcmp/softwarecatalog/kubernetes/service/HelmChartService.java b/src/main/java/kr/co/mcmp/softwarecatalog/kubernetes/service/HelmChartService.java index 4f7eedd..7ac094f 100644 --- a/src/main/java/kr/co/mcmp/softwarecatalog/kubernetes/service/HelmChartService.java +++ b/src/main/java/kr/co/mcmp/softwarecatalog/kubernetes/service/HelmChartService.java @@ -1,6 +1,10 @@ package kr.co.mcmp.softwarecatalog.kubernetes.service; +import java.io.IOException; import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import org.springframework.stereotype.Service; @@ -8,78 +12,104 @@ import com.marcnuri.helm.InstallCommand; import com.marcnuri.helm.Release; +import io.fabric8.kubernetes.client.Config; +import io.fabric8.kubernetes.client.KubernetesClient; +import kr.co.mcmp.ape.cbtumblebug.api.CbtumblebugRestApi; +import kr.co.mcmp.ape.cbtumblebug.dto.K8sClusterDto; import kr.co.mcmp.softwarecatalog.SoftwareCatalog; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; - @Service @Slf4j @RequiredArgsConstructor public class HelmChartService { + private final CbtumblebugRestApi api; - public Release deployHelmChart(String namespace, SoftwareCatalog catalog) { + public Release deployHelmChart(KubernetesClient client, String namespace, SoftwareCatalog catalog, + String clusterName) { + Path tempConfigFile = null; + K8sClusterDto dto = api.getK8sClusterByName(namespace, clusterName); + String cspType = dto.getConnectionConfig().getProviderName().toUpperCase(); + log.info("CSP Type: {}", cspType); + try { addHelmRepository(catalog); + String kubeconfig = dto.getCspViewK8sClusterDetail().getAccessInfo().getKubeconfig(); - // Helm 설치 명령어 구성 - InstallCommand installCommand = Helm.install( - catalog.getHelmChart().getRepositoryName() + "/" + catalog.getHelmChart().getChartName()) - .withName(catalog.getHelmChart().getChartName()) - .withNamespace(namespace) - .withVersion(catalog.getHelmChart().getChartVersion()) - .set("replicaCount", catalog.getMinReplicas()) - .set("image.repository", catalog.getHelmChart().getImageRepository()) - .set("image.tag", "latest") - .set("image.pullPolicy", "Always") - .set("service.port", catalog.getDefaultPort()) - .set("resources.requests.cpu", catalog.getMinCpu().toString()) - .set("resources.requests.memory", catalog.getMinMemory() + "Mi") - .set("resources.limits.cpu", catalog.getRecommendedCpu().toString()) - .set("resources.limits.memory", catalog.getRecommendedMemory() + "Mi"); - - // HPA 설정 추가 + // GCP의 경우 gke-gcloud-auth-plugin 관련 설정 제거 + if ("GCP".equalsIgnoreCase(cspType)) { + kubeconfig = kubeconfig.replaceAll("command:\\s*gke-gcloud-auth-plugin", "") + .replaceAll("apiVersion:\\s*client.authentication.k8s.io/.*\\n", "") + .replaceAll("installHint:.*\\n", "") + .replaceAll("provideClusterInfo:.*\\n", ""); + } + tempConfigFile = createTempKubeconfigFile(kubeconfig); + + InstallCommand installCommand = createInstallCommand(catalog, namespace, tempConfigFile); + applyCspSpecificSettings(installCommand, cspType); + + // HPA 설정 if (Boolean.TRUE.equals(catalog.getHpaEnabled())) { installCommand .set("autoscaling.enabled", true) .set("autoscaling.minReplicas", catalog.getMinReplicas()) .set("autoscaling.maxReplicas", catalog.getMaxReplicas()) - .set("autoscaling.targetCPUUtilizationPercentage", catalog.getCpuThreshold().intValue()) - .set("autoscaling.targetMemoryUtilizationPercentage", catalog.getMemoryThreshold().intValue()); + .set("autoscaling.targetCPUUtilizationPercentage", catalog.getCpuThreshold()) + .set("autoscaling.targetMemoryUtilizationPercentage", catalog.getMemoryThreshold()); } - - // Helm 차트 설치 실행 + Release result = installCommand.call(); - - log.info("Helm Chart '{}' 버전 '{}'가 네임스페이스 '{}'에 배포됨 (HPA: {})", - catalog.getHelmChart().getChartName(), - "latest", - namespace, - catalog.getHpaEnabled()); + log.info("Helm Chart '{}' 배포 완료 - namespace: {}, CSP: {}", + catalog.getHelmChart().getChartName(), + namespace, + cspType); return result; + } catch (Exception e) { - log.error("Helm Chart 배포 중 오류 발생", e); + log.error("Helm Chart 배포 중 오류 발생 - CSP: {}", cspType, e); throw new RuntimeException("Helm Chart 배포 실패", e); + } finally { + deleteTempFile(tempConfigFile); } } - public void uninstallHelmChart(String namespace, SoftwareCatalog catalog) { - try { - String result = Helm.uninstall(catalog.getHelmChart().getChartName()) + + private InstallCommand createInstallCommand(SoftwareCatalog catalog, String namespace, Path configFile) { + return Helm.install(catalog.getHelmChart().getRepositoryName() + "/" + catalog.getHelmChart().getChartName()) + .withKubeConfig(configFile) + .withName(catalog.getHelmChart().getChartName()) .withNamespace(namespace) - .call(); - - boolean deleted = result != null && !result.isEmpty(); - - if (deleted) { - log.info("Helm Release '{}' 가 네임스페이스 '{}'에서 삭제됨", - catalog.getHelmChart().getChartName(), namespace); - } else { - log.warn("Helm Release '{}' 삭제 실패", - catalog.getHelmChart().getChartName()); - } - } catch (Exception e) { - log.error("Helm Release 삭제 중 오류 발생", e); - throw new RuntimeException("Helm Release 삭제 실패", e); + .withVersion(catalog.getHelmChart().getChartVersion()) + .set("replicaCount", catalog.getMinReplicas()) + .set("image.repository", catalog.getHelmChart().getImageRepository()) + .set("image.tag", "latest") + .set("image.pullPolicy", "Always") + .set("service.port", catalog.getDefaultPort()) + .set("resources.requests.cpu", catalog.getMinCpu().toString()) + .set("resources.requests.memory", catalog.getMinMemory() + "Mi") + .set("resources.limits.cpu", catalog.getRecommendedCpu().toString()) + .set("resources.limits.memory", catalog.getRecommendedMemory() + "Mi") + .set("persistence.enabled", false) + .set("securityContext.enabled", false) + .set("serviceAccount.create", true) + .withTimeout(300) + .waitReady(); + } + + private void applyCspSpecificSettings(InstallCommand command, String cspType) { + log.info("Applying CSP specific settings for: {}", cspType); + switch (cspType) { + case "GCP": + command.set("gcp.auth.enabled", false) + .set("serviceAccount.annotations.iam\\.gke\\.io/gcp-service-account", "false") + .set("rbac.create", true); + break; + case "AZURE": + command.set("azure.auth.enabled", false) + .set("serviceAccount.annotations.azure\\.workload\\.identity/use", "false"); + break; + default: + log.warn("알 수 없는 CSP 타입: {}", cspType); } } @@ -90,4 +120,54 @@ private void addHelmRepository(SoftwareCatalog catalog) throws Exception { .call(); Helm.repo().update(); } -} \ No newline at end of file + + private Path createTempKubeconfigFile(String kubeconfig) throws IOException { + Path tempFile = Files.createTempFile("kubeconfig", ".yaml"); + Files.write(tempFile, kubeconfig.getBytes(StandardCharsets.UTF_8)); + return tempFile; + } + + private void deleteTempFile(Path tempFile) { + if (tempFile != null) { + try { + Files.deleteIfExists(tempFile); + } catch (IOException e) { + log.warn("임시 파일 삭제 실패: {}", tempFile, e); + } + } + } + + public void uninstallHelmChart(String namespace, SoftwareCatalog catalog, String clusterName) { + Path tempConfigFile = null; + try { + K8sClusterDto dto = api.getK8sClusterByName(namespace, clusterName); + String cspType = dto.getConnectionConfig().getProviderName().toUpperCase(); + String kubeconfig = dto.getCspViewK8sClusterDetail().getAccessInfo().getKubeconfig(); + + // GCP의 경우 gke-gcloud-auth-plugin 관련 설정 제거 + if ("GCP".equalsIgnoreCase(cspType)) { + kubeconfig = kubeconfig.replaceAll("command:\\s*gke-gcloud-auth-plugin", "") + .replaceAll("apiVersion:\\s*client.authentication.k8s.io/.*\\n", "") + .replaceAll("installHint:.*\\n", "") + .replaceAll("provideClusterInfo:.*\\n", ""); + } + tempConfigFile = createTempKubeconfigFile(kubeconfig); + + String result = Helm.uninstall(catalog.getHelmChart().getChartName()) + .withKubeConfig(tempConfigFile) + .withNamespace(namespace) + .call(); + + log.info("Helm Release '{}' 삭제 완료 - namespace: {}, CSP: {}", + catalog.getHelmChart().getChartName(), + namespace, + cspType); + + } catch (Exception e) { + log.error("Helm Release 삭제 실패", e); + throw new RuntimeException("Helm Release 삭제 실패", e); + } finally { + deleteTempFile(tempConfigFile); + } + } +} diff --git a/src/main/java/kr/co/mcmp/softwarecatalog/kubernetes/service/KubernetesDeployService.java b/src/main/java/kr/co/mcmp/softwarecatalog/kubernetes/service/KubernetesDeployService.java index 582b55b..4c492cc 100644 --- a/src/main/java/kr/co/mcmp/softwarecatalog/kubernetes/service/KubernetesDeployService.java +++ b/src/main/java/kr/co/mcmp/softwarecatalog/kubernetes/service/KubernetesDeployService.java @@ -30,36 +30,37 @@ public class KubernetesDeployService { public DeploymentHistory deployApplication(String namespace, String clusterName, SoftwareCatalog catalog, String username) { - try { - KubernetesClient client = clientFactory.getClient(namespace, clusterName); + try (KubernetesClient client = clientFactory.getClient(namespace, clusterName)) { + // KubernetesClient client = clientFactory.getClient(namespace, clusterName); namespaceService.ensureNamespaceExists(client, namespace); - Release result = helmChartService.deployHelmChart(namespace, catalog); + Release result = helmChartService.deployHelmChart(client, namespace, catalog, clusterName); String podStatus = KubernetesUtils.getPodStatus(client, namespace, catalog.getHelmChart().getChartName()); - Integer servicePort = KubernetesUtils.getServicePort(client, namespace, catalog.getHelmChart().getChartName()); + Integer servicePort = KubernetesUtils.getServicePort(client, namespace, + catalog.getHelmChart().getChartName()); return createDeploymentHistory( - namespace, - clusterName, - catalog, - username, - ActionType.INSTALL, - podStatus, - servicePort, - "SUCCESS"); - } catch (Exception e) { - log.error("애플리케이션 배포 중 오류 발생", e); - return createDeploymentHistory( - namespace, - clusterName, - catalog, - username, - ActionType.INSTALL, - "Failed", - null, - "FAILED"); - } + namespace, + clusterName, + catalog, + username, + ActionType.INSTALL, + podStatus, + servicePort, + "SUCCESS"); + } catch (Exception e) { + log.error("애플리케이션 배포 중 오류 발생", e); + return createDeploymentHistory( + namespace, + clusterName, + catalog, + username, + ActionType.INSTALL, + "Failed", + null, + "FAILED"); + } } public DeploymentHistory stopApplication(String namespace, String clusterName, SoftwareCatalog catalog, @@ -67,7 +68,7 @@ public DeploymentHistory stopApplication(String namespace, String clusterName, S try { KubernetesClient client = clientFactory.getClient(namespace, clusterName); - helmChartService.uninstallHelmChart(namespace, catalog); + helmChartService.uninstallHelmChart(namespace, catalog, clusterName); String podStatus = KubernetesUtils.getPodStatus(client, namespace, catalog.getHelmChart().getChartName()); Integer servicePort = KubernetesUtils.getServicePort(client, namespace, @@ -88,22 +89,22 @@ public DeploymentHistory stopApplication(String namespace, String clusterName, S } private DeploymentHistory createDeploymentHistory( - String namespace, String clusterName, SoftwareCatalog catalog, String username, ActionType actionType, - String podStatus, Integer servicePort, String status) { - - User user = StringUtils.isNotBlank(username) ? userRepository.findByUsername(username).orElse(null) : null; - - return DeploymentHistory.builder() - .namespace(namespace) - .clusterName(clusterName) - .catalog(catalog) - .executedBy(user) - .podStatus(podStatus) - .servicePort(servicePort) - .deploymentType(DeploymentType.K8S) - .status(status) - .actionType(actionType) - .executedAt(LocalDateTime.now()) - .build(); -} + String namespace, String clusterName, SoftwareCatalog catalog, String username, ActionType actionType, + String podStatus, Integer servicePort, String status) { + + User user = StringUtils.isNotBlank(username) ? userRepository.findByUsername(username).orElse(null) : null; + + return DeploymentHistory.builder() + .namespace(namespace) + .clusterName(clusterName) + .catalog(catalog) + .executedBy(user) + .podStatus(podStatus) + .servicePort(servicePort) + .deploymentType(DeploymentType.K8S) + .status(status) + .actionType(actionType) + .executedAt(LocalDateTime.now()) + .build(); + } } \ No newline at end of file diff --git a/src/main/java/kr/co/mcmp/softwarecatalog/kubernetes/service/KubernetesMonitoringService.java b/src/main/java/kr/co/mcmp/softwarecatalog/kubernetes/service/KubernetesMonitoringService.java index a1e8a0e..d060c68 100644 --- a/src/main/java/kr/co/mcmp/softwarecatalog/kubernetes/service/KubernetesMonitoringService.java +++ b/src/main/java/kr/co/mcmp/softwarecatalog/kubernetes/service/KubernetesMonitoringService.java @@ -40,28 +40,22 @@ public class KubernetesMonitoringService { @Scheduled(fixedRate = 60000) // 1분마다 실행 public void monitorKubernetesResources() { log.info("Starting Kubernetes resource monitoring"); - + List activeDeployments = getActiveK8sDeployments(); - + for (DeploymentHistory deployment : activeDeployments) { String namespace = deployment.getNamespace(); String clusterName = deployment.getClusterName(); - - KubernetesClient client = null; - try { - client = clientFactory.getClient(namespace, clusterName); - + + try (KubernetesClient client = clientFactory.getClient(namespace, clusterName)) { + if (!isMetricsServerInstalled(client)) { installMetricsServer(client); } - + updateApplicationStatus(deployment, client); } catch (Exception e) { log.error("Error monitoring Kubernetes resources for deployment: {}", deployment.getId(), e); - } finally { - if (client != null) { - clientFactory.releaseClient(namespace, clusterName); - } } } } @@ -90,15 +84,24 @@ private void updateApplicationStatus(DeploymentHistory deployment, KubernetesCli String appName = deployment.getCatalog().getHelmChart().getChartName(); Long catalogId = deployment.getCatalog().getId(); User user = deployment.getExecutedBy(); - + List pods = client.pods().inNamespace(namespace) - .list() - .getItems() - .stream() - .filter(pod -> pod.getMetadata().getName().startsWith(appName)) - .collect(Collectors.toList()); + .list() + .getItems() + .stream() + .filter(pod -> pod.getMetadata().getName().startsWith(appName)) + .collect(Collectors.toList()); + + log.info("Pod names in namespace: {}", + client.pods().inNamespace(namespace).list().getItems() + .stream() + .map(pod -> pod.getMetadata().getName()) + .collect(Collectors.joining(", ")) + ); + - ApplicationStatus status = statusRepository.findTopByCatalogIdOrderByCheckedAtDesc(catalogId).orElse(new ApplicationStatus()); + ApplicationStatus status = statusRepository.findTopByCatalogIdOrderByCheckedAtDesc(catalogId) + .orElse(new ApplicationStatus()); long runningPods = pods.stream() .filter(pod -> "Running".equalsIgnoreCase(pod.getStatus().getPhase())) @@ -109,7 +112,7 @@ private void updateApplicationStatus(DeploymentHistory deployment, KubernetesCli status.setCpuUsage((Double) resourceUsage.get("cpuPercentage")); status.setMemoryUsage((Double) resourceUsage.get("memoryPercentage")); status.setStatus((String) resourceUsage.get("status")); - status.setServicePort((Integer)resourceUsage.get("port")); + status.setServicePort((Integer) resourceUsage.get("port")); status.setCheckedAt(LocalDateTime.now()); status.setClusterName(clusterName); status.setNamespace(namespace); @@ -124,7 +127,7 @@ private void checkMetricsServerStatus(KubernetesClient client) { .inNamespace("kube-system") .withName("metrics-server") .get(); - + if (metricsServer == null) { log.error("Metrics Server not found. Please install it."); } else { @@ -134,7 +137,7 @@ private void checkMetricsServerStatus(KubernetesClient client) { } else { log.info("Metrics Server is running with {} ready replicas", readyReplicas); } - } + } } private Map getResourceUsagePercentage(KubernetesClient client, String namespace, String appName) { @@ -148,19 +151,18 @@ private Map getResourceUsagePercentage(KubernetesClient client, // log.info("appName : " + appName); List podMetrics = client.genericKubernetesResources(context) - .inNamespace(namespace) - // .withLabel("app", appName) - .list() - .getItems(); + .inNamespace(namespace) + // .withLabel("app", appName) + .list() + .getItems(); log.info("Found {} pod metrics in namespace {}", podMetrics.size(), namespace); - + if (podMetrics.isEmpty()) { log.warn("No pod metrics found. Checking if metrics-server is running..."); checkMetricsServerStatus(client); return Map.of("cpu", 0.0, "memory", 0.0, "status", "UNKNOWN"); } - double totalCpuUsage = 0.0; double totalMemoryUsage = 0.0; @@ -170,8 +172,9 @@ private Map getResourceUsagePercentage(KubernetesClient client, String podName = podMetric.getMetadata().getName(); if (podName.startsWith(appName.toLowerCase())) { try { - - List> containers = (List>) podMetric.getAdditionalProperties().get("containers"); + + List> containers = (List>) podMetric + .getAdditionalProperties().get("containers"); if (containers != null) { for (Map container : containers) { Map usage = (Map) container.get("usage"); @@ -188,33 +191,31 @@ private Map getResourceUsagePercentage(KubernetesClient client, log.error("Error processing pod metric: {}", e.getMessage(), e); } } - } + } double totalCpuCapacity = getTotalCpuCapacity(client); double totalMemoryCapacity = getTotalMemoryCapacity(client); - - + double cpuUsagePercentage = (totalCpuUsage / totalCpuCapacity) * 100; double memoryUsagePercentage = (totalMemoryUsage / (totalMemoryCapacity * 1024)) * 100; // 소수점 2자리까지 반올림 Double roundedCpuUsage = Math.round(cpuUsagePercentage * 100.0) / 100.0; Double roundedMemoryUsage = Math.round(memoryUsagePercentage * 100.0) / 100.0; - + // 서비스 포트 정보 수집 List ports = new ArrayList<>(); client.services().inNamespace(namespace).list().getItems().stream() - .filter(service -> service.getMetadata().getName().startsWith(appName.toLowerCase())) - .forEach(service -> { - service.getSpec().getPorts().forEach(servicePort -> { - if (servicePort.getNodePort() != null) { - ports.add(servicePort.getNodePort()); - } - }); - }); + .filter(service -> service.getMetadata().getName().startsWith(appName.toLowerCase())) + .forEach(service -> { + service.getSpec().getPorts().forEach(servicePort -> { + if (servicePort.getNodePort() != null) { + ports.add(servicePort.getNodePort()); + } + }); + }); Integer primaryPort = ports.isEmpty() ? null : ports.get(0); - Map result = new HashMap<>(); result.put("cpuPercentage", roundedCpuUsage != null ? roundedCpuUsage : 0.0); result.put("memoryPercentage", roundedMemoryUsage != null ? roundedMemoryUsage : 0.0); @@ -236,8 +237,6 @@ private double getTotalMemoryCapacity(KubernetesClient client) { .sum(); } - - private double parseCpuUsage(String cpuUsage) { if (cpuUsage.endsWith("n")) { return Double.parseDouble(cpuUsage.substring(0, cpuUsage.length() - 1)) / 1_000_000_000.0; @@ -262,14 +261,13 @@ private double parseMemoryUsage(String memoryUsage) { } } - private boolean isMetricsServerInstalled(KubernetesClient client) { return client.apps().deployments().inNamespace("kube-system").withName("metrics-server").get() != null; } private void installMetricsServer(KubernetesClient client) throws IOException { log.info("Metrics Server not found. Installing..."); - + String metricsServerYaml = ""; try { metricsServerYaml = downloadMetricsServerYaml(); @@ -277,7 +275,7 @@ private void installMetricsServer(KubernetesClient client) throws IOException { e.printStackTrace(); } client.resourceList(metricsServerYaml).createOrReplace(); - + log.info("Metrics Server installation completed"); try { waitForMetricsServerReady(client); @@ -298,8 +296,10 @@ private void waitForMetricsServerReady(KubernetesClient client) throws Interrupt int maxAttempts = 30; int attempt = 0; while (attempt < maxAttempts) { - Deployment metricsServer = client.apps().deployments().inNamespace("kube-system").withName("metrics-server").get(); - if (metricsServer != null && metricsServer.getStatus().getReadyReplicas() != null && metricsServer.getStatus().getReadyReplicas() > 0) { + Deployment metricsServer = client.apps().deployments().inNamespace("kube-system").withName("metrics-server") + .get(); + if (metricsServer != null && metricsServer.getStatus().getReadyReplicas() != null + && metricsServer.getStatus().getReadyReplicas() > 0) { log.info("Metrics Server is ready"); return; } diff --git a/src/main/java/kr/co/mcmp/softwarecatalog/kubernetes/service/KubernetesOperationService.java b/src/main/java/kr/co/mcmp/softwarecatalog/kubernetes/service/KubernetesOperationService.java index a1d9ae7..0a47423 100644 --- a/src/main/java/kr/co/mcmp/softwarecatalog/kubernetes/service/KubernetesOperationService.java +++ b/src/main/java/kr/co/mcmp/softwarecatalog/kubernetes/service/KubernetesOperationService.java @@ -73,7 +73,7 @@ public void stopApplication(String namespace, String clusterName, SoftwareCatalo public void uninstallApplication(String namespace, String clusterName, SoftwareCatalog catalog, String username) { try { - helmChartService.uninstallHelmChart(namespace, catalog); + helmChartService.uninstallHelmChart(namespace, catalog,clusterName); } catch (Exception e) { log.error("애플리케이션 제거 중 오류 발생", e); throw new RuntimeException("애플리케이션 제거 실패", e); diff --git a/src/main/java/kr/co/mcmp/softwarecatalog/kubernetes/util/KubernetesClientFactory.java b/src/main/java/kr/co/mcmp/softwarecatalog/kubernetes/util/KubernetesClientFactory.java index 0093573..fbf811c 100644 --- a/src/main/java/kr/co/mcmp/softwarecatalog/kubernetes/util/KubernetesClientFactory.java +++ b/src/main/java/kr/co/mcmp/softwarecatalog/kubernetes/util/KubernetesClientFactory.java @@ -1,15 +1,5 @@ package kr.co.mcmp.softwarecatalog.kubernetes.util; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; - -import javax.annotation.PreDestroy; - import org.springframework.stereotype.Component; import io.fabric8.kubernetes.client.Config; @@ -17,82 +7,90 @@ import io.fabric8.kubernetes.client.KubernetesClientBuilder; import kr.co.mcmp.ape.cbtumblebug.api.CbtumblebugRestApi; import kr.co.mcmp.ape.cbtumblebug.dto.K8sClusterDto; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; + + @Component @Slf4j +@RequiredArgsConstructor public class KubernetesClientFactory { - private final Map clientCache = new ConcurrentHashMap<>(); private final CbtumblebugRestApi api; - public KubernetesClientFactory(CbtumblebugRestApi api) { - this.api = api; - } - - public synchronized KubernetesClient getClient(String namespace, String clusterName) { - String cacheKey = namespace + ":" + clusterName; - return clientCache.computeIfAbsent(cacheKey, k -> createKubernetesClient(namespace, clusterName)); - } - - public synchronized void releaseClient(String namespace, String clusterName) { - String cacheKey = namespace + ":" + clusterName; - KubernetesClient client = clientCache.remove(cacheKey); - if (client != null) { - client.close(); - } - } - - private KubernetesClient createKubernetesClient(String namespace, String clusterName) { + public KubernetesClient getClient(String namespace, String clusterName) { try { K8sClusterDto dto = api.getK8sClusterByName(namespace, clusterName); + String cspType = dto.getConnectionConfig().getProviderName().toLowerCase(); String kubeconfig = extractKubeconfig(dto); - Config config = createKubernetesConfig(kubeconfig); - return new KubernetesClientBuilder().withConfig(config).build(); + + if ("gcp".equals(cspType)) { + kubeconfig = processGcpKubeconfig(kubeconfig); + } + + Config config = Config.fromKubeconfig(kubeconfig); + configureClient(config, cspType); + + return new KubernetesClientBuilder() + .withConfig(config) + .build(); } catch (Exception e) { - log.error("Kubernetes 클라이언트 생성 실패", e); + log.error("Kubernetes 클라이언트 생성 실패 - namespace: {}, cluster: {}", + namespace, clusterName, e); throw new RuntimeException("Kubernetes 클라이언트 생성 중 오류 발생", e); } } - - private String extractKubeconfig(K8sClusterDto clusterDto) { - return clusterDto.getCspViewK8sClusterDetail().getAccessInfo().getKubeconfig(); - } - private Config createKubernetesConfig(String kubeconfig) throws IOException { - Path tempConfigFile = createTempKubeconfigFile(kubeconfig); - try { - return Config.fromKubeconfig(readKubeconfigContent(tempConfigFile)); - } finally { - deleteTempFile(tempConfigFile); + private String processGcpKubeconfig(String kubeconfig) { + String[] sections = kubeconfig.split("users:"); + if (sections.length < 2) { + return kubeconfig; } - } - private Path createTempKubeconfigFile(String kubeconfig) throws IOException { - Path tempFile = Files.createTempFile("kubeconfig", ".yaml"); - Files.write(tempFile, kubeconfig.getBytes(StandardCharsets.UTF_8)); - return tempFile; - } + // users 섹션 이전 부분 유지 + StringBuilder processedConfig = new StringBuilder(sections[0]); + processedConfig.append("users:\n"); + processedConfig.append("- name: gke-user\n"); + processedConfig.append(" user:\n"); + processedConfig.append(" token: gke-token\n"); + + // current-context 이후 부분 찾아서 추가 + String remaining = sections[1]; + int contextIndex = remaining.indexOf("current-context:"); + if (contextIndex >= 0) { + processedConfig.append(remaining.substring(contextIndex)); + } - private String readKubeconfigContent(Path configFile) throws IOException { - return new String(Files.readAllBytes(configFile)); + return processedConfig.toString(); } - private void deleteTempFile(Path tempFile) { - try { - Files.deleteIfExists(tempFile); - } catch (IOException e) { - log.warn("임시 파일 삭제 실패: {}", tempFile, e); + private void configureClient(Config config, String cspType) { + config.setTrustCerts(true); + config.setConnectionTimeout(30000); + config.setRequestTimeout(30000); + // config.setWebsocketTimeout(30000); + + if ("gcp".equals(cspType)) { + config.setImpersonateUsername(null); + config.setAuthProvider(null); } } - public void closeClient(String clusterId) { - Optional.ofNullable(clientCache.remove(clusterId)) - .ifPresent(KubernetesClient::close); + private String extractKubeconfig(K8sClusterDto clusterDto) { + if (clusterDto == null || clusterDto.getCspViewK8sClusterDetail() == null + || clusterDto.getCspViewK8sClusterDetail().getAccessInfo() == null) { + throw new IllegalStateException("클러스터 정보가 올바르지 않습니다"); + } + return clusterDto.getCspViewK8sClusterDetail().getAccessInfo().getKubeconfig(); } - @PreDestroy - public void closeAllClients() { - clientCache.values().forEach(KubernetesClient::close); - clientCache.clear(); + public void releaseClient(KubernetesClient client) { + if (client != null) { + try { + client.close(); + } catch (Exception e) { + log.warn("클라이언트 해제 중 오류 발생", e); + } + } } -} \ No newline at end of file +}