diff --git a/.vscode/launch.json b/.vscode/launch.json index 196917bc..d4389f97 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -149,6 +149,47 @@ "vmArgs": "-Dlog4j2.configurationFile=log4j2.xml", "preLaunchTask": "Build and Install Operator library" }, + { + // This works with terraform config 2-04_try-now_paths_eager_start + "type": "java", + "name": "Debug Operator (eager start try-now-paths)", + "request": "launch", + "cwd": "${workspaceFolder}/java/operator/org.eclipse.theia.cloud.defaultoperator", + "mainClass": "org.eclipse.theia.cloud.defaultoperator.DefaultTheiaCloudOperatorLauncher", + "args": [ + "--keycloak", + "--keycloakURL", + "https://${input:minikubeIP}.nip.io/keycloak/", + "--keycloakRealm", + "TheiaCloud", + "--keycloakClientId", + "theia-cloud", + "--usePaths", + "--instancesPath", + "instances", + "--instancesHost", + "${input:minikubeIP}.nip.io", + "--serviceUrl", + "https://${input:minikubeIP}.nip.io/service", + "--cloudProvider", + "MINIKUBE", + "--sessionsPerUser", + "3", + "--appId", + "asdfghjkl", + "--storageClassName", + "default", + "--requestedStorage", + "250Mi", + "--bandwidthLimiter", + "WONDERSHAPER", + "--oAuth2ProxyVersion", + "v7.5.1", + "--eagerStart" + ], + "vmArgs": "-Dlog4j2.configurationFile=log4j2.xml", + "preLaunchTask": "Build and Install Operator library" + }, { // Attach to the service running (Task: Run Service) "type": "java", diff --git a/java/common/org.eclipse.theia.cloud.common/src/main/java/org/eclipse/theia/cloud/common/util/LabelsUtil.java b/java/common/org.eclipse.theia.cloud.common/src/main/java/org/eclipse/theia/cloud/common/util/LabelsUtil.java index 7acbfa22..4162c540 100644 --- a/java/common/org.eclipse.theia.cloud.common/src/main/java/org/eclipse/theia/cloud/common/util/LabelsUtil.java +++ b/java/common/org.eclipse.theia.cloud.common/src/main/java/org/eclipse/theia/cloud/common/util/LabelsUtil.java @@ -2,6 +2,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.Set; import org.eclipse.theia.cloud.common.k8s.resource.appdefinition.AppDefinitionSpec; import org.eclipse.theia.cloud.common.k8s.resource.session.SessionSpec; @@ -17,6 +18,7 @@ public class LabelsUtil { public static final String LABEL_KEY_USER = LABEL_CUSTOM_PREFIX + "/user"; public static final String LABEL_KEY_APPDEF = LABEL_CUSTOM_PREFIX + "/app-definition"; + public static final String LABEL_KEY_SESSION_NAME = LABEL_CUSTOM_PREFIX + "/session"; public static Map createSessionLabels(SessionSpec sessionSpec, AppDefinitionSpec appDefinitionSpec) { @@ -26,6 +28,16 @@ public static Map createSessionLabels(SessionSpec sessionSpec, String sanitizedUser = sessionSpec.getUser().replaceAll("@", "_at_").replaceAll("[^a-zA-Z0-9]", "_"); labels.put(LABEL_KEY_USER, sanitizedUser); labels.put(LABEL_KEY_APPDEF, appDefinitionSpec.getName()); + labels.put(LABEL_KEY_SESSION_NAME, sessionSpec.getName()); return labels; } + + /** + * Returns the set of label keys that are specific to a specific session, i.e. the user and session name keys. + * + * @return The session specific label keys. + */ + public static Set getSessionSpecificLabelKeys() { + return Set.of(LABEL_KEY_SESSION_NAME, LABEL_KEY_USER); + } } diff --git a/java/common/org.eclipse.theia.cloud.common/src/main/java/org/eclipse/theia/cloud/common/util/NamingUtil.java b/java/common/org.eclipse.theia.cloud.common/src/main/java/org/eclipse/theia/cloud/common/util/NamingUtil.java index 242468a4..2c55a2bb 100644 --- a/java/common/org.eclipse.theia.cloud.common/src/main/java/org/eclipse/theia/cloud/common/util/NamingUtil.java +++ b/java/common/org.eclipse.theia.cloud.common/src/main/java/org/eclipse/theia/cloud/common/util/NamingUtil.java @@ -31,6 +31,12 @@ public final class NamingUtil { public static final char VALID_NAME_SUFFIX = 'z'; + /** + * Prefix for names generated based on an app definition and an index. These names are typically used for objects + * related to eagerly started instance pods (e.g. services or deployments). + */ + public static final String APP_DEFINITION_INSTANCE_PREFIX = "instance-"; + private static final Locale US_LOCALE = new Locale("en", "US"); private NamingUtil() { @@ -41,7 +47,7 @@ private NamingUtil() { * @see NamingUtil#createName(AppDefinition, int, String) */ public static String createName(AppDefinition appDefinition, int instance) { - return createName(appDefinition, instance); + return createName(appDefinition, instance, null); } /** @@ -56,10 +62,10 @@ public static String createName(AppDefinition appDefinition, int instance) { * same type for an AppDefinition. *

*

- * The created name contains a "session" prefix, the session's user and app definition, the identifier (if given), - * and the last segment of the Session's UID. User, app definition and identifier are shortened to keep the name - * within Kubernetes' character limit (63) minus 6 characters. The latter allows Kubernetes to add 6 characters at - * the end of deployment pod names while the pod names pod names will still contain the whole name of the deployment + * The created name contains a "instance-" prefix, instance number, the identifier (if given), and the last segment + * of the App Definition's UID. User, app definition and identifier are shortened to keep the name within + * Kubernetes' character limit (63) minus 6 characters. The latter allows Kubernetes to add 6 characters at the end + * of deployment pod names while the pod names pod names will still contain the whole name of the deployment *

* * @param appDefinition the {@link AppDefinition} @@ -69,7 +75,7 @@ public static String createName(AppDefinition appDefinition, int instance) { * @return the name */ public static String createName(AppDefinition appDefinition, int instance, String identifier) { - String prefix = "instance-" + instance; + String prefix = APP_DEFINITION_INSTANCE_PREFIX + instance; return createName(prefix, identifier, null, appDefinition.getSpec().getName(), appDefinition.getMetadata().getUid()); } @@ -174,7 +180,7 @@ private static String createName(String prefix, String identifier, String user, // If the user is an email address, only take the part before the @ sign because // this is usually sufficient to identify the user. - String userName = user.split("@")[0]; + String userName = user != null ? user.split("@")[0] : null; int infoSegmentLength; String shortenedIdentifier = null; diff --git a/java/common/org.eclipse.theia.cloud.common/src/test/java/org/eclipse/theia/cloud/common/util/NamingUtilTests.java b/java/common/org.eclipse.theia.cloud.common/src/test/java/org/eclipse/theia/cloud/common/util/NamingUtilTests.java index d68f2d0a..939b3b26 100644 --- a/java/common/org.eclipse.theia.cloud.common/src/test/java/org/eclipse/theia/cloud/common/util/NamingUtilTests.java +++ b/java/common/org.eclipse.theia.cloud.common/src/test/java/org/eclipse/theia/cloud/common/util/NamingUtilTests.java @@ -17,6 +17,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; +import org.eclipse.theia.cloud.common.k8s.resource.appdefinition.AppDefinition; +import org.eclipse.theia.cloud.common.k8s.resource.appdefinition.AppDefinitionSpec; import org.eclipse.theia.cloud.common.k8s.resource.session.Session; import org.eclipse.theia.cloud.common.k8s.resource.session.SessionSpec; import org.eclipse.theia.cloud.common.k8s.resource.workspace.Workspace; @@ -30,6 +32,22 @@ */ class NamingUtilTests { + @Test + void createName_AppDefinitionAndInstace() { + AppDefinition appDefinition = createAppDefinition(); + + String result = NamingUtil.createName(appDefinition, 1); + assertEquals("instance-1-some-app-definiti-381261d79c23", result); + } + + @Test + void createName_AppDefinitionAndInstaceAndIdentifier() { + AppDefinition appDefinition = createAppDefinition(); + + String result = NamingUtil.createName(appDefinition, 1, "longidentifier"); + assertEquals("instance-1-longidentif-some-app-de-381261d79c23", result); + } + @Test void createName_SessionAndNullIdentifier() { Session session = createSession(); @@ -94,6 +112,21 @@ void createName_WorkspaceAndIdentifier() { assertEquals("ws-longidentif-some-userna-test-app-de-381261d79c23", result); } + private AppDefinition createAppDefinition() { + AppDefinition appDefinition = new AppDefinition(); + ObjectMeta objectMeta = new ObjectMeta(); + objectMeta.setUid("6f1a8966-4d5a-41dc-82ba-381261d79c23"); + appDefinition.setMetadata(objectMeta); + AppDefinitionSpec spec = new AppDefinitionSpec() { + @Override + public String getName() { + return "some-app-definition"; + } + }; + appDefinition.setSpec(spec); + return appDefinition; + } + private Session createSession() { Session session = new Session(); ObjectMeta objectMeta = new ObjectMeta(); diff --git a/java/operator/org.eclipse.theia.cloud.operator/src/main/java/org/eclipse/theia/cloud/operator/di/AbstractTheiaCloudOperatorModule.java b/java/operator/org.eclipse.theia.cloud.operator/src/main/java/org/eclipse/theia/cloud/operator/di/AbstractTheiaCloudOperatorModule.java index d6bdcfd6..5d0a3354 100644 --- a/java/operator/org.eclipse.theia.cloud.operator/src/main/java/org/eclipse/theia/cloud/operator/di/AbstractTheiaCloudOperatorModule.java +++ b/java/operator/org.eclipse.theia.cloud.operator/src/main/java/org/eclipse/theia/cloud/operator/di/AbstractTheiaCloudOperatorModule.java @@ -31,7 +31,7 @@ import org.eclipse.theia.cloud.operator.handler.appdef.AppDefinitionHandler; import org.eclipse.theia.cloud.operator.handler.appdef.EagerStartAppDefinitionAddedHandler; import org.eclipse.theia.cloud.operator.handler.appdef.LazyStartAppDefinitionHandler; -import org.eclipse.theia.cloud.operator.handler.session.EagerStartSessionHandler; +import org.eclipse.theia.cloud.operator.handler.session.EagerSessionHandler; import org.eclipse.theia.cloud.operator.handler.session.LazySessionHandler; import org.eclipse.theia.cloud.operator.handler.session.SessionHandler; import org.eclipse.theia.cloud.operator.handler.ws.LazyWorkspaceHandler; @@ -143,7 +143,7 @@ protected Class bindWorkspaceHandler() { protected Class bindSessionHandler() { if (arguments.isEagerStart()) { - return EagerStartSessionHandler.class; + return EagerSessionHandler.class; } else { return LazySessionHandler.class; } diff --git a/java/operator/org.eclipse.theia.cloud.operator/src/main/java/org/eclipse/theia/cloud/operator/handler/session/EagerSessionHandler.java b/java/operator/org.eclipse.theia.cloud.operator/src/main/java/org/eclipse/theia/cloud/operator/handler/session/EagerSessionHandler.java new file mode 100644 index 00000000..a627f080 --- /dev/null +++ b/java/operator/org.eclipse.theia.cloud.operator/src/main/java/org/eclipse/theia/cloud/operator/handler/session/EagerSessionHandler.java @@ -0,0 +1,449 @@ +/******************************************************************************** + * Copyright (C) 2022 EclipseSource, Lockular, Ericsson, STMicroelectronics and + * others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the Eclipse + * Public License v. 2.0 are satisfied: GNU General Public License, version 2 + * with the GNU Classpath Exception which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + ********************************************************************************/ +package org.eclipse.theia.cloud.operator.handler.session; + +import static org.eclipse.theia.cloud.common.util.LogMessageUtil.formatLogMessage; + +import java.time.Instant; +import java.util.Collections; +import java.util.List; +import java.util.Map.Entry; +import java.util.Map; +import java.util.HashMap; +import java.util.Optional; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.eclipse.theia.cloud.common.k8s.client.TheiaCloudClient; +import org.eclipse.theia.cloud.common.k8s.resource.appdefinition.AppDefinition; +import org.eclipse.theia.cloud.common.k8s.resource.appdefinition.AppDefinitionSpec; +import org.eclipse.theia.cloud.common.k8s.resource.session.Session; +import org.eclipse.theia.cloud.common.k8s.resource.session.SessionSpec; +import org.eclipse.theia.cloud.common.util.JavaUtil; +import org.eclipse.theia.cloud.common.util.LabelsUtil; +import org.eclipse.theia.cloud.operator.TheiaCloudOperatorArguments; +import org.eclipse.theia.cloud.operator.handler.AddedHandlerUtil; +import org.eclipse.theia.cloud.operator.ingress.IngressPathProvider; +import org.eclipse.theia.cloud.operator.util.K8sUtil; +import org.eclipse.theia.cloud.operator.util.TheiaCloudConfigMapUtil; +import org.eclipse.theia.cloud.operator.util.TheiaCloudDeploymentUtil; +import org.eclipse.theia.cloud.operator.util.TheiaCloudHandlerUtil; +import org.eclipse.theia.cloud.operator.util.TheiaCloudServiceUtil; + +import com.google.inject.Inject; + +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.ServiceList; +import io.fabric8.kubernetes.api.model.networking.v1.HTTPIngressPath; +import io.fabric8.kubernetes.api.model.networking.v1.HTTPIngressRuleValue; +import io.fabric8.kubernetes.api.model.networking.v1.Ingress; +import io.fabric8.kubernetes.api.model.networking.v1.IngressBackend; +import io.fabric8.kubernetes.api.model.networking.v1.IngressRule; +import io.fabric8.kubernetes.api.model.networking.v1.IngressServiceBackend; +import io.fabric8.kubernetes.api.model.networking.v1.ServiceBackendPort; +import io.fabric8.kubernetes.client.KubernetesClientException; +import io.fabric8.kubernetes.client.NamespacedKubernetesClient; +import io.fabric8.kubernetes.client.dsl.FilterWatchListDeletable; +import io.fabric8.kubernetes.client.dsl.PodResource; +import io.fabric8.kubernetes.client.dsl.ServiceResource; + +/** + * A {@link SessionAddedHandler} that relies on the fact that the app definition handler created spare deployments to + * use. + */ +public class EagerSessionHandler implements SessionHandler { + + public static final String EAGER_START_REFRESH_ANNOTATION = "theia-cloud.io/eager-start-refresh"; + + private static final Logger LOGGER = LogManager.getLogger(EagerSessionHandler.class); + + @Inject + private TheiaCloudClient client; + + @Inject + protected IngressPathProvider ingressPathProvider; + + @Inject + protected TheiaCloudOperatorArguments arguments; + + @Override + public boolean sessionAdded(Session session, String correlationId) { + SessionSpec spec = session.getSpec(); + LOGGER.info(formatLogMessage(correlationId, "Handling sessionAdded " + spec)); + + String sessionResourceName = session.getMetadata().getName(); + String sessionResourceUID = session.getMetadata().getUid(); + + String appDefinitionID = spec.getAppDefinition(); + String userEmail = spec.getUser(); + + /* find app definition for session */ + Optional appDefinition = client.appDefinitions().get(appDefinitionID); + if (appDefinition.isEmpty()) { + LOGGER.error(formatLogMessage(correlationId, "No App Definition with name " + appDefinitionID + " found.")); + return false; + } + + String appDefinitionResourceName = appDefinition.get().getMetadata().getName(); + String appDefinitionResourceUID = appDefinition.get().getMetadata().getUid(); + int port = appDefinition.get().getSpec().getPort(); + + /* find ingress */ + Optional ingress = K8sUtil.getExistingIngress(client.kubernetes(), client.namespace(), + appDefinitionResourceName, appDefinitionResourceUID); + if (ingress.isEmpty()) { + LOGGER.error( + formatLogMessage(correlationId, "No Ingress for app definition " + appDefinitionID + " found.")); + return false; + } + + /* get a service to use */ + Entry, Boolean> reserveServiceResult = reserveService(client.kubernetes(), client.namespace(), + appDefinitionResourceName, appDefinitionResourceUID, appDefinitionID, sessionResourceName, + sessionResourceUID, correlationId); + if (reserveServiceResult.getValue()) { + LOGGER.info(formatLogMessage(correlationId, "Found an already reserved service")); + return true; + } + Optional serviceToUse = reserveServiceResult.getKey(); + if (serviceToUse.isEmpty()) { + LOGGER.error( + formatLogMessage(correlationId, "No Service for app definition " + appDefinitionID + " found.")); + return false; + } + + try { + client.services().inNamespace(client.namespace()).withName(serviceToUse.get().getMetadata().getName()) + .edit(service -> { + LOGGER.debug("Setting session labels"); + Map labels = service.getMetadata().getLabels(); + if (labels == null) { + labels = new HashMap<>(); + service.getMetadata().setLabels(labels); + } + Map newLabels = LabelsUtil.createSessionLabels(spec, + appDefinition.get().getSpec()); + labels.putAll(newLabels); + return service; + }); + } catch (KubernetesClientException e) { + LOGGER.error(formatLogMessage(correlationId, + "Error while adding labels to service " + (serviceToUse.get().getMetadata().getName())), e); + return false; + } + + /* get the deployment for the service and add as owner */ + Integer instance = TheiaCloudServiceUtil.getId(correlationId, appDefinition.get(), serviceToUse.get()); + if (instance == null) { + LOGGER.error(formatLogMessage(correlationId, "Error while getting instance from Service")); + return false; + } + + final String deploymentName = TheiaCloudDeploymentUtil.getDeploymentName(appDefinition.get(), instance); + try { + client.kubernetes().apps().deployments().withName(deploymentName).edit(deployment -> TheiaCloudHandlerUtil + .addOwnerReferenceToItem(correlationId, sessionResourceName, sessionResourceUID, deployment)); + } catch (KubernetesClientException e) { + LOGGER.error(formatLogMessage(correlationId, "Error while editing deployment " + + (appDefinitionID + TheiaCloudDeploymentUtil.DEPLOYMENT_NAME + instance)), e); + return false; + } + + if (arguments.isUseKeycloak()) { + /* add user to allowed emails */ + try { + client.kubernetes().configMaps() + .withName(TheiaCloudConfigMapUtil.getEmailConfigName(appDefinition.get(), instance)) + .edit(configmap -> { + configmap.setData(Collections + .singletonMap(AddedHandlerUtil.FILENAME_AUTHENTICATED_EMAILS_LIST, userEmail)); + return configmap; + }); + + } catch (KubernetesClientException e) { + LOGGER.error( + formatLogMessage(correlationId, + "Error while editing email configmap " + + (appDefinitionID + TheiaCloudConfigMapUtil.CONFIGMAP_EMAIL_NAME + instance)), + e); + return false; + } + + // Add/update annotation to the session pod to trigger a sync with the Kubelet. + // Otherwise, the pod might not be updated with the new email list for the OAuth proxy in time. + // This is the case because ConfigMap changes are not propagated to the pod immediately but during a + // periodic sync. See + // https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically + // NOTE that this is still not a one hundred percent guarantee that the pod is updated in time. + try { + LOGGER.info(formatLogMessage(correlationId, "Adding update annotation to pods...")); + client.kubernetes().pods().list().getItems().forEach(pod -> { + // Use startsWith because the actual owner is the deployment's ReplicaSet + // whose name starts with the deployment name + if (pod.getMetadata().getOwnerReferences().stream() + .anyMatch(or -> or.getName().startsWith(deploymentName))) { + + LOGGER.debug(formatLogMessage(correlationId, + "Adding update annotation to pod " + pod.getMetadata().getName())); + pod.getMetadata().getAnnotations().put(EAGER_START_REFRESH_ANNOTATION, + Instant.now().toString()); + // Apply the changes + PodResource podResource = client.pods().withName(pod.getMetadata().getName()); + podResource.edit(p -> pod); + LOGGER.debug(formatLogMessage(correlationId, + "Added update annotation to pod " + pod.getMetadata().getName())); + } + }); + } catch (KubernetesClientException e) { + LOGGER.error(formatLogMessage(correlationId, "Error while editing pod annotations"), e); + return false; + } + } + + /* adjust the ingress */ + String host; + try { + host = updateIngress(ingress, serviceToUse, appDefinitionID, instance, port, appDefinition.get(), + correlationId); + } catch (KubernetesClientException e) { + LOGGER.error(formatLogMessage(correlationId, + "Error while editing ingress " + ingress.get().getMetadata().getName()), e); + return false; + } + + /* Update session resource */ + try { + AddedHandlerUtil.updateSessionURLAsync(client.sessions(), session, client.namespace(), host, correlationId); + } catch (KubernetesClientException e) { + LOGGER.error( + formatLogMessage(correlationId, "Error while editing session " + session.getMetadata().getName()), + e); + return false; + } + + return true; + } + + protected synchronized Entry, Boolean> reserveService(NamespacedKubernetesClient client, + String namespace, String appDefinitionResourceName, String appDefinitionResourceUID, String appDefinitionID, + String sessionResourceName, String sessionResourceUID, String correlationId) { + List existingServices = K8sUtil.getExistingServices(client, namespace, appDefinitionResourceName, + appDefinitionResourceUID); + + Optional alreadyReservedService = TheiaCloudServiceUtil.getServiceOwnedBySession(sessionResourceName, + sessionResourceUID, existingServices); + if (alreadyReservedService.isPresent()) { + return JavaUtil.tuple(alreadyReservedService, true); + } + + Optional serviceToUse = TheiaCloudServiceUtil.getUnusedService(existingServices, + appDefinitionResourceUID); + if (serviceToUse.isEmpty()) { + return JavaUtil.tuple(serviceToUse, false); + } + + /* add our session as owner to the service */ + try { + client.services().inNamespace(namespace).withName(serviceToUse.get().getMetadata().getName()) + .edit(service -> TheiaCloudHandlerUtil.addOwnerReferenceToItem(correlationId, sessionResourceName, + sessionResourceUID, service)); + } catch (KubernetesClientException e) { + LOGGER.error(formatLogMessage(correlationId, + "Error while editing service " + (serviceToUse.get().getMetadata().getName())), e); + return JavaUtil.tuple(Optional.empty(), false); + } + return JavaUtil.tuple(serviceToUse, false); + } + + protected synchronized String updateIngress(Optional ingress, Optional serviceToUse, + String appDefinitionID, int instance, int port, AppDefinition appDefinition, String correlationId) { + final String host = arguments.getInstancesHost(); + String path = ingressPathProvider.getPath(appDefinition, instance); + client.ingresses().edit(correlationId, ingress.get().getMetadata().getName(), + ingressToUpdate -> addIngressRule(ingressToUpdate, serviceToUse.get(), host, port, path)); + return host + path + "/"; + } + + protected Ingress addIngressRule(Ingress ingress, Service serviceToUse, String host, int port, String path) { + IngressRule ingressRule = new IngressRule(); + ingress.getSpec().getRules().add(ingressRule); + + ingressRule.setHost(host); + + HTTPIngressRuleValue http = new HTTPIngressRuleValue(); + ingressRule.setHttp(http); + + HTTPIngressPath httpIngressPath = new HTTPIngressPath(); + http.getPaths().add(httpIngressPath); + httpIngressPath.setPath(path + AddedHandlerUtil.INGRESS_REWRITE_PATH); + httpIngressPath.setPathType("Prefix"); + + IngressBackend ingressBackend = new IngressBackend(); + httpIngressPath.setBackend(ingressBackend); + + IngressServiceBackend ingressServiceBackend = new IngressServiceBackend(); + ingressBackend.setService(ingressServiceBackend); + ingressServiceBackend.setName(serviceToUse.getMetadata().getName()); + + ServiceBackendPort serviceBackendPort = new ServiceBackendPort(); + ingressServiceBackend.setPort(serviceBackendPort); + serviceBackendPort.setNumber(port); + + return ingress; + } + + @Override + public boolean sessionDeleted(Session session, String correlationId) { + SessionSpec spec = session.getSpec(); + LOGGER.info(formatLogMessage(correlationId, "Handling sessionDeleted " + spec)); + + // Find app definition for session. If it's not there anymore, we don't need to clean up because the resources + // are deleted by Kubernetes garbage collection. + String appDefinitionID = spec.getAppDefinition(); + Optional appDefinition = client.appDefinitions().get(appDefinitionID); + if (appDefinition.isEmpty()) { + LOGGER.info(formatLogMessage(correlationId, "No App Definition with name " + appDefinitionID + + " found. Thus, no cleanup is needed because associated resources are deleted by Kubernets garbage collecion.")); + return true; + } + AppDefinitionSpec appDefinitionSpec = appDefinition.get().getSpec(); + + // Find service by first filtering all services by the session's corresponding session labels (as added in + // sessionCreated) and then checking if the service has an owner reference to the session + String sessionResourceName = session.getMetadata().getName(); + String sessionResourceUID = session.getMetadata().getUid(); + Map sessionLabels = LabelsUtil.createSessionLabels(spec, appDefinitionSpec); + // Filtering by withLabels(sessionLabels) because the method requires an exact match of the labels. + // Additional labels on the service prevent a match and the service has an additional app label. + // Thus, filter by each session label separately. + // We rely on the fact that the session labels are unique for each session. + // We cannot rely on owner references because they might have been cleaned up automatically by Kubernetes. + // While this should not happen, it did on Minikube. + FilterWatchListDeletable> servicesFilter = client.services(); + for (Entry entry : sessionLabels.entrySet()) { + servicesFilter = servicesFilter.withLabel(entry.getKey(), entry.getValue()); + } + List services = servicesFilter.list().getItems(); + if (services.isEmpty()) { + LOGGER.error(formatLogMessage(correlationId, "No Service owned by session " + spec.getName() + " found.")); + return false; + } else if (services.size() > 1) { + LOGGER.error(formatLogMessage(correlationId, + "Multiple Services owned by session " + spec.getName() + " found. This should never happen.")); + return false; + } + Service ownedService = services.get(0); + String serviceName = ownedService.getMetadata().getName(); + + // Remove owner reference and user specific labels from the service + Service cleanedService; + try { + cleanedService = client.services().withName(serviceName).edit(service -> { + TheiaCloudHandlerUtil.removeOwnerReferenceFromItem(correlationId, sessionResourceName, + sessionResourceUID, service); + service.getMetadata().getLabels().keySet().removeAll(LabelsUtil.getSessionSpecificLabelKeys()); + return service; + }); + LOGGER.info(formatLogMessage(correlationId, + "Removed owner reference and user-specific session labels from service: " + serviceName)); + } catch (KubernetesClientException e) { + LOGGER.error(formatLogMessage(correlationId, "Error while editing service " + serviceName), e); + return false; + } + Integer instance = TheiaCloudServiceUtil.getId(correlationId, appDefinition.get(), cleanedService); + + // Cleanup ingress rule to prevent further traffic to the session pod + Optional ingress = K8sUtil.getExistingIngress(client.kubernetes(), client.namespace(), + appDefinition.get().getMetadata().getName(), appDefinition.get().getMetadata().getUid()); + if (ingress.isEmpty()) { + LOGGER.error( + formatLogMessage(correlationId, "No Ingress for app definition " + appDefinitionID + " found.")); + return false; + } + // Remove ingress rule + try { + removeIngressRule(correlationId, appDefinition.get(), ingress.get(), instance); + } catch (KubernetesClientException e) { + LOGGER.error(formatLogMessage(correlationId, + "Error while editing ingress " + ingress.get().getMetadata().getName()), e); + return false; + } + + // Remove owner reference from deployment + if (instance == null) { + LOGGER.error(formatLogMessage(correlationId, "Error while getting instance from Service")); + return false; + } + final String deploymentName = TheiaCloudDeploymentUtil.getDeploymentName(appDefinition.get(), instance); + try { + client.kubernetes().apps().deployments().withName(deploymentName).edit(deployment -> TheiaCloudHandlerUtil + .removeOwnerReferenceFromItem(correlationId, sessionResourceName, sessionResourceUID, deployment)); + } catch (KubernetesClientException e) { + LOGGER.error(formatLogMessage(correlationId, "Error while editing deployment " + + (appDefinitionID + TheiaCloudDeploymentUtil.DEPLOYMENT_NAME + instance)), e); + return false; + } + + // Remove user from allowed emails in config map + try { + client.kubernetes().configMaps() + .withName(TheiaCloudConfigMapUtil.getEmailConfigName(appDefinition.get(), instance)) + .edit(configmap -> { + configmap.setData( + Collections.singletonMap(AddedHandlerUtil.FILENAME_AUTHENTICATED_EMAILS_LIST, null)); + return configmap; + }); + } catch (KubernetesClientException e) { + LOGGER.error(formatLogMessage(correlationId, "Error while editing email configmap " + + (appDefinitionID + TheiaCloudConfigMapUtil.CONFIGMAP_EMAIL_NAME + instance)), e); + return false; + } + + // Delete the pod to clean temporary workspace files. The deployment recreates a fresh pod automatically. + try { + Optional pod = client.kubernetes().pods().list().getItems().stream() + .filter(p -> p.getMetadata().getName().startsWith(deploymentName)).findAny(); + if (pod.isPresent()) { + LOGGER.info(formatLogMessage(correlationId, "Deleting pod " + pod.get().getMetadata().getName())); + client.pods().withName(pod.get().getMetadata().getName()).delete(); + } + } catch (KubernetesClientException e) { + LOGGER.error(formatLogMessage(correlationId, "Error while deleting pod"), e); + return false; + } + + return true; + } + + protected synchronized void removeIngressRule(String correlationId, AppDefinition appDefinition, Ingress ingress, + Integer instance) throws KubernetesClientException { + final String ruleHttpPath = ingressPathProvider.getPath(appDefinition, instance) + + AddedHandlerUtil.INGRESS_REWRITE_PATH; + client.ingresses().resource(ingress.getMetadata().getName()).edit(ingressToUpdate -> { + ingressToUpdate.getSpec().getRules().removeIf(rule -> { + if (rule.getHttp() == null) { + LOGGER.warn(formatLogMessage(correlationId, + "Error while removing ingress rule: The rule's HTTP block is null")); + return false; + } + return rule.getHttp().getPaths().stream().anyMatch(httpPath -> ruleHttpPath.equals(httpPath.getPath())); + }); + return ingressToUpdate; + }); + } +} diff --git a/java/operator/org.eclipse.theia.cloud.operator/src/main/java/org/eclipse/theia/cloud/operator/handler/session/EagerStartSessionHandler.java b/java/operator/org.eclipse.theia.cloud.operator/src/main/java/org/eclipse/theia/cloud/operator/handler/session/EagerStartSessionHandler.java deleted file mode 100644 index 0cb2a107..00000000 --- a/java/operator/org.eclipse.theia.cloud.operator/src/main/java/org/eclipse/theia/cloud/operator/handler/session/EagerStartSessionHandler.java +++ /dev/null @@ -1,269 +0,0 @@ -/******************************************************************************** - * Copyright (C) 2022 EclipseSource, Lockular, Ericsson, STMicroelectronics and - * others. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License v. 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0. - * - * This Source Code may also be made available under the following Secondary - * Licenses when the conditions for such availability set forth in the Eclipse - * Public License v. 2.0 are satisfied: GNU General Public License, version 2 - * with the GNU Classpath Exception which is available at - * https://www.gnu.org/software/classpath/license.html. - * - * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 - ********************************************************************************/ -package org.eclipse.theia.cloud.operator.handler.session; - -import static org.eclipse.theia.cloud.common.util.LogMessageUtil.formatLogMessage; - -import java.util.Collections; -import java.util.List; -import java.util.Map.Entry; -import java.util.Map; -import java.util.HashMap; -import java.util.Optional; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.eclipse.theia.cloud.common.k8s.client.TheiaCloudClient; -import org.eclipse.theia.cloud.common.k8s.resource.appdefinition.AppDefinition; -import org.eclipse.theia.cloud.common.k8s.resource.session.Session; -import org.eclipse.theia.cloud.common.k8s.resource.session.SessionSpec; -import org.eclipse.theia.cloud.common.util.JavaUtil; -import org.eclipse.theia.cloud.common.util.LabelsUtil; -import org.eclipse.theia.cloud.operator.TheiaCloudOperatorArguments; -import org.eclipse.theia.cloud.operator.handler.AddedHandlerUtil; -import org.eclipse.theia.cloud.operator.ingress.IngressPathProvider; -import org.eclipse.theia.cloud.operator.util.K8sUtil; -import org.eclipse.theia.cloud.operator.util.TheiaCloudConfigMapUtil; -import org.eclipse.theia.cloud.operator.util.TheiaCloudDeploymentUtil; -import org.eclipse.theia.cloud.operator.util.TheiaCloudHandlerUtil; -import org.eclipse.theia.cloud.operator.util.TheiaCloudServiceUtil; - -import com.google.inject.Inject; - -import io.fabric8.kubernetes.api.model.Service; -import io.fabric8.kubernetes.api.model.networking.v1.HTTPIngressPath; -import io.fabric8.kubernetes.api.model.networking.v1.HTTPIngressRuleValue; -import io.fabric8.kubernetes.api.model.networking.v1.Ingress; -import io.fabric8.kubernetes.api.model.networking.v1.IngressBackend; -import io.fabric8.kubernetes.api.model.networking.v1.IngressRule; -import io.fabric8.kubernetes.api.model.networking.v1.IngressServiceBackend; -import io.fabric8.kubernetes.api.model.networking.v1.ServiceBackendPort; -import io.fabric8.kubernetes.client.KubernetesClientException; -import io.fabric8.kubernetes.client.NamespacedKubernetesClient; - -/** - * A {@link SessionAddedHandler} that relies on the fact that the app definition handler created spare deployments to - * use. - */ -public class EagerStartSessionHandler implements SessionHandler { - - private static final Logger LOGGER = LogManager.getLogger(EagerStartSessionHandler.class); - - @Inject - private TheiaCloudClient client; - - @Inject - protected IngressPathProvider ingressPathProvider; - - @Inject - protected TheiaCloudOperatorArguments arguments; - - @Override - public boolean sessionAdded(Session session, String correlationId) { - SessionSpec spec = session.getSpec(); - LOGGER.info(formatLogMessage(correlationId, "Handling " + spec)); - - String sessionResourceName = session.getMetadata().getName(); - String sessionResourceUID = session.getMetadata().getUid(); - - String appDefinitionID = spec.getAppDefinition(); - String userEmail = spec.getUser(); - - /* find app definition for session */ - Optional appDefinition = client.appDefinitions().get(appDefinitionID); - if (appDefinition.isEmpty()) { - LOGGER.error(formatLogMessage(correlationId, "No App Definition with name " + appDefinitionID + " found.")); - return false; - } - - String appDefinitionResourceName = appDefinition.get().getMetadata().getName(); - String appDefinitionResourceUID = appDefinition.get().getMetadata().getUid(); - int port = appDefinition.get().getSpec().getPort(); - - /* find ingress */ - Optional ingress = K8sUtil.getExistingIngress(client.kubernetes(), client.namespace(), - appDefinitionResourceName, appDefinitionResourceUID); - if (ingress.isEmpty()) { - LOGGER.error( - formatLogMessage(correlationId, "No Ingress for app definition " + appDefinitionID + " found.")); - return false; - } - - /* get a service to use */ - Entry, Boolean> reserveServiceResult = reserveService(client.kubernetes(), client.namespace(), - appDefinitionResourceName, appDefinitionResourceUID, appDefinitionID, sessionResourceName, - sessionResourceUID, correlationId); - if (reserveServiceResult.getValue()) { - LOGGER.info(formatLogMessage(correlationId, "Found an already reserved service")); - return true; - } - Optional serviceToUse = reserveServiceResult.getKey(); - if (serviceToUse.isEmpty()) { - LOGGER.error( - formatLogMessage(correlationId, "No Service for app definition " + appDefinitionID + " found.")); - return false; - } - - try { - client.services().inNamespace(client.namespace()).withName(serviceToUse.get().getMetadata().getName()) - .edit(service -> { - LOGGER.debug("Setting session labels"); - Map labels = service.getMetadata().getLabels(); - if (labels == null) { - labels = new HashMap<>(); - service.getMetadata().setLabels(labels); - } - Map newLabels = LabelsUtil.createSessionLabels(spec, appDefinition.get().getSpec()); - labels.putAll(newLabels); - return service; - }); - } catch (KubernetesClientException e) { - LOGGER.error(formatLogMessage(correlationId, - "Error while adding labels to service " + (serviceToUse.get().getMetadata().getName())), e); - return false; - } - - /* get the deployment for the service and add as owner */ - Integer instance = TheiaCloudServiceUtil.getId(correlationId, appDefinition.get(), serviceToUse.get()); - if (instance == null) { - LOGGER.error(formatLogMessage(correlationId, "Error while getting instance from Service")); - return false; - } - - try { - client.kubernetes().apps().deployments() - .withName(TheiaCloudDeploymentUtil.getDeploymentName(appDefinition.get(), instance)) - .edit(deployment -> TheiaCloudHandlerUtil.addOwnerReferenceToItem(correlationId, - sessionResourceName, sessionResourceUID, deployment)); - } catch (KubernetesClientException e) { - LOGGER.error(formatLogMessage(correlationId, "Error while editing deployment " - + (appDefinitionID + TheiaCloudDeploymentUtil.DEPLOYMENT_NAME + instance)), e); - return false; - } - - if (arguments.isUseKeycloak()) { - /* add user to allowed emails */ - try { - client.kubernetes().configMaps() - .withName(TheiaCloudConfigMapUtil.getEmailConfigName(appDefinition.get(), instance)) - .edit(configmap -> { - configmap.setData(Collections - .singletonMap(AddedHandlerUtil.FILENAME_AUTHENTICATED_EMAILS_LIST, userEmail)); - return configmap; - }); - } catch (KubernetesClientException e) { - LOGGER.error( - formatLogMessage(correlationId, - "Error while editing email configmap " - + (appDefinitionID + TheiaCloudConfigMapUtil.CONFIGMAP_EMAIL_NAME + instance)), - e); - return false; - } - } - - /* adjust the ingress */ - String host; - try { - host = updateIngress(ingress, serviceToUse, appDefinitionID, instance, port, appDefinition.get(), - correlationId); - } catch (KubernetesClientException e) { - LOGGER.error(formatLogMessage(correlationId, - "Error while editing ingress " + ingress.get().getMetadata().getName()), e); - return false; - } - - /* Update session resource */ - try { - AddedHandlerUtil.updateSessionURLAsync(client.sessions(), session, client.namespace(), host, correlationId); - } catch (KubernetesClientException e) { - LOGGER.error( - formatLogMessage(correlationId, "Error while editing session " + session.getMetadata().getName()), - e); - return false; - } - - return true; - } - - protected synchronized Entry, Boolean> reserveService(NamespacedKubernetesClient client, - String namespace, String appDefinitionResourceName, String appDefinitionResourceUID, String appDefinitionID, - String sessionResourceName, String sessionResourceUID, String correlationId) { - List existingServices = K8sUtil.getExistingServices(client, namespace, appDefinitionResourceName, - appDefinitionResourceUID); - - Optional alreadyReservedService = TheiaCloudServiceUtil.getServiceOwnedBySession(sessionResourceName, - sessionResourceUID, existingServices); - if (alreadyReservedService.isPresent()) { - return JavaUtil.tuple(alreadyReservedService, true); - } - - Optional serviceToUse = TheiaCloudServiceUtil.getUnusedService(existingServices); - if (serviceToUse.isEmpty()) { - return JavaUtil.tuple(serviceToUse, false); - } - - /* add our session as owner to the service */ - try { - client.services().inNamespace(namespace).withName(serviceToUse.get().getMetadata().getName()) - .edit(service -> TheiaCloudHandlerUtil.addOwnerReferenceToItem(correlationId, sessionResourceName, - sessionResourceUID, service)); - } catch (KubernetesClientException e) { - LOGGER.error(formatLogMessage(correlationId, - "Error while editing service " + (serviceToUse.get().getMetadata().getName())), e); - return JavaUtil.tuple(Optional.empty(), false); - } - return JavaUtil.tuple(serviceToUse, false); - } - - protected synchronized String updateIngress(Optional ingress, Optional serviceToUse, - String appDefinitionID, int instance, int port, AppDefinition appDefinition, String correlationId) { - final String host = arguments.getInstancesHost(); - String path = ingressPathProvider.getPath(appDefinition, instance); - client.ingresses().edit(correlationId, ingress.get().getMetadata().getName(), - ingressToUpdate -> addIngressRule(ingressToUpdate, serviceToUse.get(), host, port, path)); - return host + path + "/"; - } - - protected Ingress addIngressRule(Ingress ingress, Service serviceToUse, String host, int port, String path) { - IngressRule ingressRule = new IngressRule(); - ingress.getSpec().getRules().add(ingressRule); - - ingressRule.setHost(host); - - HTTPIngressRuleValue http = new HTTPIngressRuleValue(); - ingressRule.setHttp(http); - - HTTPIngressPath httpIngressPath = new HTTPIngressPath(); - http.getPaths().add(httpIngressPath); - httpIngressPath.setPath(path + AddedHandlerUtil.INGRESS_REWRITE_PATH); - httpIngressPath.setPathType("Prefix"); - - IngressBackend ingressBackend = new IngressBackend(); - httpIngressPath.setBackend(ingressBackend); - - IngressServiceBackend ingressServiceBackend = new IngressServiceBackend(); - ingressBackend.setService(ingressServiceBackend); - ingressServiceBackend.setName(serviceToUse.getMetadata().getName()); - - ServiceBackendPort serviceBackendPort = new ServiceBackendPort(); - ingressServiceBackend.setPort(serviceBackendPort); - serviceBackendPort.setNumber(port); - - return ingress; - } - -} diff --git a/java/operator/org.eclipse.theia.cloud.operator/src/main/java/org/eclipse/theia/cloud/operator/util/TheiaCloudHandlerUtil.java b/java/operator/org.eclipse.theia.cloud.operator/src/main/java/org/eclipse/theia/cloud/operator/util/TheiaCloudHandlerUtil.java index 974a4ec4..36f4e05e 100644 --- a/java/operator/org.eclipse.theia.cloud.operator/src/main/java/org/eclipse/theia/cloud/operator/util/TheiaCloudHandlerUtil.java +++ b/java/operator/org.eclipse.theia.cloud.operator/src/main/java/org/eclipse/theia/cloud/operator/util/TheiaCloudHandlerUtil.java @@ -73,6 +73,20 @@ public static T addOwnerReferenceToItem(String correlati return item; } + /** + * Removes the owner reference from the item if it is present. Does nothing otherwise. + */ + public static T removeOwnerReferenceFromItem(String correlationId, + String sessionResourceName, String sessionResourceUID, T item) { + LOGGER.info( + formatLogMessage(correlationId, "Removing the owner reference from " + item.getMetadata().getName())); + item.getMetadata().getOwnerReferences().removeIf(ownerReference -> { + return ownerReference.getName().equals(sessionResourceName) + && ownerReference.getUid().equals(sessionResourceUID); + }); + return item; + } + public static OwnerReference createOwnerReference(String sessionResourceName, String sessionResourceUID) { OwnerReference ownerReference = new OwnerReference(); ownerReference.setApiVersion(HasMetadata.getApiVersion(Session.class)); diff --git a/java/operator/org.eclipse.theia.cloud.operator/src/main/java/org/eclipse/theia/cloud/operator/util/TheiaCloudK8sUtil.java b/java/operator/org.eclipse.theia.cloud.operator/src/main/java/org/eclipse/theia/cloud/operator/util/TheiaCloudK8sUtil.java index 80d145f5..4b5cffa4 100644 --- a/java/operator/org.eclipse.theia.cloud.operator/src/main/java/org/eclipse/theia/cloud/operator/util/TheiaCloudK8sUtil.java +++ b/java/operator/org.eclipse.theia.cloud.operator/src/main/java/org/eclipse/theia/cloud/operator/util/TheiaCloudK8sUtil.java @@ -21,11 +21,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.eclipse.theia.cloud.common.k8s.resource.OperatorStatus; +import org.eclipse.theia.cloud.common.k8s.resource.appdefinition.AppDefinition; import org.eclipse.theia.cloud.common.k8s.resource.appdefinition.AppDefinitionSpec; import org.eclipse.theia.cloud.common.k8s.resource.session.Session; import org.eclipse.theia.cloud.common.k8s.resource.session.SessionSpec; import org.eclipse.theia.cloud.common.k8s.resource.session.SessionSpecResourceList; -import org.eclipse.theia.cloud.common.util.NamingUtil; import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.client.NamespacedKubernetesClient; @@ -69,14 +69,20 @@ public static boolean checkIfMaxInstancesReached(NamespacedKubernetesClient clie return currentInstances > appDefinitionSpec.getMaxInstances(); } + /** + * Extracts the instance id from the name of a Kubernetes object whose name was generated based on an app definition + * and an id. + * + * @param metadata The object's metadata + * @return the extracted identifier + * @see org.eclipse.theia.cloud.common.util.NamingUtil#createName(AppDefinition, int) + * @see org.eclipse.theia.cloud.common.util.NamingUtil#createName(AppDefinition, int, String) + */ public static String extractIdFromName(ObjectMeta metadata) { + // Generated name is of the form "instance--" String name = metadata.getName(); String[] split = name.split("-"); - String instance = split.length == 0 ? "" : split[0]; - // kubernetes names must not start with letter, remove automatically added - // prefix - instance = instance.length() == 0 ? "" - : instance.charAt(0) == NamingUtil.VALID_NAME_PREFIX ? instance.substring(1) : instance; + String instance = split.length < 2 ? "" : split[1]; return instance; } diff --git a/java/operator/org.eclipse.theia.cloud.operator/src/main/java/org/eclipse/theia/cloud/operator/util/TheiaCloudServiceUtil.java b/java/operator/org.eclipse.theia.cloud.operator/src/main/java/org/eclipse/theia/cloud/operator/util/TheiaCloudServiceUtil.java index f23cdead..db3570b9 100644 --- a/java/operator/org.eclipse.theia.cloud.operator/src/main/java/org/eclipse/theia/cloud/operator/util/TheiaCloudServiceUtil.java +++ b/java/operator/org.eclipse.theia.cloud.operator/src/main/java/org/eclipse/theia/cloud/operator/util/TheiaCloudServiceUtil.java @@ -132,6 +132,15 @@ public static boolean isUnusedService(Service service) { return service.getMetadata().getOwnerReferences().size() == 1; } + /** + * Returns an unused service. + * + * @param existingServices + * @return + * @deprecated Use {@link #getUnusedService(List, String)} instead: Services should be owned by the corresponding + * app definition. + */ + @Deprecated public static Optional getUnusedService(List existingServices) { Optional serviceToUse = existingServices.stream()// .filter(TheiaCloudServiceUtil::isUnusedService)// @@ -139,4 +148,25 @@ public static Optional getUnusedService(List existingServices) return serviceToUse; } + /** + * Returns an unused service that is owned by the given app definition. + * + * @param existingServices The list of services to search in. + * @param appDefinitionResourceUID The UID of the app definition that should own the service. + * @return The unused service that is owned by the given app definition or nothing if none is available. + */ + public static Optional getUnusedService(List existingServices, String appDefinitionResourceUID) { + Optional serviceToUse = existingServices.stream()// + .filter(TheiaCloudServiceUtil::isUnusedService)// + .filter(service -> { + for (OwnerReference ownerReference : service.getMetadata().getOwnerReferences()) { + if (appDefinitionResourceUID.equals(ownerReference.getUid())) { + return true; + } + } + return false; + })// + .findAny(); + return serviceToUse; + }; } diff --git a/java/service/org.eclipse.theia.cloud.service/src/main/java/org/eclipse/theia/cloud/service/BaseResource.java b/java/service/org.eclipse.theia.cloud.service/src/main/java/org/eclipse/theia/cloud/service/BaseResource.java index a86692a4..c6d0b059 100644 --- a/java/service/org.eclipse.theia.cloud.service/src/main/java/org/eclipse/theia/cloud/service/BaseResource.java +++ b/java/service/org.eclipse.theia.cloud.service/src/main/java/org/eclipse/theia/cloud/service/BaseResource.java @@ -100,6 +100,10 @@ public void warn(String correlationId, String message) { logger.warn(LogMessageUtil.formatLogMessage(correlationId, message)); } + public void warn(String correlationId, String message, Throwable throwable) { + logger.warn(LogMessageUtil.formatLogMessage(correlationId, message), throwable); + } + public void error(String correlationId, String message) { logger.error(LogMessageUtil.formatLogMessage(correlationId, message)); } diff --git a/java/service/org.eclipse.theia.cloud.service/src/main/java/org/eclipse/theia/cloud/service/K8sUtil.java b/java/service/org.eclipse.theia.cloud.service/src/main/java/org/eclipse/theia/cloud/service/K8sUtil.java index 99736966..b78f0ac1 100644 --- a/java/service/org.eclipse.theia.cloud.service/src/main/java/org/eclipse/theia/cloud/service/K8sUtil.java +++ b/java/service/org.eclipse.theia.cloud.service/src/main/java/org/eclipse/theia/cloud/service/K8sUtil.java @@ -18,6 +18,7 @@ import static org.eclipse.theia.cloud.common.util.WorkspaceUtil.getSessionName; +import java.text.MessageFormat; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -37,11 +38,11 @@ import org.eclipse.theia.cloud.service.workspace.UserWorkspace; import org.jboss.logging.Logger; -import io.fabric8.kubernetes.api.model.Container; -import io.fabric8.kubernetes.api.model.EnvVar; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.PodList; import io.fabric8.kubernetes.api.model.Quantity; +import io.fabric8.kubernetes.api.model.apps.Deployment; +import io.fabric8.kubernetes.api.model.apps.ReplicaSet; import io.fabric8.kubernetes.api.model.metrics.v1beta1.ContainerMetrics; import io.fabric8.kubernetes.api.model.metrics.v1beta1.PodMetrics; import io.fabric8.kubernetes.client.KubernetesClientException; @@ -154,11 +155,15 @@ public List listWorkspaces(String user) { public SessionPerformance reportPerformance(String sessionName) { Optional optionalSession = CLIENT.sessions().get(sessionName); if (optionalSession.isEmpty()) { + logger.warn(MessageFormat.format("Cannot get performance data for session {0} because it does not exist.", + sessionName)); return null; } Session session = optionalSession.get(); Optional optionalPod = getPodForSession(session); if (optionalPod.isEmpty()) { + logger.warn(MessageFormat.format("Cannot get performance data for session {0} because no pod was found.", + sessionName)); return null; } PodMetrics test = CLIENT.kubernetes().top().pods().metrics(CLIENT.namespace(), @@ -166,6 +171,9 @@ public SessionPerformance reportPerformance(String sessionName) { Optional optionalContainer = test.getContainers().stream() .filter(con -> con.getName().equals(session.getSpec().getAppDefinition())).findFirst(); if (optionalContainer.isEmpty()) { + logger.warn(MessageFormat.format( + "Cannot get performance data for session {0} because the app container was not found in the pod.", + sessionName)); return null; } ContainerMetrics container = optionalContainer.get(); @@ -179,20 +187,39 @@ public Optional getPodForSession(Session session) { return podlist.getItems().stream().filter(pod -> isPodFromSession(pod, session)).findFirst(); } + /** + * Checks whether a Pod belongs to a Session by resolving the Pod's Deployment and checking whether the Deployment + * is owned by the Session. + */ private boolean isPodFromSession(Pod pod, Session session) { - Optional optionalContainer = pod.getSpec().getContainers().stream() - .filter(con -> con.getName().equals(session.getSpec().getAppDefinition())).findFirst(); - if (optionalContainer.isEmpty()) { - return false; - } - Container container = optionalContainer.get(); - Optional optionalEnv = container.getEnv().stream() - .filter(env -> env.getName().equals("THEIACLOUD_SESSION_NAME")).findFirst(); - if (optionalEnv.isEmpty()) { - return false; - } - EnvVar env = optionalEnv.get(); - return env.getValue().equals(session.getSpec().getName()); + return getDeploymentForPod(pod)// + .flatMap(deployment -> deployment.getOwnerReferenceFor(session.getMetadata().getUid()))// + .isPresent(); + } + + /** + *

+ * Returns the Deployment associated with a given Pod. + *

+ *

+ * The deployment is retrieved by following the owner references of the Pod. The Pod's owner references are filtered + * to find the ReplicaSet. Then, the ReplicaSet's owner references are filtered to find the Deployment and return + * it. + *

+ * + * @param Pod the Pod for which to retrieve the Deployment. + * @return the Deployment associated with the given Pod or an empty Optional if not found. + */ + private Optional getDeploymentForPod(Pod pod) { + Optional replicaSet = pod.getMetadata().getOwnerReferences().stream() + .filter(ownerReference -> "ReplicaSet".equals(ownerReference.getKind())) + .map(ownerReference -> CLIENT.apps().replicaSets().withName(ownerReference.getName()).get()) + .filter(Objects::nonNull).findFirst(); + + return replicaSet.flatMap(rs -> rs.getMetadata().getOwnerReferences().stream() + .filter(rsOwnerReference -> "Deployment".equals(rsOwnerReference.getKind())) + .map(rsOwnerReference -> CLIENT.apps().deployments().withName(rsOwnerReference.getName()).get()) + .filter(Objects::nonNull).findFirst()); } public boolean hasAppDefinition(String appDefinition) { @@ -212,9 +239,11 @@ public boolean isMaxInstancesReached(String appDefString) { if (maxInstances < 0) { return false; // max instances is set to negative, so we can ignore it } - long podsOfAppDef = CLIENT.sessions().list().stream() // All sessions + + long sessionsOfAppDef = CLIENT.sessions().list().stream() // All sessions .filter(s -> s.getSpec().getAppDefinition().equals(appDefString)) // That are from the appDefinition - .filter(s -> getPodForSession(s).isPresent()).count(); // That already have a pod - return podsOfAppDef >= maxInstances; + .filter(s -> s.getStatus() == null || !s.getStatus().hasError()) // That are not in error state + .count(); + return sessionsOfAppDef >= maxInstances; } } diff --git a/java/service/org.eclipse.theia.cloud.service/src/main/java/org/eclipse/theia/cloud/service/session/SessionResource.java b/java/service/org.eclipse.theia.cloud.service/src/main/java/org/eclipse/theia/cloud/service/session/SessionResource.java index 17019f17..1c4e7ad5 100644 --- a/java/service/org.eclipse.theia.cloud.service/src/main/java/org/eclipse/theia/cloud/service/session/SessionResource.java +++ b/java/service/org.eclipse.theia.cloud.service/src/main/java/org/eclipse/theia/cloud/service/session/SessionResource.java @@ -162,7 +162,7 @@ public SessionPerformance performance(@PathParam("appId") String appId, try { performance = k8sUtil.reportPerformance(sessionName); } catch (Exception e) { - trace(correlationId, "", e); + warn(correlationId, "", e); performance = null; } if (performance == null) { diff --git a/node/testing-page/src/App.tsx b/node/testing-page/src/App.tsx index bc4a26cd..ffd9a2e8 100644 --- a/node/testing-page/src/App.tsx +++ b/node/testing-page/src/App.tsx @@ -12,7 +12,8 @@ import { WorkspaceDeletionRequest, WorkspaceListRequest, PingRequest, - LaunchRequest + LaunchRequest, + SessionPerformanceRequest } from '@eclipse-theiacloud/common'; const KEYCLOAK_CONFIG: KeycloakConfig = { @@ -159,6 +160,14 @@ function App() { }; return TheiaCloud.Session.stopSession(request, generateRequestOptions(accessToken)); }; + const reportSessionPerformance = (user: string, accessToken?: string) => { + const request: SessionPerformanceRequest = { + appId: APP_ID, + sessionName: resourceName, + serviceUrl: SERVICE_URL + }; + return TheiaCloud.Session.getSessionPerformance(request, generateRequestOptions(accessToken)); + }; // App definition requests const listAppDefinitions = (user: string, accessToken?: string) => { @@ -197,6 +206,7 @@ function App() { +

diff --git a/terraform/test-configurations/2-04_try-now_paths_eager-start/.terraform.lock.hcl b/terraform/test-configurations/2-04_try-now_paths_eager-start/.terraform.lock.hcl new file mode 100644 index 00000000..0b52d686 --- /dev/null +++ b/terraform/test-configurations/2-04_try-now_paths_eager-start/.terraform.lock.hcl @@ -0,0 +1,39 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/gavinbunney/kubectl" { + version = "1.14.0" + constraints = ">= 1.14.0" + hashes = [ + "h1:gLFn+RvP37sVzp9qnFCwngRjjFV649r6apjxvJ1E/SE=", + "zh:0350f3122ff711984bbc36f6093c1fe19043173fad5a904bce27f86afe3cc858", + "zh:07ca36c7aa7533e8325b38232c77c04d6ef1081cb0bac9d56e8ccd51f12f2030", + "zh:0c351afd91d9e994a71fe64bbd1662d0024006b3493bb61d46c23ea3e42a7cf5", + "zh:39f1a0aa1d589a7e815b62b5aa11041040903b061672c4cfc7de38622866cbc4", + "zh:428d3a321043b78e23c91a8d641f2d08d6b97f74c195c654f04d2c455e017de5", + "zh:4baf5b1de2dfe9968cc0f57fd4be5a741deb5b34ee0989519267697af5f3eee5", + "zh:6131a927f9dffa014ab5ca5364ac965fe9b19830d2bbf916a5b2865b956fdfcf", + "zh:c62e0c9fd052cbf68c5c2612af4f6408c61c7e37b615dc347918d2442dd05e93", + "zh:f0beffd7ce78f49ead612e4b1aefb7cb6a461d040428f514f4f9cc4e5698ac65", + ] +} + +provider "registry.terraform.io/hashicorp/helm" { + version = "2.9.0" + constraints = ">= 2.9.0" + hashes = [ + "h1:D5BLFN82WndhQZQleXE5rO0hUDnlyqb60XeUJKDhuo4=", + "zh:1471cb45908b426104687c962007b2980cfde294fa3530fabc4798ce9fb6c20c", + "zh:1572e9cec20591ec08ece797b3630802be816a5adde36ca91a93359f2430b130", + "zh:1b10ae03cf5ab1ae21ffaac2251de99797294ae4242b156b3b0beebbdbcb7e0f", + "zh:3bd043b68de967d8d0b549d3f71485193d81167d5656f5507d743dedfe60e352", + "zh:538911921c729185900176cc22eb8edcb822bc8d22b9ebb48103a1d9bb53cc38", + "zh:69a6a2d40c0463662c3fb1621e37a3ee65024ea4479adf4d5f7f19fb0dea48c2", + "zh:94b58daa0c351a49d01f6d8f1caae46c95c2d6c3f29753e2b9ea3e3c0e7c9ab4", + "zh:9d0543331a4a32241e1ab5457f30b41df745acb235a0391205c725a5311e4809", + "zh:a6789306524ca121512a95e873e3949b4175114a6c5db32bed2df2551a79368f", + "zh:d146b94cd9502cca7f2044797a328d71c7ec2a98e2d138270d8a28c872f04289", + "zh:d14ccd14511f0446eacf43a9243f22de7c1427ceb059cf67d7bf9803be2cb15d", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/terraform/test-configurations/2-04_try-now_paths_eager-start/README.md b/terraform/test-configurations/2-04_try-now_paths_eager-start/README.md new file mode 100644 index 00000000..7869d22b --- /dev/null +++ b/terraform/test-configurations/2-04_try-now_paths_eager-start/README.md @@ -0,0 +1,12 @@ +# Eager start test config + +This contains a test configuration for eagerly started pods. +It does not use persistent workspaces. + +It installs two appdefinitions: + +- The default app defintion with 0 mininum instances and 10 maximum instances. + - No pre-warmed pod is started for this +- A CDT cloud app definition with name `cdt-cloud-demo` with 1 minimum instance and 10 maximum instances. + - 1 pre-warmed pod is started for this. + - This may be adjusted by increasing the `minInstances` property of the app definition in [theia_cloud.tf](./theia_cloud.tf). diff --git a/terraform/test-configurations/2-04_try-now_paths_eager-start/outputs.tf b/terraform/test-configurations/2-04_try-now_paths_eager-start/outputs.tf new file mode 100644 index 00000000..606fd7e6 --- /dev/null +++ b/terraform/test-configurations/2-04_try-now_paths_eager-start/outputs.tf @@ -0,0 +1,15 @@ +output "service" { + value = "https://${data.terraform_remote_state.minikube.outputs.hostname}/service" +} + +output "instance" { + value = "https://${data.terraform_remote_state.minikube.outputs.hostname}/instances" +} + +output "keycloak" { + value = "https://${data.terraform_remote_state.minikube.outputs.hostname}/keycloak/" +} + +output "landing" { + value = "https://${data.terraform_remote_state.minikube.outputs.hostname}/try" +} diff --git a/terraform/test-configurations/2-04_try-now_paths_eager-start/theia_cloud.tf b/terraform/test-configurations/2-04_try-now_paths_eager-start/theia_cloud.tf new file mode 100644 index 00000000..092c5ea0 --- /dev/null +++ b/terraform/test-configurations/2-04_try-now_paths_eager-start/theia_cloud.tf @@ -0,0 +1,126 @@ +data "terraform_remote_state" "minikube" { + backend = "local" + + config = { + path = "${path.module}/../0_minikube-setup/terraform.tfstate" + } +} + +provider "helm" { + kubernetes { + host = data.terraform_remote_state.minikube.outputs.host + client_certificate = data.terraform_remote_state.minikube.outputs.client_certificate + client_key = data.terraform_remote_state.minikube.outputs.client_key + cluster_ca_certificate = data.terraform_remote_state.minikube.outputs.cluster_ca_certificate + } +} + +provider "kubectl" { + load_config_file = false + host = data.terraform_remote_state.minikube.outputs.host + client_certificate = data.terraform_remote_state.minikube.outputs.client_certificate + client_key = data.terraform_remote_state.minikube.outputs.client_key + cluster_ca_certificate = data.terraform_remote_state.minikube.outputs.cluster_ca_certificate +} + +resource "helm_release" "theia-cloud" { + name = "theia-cloud" + chart = "../../../../theia-cloud-helm/charts/theia-cloud" + namespace = "theia-cloud" + create_namespace = true + + values = [ + "${file("${path.module}/../../values/valuesDemo.yaml")}" + ] + + set { + name = "hosts.usePaths" + # Need to hand in boolean as string as terraform converts boolean to 1 resp. 0. + # See https://github.com/hashicorp/terraform-provider-helm/issues/208 + value = "true" + } + + set { + name = "ingress.addTLSSecretName" + value = "true" + } + + set { + name = "hosts.configuration.service" + value = "service" + } + + set { + name = "hosts.configuration.landing" + value = "try" + } + + set { + name = "hosts.configuration.instance" + value = "instances" + } + + + set { + name = "hosts.configuration.baseHost" + value = data.terraform_remote_state.minikube.outputs.hostname + } + + set { + name = "keycloak.authUrl" + value = "https://${data.terraform_remote_state.minikube.outputs.hostname}/keycloak/" + } + + set { + name = "operator.cloudProvider" + value = "MINIKUBE" + } + + set { + name = "operator.eagerStart" + value = true + } + + set { + name = "ingress.clusterIssuer" + value = "theia-cloud-selfsigned-issuer" + } + + set { + name = "ingress.theiaCloudCommonName" + value = true + } + + # Only pull missing images. This is needed to use images built locally in Minikube + set { + name = "imagePullPolicy" + value = "IfNotPresent" + } +} + +resource "kubectl_manifest" "cdt-cloud-demo" { + depends_on = [helm_release.theia-cloud] + yaml_body = <<-EOF + apiVersion: theia.cloud/v1beta10 + kind: AppDefinition + metadata: + name: cdt-cloud-demo + namespace: theia-cloud + spec: + downlinkLimit: 30000 + image: theiacloud/cdt-cloud:v1.43.1 + imagePullPolicy: IfNotPresent + ingressname: theia-cloud-demo-ws-ingress + limitsCpu: "2" + limitsMemory: 1200M + maxInstances: 10 + minInstances: 1 + name: cdt-cloud-demo + port: 3000 + requestsCpu: 100m + requestsMemory: 1000M + timeout: 30 + uid: 101 + uplinkLimit: 30000 + EOF +} diff --git a/terraform/test-configurations/2-04_try-now_paths_eager-start/versions.tf b/terraform/test-configurations/2-04_try-now_paths_eager-start/versions.tf new file mode 100644 index 00000000..32efe434 --- /dev/null +++ b/terraform/test-configurations/2-04_try-now_paths_eager-start/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_providers { + helm = { + source = "hashicorp/helm" + version = ">= 2.9.0" + } + kubectl = { + source = "gavinbunney/kubectl" + version = ">= 1.14.0" + } + } + + required_version = ">= 1.4.0" +} diff --git a/terraform/test-configurations/test.md b/terraform/test-configurations/test.md index 035956d7..3fb61dac 100644 --- a/terraform/test-configurations/test.md +++ b/terraform/test-configurations/test.md @@ -21,4 +21,6 @@ terraform state rm kubernetes_persistent_volume.minikube Pick an installation in one of below directories and run `terraform init` and `terraform apply`. - `2-01_try-now` installs a local version of -- `2-02_monitor-vscode` installs a setup that allows to test the vscode monitor with and without authentication +- `2-02_monitor` installs a setup that allows to test the monitor (VSCode extension or Theia extension based) with and without authentication +- `2-03_try-now_paths` installs a local version of using paths instead of subdomains. +- `2-04_try-now_paths_eager-start` installs a local version of using paths and eager instead of lazy starting of pods. See its [README](./2-04_try-now_paths_eager-start/README.md) for more details.