Skip to content

Commit

Permalink
Improve session pod labels (#362)
Browse files Browse the repository at this point in the history 
org.eclipse.theia.cloud.common.util.LabelsUtil is added to encapsulate
creation of labels.

To indicate that a pod is a user session, its
'app.kubernetes.io/component' label is set to 'session'.

For custom labels, we use the prefix 'theia-cloud.io'.

This is currently used for:
- Identifying a session pods associated with a user:
  'theia-cloud.io/user' = '$username'
- Identifying a session pod by appdefinition name:
  'theia-cloud.io/app-definition' = $appdefinitionname

Signed-off-by: Olaf Lessenich <[email protected]>
Co-authored-by: Johannes Faltermeier <[email protected]>
Co-authored-by: Lucas Koehler <[email protected]>
  • Loading branch information
@xai @jfaltermeier @lucas-koehler
3 people authored Nov 28, 2024
1 parent 84d00f6 commit 518c317
Show file tree
Hide file tree
Showing 5 changed files with 135 additions and 33 deletions.
31 changes: 31 additions & 0 deletions ...ipse.theia.cloud.common/src/main/java/org/eclipse/theia/cloud/common/util/LabelsUtil.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package org.eclipse.theia.cloud.common.util;

import java.util.HashMap;
import java.util.Map;

import org.eclipse.theia.cloud.common.k8s.resource.appdefinition.AppDefinitionSpec;
import org.eclipse.theia.cloud.common.k8s.resource.session.SessionSpec;

public class LabelsUtil {
public static final String LABEL_CUSTOM_PREFIX = "theia-cloud.io";

public static final String LABEL_KEY_SESSION = "app.kubernetes.io/component";
public static final String LABEL_VALUE_SESSION = "session";

public static final String LABEL_KEY_THEIACLOUD = "app.kubernetes.io/part-of";
public static final String LABEL_VALUE_THEIACLOUD = "theia-cloud";

public static final String LABEL_KEY_USER = LABEL_CUSTOM_PREFIX + "/user";
public static final String LABEL_KEY_APPDEF = LABEL_CUSTOM_PREFIX + "/app-definition";

public static Map<String, String> createSessionLabels(SessionSpec sessionSpec,
AppDefinitionSpec appDefinitionSpec) {
Map<String, String> labels = new HashMap<>();
labels.put(LABEL_KEY_SESSION, LABEL_VALUE_SESSION);
labels.put(LABEL_KEY_THEIACLOUD, LABEL_VALUE_THEIACLOUD);
String sanitizedUser = sessionSpec.getUser().replaceAll("@", "_at_").replaceAll("[^a-zA-Z0-9]", "_");
labels.put(LABEL_KEY_USER, sanitizedUser);
labels.put(LABEL_KEY_APPDEF, appDefinitionSpec.getName());
return labels;
}
}
29 changes: 19 additions & 10 deletions .../org/eclipse/theia/cloud/operator/handler/appdef/EagerStartAppDefinitionAddedHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.net.URISyntaxException;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
import java.util.Set;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -104,10 +105,12 @@ public boolean appDefinitionAdded(AppDefinition appDefinition, String correlatio
Set<Integer> missingServiceIds = TheiaCloudServiceUtil.computeIdsOfMissingServices(appDefinition, correlationId,
instances, existingServices);

Map<String, String> labelsToAdd = new HashMap<String, String>();

/* Create missing services for this app definition */
for (int instance : missingServiceIds) {
createAndApplyService(client.kubernetes(), client.namespace(), correlationId, appDefinitionResourceName,
appDefinitionResourceUID, instance, appDefinition, arguments.isUseKeycloak());
appDefinitionResourceUID, instance, appDefinition, arguments.isUseKeycloak(), labelsToAdd);
}

if (arguments.isUseKeycloak()) {
Expand All @@ -130,11 +133,13 @@ public boolean appDefinitionAdded(AppDefinition appDefinition, String correlatio
/* Create missing configmaps for this app definition */
for (int instance : missingProxyIds) {
createAndApplyProxyConfigMap(client.kubernetes(), client.namespace(), correlationId,
appDefinitionResourceName, appDefinitionResourceUID, instance, appDefinition);
appDefinitionResourceName, appDefinitionResourceUID, instance, appDefinition,
labelsToAdd);
}
for (int instance : missingEmailIds) {
createAndApplyEmailConfigMap(client.kubernetes(), client.namespace(), correlationId,
appDefinitionResourceName, appDefinitionResourceUID, instance, appDefinition);
appDefinitionResourceName, appDefinitionResourceUID, instance, appDefinition,
labelsToAdd);
}
}

Expand All @@ -149,14 +154,14 @@ public boolean appDefinitionAdded(AppDefinition appDefinition, String correlatio
/* Create missing deployments for this app definition */
for (int instance : missingDeploymentIds) {
createAndApplyDeployment(client.kubernetes(), client.namespace(), correlationId, appDefinitionResourceName,
appDefinitionResourceUID, instance, appDefinition, arguments.isUseKeycloak());
appDefinitionResourceUID, instance, appDefinition, arguments.isUseKeycloak(), labelsToAdd);
}
return true;
}

protected void createAndApplyService(NamespacedKubernetesClient client, String namespace, String correlationId,
String appDefinitionResourceName, String appDefinitionResourceUID, int instance,
AppDefinition appDefinition, boolean useOAuth2Proxy) {
AppDefinition appDefinition, boolean useOAuth2Proxy, Map<String, String> labelsToAdd) {
Map<String, String> replacements = TheiaCloudServiceUtil.getServiceReplacements(namespace, appDefinition,
instance);
String templateYaml = useOAuth2Proxy ? AddedHandlerUtil.TEMPLATE_SERVICE_YAML
Expand All @@ -172,12 +177,13 @@ protected void createAndApplyService(NamespacedKubernetesClient client, String n
return;
}
K8sUtil.loadAndCreateServiceWithOwnerReference(client, namespace, correlationId, serviceYaml, AppDefinition.API,
AppDefinition.KIND, appDefinitionResourceName, appDefinitionResourceUID, 0);
AppDefinition.KIND, appDefinitionResourceName, appDefinitionResourceUID, 0,
labelsToAdd);
}

protected void createAndApplyDeployment(NamespacedKubernetesClient client, String namespace, String correlationId,
String appDefinitionResourceName, String appDefinitionResourceUID, int instance,
AppDefinition appDefinition, boolean useOAuth2Proxy) {
AppDefinition appDefinition, boolean useOAuth2Proxy, Map<String, String> labelsToAdd) {
Map<String, String> replacements = deploymentReplacements.getReplacements(namespace, appDefinition, instance);
String templateYaml = useOAuth2Proxy ? AddedHandlerUtil.TEMPLATE_DEPLOYMENT_YAML
: AddedHandlerUtil.TEMPLATE_DEPLOYMENT_WITHOUT_AOUTH2_PROXY_YAML;
Expand All @@ -193,6 +199,7 @@ protected void createAndApplyDeployment(NamespacedKubernetesClient client, Strin
}
K8sUtil.loadAndCreateDeploymentWithOwnerReference(client, namespace, correlationId, deploymentYaml,
AppDefinition.API, AppDefinition.KIND, appDefinitionResourceName, appDefinitionResourceUID, 0,
labelsToAdd,
deployment -> {
bandwidthLimiter.limit(deployment, appDefinition.getSpec().getDownlinkLimit(),
appDefinition.getSpec().getUplinkLimit(), correlationId);
Expand All @@ -206,7 +213,7 @@ protected void createAndApplyDeployment(NamespacedKubernetesClient client, Strin

protected void createAndApplyProxyConfigMap(NamespacedKubernetesClient client, String namespace,
String correlationId, String appDefinitionResourceName, String appDefinitionResourceUID, int instance,
AppDefinition appDefinition) {
AppDefinition appDefinition, Map<String, String> labelsToAdd) {
Map<String, String> replacements = TheiaCloudConfigMapUtil.getProxyConfigMapReplacements(namespace,
appDefinition, instance);
String configMapYaml;
Expand All @@ -221,6 +228,7 @@ protected void createAndApplyProxyConfigMap(NamespacedKubernetesClient client, S
}
K8sUtil.loadAndCreateConfigMapWithOwnerReference(client, namespace, correlationId, configMapYaml,
AppDefinition.API, AppDefinition.KIND, appDefinitionResourceName, appDefinitionResourceUID, 0,
labelsToAdd,
configMap -> {
String host = arguments.getInstancesHost() + ingressPathProvider.getPath(appDefinition, instance);
int port = appDefinition.getSpec().getPort();
Expand All @@ -230,7 +238,7 @@ protected void createAndApplyProxyConfigMap(NamespacedKubernetesClient client, S

protected void createAndApplyEmailConfigMap(NamespacedKubernetesClient client, String namespace,
String correlationId, String appDefinitionResourceName, String appDefinitionResourceUID, int instance,
AppDefinition appDefinition) {
AppDefinition appDefinition, Map<String, String> labelsToAdd) {
Map<String, String> replacements = TheiaCloudConfigMapUtil.getEmailConfigMapReplacements(namespace,
appDefinition, instance);
String configMapYaml;
Expand All @@ -244,7 +252,8 @@ protected void createAndApplyEmailConfigMap(NamespacedKubernetesClient client, S
return;
}
K8sUtil.loadAndCreateConfigMapWithOwnerReference(client, namespace, correlationId, configMapYaml,
AppDefinition.API, AppDefinition.KIND, appDefinitionResourceName, appDefinitionResourceUID, 0);
AppDefinition.API, AppDefinition.KIND, appDefinitionResourceName, appDefinitionResourceUID, 0,
labelsToAdd);
}

}
22 changes: 22 additions & 0 deletions .../main/java/org/eclipse/theia/cloud/operator/handler/session/EagerStartSessionHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
import java.util.Collections;
import java.util.List;
import java.util.Map.Entry;
import java.util.Map;
import java.util.HashMap;
import java.util.Optional;

import org.apache.logging.log4j.LogManager;
Expand All @@ -30,6 +32,7 @@
import org.eclipse.theia.cloud.common.k8s.resource.session.Session;
import org.eclipse.theia.cloud.common.k8s.resource.session.SessionSpec;
import org.eclipse.theia.cloud.common.util.JavaUtil;
import org.eclipse.theia.cloud.common.util.LabelsUtil;
import org.eclipse.theia.cloud.operator.TheiaCloudOperatorArguments;
import org.eclipse.theia.cloud.operator.handler.AddedHandlerUtil;
import org.eclipse.theia.cloud.operator.ingress.IngressPathProvider;
Expand Down Expand Up @@ -115,6 +118,25 @@ public boolean sessionAdded(Session session, String correlationId) {
return false;
}

try {
client.services().inNamespace(client.namespace()).withName(serviceToUse.get().getMetadata().getName())
.edit(service -> {
LOGGER.debug("Setting session labels");
Map<String, String> labels = service.getMetadata().getLabels();
if (labels == null) {
labels = new HashMap<>();
service.getMetadata().setLabels(labels);
}
Map<String, String> newLabels = LabelsUtil.createSessionLabels(spec, appDefinition.get().getSpec());
labels.putAll(newLabels);
return service;
});
} catch (KubernetesClientException e) {
LOGGER.error(formatLogMessage(correlationId,
"Error while adding labels to service " + (serviceToUse.get().getMetadata().getName())), e);
return false;
}

/* get the deployment for the service and add as owner */
Integer instance = TheiaCloudServiceUtil.getId(correlationId, appDefinition.get(), serviceToUse.get());
if (instance == null) {
Expand Down
44 changes: 32 additions & 12 deletions ...or/src/main/java/org/eclipse/theia/cloud/operator/handler/session/LazySessionHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
import java.util.Optional;

import org.apache.logging.log4j.LogManager;
Expand All @@ -39,6 +40,7 @@
import org.eclipse.theia.cloud.common.k8s.resource.session.SessionSpec;
import org.eclipse.theia.cloud.common.k8s.resource.session.SessionStatus;
import org.eclipse.theia.cloud.common.k8s.resource.workspace.Workspace;
import org.eclipse.theia.cloud.common.util.LabelsUtil;
import org.eclipse.theia.cloud.common.util.TheiaCloudError;
import org.eclipse.theia.cloud.common.util.WorkspaceUtil;
import org.eclipse.theia.cloud.operator.TheiaCloudOperatorArguments;
Expand Down Expand Up @@ -155,6 +157,10 @@ protected boolean doSessionAdded(Session session, String correlationId) {
return false;
}
AppDefinition appDefinition = optionalAppDefinition.get();
AppDefinitionSpec appDefinitionSpec = appDefinition.getSpec();

/* label maps */
Map<String, String> labelsToAdd = LabelsUtil.createSessionLabels(session.getSpec(), appDefinitionSpec);

if (hasMaxInstancesReached(appDefinition, session, correlationId)) {
client.sessions().updateStatus(correlationId, session, s -> {
Expand Down Expand Up @@ -200,7 +206,7 @@ protected boolean doSessionAdded(Session session, String correlationId) {
}

Optional<Service> serviceToUse = createAndApplyService(correlationId, sessionResourceName, sessionResourceUID,
session, appDefinition.getSpec(), arguments.isUseKeycloak());
session, appDefinitionSpec, arguments.isUseKeycloak(), labelsToAdd);
if (serviceToUse.isEmpty()) {
LOGGER.error(formatLogMessage(correlationId, "Unable to create service for session " + sessionSpec));
client.sessions().updateStatus(correlationId, session, s -> {
Expand All @@ -226,9 +232,9 @@ protected boolean doSessionAdded(Session session, String correlationId) {
// this handler
return true;
}
createAndApplyEmailConfigMap(correlationId, sessionResourceName, sessionResourceUID, session);
createAndApplyEmailConfigMap(correlationId, sessionResourceName, sessionResourceUID, session, labelsToAdd);
createAndApplyProxyConfigMap(correlationId, sessionResourceName, sessionResourceUID, session,
appDefinition);
appDefinition, labelsToAdd);
}

/* Create deployment for this session */
Expand All @@ -247,7 +253,7 @@ protected boolean doSessionAdded(Session session, String correlationId) {

Optional<String> storageName = getStorageName(session, correlationId);
createAndApplyDeployment(correlationId, sessionResourceName, sessionResourceUID, session, appDefinition,
storageName, arguments.isUseKeycloak());
storageName, arguments.isUseKeycloak(), labelsToAdd);

/* adjust the ingress */
String host;
Expand Down Expand Up @@ -371,7 +377,8 @@ protected Optional<String> getStorageName(Session session, String correlationId)
}

protected Optional<Service> createAndApplyService(String correlationId, String sessionResourceName,
String sessionResourceUID, Session session, AppDefinitionSpec appDefinitionSpec, boolean useOAuth2Proxy) {
String sessionResourceUID, Session session, AppDefinitionSpec appDefinitionSpec, boolean useOAuth2Proxy,
Map<String, String> labelsToAdd) {
Map<String, String> replacements = TheiaCloudServiceUtil.getServiceReplacements(client.namespace(), session,
appDefinitionSpec);
String templateYaml = useOAuth2Proxy ? AddedHandlerUtil.TEMPLATE_SERVICE_YAML
Expand All @@ -385,11 +392,11 @@ protected Optional<Service> createAndApplyService(String correlationId, String s
return Optional.empty();
}
return K8sUtil.loadAndCreateServiceWithOwnerReference(client.kubernetes(), client.namespace(), correlationId,
serviceYaml, Session.API, Session.KIND, sessionResourceName, sessionResourceUID, 0);
serviceYaml, Session.API, Session.KIND, sessionResourceName, sessionResourceUID, 0, labelsToAdd);
}

protected void createAndApplyEmailConfigMap(String correlationId, String sessionResourceName,
String sessionResourceUID, Session session) {
String sessionResourceUID, Session session, Map<String, String> labelsToAdd) {
Map<String, String> replacements = TheiaCloudConfigMapUtil.getEmailConfigMapReplacements(client.namespace(),
session);
String configMapYaml;
Expand All @@ -401,14 +408,15 @@ protected void createAndApplyEmailConfigMap(String correlationId, String session
return;
}
K8sUtil.loadAndCreateConfigMapWithOwnerReference(client.kubernetes(), client.namespace(), correlationId,
configMapYaml, Session.API, Session.KIND, sessionResourceName, sessionResourceUID, 0, configmap -> {
configMapYaml, Session.API, Session.KIND, sessionResourceName, sessionResourceUID, 0,
labelsToAdd, configmap -> {
configmap.setData(Collections.singletonMap(AddedHandlerUtil.FILENAME_AUTHENTICATED_EMAILS_LIST,
session.getSpec().getUser()));
});
}

protected void createAndApplyProxyConfigMap(String correlationId, String sessionResourceName,
String sessionResourceUID, Session session, AppDefinition appDefinition) {
String sessionResourceUID, Session session, AppDefinition appDefinition, Map<String, String> labelsToAdd) {
Map<String, String> replacements = TheiaCloudConfigMapUtil.getProxyConfigMapReplacements(client.namespace(),
session);
String configMapYaml;
Expand All @@ -420,7 +428,8 @@ protected void createAndApplyProxyConfigMap(String correlationId, String session
return;
}
K8sUtil.loadAndCreateConfigMapWithOwnerReference(client.kubernetes(), client.namespace(), correlationId,
configMapYaml, Session.API, Session.KIND, sessionResourceName, sessionResourceUID, 0, configMap -> {
configMapYaml, Session.API, Session.KIND, sessionResourceName, sessionResourceUID, 0,
labelsToAdd, configMap -> {
String host = arguments.getInstancesHost() + ingressPathProvider.getPath(appDefinition, session);
int port = appDefinition.getSpec().getPort();
AddedHandlerUtil.updateProxyConfigMap(client.kubernetes(), client.namespace(), configMap, host,
Expand All @@ -429,7 +438,8 @@ protected void createAndApplyProxyConfigMap(String correlationId, String session
}

protected void createAndApplyDeployment(String correlationId, String sessionResourceName, String sessionResourceUID,
Session session, AppDefinition appDefinition, Optional<String> pvName, boolean useOAuth2Proxy) {
Session session, AppDefinition appDefinition, Optional<String> pvName, boolean useOAuth2Proxy,
Map<String, String> labelsToAdd) {
Map<String, String> replacements = deploymentReplacements.getReplacements(client.namespace(), appDefinition,
session);
String templateYaml = useOAuth2Proxy ? AddedHandlerUtil.TEMPLATE_DEPLOYMENT_YAML
Expand All @@ -443,7 +453,17 @@ protected void createAndApplyDeployment(String correlationId, String sessionReso
return;
}
K8sUtil.loadAndCreateDeploymentWithOwnerReference(client.kubernetes(), client.namespace(), correlationId,
deploymentYaml, Session.API, Session.KIND, sessionResourceName, sessionResourceUID, 0, deployment -> {
deploymentYaml, Session.API, Session.KIND, sessionResourceName, sessionResourceUID, 0,
labelsToAdd, deployment -> {

LOGGER.debug("Setting session labels");
Map<String, String> labels = deployment.getSpec().getTemplate().getMetadata().getLabels();
if (labels == null) {
labels = new HashMap<>();
deployment.getSpec().getTemplate().getMetadata().setLabels(labels);
}
labels.putAll(labelsToAdd);

pvName.ifPresent(name -> addVolumeClaim(deployment, name, appDefinition.getSpec()));
bandwidthLimiter.limit(deployment, appDefinition.getSpec().getDownlinkLimit(),
appDefinition.getSpec().getUplinkLimit(), correlationId);
Expand Down
Loading

0 comments on commit 518c317

Please sign in to comment.