issue-628 Changes to handle failure during clustered write-behind

bump versions Cleanup unused Jackson stuff Use platform-layout instead of platform-kit since platform 5.10.17 Jackson => Gson Switch to gradle nexus publish plugin platform version bump up issue-628 Changes to handle failure during clustered write-behind issue-628 Changed test case to use hamcrest libraries instead of junit issue-628 Minor formatting changes issue-628 Incorporated review comments by Chris issue-628 Incorporated review comments issue-628 Incorporated review comments issue-628 fix for current issue for handling failures of clustered write-behind issue-628 Reverting previous change issue-628 Another approach to handle the failures in clustered write behind issue-628 Reverting previous change issue-628 Increases test case timeout issue-628 Increased test case timeout issue-628 Increased test case timeout issue-628 Increased test case timeout issue-628 Increases test case timeout issue-628 Increases test case timeout issue-628 Changed test case timeout issue-628 Changed test case timeout issue-628 Changed test case timeout issue-628 Changed test case timeout issue-628 Changed test case timeout issue-628 Changed test case timeout issue-628 Changed test case timeout issue-628 Changed test case timeout issue-628 Changed test case timeout issue-628 Changed test case timeout issue-628 Changed test case timeout issue-628 Changed test case timeout issue-628 Changed test case timeout issue-628 Changed test case timeout issue-628 Changed test case timeout
ehcache · Aug 2, 2023 · 5b27a2a · 5b27a2a
1 parent ad2655f
commit 5b27a2a
Show file tree

Hide file tree

Showing 16 changed files with 251 additions and 126 deletions.
diff --git a/build-logic/src/main/java/org/ehcache/build/conventions/DeployConvention.java b/build-logic/src/main/java/org/ehcache/build/conventions/DeployConvention.java
@@ -80,23 +80,6 @@ public void apply(Project project) {
           dev.getOrganizationUrl().set("http://ehcache.org");
         }));
       }));
-      publishing.repositories(repositories -> repositories.maven(maven -> {
-        if (IS_RELEASE.test(project)) {
-          maven.setUrl(project.property("deployUrl"));
-          maven.credentials(creds -> {
-            creds.setUsername(project.property("deployUser").toString());
-            creds.setPassword(project.property("deployPwd").toString());
-          });
-          maven.setAllowInsecureProtocol(true);
-        } else {
-          maven.setName("sonatype-nexus-snapshot");
-          maven.setUrl("https://oss.sonatype.org/content/repositories/snapshots");
-          maven.credentials(creds -> {
-            creds.setUsername(project.property("sonatypeUser").toString());
-            creds.setPassword(project.property("sonatypePwd").toString());
-          });
-        }
-      }));
     });
 
     project.getExtensions().configure(SigningExtension.class, signing -> {

diff --git a/build.gradle b/build.gradle
@@ -16,8 +16,8 @@
 
 plugins {
   // This adds tasks to auto close or release nexus staging repos
-  // see https://github.com/Codearte/gradle-nexus-staging-plugin/
-  id 'io.codearte.nexus-staging'
+  // see https://github.com/gradle-nexus/publish-plugin
+  id 'io.github.gradle-nexus.publish-plugin'
   //OWASP Security Vulnerability Detection
   id 'org.owasp.dependencycheck'
 }
@@ -30,38 +30,26 @@ allprojects {
   version = findProperty('overrideVersion') ?: ehcacheVersion
 }
 
-if (deployUrl.contains('nexus')) {
-  //internal terracotta config, shorten url for this plugin to end at local/
-  project.nexusStaging {
-    serverUrl = deployUrl.replaceAll(~/local\/.*$/, "local/")
-    packageGroup = 'Ehcache OS' //internal staging repository name
+nexusPublishing {
+  repositories {
+    sonatype {
+      username = sonatypeUser
+      password = sonatypePwd
+      packageGroup = 'org.ehcache' //Sonatype staging repository name
+    }
+    nexus {
+      username = tcDeployUser
+      password = tcDeployPassword
+      packageGroup = 'Ehcache OS' //internal staging repository name
+      nexusUrl.set(uri('https://nexus.terracotta.eur.ad.sag:8443/service/local/'))
+      snapshotRepositoryUrl.set(uri("https://nexus.terracotta.eur.ad.sag:8443/content/repositories/terracotta-os-snapshots/"))
+    }
   }
-  ext {
-    deployUser = tcDeployUser
-    deployPwd = tcDeployPassword
-  }
-} else {
-  project.nexusStaging {
-    packageGroup = 'org.ehcache' //Sonatype staging repository name
-  }
-  ext {
-    deployUser = sonatypeUser
-    deployPwd = sonatypePwd
-  }
-}
-
-nexusStaging {
-  username = project.ext.deployUser
-  password = project.ext.deployPwd
-  logger.debug("Nexus Staging: Using login ${username} and url ${serverUrl}")
   // Sonatype is often very slow in these operations:
-  delayBetweenRetriesInMillis = (findProperty('delayBetweenRetriesInMillis') ?: '10000') as int
-  numberOfRetries = (findProperty('numberOfRetries') ?: '100') as int
-}
-
-tasks.named('closeAndReleaseRepository') {
-  // Disable automatic promotion for added safety
-  enabled = false;
+  transitionCheckOptions {
+    delayBetween = Duration.ofSeconds((findProperty('delayBetweenRetriesInSeconds') ?: '10') as int)
+    maxRetries = (findProperty('numberOfRetries') ?: '100') as int
+  }
 }
 
 assert (JavaVersion.current().isJava8Compatible()) : 'The Ehcache 3 build requires Java 8+ to run'

diff --git a/.../org/ehcache/clustered/client/internal/loaderwriter/writebehind/ClusteredWriteBehind.java b/.../org/ehcache/clustered/client/internal/loaderwriter/writebehind/ClusteredWriteBehind.java
@@ -75,10 +75,13 @@ private void process(long hash) {
                 operation);
               try {
                 if (result != null) {
+                  try{
                   if (result != currentState.get(key) && !(operation instanceof PutOperation)) {
-                    cacheLoaderWriter.write(result.getKey(), result.getValue());
-                  }
+                      cacheLoaderWriter.write(result.getKey(), result.getValue());
+                    }
                   currentState.put(key, result.asOperationExpiringAt(result.expirationTime()));
+                  }catch (Exception ignored){
+                  }
                 } else {
                   if (currentState.get(key) != null && (operation instanceof RemoveOperation
                     || operation instanceof ConditionalRemoveOperation)) {
@@ -95,7 +98,6 @@ private void process(long hash) {
             for (PutOperation<K, V> operation : currentState.values()) {
               builder = builder.add(codec.encode(operation));
             }
-
             clusteredWriteBehindStore.replaceAtHead(hash, chain, builder.build());
           }
         } finally {

diff --git a/clustered/ehcache-clustered/build.gradle b/clustered/ehcache-clustered/build.gradle
@@ -90,7 +90,7 @@ dependencies {
   serverLibs project(':clustered:server:ehcache-service')
 
   kit "org.terracotta.internal:terracotta-kit:$terracottaCoreVersion@tar.gz"
-  kit "org.terracotta:platform-kit:$terracottaPlatformVersion@tar.gz"
+  kit "org.terracotta:platform-layout:$terracottaPlatformVersion@tar.gz"
 }
 
 task copyDocs(type: Sync) {
@@ -107,7 +107,7 @@ tasks.named('jar') {
   osgi {
     instruction Constants.BUNDLE_SYMBOLICNAME, 'org.ehcache.clustered'
     instruction Constants.EXPORT_PACKAGE, '!com.tc.*, !com.terracotta.*, !org.terracotta.*, !org.ehcache.*.internal.*, !sun.misc, org.ehcache.clustered.client.*, org.ehcache.clustered.common.*'
-    instruction Constants.IMPORT_PACKAGE, '!sun.misc.*, org.ehcache.xml.*;resolution:=optional, jdk.jfr.*;resolution:=optional, !com.fasterxml.jackson.*, !org.terracotta.json, javax.xml.bind*;version="[2.2,3)", *'
+    instruction Constants.IMPORT_PACKAGE, '!sun.misc.*, org.ehcache.xml.*;resolution:=optional, jdk.jfr.*;resolution:=optional, javax.xml.bind*;version="[2.2,3)", *'
   }
 }
 

diff --git a/clustered/integration-test/build.gradle b/clustered/integration-test/build.gradle
@@ -41,12 +41,6 @@ dependencies {
   testImplementation project(':ehcache-management')
   testImplementation "org.terracotta.management:nms-entity-client:$terracottaPlatformVersion"
   testImplementation "org.terracotta.management:nms-agent-entity-client:$terracottaPlatformVersion"
-  /*
-   * Jackson BOMs are broken in a bunch of ways. One day we might be able to use the constraints provided through the
-   * platform dependencies, today is not that day.
-   */
-  testImplementation enforcedPlatform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")
-  testImplementation 'com.fasterxml.jackson.core:jackson-databind'
   testImplementation "org.terracotta:terracotta-utilities-port-chooser:$terracottaUtilitiesVersion"
   testImplementation("org.terracotta:galvan-platform-support:$terracottaPlatformVersion") {
     exclude group: 'org.terracotta', module: 'terracotta-utilities-port-chooser'

diff --git a/...test/src/test/java/org/ehcache/clustered/management/AbstractClusteringManagementTest.java b/...test/src/test/java/org/ehcache/clustered/management/AbstractClusteringManagementTest.java
@@ -15,8 +15,6 @@
  */
 package org.ehcache.clustered.management;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
 import org.ehcache.CacheManager;
 import org.ehcache.Status;
 import org.ehcache.clustered.util.BeforeAll;
@@ -84,7 +82,6 @@ public abstract class AbstractClusteringManagementTest {
   protected static CacheManager cacheManager;
   protected static ClientIdentifier ehcacheClientIdentifier;
   protected static ServerEntityIdentifier clusterTierManagerEntityIdentifier;
-  protected static final ObjectMapper mapper = new ObjectMapper().configure(SerializationFeature.INDENT_OUTPUT, true);
 
   @ClassRule
   public static final ClusterWithManagement CLUSTER = new ClusterWithManagement(newCluster(2)

diff --git a/...-test/src/test/java/org/ehcache/clustered/management/ClusteringManagementServiceTest.java b/...-test/src/test/java/org/ehcache/clustered/management/ClusteringManagementServiceTest.java
@@ -20,7 +20,6 @@
 import org.ehcache.config.units.MemoryUnit;
 import org.hamcrest.MatcherAssert;
 import org.junit.FixMethodOrder;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runners.MethodSorters;
 import org.terracotta.management.model.capabilities.Capability;
@@ -133,14 +132,6 @@ public class ClusteringManagementServiceTest extends AbstractClusteringManagemen
     new StatisticDescriptor("OffHeapResource:AllocatedMemory", "GAUGE")
   );
 
-  @Test
-  @Ignore("This is not a test, but something useful to show a json print of a cluster topology with all management metadata inside")
-  public void test_A_topology() throws Exception {
-    Cluster cluster = CLUSTER.getNmsService().readTopology();
-    String json = mapper.writeValueAsString(cluster.toMap());
-    //System.out.println(json);
-  }
-
   @Test
   public void test_A_client_tags_exposed() throws Exception {
     MatcherAssert.assertThat(() -> {

diff --git a/...-test/src/test/java/org/ehcache/clustered/management/ManagementClusterConnectionTest.java b/...-test/src/test/java/org/ehcache/clustered/management/ManagementClusterConnectionTest.java
@@ -15,8 +15,6 @@
  */
 package org.ehcache.clustered.management;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
 import org.ehcache.Cache;
 import org.ehcache.CacheManager;
 import org.ehcache.Status;
@@ -59,7 +57,6 @@
 public class ManagementClusterConnectionTest {
 
   protected static CacheManager cacheManager;
-  protected static ObjectMapper mapper = new ObjectMapper();
 
   private static TCPProxyManager proxyManager;
   private static final Map<String, Long> resources;
@@ -79,9 +76,6 @@ public class ManagementClusterConnectionTest {
 
   @BeforeClass
   public static void beforeClass() throws Exception {
-
-    mapper.configure(SerializationFeature.INDENT_OUTPUT, true);
-
     CLUSTER.get().getCluster().getClusterControl().waitForActive();
 
     proxyManager = TCPProxyManager.create(CLUSTER.get().getCluster().getConnectionURI());

diff --git a/...n-test/src/test/java/org/ehcache/clustered/writebehind/BasicClusteredWriteBehindTest.java b/...n-test/src/test/java/org/ehcache/clustered/writebehind/BasicClusteredWriteBehindTest.java
@@ -29,13 +29,19 @@
 import java.lang.management.ManagementFactory;
 import java.lang.management.ThreadInfo;
 import java.lang.management.ThreadMXBean;
+import java.time.Duration;
 import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 import static org.ehcache.testing.StandardCluster.clusterPath;
 import static org.ehcache.testing.StandardCluster.newCluster;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.terracotta.utilities.test.matchers.Eventually.within;
 
 
 public class BasicClusteredWriteBehindTest extends WriteBehindTestBase {
@@ -127,4 +133,32 @@ public void testClusteredWriteBehindLoading() throws Exception {
 
     doThreadDump = false;
   }
+
+  @Test(timeout = 120000)
+  public void testBasicClusteredWriteBehindWithFailure() {
+    cacheManager.close();
+    cacheManager = null;
+    cacheManager = createCacheManagerWithLoaderWriterWithFailure(CLUSTER.getConnectionURI());
+    final Cache<Long, String> localCache = cacheManager.getCache(testName.getMethodName(), Long.class, String.class);
+    localCache.put(1L, String.valueOf(1));
+    localCache.put(2L, String.valueOf(2));
+    localCache.put(3L, String.valueOf(3));
+    localCache.put(4L, String.valueOf(4));
+
+    assertThat(() -> localCache.get(1L), within(Duration.ofSeconds(100)).matches(is(nullValue())));
+    assertThat(() -> localCache.get(3L), within(Duration.ofSeconds(100)).matches(is(nullValue())));
+
+    Map<Long, List<String>> records = getEvenNumberLoaderWriter().getRecords();
+
+    // Error will be thrown for odd keys, hence only 2 entries are expected here.
+    assertThat(() -> records.size(), within(Duration.ofSeconds(2)).matches(is(2)));
+
+    // Verify that values with only even entries are present in records.
+    List<String> keyRecords = records.get(2L);
+    assertThat(keyRecords.get(0), is("2"));
+    keyRecords = records.get(4L);
+    assertThat(keyRecords.get(0), is("4"));
+
+    doThreadDump = false;
+  }
 }
diff --git a/...egration-test/src/test/java/org/ehcache/clustered/writebehind/EvenNumberLoaderWriter.java b/...egration-test/src/test/java/org/ehcache/clustered/writebehind/EvenNumberLoaderWriter.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright Terracotta, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.ehcache.clustered.writebehind;
+
+import org.ehcache.spi.loaderwriter.CacheLoaderWriter;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+class EvenNumberLoaderWriter<Long, String> implements CacheLoaderWriter<Long, String> {
+
+  private final Map<Long, List<String>> records = new HashMap<>();
+
+  @Override
+  public synchronized String load(Long key) {
+    List<String> list = records.get(key);
+    return list == null ? null : list.get(list.size() - 1);
+  }
+
+  @Override
+  public synchronized void write(Long key, String value) throws Exception {
+    if (Integer.parseInt(key.toString()) % 2 != 0) {
+      throw new RuntimeException("Only even keys can be inserted");
+    }
+    record(key, value);
+  }
+
+  @Override
+  public void delete(Long key) {
+  }
+
+  private void record(Long key, String value) {
+    records.computeIfAbsent(key, k -> new ArrayList<>()).add(value);
+  }
+
+  synchronized Map<Long, List<String>> getRecords() {
+    return Collections.unmodifiableMap(records);
+  }
+}
diff --git a/...integration-test/src/test/java/org/ehcache/clustered/writebehind/WriteBehindTestBase.java b/...integration-test/src/test/java/org/ehcache/clustered/writebehind/WriteBehindTestBase.java
@@ -29,6 +29,7 @@
 import org.ehcache.config.units.EntryUnit;
 import org.ehcache.config.units.MemoryUnit;
 import org.ehcache.core.internal.resilience.ThrowingResilienceStrategy;
+import org.ehcache.spi.loaderwriter.CacheLoaderWriter;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.rules.TestName;
@@ -56,6 +57,7 @@ public class WriteBehindTestBase {
   public final TestName testName = new TestName();
 
   private RecordingLoaderWriter<Long, String> loaderWriter;
+  private EvenNumberLoaderWriter<Long, String> evenNumberLoaderWriter;
 
   @Before
   public void setUp() throws Exception {
@@ -85,12 +87,16 @@ void assertValue(Cache<Long, String> cache, String value) {
   }
 
   PersistentCacheManager createCacheManager(URI clusterUri) {
+    return getPersistentCacheManager(clusterUri, loaderWriter);
+  }
+
+  private PersistentCacheManager getPersistentCacheManager(URI clusterUri, CacheLoaderWriter<Long, String> localLoaderWriter) {
     CacheConfiguration<Long, String> cacheConfiguration =
       newCacheConfigurationBuilder(Long.class, String.class, ResourcePoolsBuilder.newResourcePoolsBuilder()
-                                                                                 .heap(10, EntryUnit.ENTRIES)
-                                                                                 .offheap(1, MemoryUnit.MB)
-                                                                                 .with(ClusteredResourcePoolBuilder.clusteredDedicated("primary-server-resource", 2, MemoryUnit.MB)))
-        .withLoaderWriter(loaderWriter)
+        .heap(10, EntryUnit.ENTRIES)
+        .offheap(1, MemoryUnit.MB)
+        .with(ClusteredResourcePoolBuilder.clusteredDedicated("primary-server-resource", 2, MemoryUnit.MB)))
+        .withLoaderWriter(localLoaderWriter)
         .withService(WriteBehindConfigurationBuilder.newUnBatchedWriteBehindConfiguration())
         .withResilienceStrategy(new ThrowingResilienceStrategy<>())
         .withService(new ClusteredStoreConfiguration(Consistency.STRONG))
@@ -102,4 +108,13 @@ PersistentCacheManager createCacheManager(URI clusterUri) {
       .withCache(testName.getMethodName(), cacheConfiguration)
       .build(true);
   }
+
+  PersistentCacheManager createCacheManagerWithLoaderWriterWithFailure(URI clusterUri) {
+    evenNumberLoaderWriter = new EvenNumberLoaderWriter<Long, String>();
+    return getPersistentCacheManager(clusterUri, evenNumberLoaderWriter);
+  }
+
+  protected EvenNumberLoaderWriter<Long, String> getEvenNumberLoaderWriter() {
+    return this.evenNumberLoaderWriter;
+  }
 }
diff --git a/ehcache-management/build.gradle b/ehcache-management/build.gradle
@@ -43,11 +43,6 @@ dependencies {
 
   testImplementation "org.terracotta.management:management-registry:$terracottaPlatformVersion"
   testImplementation project(':ehcache-xml')
-  /*
-   * Jackson BOMs are broken in a bunch of ways. One day we might be able to use the constraints provided through the
-   * platform dependencies, today is not that day.
-   */
-  testImplementation enforcedPlatform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")
-  testImplementation 'com.fasterxml.jackson.core:jackson-databind'
+  testImplementation "org.terracotta.common:common-json-support:$terracottaPlatformVersion"
   testImplementation testFixtures(project(':ehcache-xml'))
 }