Fixing ehcache flaky test (opensearch-project#12764)

* Fixing ehcache flaky test

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

* Adding a ehcache issue reference for thread leak issue

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

* Updating comment

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>

---------

Signed-off-by: Sagar Upadhyaya <sagar.upadhyaya.121@gmail.com>
Signed-off-by: Shivansh Arora <hishiv@amazon.com>

Author: sgup432

plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java

@@ -8,6 +8,8 @@
 
 package org.opensearch.cache.store.disk;
 
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
 import org.opensearch.cache.EhcacheDiskCacheSettings;
 import org.opensearch.common.Randomness;
 import org.opensearch.common.cache.CacheType;
@@ -47,6 +49,7 @@
 import static org.opensearch.cache.EhcacheDiskCacheSettings.DISK_STORAGE_PATH_KEY;
 import static org.hamcrest.CoreMatchers.instanceOf;
 
+@ThreadLeakFilters(filters = { EhcacheThreadLeakFilter.class })
 public class EhCacheDiskCacheTests extends OpenSearchSingleNodeTestCase {
 
     private static final int CACHE_SIZE_IN_BYTES = 1024 * 101;
@@ -633,6 +636,47 @@ public void testBasicGetAndPutBytesReference() throws Exception {
         }
     }
 
+    public void testInvalidate() throws Exception {
+        Settings settings = Settings.builder().build();
+        MockRemovalListener<String, String> removalListener = new MockRemovalListener<>();
+        try (NodeEnvironment env = newNodeEnvironment(settings)) {
+            ICache<String, String> ehcacheTest = new EhcacheDiskCache.Builder<String, String>().setThreadPoolAlias("ehcacheTest")
+                .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
+                .setIsEventListenerModeSync(true)
+                .setKeyType(String.class)
+                .setKeySerializer(new StringSerializer())
+                .setValueSerializer(new StringSerializer())
+                .setValueType(String.class)
+                .setCacheType(CacheType.INDICES_REQUEST_CACHE)
+                .setSettings(settings)
+                .setExpireAfterAccess(TimeValue.MAX_VALUE)
+                .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
+                .setRemovalListener(removalListener)
+                .build();
+            int randomKeys = randomIntBetween(10, 100);
+            Map<String, String> keyValueMap = new HashMap<>();
+            for (int i = 0; i < randomKeys; i++) {
+                keyValueMap.put(UUID.randomUUID().toString(), UUID.randomUUID().toString());
+            }
+            for (Map.Entry<String, String> entry : keyValueMap.entrySet()) {
+                ehcacheTest.put(entry.getKey(), entry.getValue());
+            }
+            assertEquals(keyValueMap.size(), ehcacheTest.count());
+            List<String> removedKeyList = new ArrayList<>();
+            for (Map.Entry<String, String> entry : keyValueMap.entrySet()) {
+                if (randomBoolean()) {
+                    removedKeyList.add(entry.getKey());
+                    ehcacheTest.invalidate(entry.getKey());
+                }
+            }
+            for (String removedKey : removedKeyList) {
+                assertNull(ehcacheTest.get(removedKey));
+            }
+            assertEquals(keyValueMap.size() - removedKeyList.size(), ehcacheTest.count());
+            ehcacheTest.close();
+        }
+    }
+
     private static String generateRandomString(int length) {
         String characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
         StringBuilder randomString = new StringBuilder(length);

plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhcacheThreadLeakFilter.java

@@ -0,0 +1,29 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cache.store.disk;
+
+import com.carrotsearch.randomizedtesting.ThreadFilter;
+
+/**
+ * In Ehcache(as of 3.10.8), while calling remove/invalidate() on entries causes to start a daemon thread in the
+ * background to clean up the stale offheap memory associated with the disk cache. And this thread is not closed even
+ * after we try to close the cache or cache manager. Considering that it requires a node restart to switch between
+ * different cache plugins, this shouldn't be a problem for now.
+ *
+ * See: https://github.com/ehcache/ehcache3/issues/3204
+ */
+public class EhcacheThreadLeakFilter implements ThreadFilter {
+
+    private static final String OFFENDING_THREAD_NAME = "MappedByteBufferSource";
+
+    @Override
+    public boolean reject(Thread t) {
+        return t.getName().startsWith(OFFENDING_THREAD_NAME);
+    }
+}