Updated to provide mechanism to ensure table cache consistency

core/connection-pool/src/main/java/datawave/core/common/cache/AccumuloTableCache.java

@@ -22,7 +22,9 @@ public interface AccumuloTableCache extends AutoCloseable {
 
     void submitReloadTasks();
 
-    public void reloadTableCache(String tableName);
+    void reloadTableCache(String tableName);
 
-    public List<TableCacheDescription> getTableCaches();
+    List<TableCacheDescription> getTableCaches();
+
+    boolean isAvailable();
 }

core/connection-pool/src/main/java/datawave/core/common/cache/AccumuloTableCacheImpl.java

@@ -241,4 +241,14 @@ public List<TableCacheDescription> getTableCaches() {
         }
         return tableCaches;
     }
+
+    @Override
+    public boolean isAvailable() {
+        for (Entry<String,TableCache> entry : details.entrySet()) {
+            if (!entry.getValue().isAvailable()) {
+                return false;
+            }
+        }
+        return true;
+    }
 }

core/connection-pool/src/main/java/datawave/core/common/cache/BaseTableCache.java

@@ -18,7 +18,6 @@
 import org.apache.accumulo.core.client.BatchWriterConfig;
 import org.apache.accumulo.core.client.IteratorSetting;
 import org.apache.accumulo.core.client.NamespaceExistsException;
-import org.apache.accumulo.core.client.TableNotFoundException;
 import org.apache.accumulo.core.client.admin.NamespaceOperations;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Mutation;
@@ -34,6 +33,7 @@
 
 import datawave.accumulo.inmemory.InMemoryAccumuloClient;
 import datawave.accumulo.inmemory.InMemoryInstance;
+import datawave.accumulo.inmemory.InMemoryTableOperations;
 import datawave.core.common.connection.AccumuloConnectionFactory;
 import datawave.webservice.common.connection.WrappedAccumuloClient;
 
@@ -51,6 +51,7 @@ public class BaseTableCache implements Serializable, TableCache {
     private long maxRows = Long.MAX_VALUE;
 
     /** set programatically **/
+    private boolean available = false;
     private Date lastRefresh = new Date(0);
     private AccumuloConnectionFactory connectionFactory = null;
     private transient InMemoryInstance instance = null;
@@ -84,6 +85,11 @@ public Date getLastRefresh() {
         return lastRefresh;
     }
 
+    @Override
+    public boolean isAvailable() {
+        return available;
+    }
+
     @Override
     public AccumuloConnectionFactory getConnectionFactory() {
         return connectionFactory;
@@ -216,13 +222,9 @@ public Boolean call() throws Exception {
                 count++;
             }
             this.lastRefresh = new Date();
-            try {
-                instanceClient.tableOperations().delete(tableName);
-            } catch (TableNotFoundException e) {
-                // the table will not exist the first time this is run
-            }
-            instanceClient.tableOperations().rename(tempTableName, tableName);
+            ((InMemoryTableOperations) instanceClient.tableOperations()).rename(tempTableName, tableName, true);
             log.info("Cached {} k,v for table: {}", count, tableName);
+            this.available = true;
         } catch (Exception e) {
             log.error(e.getMessage(), e);
             throw e;

core/connection-pool/src/main/java/datawave/core/common/cache/TableCache.java

@@ -20,6 +20,8 @@ public interface TableCache extends Callable<Boolean>, Serializable {
 
     Date getLastRefresh();
 
+    boolean isAvailable();
+
     AccumuloConnectionFactory getConnectionFactory();
 
     InMemoryInstance getInstance();

microservices/services/dictionary/service/pom.xml

@@ -29,7 +29,7 @@
         <version.datawave.dictionary-api>4.0.5</version.datawave.dictionary-api>
         <version.datawave.starter>4.0.5</version.datawave.starter>
         <version.datawave.starter-metadata>3.0.2</version.datawave.starter-metadata>
-        <version.in-memory-accumulo>4.0.0</version.in-memory-accumulo>
+        <version.in-memory-accumulo>4.0.2</version.in-memory-accumulo>
         <version.webjars.datatables>1.11.4</version.webjars.datatables>
         <version.webjars.jquery>3.6.0</version.webjars.jquery>
         <version.webjars.locator>0.50</version.webjars.locator>

microservices/services/modification/service/pom.xml

@@ -22,7 +22,7 @@
         <version.datawave>7.1.3</version.datawave>
         <version.datawave.modification-api>1.0.0</version.datawave.modification-api>
         <version.datawave.starter-metadata>3.0.2</version.datawave.starter-metadata>
-        <version.in-memory-accumulo>4.0.0</version.in-memory-accumulo>
+        <version.in-memory-accumulo>4.0.2</version.in-memory-accumulo>
         <version.zookeeper>3.8.0</version.zookeeper>
     </properties>
     <dependencyManagement>

microservices/services/query-executor/service/pom.xml

@@ -24,7 +24,7 @@
         <version.datawave>7.3.0</version.datawave>
         <version.datawave.query-executor-api>1.0.7</version.datawave.query-executor-api>
         <version.datawave.starter-query>1.0.8</version.datawave.starter-query>
-        <version.in-memory-accumulo>4.0.0</version.in-memory-accumulo>
+        <version.in-memory-accumulo>4.0.2</version.in-memory-accumulo>
         <version.zookeeper>3.8.0</version.zookeeper>
     </properties>
     <dependencyManagement>

microservices/services/query-metric/service/pom.xml

@@ -31,7 +31,7 @@
         <version.datawave.starter-metadata>3.0.2</version.datawave.starter-metadata>
         <version.datawave.starter-metrics>3.0.3</version.datawave.starter-metrics>
         <version.hazelcast-kubernetes>2.2.3</version.hazelcast-kubernetes>
-        <version.in-memory-accumulo>4.0.0</version.in-memory-accumulo>
+        <version.in-memory-accumulo>4.0.2</version.in-memory-accumulo>
         <version.protobuf-java>3.16.3</version.protobuf-java>
         <version.slf4j>2.0.12</version.slf4j>
         <version.validation-api>2.0.2</version.validation-api>

microservices/starters/datawave/pom.xml

@@ -33,7 +33,7 @@
         <version.datawave.metadata-utils>4.0.16</version.datawave.metadata-utils>
         <version.datawave.metrics-reporter>3.0.0</version.datawave.metrics-reporter>
         <version.dropwizard-metrics>4.1.2</version.dropwizard-metrics>
-        <version.in-memory-accumulo>4.0.0</version.in-memory-accumulo>
+        <version.in-memory-accumulo>4.0.2</version.in-memory-accumulo>
         <version.jackson>2.10.1</version.jackson>
         <version.jzlib>1.1.3</version.jzlib>
         <version.metrics-spring>3.1.3</version.metrics-spring>

microservices/starters/metadata/pom.xml

@@ -26,7 +26,7 @@
         <version.datawave>7.1.3</version.datawave>
         <version.datawave.metadata-utils>4.0.16</version.datawave.metadata-utils>
         <version.datawave.starter>4.0.5</version.datawave.starter>
-        <version.in-memory-accumulo>4.0.0</version.in-memory-accumulo>
+        <version.in-memory-accumulo>4.0.2</version.in-memory-accumulo>
     </properties>
     <dependencyManagement>
         <dependencies>

pom.xml

@@ -98,7 +98,7 @@
         <version.hamcrest>1.3</version.hamcrest>
         <version.httpcomponents-httpclient>4.5.13</version.httpcomponents-httpclient>
         <version.httpcomponents-httpcore>4.4.8</version.httpcomponents-httpcore>
-        <version.in-memory-accumulo>4.0.1</version.in-memory-accumulo>
+        <version.in-memory-accumulo>4.0.2</version.in-memory-accumulo>
         <version.infinispan>9.4.21.Final</version.infinispan>
         <version.jackson>2.10.0.pr1</version.jackson>
         <version.jackson-mapper-asl>1.9.13</version.jackson-mapper-asl>

web-services/common/src/main/java/datawave/webservice/common/cache/AccumuloTableCacheBean.java

@@ -199,6 +199,11 @@ public AccumuloTableCacheStatus getStatus() {
         return response;
     }
 
+    @Override
+    public boolean isAvailable() {
+        return tableCache.isAvailable();
+    }
+
     @Override
     public List<TableCacheDescription> getTableCaches() {
         return tableCache.getTableCaches();

web-services/common/src/main/java/datawave/webservice/common/health/HealthBean.java

@@ -46,6 +46,7 @@
 import com.sun.management.OperatingSystemMXBean;
 
 import datawave.configuration.DatawaveEmbeddedProjectStageHolder;
+import datawave.core.common.cache.AccumuloTableCache;
 import datawave.core.common.connection.AccumuloConnectionFactory;
 import datawave.webservice.result.GenericResponse;
 
@@ -65,6 +66,9 @@ public class HealthBean {
     private static boolean shutdownInProgress = false;
     private static String status = "ready";
 
+    @Inject
+    private AccumuloTableCache tableCache;
+
     @Inject
     private AccumuloConnectionFactory accumuloConnectionFactoryBean;
 
@@ -97,7 +101,10 @@ public Response health() {
         ServerHealth health = new ServerHealth();
         health.status = status;
         health.connectionUsagePercent = accumuloConnectionFactoryBean.getConnectionUsagePercent();
-        if (health.connectionUsagePercent >= maxUsedPercent) {
+        if (!tableCache.isAvailable()) {
+            health.details = "Accumulo table cache is not ready yet";
+            return Response.status(Status.SERVICE_UNAVAILABLE).entity(health).build();
+        } else if (health.connectionUsagePercent >= maxUsedPercent) {
             health.details = health.connectionUsagePercent + " of connections used [>= max of " + maxUsedPercent + " allowed]";
             return Response.status(Status.SERVICE_UNAVAILABLE).entity(health).build();
         } else if (shutdownInProgress) {