Updated DatePartitionedQueryPlanner to succeed as long as one

core/base-rest-responses/src/main/java/datawave/webservice/result/BaseResponse.java

@@ -51,7 +51,14 @@ public void setOperationTimeMS(long operationTimeMS) {
     }
 
     public void setMessages(List<String> messages) {
-        this.messages = messages;
+        if (messages == null) {
+            this.messages = null;
+        } else if (this.messages == null) {
+            this.messages = new ArrayList<>(messages);
+        } else {
+            this.messages.clear();
+            this.messages.addAll(messages);
+        }
     }
 
     public void setExceptions(LinkedList<QueryExceptionType> exceptions) {

core/utils/type-utils/src/main/java/datawave/webservice/query/util/QueryUncaughtExceptionHandler.java

@@ -1,11 +1,15 @@
 package datawave.webservice.query.util;
 
 import java.lang.Thread.UncaughtExceptionHandler;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 
 public class QueryUncaughtExceptionHandler implements UncaughtExceptionHandler {
 
     private Thread thread;
     private Throwable throwable;
+    private List<String> messages = Collections.synchronizedList(new ArrayList<>());
 
     @Override
     public void uncaughtException(Thread t, Throwable e) {
@@ -25,4 +29,14 @@ public Thread getThread() {
     public Throwable getThrowable() {
         return throwable;
     }
+
+    public void addMessage(String message) {
+        messages.add(message);
+    }
+
+    public List<String> getMessages() {
+        synchronized (messages) {
+            return Collections.unmodifiableList(new ArrayList<>(messages));
+        }
+    }
 }

warehouse/query-core/src/main/java/datawave/query/planner/DatePartitionedQueryPlanner.java

@@ -2,13 +2,15 @@
 
 import java.io.IOException;
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.SortedMap;
@@ -321,6 +323,7 @@ public CloseableIterable<QueryData> process(GenericQueryConfiguration genericCon
             this.plannedScript = this.initialPlan;
         } else {
             DatePartitionedQueryIterable results = new DatePartitionedQueryIterable();
+            List<Exception> exceptions = new ArrayList<>();
 
             for (Map.Entry<Pair<Date,Date>,Set<String>> dateRange : dateRanges.entrySet()) {
                 String subBeginDate = dateFormat.format(dateRange.getKey().getLeft());
@@ -342,8 +345,7 @@ public CloseableIterable<QueryData> process(GenericQueryConfiguration genericCon
                     }
                 } catch (DatawaveQueryException e) {
                     log.warn("Exception occurred when processing sub-plan against date range (" + subBeginDate + "-" + subEndDate + ")", e);
-
-                    throw e;
+                    exceptions.add(e);
                 } finally {
                     // append the new timers for logging at the end
                     originalConfig.appendTimers(configCopy.getTimers());
@@ -356,6 +358,15 @@ public CloseableIterable<QueryData> process(GenericQueryConfiguration genericCon
                 }
             }
 
+            // if every plan failed, then pass an exception up
+            if (exceptions.size() == dateRanges.size()) {
+                DatawaveQueryException e = new DatawaveQueryException("Query failed creation");
+                for (Exception reason : exceptions) {
+                    e.addSuppressed(reason);
+                }
+                throw e;
+            }
+
             // reset the iterator to be our federated iterator
             iterator = results;
         }

warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java

@@ -3002,6 +3002,11 @@ public Tuple2<CloseableIterable<QueryPlan>,Boolean> getQueryRanges(ScannerFactor
                 throw new InvalidQueryException(qe);
             }
             log.warn("After expanding the query, it is determined that the query cannot be executed against the field index and a full table scan is required");
+            if (config.getQuery().getUncaughtExceptionHandler() != null) {
+                config.getQuery().getUncaughtExceptionHandler().addMessage(state.reason);
+                config.getQuery().getUncaughtExceptionHandler().addMessage(
+                                "After expanding the query, it is determined that the query cannot be executed against the field index and a full table scan is required");
+            }
             needsFullTable = true;
             fullTableScanReason = state.reason;
         }

warehouse/query-core/src/test/java/datawave/query/planner/DatePartitionedQueryPlannerTest.java

@@ -51,7 +51,7 @@
 import datawave.ingest.data.TypeRegistry;
 import datawave.microservice.query.QueryImpl;
 import datawave.query.QueryTestTableHelper;
-import datawave.query.exceptions.FullTableScansDisallowedException;
+import datawave.query.exceptions.DatawaveQueryException;
 import datawave.query.function.deserializer.KryoDocumentDeserializer;
 import datawave.query.jexl.JexlASTHelper;
 import datawave.query.jexl.visitors.JexlStringBuildingVisitor;
@@ -382,10 +382,20 @@ private void assertSubrangesCorrect(AccumuloClient client) throws Exception {
     }
 
     private AccumuloClient assertQueryResults() throws Exception {
-        return assertQueryResults(false);
+        return assertQueryResults(false, false);
     }
 
-    private AccumuloClient assertQueryResults(boolean fullTableScanRequired) throws Exception {
+    /**
+     * Assert the query results
+     *
+     * @param fullTableScanRequired
+     *            This denotes that to get all results, full table scan will be required
+     * @param partialSuccessExpected
+     *            This denotes that with full table scan disabled, success is expected anyway (at least one date range will succeed).
+     * @return The accumulo client used
+     * @throws Exception
+     */
+    private AccumuloClient assertQueryResults(boolean fullTableScanRequired, boolean partialSuccessExpected) throws Exception {
         // setup the full table scan enabled flag
         this.logic.setFullTableScanEnabled(fullTableScanRequired);
 
@@ -438,9 +448,13 @@ private AccumuloClient assertQueryResults(boolean fullTableScanRequired) throws
             try {
                 logic.setFullTableScanEnabled(false);
                 logic.initialize(client, settings, authSet);
-                Assert.fail("Expected full table scan to be required");
-            } catch (FullTableScansDisallowedException e) {
-                // expected
+                if (!partialSuccessExpected) {
+                    Assert.fail("Expected full table scan to be required");
+                }
+            } catch (DatawaveQueryException e) {
+                if (partialSuccessExpected) {
+                    Assert.fail("Expected success even with failed date ranges");
+                }
             }
         }
 
@@ -596,7 +610,7 @@ public void testFieldIndexHolesWithinDateRange() throws Exception {
         expectEvents("20130104", IndexFieldHoleDataIngest.corleoneUID, IndexFieldHoleDataIngest.caponeUID, IndexFieldHoleDataIngest.sopranoUID);
         expectEvents("20130105", IndexFieldHoleDataIngest.corleoneUID, IndexFieldHoleDataIngest.caponeUID, IndexFieldHoleDataIngest.sopranoUID);
 
-        assertSubrangesCorrect(assertQueryResults(true));
+        assertSubrangesCorrect(assertQueryResults(true, true));
     }
 
     /**
@@ -624,7 +638,7 @@ public void testFieldIndexHolesPartiallyWithinDateRange() throws Exception {
         expectEvents("20130103", IndexFieldHoleDataIngest.corleoneUID, IndexFieldHoleDataIngest.caponeUID, IndexFieldHoleDataIngest.sopranoUID);
         expectEvents("20130104", IndexFieldHoleDataIngest.corleoneUID, IndexFieldHoleDataIngest.caponeUID, IndexFieldHoleDataIngest.sopranoUID);
 
-        assertSubrangesCorrect(assertQueryResults(true));
+        assertSubrangesCorrect(assertQueryResults(true, true));
     }
 
     /**
@@ -700,7 +714,7 @@ public void testOverlappingFieldIndexHolesForDifferentFields() throws Exception
         expectEvents("20130104", IndexFieldHoleDataIngest.corleoneUID, IndexFieldHoleDataIngest.caponeUID, IndexFieldHoleDataIngest.sopranoUID);
         expectEvents("20130105", IndexFieldHoleDataIngest.corleoneUID, IndexFieldHoleDataIngest.caponeUID, IndexFieldHoleDataIngest.sopranoUID);
 
-        assertSubrangesCorrect(assertQueryResults(true));
+        assertSubrangesCorrect(assertQueryResults(true, true));
     }
 
     /**
@@ -732,7 +746,7 @@ public void testOverlappingFieldIndexHolesForDifferentFieldsNoSingleDates() thro
         expectEvents("20130104", IndexFieldHoleDataIngest.corleoneUID, IndexFieldHoleDataIngest.caponeUID, IndexFieldHoleDataIngest.sopranoUID);
         expectEvents("20130105", IndexFieldHoleDataIngest.corleoneUID, IndexFieldHoleDataIngest.caponeUID, IndexFieldHoleDataIngest.sopranoUID);
 
-        assertSubrangesCorrect(assertQueryResults(true));
+        assertSubrangesCorrect(assertQueryResults(true, true));
     }
 
     /**
@@ -761,7 +775,7 @@ public void testMissingIndexedData() throws Exception {
         expectEvents("20130104", IndexFieldHoleDataIngest.corleoneUID, IndexFieldHoleDataIngest.caponeUID, IndexFieldHoleDataIngest.sopranoUID);
         expectEvents("20130105", IndexFieldHoleDataIngest.corleoneUID, IndexFieldHoleDataIngest.caponeUID, IndexFieldHoleDataIngest.sopranoUID);
 
-        assertSubrangesCorrect(assertQueryResults(true));
+        assertSubrangesCorrect(assertQueryResults(true, true));
     }
 
     /**
@@ -790,7 +804,7 @@ public void testFieldIndexHolesAtStartOfRange() throws Exception {
         expectEvents("20130104", IndexFieldHoleDataIngest.corleoneUID, IndexFieldHoleDataIngest.caponeUID, IndexFieldHoleDataIngest.sopranoUID);
         expectEvents("20130105", IndexFieldHoleDataIngest.corleoneUID, IndexFieldHoleDataIngest.caponeUID, IndexFieldHoleDataIngest.sopranoUID);
 
-        assertSubrangesCorrect(assertQueryResults(true));
+        assertSubrangesCorrect(assertQueryResults(true, true));
     }
 
     @Test
@@ -819,7 +833,7 @@ public void testFieldIndexHolesAtEndOfRange() throws Exception {
         expectEvents("20130104", IndexFieldHoleDataIngest.corleoneUID, IndexFieldHoleDataIngest.caponeUID, IndexFieldHoleDataIngest.sopranoUID);
         expectEvents("20130105", IndexFieldHoleDataIngest.corleoneUID, IndexFieldHoleDataIngest.caponeUID, IndexFieldHoleDataIngest.sopranoUID);
 
-        assertSubrangesCorrect(assertQueryResults(true));
+        assertSubrangesCorrect(assertQueryResults(true, true));
     }
 
     /**
@@ -878,6 +892,37 @@ public void testFieldIndexMinThresholdWithSomeNotMeeting() throws Exception {
         expectEvents("20130104", IndexFieldHoleDataIngest.corleoneUID, IndexFieldHoleDataIngest.caponeUID, IndexFieldHoleDataIngest.sopranoUID);
         expectEvents("20130105", IndexFieldHoleDataIngest.corleoneUID, IndexFieldHoleDataIngest.caponeUID, IndexFieldHoleDataIngest.sopranoUID);
 
-        assertSubrangesCorrect(assertQueryResults(true));
+        assertSubrangesCorrect(assertQueryResults(true, true));
     }
+
+    /**
+     * Test a query that targets fields with field index holes for different fields that are consecutive to each other, completely covers the date range with a
+     * query such that no date range can succeed without a full table scan.
+     */
+    @Test
+    public void testConsecutiveAllFieldIndexHolesForDifferentFields() throws Exception {
+        configureEvent(IndexFieldHoleDataIngest.EventConfig.forDate("20130101").withMetadataCount("GENDER", 10L, 2L));
+        configureEvent(IndexFieldHoleDataIngest.EventConfig.forDate("20130102").withMetadataCount("GENDER", 10L, 2L));
+        configureEvent(IndexFieldHoleDataIngest.EventConfig.forDate("20130103").withMetadataCount("GENDER", 10L, 2L));
+        configureEvent(IndexFieldHoleDataIngest.EventConfig.forDate("20130104").withMetadataCount("UUID", 10L, 2L));
+        configureEvent(IndexFieldHoleDataIngest.EventConfig.forDate("20130105").withMetadataCount("UUID", 10L, 2L));
+
+        givenQuery("(UUID =~ 'C.*' || GEN == 'MALE')");
+        givenStartDate("20130101");
+        givenEndDate("20130105");
+        givenPlan("(GENDER == 'male' || GENERE == 'male' || UUID == 'capone' || UUID == 'corleone')");
+
+        expectPlan(start("20130101"), end("20130103"), "GENERE == 'male' || UUID == 'capone' || UUID == 'corleone' || ((_Eval_ = true) && (GENDER == 'male'))");
+        expectPlan(start("20130104"), end("20130105"),
+                        "GENDER == 'male' || GENERE == 'male' || ((_Eval_ = true) && (UUID == 'capone')) || ((_Eval_ = true) && (UUID == 'corleone'))");
+
+        expectEvents("20130101", IndexFieldHoleDataIngest.corleoneUID, IndexFieldHoleDataIngest.caponeUID, IndexFieldHoleDataIngest.sopranoUID);
+        expectEvents("20130102", IndexFieldHoleDataIngest.corleoneUID, IndexFieldHoleDataIngest.caponeUID, IndexFieldHoleDataIngest.sopranoUID);
+        expectEvents("20130103", IndexFieldHoleDataIngest.corleoneUID, IndexFieldHoleDataIngest.caponeUID, IndexFieldHoleDataIngest.sopranoUID);
+        expectEvents("20130104", IndexFieldHoleDataIngest.corleoneUID, IndexFieldHoleDataIngest.caponeUID, IndexFieldHoleDataIngest.sopranoUID);
+        expectEvents("20130105", IndexFieldHoleDataIngest.corleoneUID, IndexFieldHoleDataIngest.caponeUID, IndexFieldHoleDataIngest.sopranoUID);
+
+        assertSubrangesCorrect(assertQueryResults(true, false));
+    }
+
 }

warehouse/query-core/src/test/java/datawave/query/planner/ThreadedRangeBundlerIteratorTest.java

@@ -177,7 +177,7 @@ public void disableBundlingFastProducerTest() throws IOException {
 
     @Test
     public void blockForNumRangesToBufferTest() {
-        int delay = 200;
+        int delay = 300;
 
         Iterator<QueryPlan> itr = plans.iterator();
 
@@ -223,7 +223,7 @@ public void failFastOnExhaustedTest() throws IOException {
         assertFalse(trbi.hasNext());
         long end = System.currentTimeMillis();
         // arbitrary fast time less than any previous poll time, actual time probably 1 but to keep this unit test predictable
-        assertTrue(end - start < 5);
+        assertTrue(end - start < 20);
         verifyAll();
     }