Skip to content

Commit adc3210

Browse files
jonathan-buttnerjonathan-buttner
committed
Merge branch 'main' of github.com:elastic/elasticsearch into custom-inference-service
2 parents eb63e8b + 7c3e850 commit adc3210

File tree

458 files changed

+11909
-4498
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

458 files changed

+11909
-4498
lines changed

‎.buildkite/pipelines/intake.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ steps:
5656
timeout_in_minutes: 300
5757
matrix:
5858
setup:
59-
BWC_VERSION: ["8.17.6", "8.18.0", "8.19.0", "9.0.1", "9.1.0"]
59+
BWC_VERSION: ["8.17.6", "8.18.1", "8.19.0", "9.0.1", "9.1.0"]
6060
agents:
6161
provider: gcp
6262
image: family/elasticsearch-ubuntu-2004

‎.buildkite/pipelines/periodic-packaging.yml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -287,8 +287,8 @@ steps:
287287
env:
288288
BWC_VERSION: 8.15.5
289289

290-
- label: "{{matrix.image}} / 8.16.7 / packaging-tests-upgrade"
291-
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.7
290+
- label: "{{matrix.image}} / 8.16.6 / packaging-tests-upgrade"
291+
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.6
292292
timeout_in_minutes: 300
293293
matrix:
294294
setup:
@@ -301,7 +301,7 @@ steps:
301301
machineType: custom-16-32768
302302
buildDirectory: /dev/shm/bk
303303
env:
304-
BWC_VERSION: 8.16.7
304+
BWC_VERSION: 8.16.6
305305

306306
- label: "{{matrix.image}} / 8.17.6 / packaging-tests-upgrade"
307307
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.6
@@ -319,8 +319,8 @@ steps:
319319
env:
320320
BWC_VERSION: 8.17.6
321321

322-
- label: "{{matrix.image}} / 8.18.0 / packaging-tests-upgrade"
323-
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.0
322+
- label: "{{matrix.image}} / 8.18.1 / packaging-tests-upgrade"
323+
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.1
324324
timeout_in_minutes: 300
325325
matrix:
326326
setup:
@@ -333,7 +333,7 @@ steps:
333333
machineType: custom-16-32768
334334
buildDirectory: /dev/shm/bk
335335
env:
336-
BWC_VERSION: 8.18.0
336+
BWC_VERSION: 8.18.1
337337

338338
- label: "{{matrix.image}} / 8.19.0 / packaging-tests-upgrade"
339339
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.0

‎.buildkite/pipelines/periodic.yml

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -306,8 +306,8 @@ steps:
306306
- signal_reason: agent_stop
307307
limit: 3
308308

309-
- label: 8.16.7 / bwc
310-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.7#bwcTest
309+
- label: 8.16.6 / bwc
310+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.6#bwcTest
311311
timeout_in_minutes: 300
312312
agents:
313313
provider: gcp
@@ -316,7 +316,7 @@ steps:
316316
buildDirectory: /dev/shm/bk
317317
preemptible: true
318318
env:
319-
BWC_VERSION: 8.16.7
319+
BWC_VERSION: 8.16.6
320320
retry:
321321
automatic:
322322
- exit_status: "-1"
@@ -344,8 +344,8 @@ steps:
344344
- signal_reason: agent_stop
345345
limit: 3
346346

347-
- label: 8.18.0 / bwc
348-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.0#bwcTest
347+
- label: 8.18.1 / bwc
348+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.1#bwcTest
349349
timeout_in_minutes: 300
350350
agents:
351351
provider: gcp
@@ -354,7 +354,7 @@ steps:
354354
buildDirectory: /dev/shm/bk
355355
preemptible: true
356356
env:
357-
BWC_VERSION: 8.18.0
357+
BWC_VERSION: 8.18.1
358358
retry:
359359
automatic:
360360
- exit_status: "-1"
@@ -486,7 +486,7 @@ steps:
486486
setup:
487487
ES_RUNTIME_JAVA:
488488
- openjdk21
489-
BWC_VERSION: ["8.17.6", "8.18.0", "8.19.0", "9.0.1", "9.1.0"]
489+
BWC_VERSION: ["8.17.6", "8.18.1", "8.19.0", "9.0.1", "9.1.0"]
490490
agents:
491491
provider: gcp
492492
image: family/elasticsearch-ubuntu-2004
@@ -533,7 +533,7 @@ steps:
533533
ES_RUNTIME_JAVA:
534534
- openjdk21
535535
- openjdk23
536-
BWC_VERSION: ["8.17.6", "8.18.0", "8.19.0", "9.0.1", "9.1.0"]
536+
BWC_VERSION: ["8.17.6", "8.18.1", "8.19.0", "9.0.1", "9.1.0"]
537537
agents:
538538
provider: gcp
539539
image: family/elasticsearch-ubuntu-2004

‎.buildkite/scripts/dra-workflow.trigger.sh

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,6 @@ echo "steps:"
77
source .buildkite/scripts/branches.sh
88

99
for BRANCH in "${BRANCHES[@]}"; do
10-
if [[ "$BRANCH" == "9.0" ]]; then
11-
continue
12-
fi
13-
1410
INTAKE_PIPELINE_SLUG="elasticsearch-intake"
1511
BUILD_JSON=$(curl -sH "Authorization: Bearer ${BUILDKITE_API_TOKEN}" "https://api.buildkite.com/v2/organizations/elastic/pipelines/${INTAKE_PIPELINE_SLUG}/builds?branch=${BRANCH}&state=passed&per_page=1" | jq '.[0] | {commit: .commit, url: .web_url}')
1612
LAST_GOOD_COMMIT=$(echo "${BUILD_JSON}" | jq -r '.commit')

‎.ci/bwcVersions

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@ BWC_VERSION:
1515
- "8.13.4"
1616
- "8.14.3"
1717
- "8.15.5"
18-
- "8.16.7"
18+
- "8.16.6"
1919
- "8.17.6"
20-
- "8.18.0"
20+
- "8.18.1"
2121
- "8.19.0"
2222
- "9.0.1"
2323
- "9.1.0"

‎.ci/snapshotBwcVersions

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
BWC_VERSION:
22
- "8.17.6"
3-
- "8.18.0"
3+
- "8.18.1"
44
- "8.19.0"
55
- "9.0.1"
66
- "9.1.0"

‎benchmarks/run.sh

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
#
3+
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
4+
# or more contributor license agreements. Licensed under the "Elastic License
5+
# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
6+
# Public License v 1"; you may not use this file except in compliance with, at
7+
# your election, the "Elastic License 2.0", the "GNU Affero General Public
8+
# License v3.0 only", or the "Server Side Public License, v 1".
9+
#
10+
11+
EXTRA=""
12+
POSITIONAL_ARGS=()
13+
while [[ $# -gt 0 ]]; do
14+
case $1 in
15+
--test)
16+
# Get inaccurate results quickly by shortening all measurements
17+
# to 50ms each and skip self tests.
18+
EXTRA="-r 50ms -w 50ms -jvmArgsAppend -DskipSelfTest=true"
19+
shift
20+
;;
21+
*)
22+
POSITIONAL_ARGS+=("$1")
23+
shift
24+
;;
25+
esac
26+
done
27+
28+
set -- "${POSITIONAL_ARGS[@]}"
29+
30+
run() {
31+
../gradlew run --args "$2 -rf json $EXTRA"
32+
mv jmh-result.json build/benchmarks/$1.json
33+
}
34+
35+
cd "$(dirname "$0")"
36+
mkdir -p build/benchmarks
37+
run 'esql_agg' 'AggregatorBenchmark -pgrouping=none,longs -pfilter=none -pblockType=vector_longs,half_null_longs'
38+
run 'esql_block_keep_mask' 'BlockKeepMaskBenchmark -pdataTypeAndBlockKind=BytesRef/array,BytesRef/vector,long/array,long/vector'
39+
run 'esql_block_read' 'BlockReadBenchmark -paccessType=sequential'
40+
run 'esql_eval' 'EvalBenchmark'
41+
run 'esql_parse_ip' 'ParseIpBenchmark'
42+
run 'esql_topn' 'TopNBenchmark'
43+
run 'esql_values_agg' 'ValuesAggregatorBenchmark'
44+
run 'esql_values_source_reader' 'ValuesSourceReaderBenchmark'

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,12 @@ public class AggregatorBenchmark {
113113

114114
static {
115115
// Smoke test all the expected values and force loading subclasses more like prod
116+
if (false == "true".equals(System.getProperty("skipSelfTest"))) {
117+
selfTest();
118+
}
119+
}
120+
121+
static void selfTest() {
116122
try {
117123
for (String grouping : AggregatorBenchmark.class.getField("grouping").getAnnotationsByType(Param.class)[0].value()) {
118124
for (String op : AggregatorBenchmark.class.getField("op").getAnnotationsByType(Param.class)[0].value()) {

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockBenchmark.java

Lines changed: 23 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import java.util.ArrayList;
3838
import java.util.BitSet;
3939
import java.util.Random;
40+
import java.util.stream.IntStream;
4041

4142
public class BlockBenchmark {
4243
/**
@@ -112,15 +113,15 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
112113
blocks[blockIndex] = blockFactory.newBooleanArrayBlock(
113114
values,
114115
totalPositions,
115-
null,
116+
IntStream.rangeClosed(0, totalPositions).toArray(),
116117
null,
117118
Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING
118119
);
119120
}
120121
case "array-multivalue-null" -> {
121122
int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
122123
int positionCount = firstValueIndexes.length - 1;
123-
BitSet nulls = randomNulls(positionCount);
124+
BitSet nulls = nullsFromFirstValues(firstValueIndexes);
124125

125126
blocks[blockIndex] = blockFactory.newBooleanArrayBlock(
126127
values,
@@ -141,7 +142,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
141142
blocks[blockIndex] = new BooleanBigArrayBlock(
142143
valuesBigArray,
143144
totalPositions,
144-
null,
145+
IntStream.rangeClosed(0, totalPositions).toArray(),
145146
null,
146147
Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING,
147148
blockFactory
@@ -150,7 +151,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
150151
case "big-array-multivalue-null" -> {
151152
int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
152153
int positionCount = firstValueIndexes.length - 1;
153-
BitSet nulls = randomNulls(positionCount);
154+
BitSet nulls = nullsFromFirstValues(firstValueIndexes);
154155
BitArray valuesBigArray = new BitArray(totalPositions, BigArrays.NON_RECYCLING_INSTANCE);
155156
for (int i = 0; i < values.length; i++) {
156157
if (values[i]) {
@@ -211,15 +212,15 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
211212
blocks[blockIndex] = blockFactory.newBytesRefArrayBlock(
212213
values,
213214
totalPositions,
214-
null,
215+
IntStream.rangeClosed(0, totalPositions).toArray(),
215216
null,
216217
Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING
217218
);
218219
}
219220
case "array-multivalue-null" -> {
220221
int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
221222
int positionCount = firstValueIndexes.length - 1;
222-
BitSet nulls = randomNulls(positionCount);
223+
BitSet nulls = nullsFromFirstValues(firstValueIndexes);
223224

224225
blocks[blockIndex] = blockFactory.newBytesRefArrayBlock(
225226
values,
@@ -257,15 +258,15 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
257258
blocks[blockIndex] = blockFactory.newDoubleArrayBlock(
258259
values,
259260
totalPositions,
260-
null,
261+
IntStream.rangeClosed(0, totalPositions).toArray(),
261262
null,
262263
Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING
263264
);
264265
}
265266
case "array-multivalue-null" -> {
266267
int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
267268
int positionCount = firstValueIndexes.length - 1;
268-
BitSet nulls = randomNulls(positionCount);
269+
BitSet nulls = nullsFromFirstValues(firstValueIndexes);
269270

270271
blocks[blockIndex] = blockFactory.newDoubleArrayBlock(
271272
values,
@@ -284,7 +285,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
284285
blocks[blockIndex] = new DoubleBigArrayBlock(
285286
valuesBigArray,
286287
totalPositions,
287-
null,
288+
IntStream.rangeClosed(0, totalPositions).toArray(),
288289
null,
289290
Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING,
290291
blockFactory
@@ -293,7 +294,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
293294
case "big-array-multivalue-null" -> {
294295
int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
295296
int positionCount = firstValueIndexes.length - 1;
296-
BitSet nulls = randomNulls(positionCount);
297+
BitSet nulls = nullsFromFirstValues(firstValueIndexes);
297298
DoubleArray valuesBigArray = blockFactory.bigArrays().newDoubleArray(totalPositions, false);
298299
for (int i = 0; i < values.length; i++) {
299300
valuesBigArray.set(i, values[i]);
@@ -344,15 +345,15 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
344345
blocks[blockIndex] = blockFactory.newIntArrayBlock(
345346
values,
346347
totalPositions,
347-
null,
348+
IntStream.rangeClosed(0, totalPositions).toArray(),
348349
null,
349350
Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING
350351
);
351352
}
352353
case "array-multivalue-null" -> {
353354
int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
354355
int positionCount = firstValueIndexes.length - 1;
355-
BitSet nulls = randomNulls(positionCount);
356+
BitSet nulls = nullsFromFirstValues(firstValueIndexes);
356357

357358
blocks[blockIndex] = blockFactory.newIntArrayBlock(
358359
values,
@@ -371,7 +372,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
371372
blocks[blockIndex] = new IntBigArrayBlock(
372373
valuesBigArray,
373374
totalPositions,
374-
null,
375+
IntStream.rangeClosed(0, totalPositions).toArray(),
375376
null,
376377
Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING,
377378
blockFactory
@@ -380,7 +381,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
380381
case "big-array-multivalue-null" -> {
381382
int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
382383
int positionCount = firstValueIndexes.length - 1;
383-
BitSet nulls = randomNulls(positionCount);
384+
BitSet nulls = nullsFromFirstValues(firstValueIndexes);
384385
IntArray valuesBigArray = blockFactory.bigArrays().newIntArray(totalPositions, false);
385386
for (int i = 0; i < values.length; i++) {
386387
valuesBigArray.set(i, values[i]);
@@ -431,15 +432,15 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
431432
blocks[blockIndex] = blockFactory.newLongArrayBlock(
432433
values,
433434
totalPositions,
434-
null,
435+
IntStream.rangeClosed(0, totalPositions).toArray(),
435436
null,
436437
Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING
437438
);
438439
}
439440
case "array-multivalue-null" -> {
440441
int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
441442
int positionCount = firstValueIndexes.length - 1;
442-
BitSet nulls = randomNulls(positionCount);
443+
BitSet nulls = nullsFromFirstValues(firstValueIndexes);
443444

444445
blocks[blockIndex] = blockFactory.newLongArrayBlock(
445446
values,
@@ -458,7 +459,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
458459
blocks[blockIndex] = new LongBigArrayBlock(
459460
valuesBigArray,
460461
totalPositions,
461-
null,
462+
IntStream.rangeClosed(0, totalPositions).toArray(),
462463
null,
463464
Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING,
464465
blockFactory
@@ -467,7 +468,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
467468
case "big-array-multivalue-null" -> {
468469
int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
469470
int positionCount = firstValueIndexes.length - 1;
470-
BitSet nulls = randomNulls(positionCount);
471+
BitSet nulls = nullsFromFirstValues(firstValueIndexes);
471472
LongArray valuesBigArray = blockFactory.bigArrays().newLongArray(totalPositions, false);
472473
for (int i = 0; i < values.length; i++) {
473474
valuesBigArray.set(i, values[i]);
@@ -526,10 +527,10 @@ private static int[] randomFirstValueIndexes(int totalPositions) {
526527
return firstValueIndexes.stream().mapToInt(x -> x).toArray();
527528
}
528529

529-
private static BitSet randomNulls(int positionCount) {
530-
BitSet nulls = new BitSet(positionCount);
531-
for (int i = 0; i < positionCount; i++) {
532-
if (random.nextDouble() < NULL_PERCENTAGE) {
530+
private static BitSet nullsFromFirstValues(int[] firstValueIndexes) {
531+
BitSet nulls = new BitSet(firstValueIndexes.length - 1);
532+
for (int i = 0; i < firstValueIndexes.length - 1; i++) {
533+
if (firstValueIndexes[i + 1] - firstValueIndexes[i] == 1 && random.nextDouble() < NULL_PERCENTAGE) {
533534
nulls.set(i);
534535
}
535536
}

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockKeepMaskBenchmark.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,13 @@
4242
@Fork(1)
4343
public class BlockKeepMaskBenchmark extends BlockBenchmark {
4444
static {
45-
// Smoke test all the expected values and force loading subclasses more like prod
45+
if (false == "true".equals(System.getProperty("skipSelfTest"))) {
46+
// Smoke test all the expected values and force loading subclasses more like prod
47+
selfTest();
48+
}
49+
}
50+
51+
static void selfTest() {
4652
int totalPositions = 10;
4753
for (String paramString : RELEVANT_TYPE_BLOCK_COMBINATIONS) {
4854
String[] params = paramString.split("/");

0 commit comments

Comments
 (0)