From 980723ef83556263ff2fe00fab0ae0d6af24e94a Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 24 Jun 2025 22:49:43 +0200 Subject: [PATCH 01/28] poc --- .../org/elasticsearch/FlatIndicesRequest.java | 18 +++ .../action/search/SearchRequest.java | 12 +- .../security/authz/AuthorizationEngine.java | 5 + .../core/security/SerializationDemoTests.java | 136 ++++++++++++++++++ .../authz/IndicesAndAliasesResolver.java | 53 ++++++- .../xpack/security/authz/RBACEngine.java | 14 +- 6 files changed, 235 insertions(+), 3 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/FlatIndicesRequest.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/SerializationDemoTests.java diff --git a/server/src/main/java/org/elasticsearch/FlatIndicesRequest.java b/server/src/main/java/org/elasticsearch/FlatIndicesRequest.java new file mode 100644 index 0000000000000..8e9785d3f82fa --- /dev/null +++ b/server/src/main/java/org/elasticsearch/FlatIndicesRequest.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch; + +import org.elasticsearch.action.IndicesRequest; + +import java.util.List; + +public interface FlatIndicesRequest extends IndicesRequest { + void indices(List indices); +} diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index fda2df81d3f94..60b01bda53b28 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.search; +import org.elasticsearch.FlatIndicesRequest; import org.elasticsearch.TransportVersions; import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; @@ -53,7 +54,11 @@ * @see Client#search(SearchRequest) * @see SearchResponse */ -public class SearchRequest extends LegacyActionRequest implements IndicesRequest.Replaceable, Rewriteable { +public class SearchRequest extends LegacyActionRequest + implements + FlatIndicesRequest, + IndicesRequest.Replaceable, + Rewriteable { public static final ToXContent.Params FORMAT_PARAMS = new ToXContent.MapParams(Collections.singletonMap("pretty", "false")); @@ -853,4 +858,9 @@ public String toString() { + source + '}'; } + + @Override + public void indices(List indices) { + indices(indices.toArray(Strings.EMPTY_ARRAY)); + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java index 2c831645d0e69..e2ffcf7480381 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java @@ -299,6 +299,11 @@ interface AuthorizedIndices { * Checks if an index-like resource name is authorized, for an action by a user. The resource might or might not exist. */ boolean check(String name, IndexComponentSelector selector); + + // Does not belong here + default boolean checkProject(String projectId) { + return false; + } } /** diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/SerializationDemoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/SerializationDemoTests.java new file mode 100644 index 0000000000000..caabd1d1ee0da --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/SerializationDemoTests.java @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.security; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentType; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.TransportVersions.PARTIAL_DATA_DEMO; +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class SerializationDemoTests extends ESTestCase { + + record SearchResult(boolean success, @Nullable List results, @Nullable List failures) + implements + Writeable, + // ToXContentFragment also exists + ToXContentObject { + + private static final ConstructingObjectParser PARSER = buildParser(); + + @SuppressWarnings("unchecked") + private static ConstructingObjectParser buildParser() { + final ConstructingObjectParser parser = new ConstructingObjectParser<>( + "search_result", + true, + a -> new SearchResult((boolean) a[0], (List) a[1], (List) a[2]) + ); + parser.declareBoolean(constructorArg(), new ParseField("success")); + parser.declareStringArray(optionalConstructorArg(), new ParseField("results")); + parser.declareStringArray(optionalConstructorArg(), new ParseField("failures")); + return parser; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeBoolean(success); + out.writeOptionalCollection(results, StreamOutput::writeString); + // Elasticsearch supports rolling upgrades across 1 major version and within major versions. + // For example 7.17 needs to be able to communicate with 8.4 nodes, and 8.1 nodes need to be able to talk with 8.4 nodes. + // Serverless removed the notion of transport versions being tied cleanly to ES versions since we release to serverless + // every week and have rolling upgrades + if (out.getTransportVersion().onOrAfter(PARTIAL_DATA_DEMO)) { + out.writeOptionalCollection(failures, StreamOutput::writeString); + } + } + + SearchResult(StreamInput input) throws IOException { + this( + input.readBoolean(), + input.readOptionalCollectionAsList(StreamInput::readString), + input.getTransportVersion().onOrAfter(PARTIAL_DATA_DEMO) + ? input.readOptionalCollectionAsList(StreamInput::readString) + : List.of() + ); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + var xcb = builder.startObject().field("success", success); + if (results != null) { + xcb = xcb.field("results", results); + } + if (failures != null) { + xcb = xcb.field("failures", failures); + } + return xcb.endObject(); + } + + public SearchResult fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + } + + public void testRoundTripTransportSerialization() throws IOException { + var result = new SearchResult(true, List.of("hit1"), List.of()); + + try (var out = new BytesStreamOutput()) { + result.writeTo(out); + var received = new SearchResult(out.bytes().streamInput()); + + System.out.println("Original: " + result); + System.out.println("Received: " + received); + } + } + + public void testToXContent() { + var result = new SearchResult(true, List.of("hit1", "hit2"), List.of("failure1")); + + try (var builder = XContentBuilder.builder(XContentType.JSON.xContent())) { + result.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.flush(); + String json = Strings.toString(builder); + System.out.println("JSON Output: " + json); + // test from XContent + try ( + var parser = XContentType.JSON.xContent() + .createParser( + NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + new ByteArrayInputStream(json.getBytes()) + ) + ) { + var parsedResult = result.fromXContent(parser); + System.out.println("Parsed Result: " + parsedResult); + } + } catch (IOException e) { + fail("Failed to convert to XContent: " + e.getMessage()); + } + } + +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java index ff39fd587dc3a..f3573d8fb6518 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java @@ -6,6 +6,9 @@ */ package org.elasticsearch.xpack.security.authz; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.FlatIndicesRequest; import org.elasticsearch.action.AliasesRequest; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; @@ -55,6 +58,8 @@ class IndicesAndAliasesResolver { + private static final Logger logger = LogManager.getLogger(IndicesAndAliasesResolver.class); + private final IndexNameExpressionResolver nameExpressionResolver; private final IndexAbstractionResolver indexAbstractionResolver; private final RemoteClusterResolver remoteClusterResolver; @@ -103,7 +108,6 @@ class IndicesAndAliasesResolver { * resolving wildcards. *

*/ - ResolvedIndices resolve( String action, TransportRequest request, @@ -124,9 +128,52 @@ ResolvedIndices resolve( if (request instanceof IndicesRequest == false) { throw new IllegalStateException("Request [" + request + "] is not an Indices request, but should be."); } + + if (request instanceof FlatIndicesRequest flatIndicesRequest) { + rewrite(flatIndicesRequest, authorizedIndices); + } + return resolveIndicesAndAliases(action, (IndicesRequest) request, projectMetadata, authorizedIndices); } + void rewrite(FlatIndicesRequest request, AuthorizationEngine.AuthorizedIndices authorizedIndices) { + var clusters = remoteClusterResolver.clusters(); + logger.info("Clusters available for remote indices: {}", clusters); + // no remotes, nothing to rewrite... + if (clusters.isEmpty()) { + logger.info("Skipping..."); + return; + } + + var indices = request.indices(); + // empty indices actually means search everything so would need to also rewrite that + + var authorizedClusters = new HashSet(); + for (var cluster : clusters) { + if (authorizedIndices.checkProject(cluster)) { + logger.info("Remote cluster [{}] authorized", cluster); + authorizedClusters.add(cluster); + } + } + + // TODO do not rewrite twice + List rewrittenIndices = new ArrayList<>(indices.length); + ResolvedIndices resolved = remoteClusterResolver.splitLocalAndRemoteIndexNames(indices); + for (var local : resolved.getLocal()) { + String rewritten = local; + for (var cluster : authorizedClusters) { + rewritten += "," + RemoteClusterAware.buildRemoteIndexName(cluster, local); + rewrittenIndices.add(rewritten); + } + logger.info("Rewrote [{}] to [{}]", local, rewritten); + } + if (resolved.getRemote().isEmpty() == false) { + rewrittenIndices.addAll(resolved.getRemote()); + } + request.indices(rewrittenIndices); + // skipping mixed expressions, _local expressions and all that jazz + } + /** * Attempt to resolve requested indices without expanding any wildcards. * @return The {@link ResolvedIndices} or null if wildcard expansion must be performed. @@ -569,5 +616,9 @@ ResolvedIndices splitLocalAndRemoteIndexNames(String... indices) { .toList(); return new ResolvedIndices(local == null ? List.of() : local, remote); } + + Set clusters() { + return Collections.unmodifiableSet(clusters); + } } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java index 1b99bd6888c4f..2ef0319e4729a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java @@ -998,6 +998,9 @@ static AuthorizedIndices resolveAuthorizedIndicesFromRole( } // we don't support granting access to a backing index with a failure selector via the parent data stream } return predicate.test(indexAbstraction, selector); + }, name -> { + // just some bogus predicate that lets us differentiate between roles + return Arrays.asList(role.names()).contains("remote_searcher"); }); } @@ -1125,15 +1128,18 @@ static final class AuthorizedIndices implements AuthorizationEngine.AuthorizedIn private final CachedSupplier> authorizedAndAvailableDataResources; private final CachedSupplier> authorizedAndAvailableFailuresResources; private final BiPredicate isAuthorizedPredicate; + private final Predicate projectPredicate; AuthorizedIndices( Supplier> authorizedAndAvailableDataResources, Supplier> authorizedAndAvailableFailuresResources, - BiPredicate isAuthorizedPredicate + BiPredicate isAuthorizedPredicate, + Predicate projectPredicate ) { this.authorizedAndAvailableDataResources = CachedSupplier.wrap(authorizedAndAvailableDataResources); this.authorizedAndAvailableFailuresResources = CachedSupplier.wrap(authorizedAndAvailableFailuresResources); this.isAuthorizedPredicate = Objects.requireNonNull(isAuthorizedPredicate); + this.projectPredicate = projectPredicate; } @Override @@ -1149,5 +1155,11 @@ public boolean check(String name, IndexComponentSelector selector) { Objects.requireNonNull(selector, "must specify a selector for authorization check"); return isAuthorizedPredicate.test(name, selector); } + + @Override + public boolean checkProject(String name) { + Objects.requireNonNull(name, "must specify a project name for authorization check"); + return projectPredicate.test(name); + } } } From 4ff5e6eb8cbacedda35dc0a916a604624b823437 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 24 Jun 2025 22:50:15 +0200 Subject: [PATCH 02/28] poc --- .../core/security/SerializationDemoTests.java | 136 ------------------ 1 file changed, 136 deletions(-) delete mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/SerializationDemoTests.java diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/SerializationDemoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/SerializationDemoTests.java deleted file mode 100644 index caabd1d1ee0da..0000000000000 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/SerializationDemoTests.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.core.security; - -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.core.Nullable; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xcontent.ConstructingObjectParser; -import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xcontent.ParseField; -import org.elasticsearch.xcontent.ToXContent; -import org.elasticsearch.xcontent.ToXContentObject; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.XContentType; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.util.List; - -import static org.elasticsearch.TransportVersions.PARTIAL_DATA_DEMO; -import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; -import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; - -public class SerializationDemoTests extends ESTestCase { - - record SearchResult(boolean success, @Nullable List results, @Nullable List failures) - implements - Writeable, - // ToXContentFragment also exists - ToXContentObject { - - private static final ConstructingObjectParser PARSER = buildParser(); - - @SuppressWarnings("unchecked") - private static ConstructingObjectParser buildParser() { - final ConstructingObjectParser parser = new ConstructingObjectParser<>( - "search_result", - true, - a -> new SearchResult((boolean) a[0], (List) a[1], (List) a[2]) - ); - parser.declareBoolean(constructorArg(), new ParseField("success")); - parser.declareStringArray(optionalConstructorArg(), new ParseField("results")); - parser.declareStringArray(optionalConstructorArg(), new ParseField("failures")); - return parser; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeBoolean(success); - out.writeOptionalCollection(results, StreamOutput::writeString); - // Elasticsearch supports rolling upgrades across 1 major version and within major versions. - // For example 7.17 needs to be able to communicate with 8.4 nodes, and 8.1 nodes need to be able to talk with 8.4 nodes. - // Serverless removed the notion of transport versions being tied cleanly to ES versions since we release to serverless - // every week and have rolling upgrades - if (out.getTransportVersion().onOrAfter(PARTIAL_DATA_DEMO)) { - out.writeOptionalCollection(failures, StreamOutput::writeString); - } - } - - SearchResult(StreamInput input) throws IOException { - this( - input.readBoolean(), - input.readOptionalCollectionAsList(StreamInput::readString), - input.getTransportVersion().onOrAfter(PARTIAL_DATA_DEMO) - ? input.readOptionalCollectionAsList(StreamInput::readString) - : List.of() - ); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - var xcb = builder.startObject().field("success", success); - if (results != null) { - xcb = xcb.field("results", results); - } - if (failures != null) { - xcb = xcb.field("failures", failures); - } - return xcb.endObject(); - } - - public SearchResult fromXContent(XContentParser parser) { - return PARSER.apply(parser, null); - } - - } - - public void testRoundTripTransportSerialization() throws IOException { - var result = new SearchResult(true, List.of("hit1"), List.of()); - - try (var out = new BytesStreamOutput()) { - result.writeTo(out); - var received = new SearchResult(out.bytes().streamInput()); - - System.out.println("Original: " + result); - System.out.println("Received: " + received); - } - } - - public void testToXContent() { - var result = new SearchResult(true, List.of("hit1", "hit2"), List.of("failure1")); - - try (var builder = XContentBuilder.builder(XContentType.JSON.xContent())) { - result.toXContent(builder, ToXContent.EMPTY_PARAMS); - builder.flush(); - String json = Strings.toString(builder); - System.out.println("JSON Output: " + json); - // test from XContent - try ( - var parser = XContentType.JSON.xContent() - .createParser( - NamedXContentRegistry.EMPTY, - LoggingDeprecationHandler.INSTANCE, - new ByteArrayInputStream(json.getBytes()) - ) - ) { - var parsedResult = result.fromXContent(parser); - System.out.println("Parsed Result: " + parsedResult); - } - } catch (IOException e) { - fail("Failed to convert to XContent: " + e.getMessage()); - } - } - -} From 3df7a1df89671fa0df007c889e5dfbba9922b547 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 26 Jun 2025 13:45:37 +0200 Subject: [PATCH 03/28] Fix ups --- .../java/org/elasticsearch/FlatIndicesRequest.java | 6 +++++- .../elasticsearch/action/search/SearchRequest.java | 14 ++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/FlatIndicesRequest.java b/server/src/main/java/org/elasticsearch/FlatIndicesRequest.java index 8e9785d3f82fa..2d4594c05c6f1 100644 --- a/server/src/main/java/org/elasticsearch/FlatIndicesRequest.java +++ b/server/src/main/java/org/elasticsearch/FlatIndicesRequest.java @@ -14,5 +14,9 @@ import java.util.List; public interface FlatIndicesRequest extends IndicesRequest { - void indices(List indices); + boolean requiresRewrite(); + + void indexExpressions(List indexExpressions); + + record IndexExpression(String original, List rewritten) {} } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 60b01bda53b28..cd5c0ad466bc7 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -75,6 +75,9 @@ public class SearchRequest extends LegacyActionRequest private String[] indices = Strings.EMPTY_ARRAY; + @Nullable + private List indexExpressions; + @Nullable private String routing; @Nullable @@ -860,7 +863,14 @@ public String toString() { } @Override - public void indices(List indices) { - indices(indices.toArray(Strings.EMPTY_ARRAY)); + public boolean requiresRewrite() { + return indexExpressions == null; + } + + @Override + public void indexExpressions(List indexExpressions) { + assert requiresRewrite(); + this.indexExpressions = indexExpressions; + indices(indexExpressions.stream().flatMap(indexExpression -> indexExpression.rewritten().stream()).toArray(String[]::new)); } } From c21a0a9d89845009cd722ee04e00be3a77956c29 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 26 Jun 2025 13:57:33 +0200 Subject: [PATCH 04/28] The missing commit --- .../authz/IndicesAndAliasesResolver.java | 34 ++++++++++++------- .../xpack/security/authz/RBACEngine.java | 5 +-- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java index f3573d8fb6518..e895c8be3ce69 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java @@ -129,7 +129,7 @@ ResolvedIndices resolve( throw new IllegalStateException("Request [" + request + "] is not an Indices request, but should be."); } - if (request instanceof FlatIndicesRequest flatIndicesRequest) { + if (request instanceof FlatIndicesRequest flatIndicesRequest && flatIndicesRequest.requiresRewrite()) { rewrite(flatIndicesRequest, authorizedIndices); } @@ -137,11 +137,13 @@ ResolvedIndices resolve( } void rewrite(FlatIndicesRequest request, AuthorizationEngine.AuthorizedIndices authorizedIndices) { + assert request.requiresRewrite(); + var clusters = remoteClusterResolver.clusters(); logger.info("Clusters available for remote indices: {}", clusters); // no remotes, nothing to rewrite... if (clusters.isEmpty()) { - logger.info("Skipping..."); + logger.info("No remote clusters linked, skipping..."); return; } @@ -156,22 +158,30 @@ void rewrite(FlatIndicesRequest request, AuthorizationEngine.AuthorizedIndices a } } - // TODO do not rewrite twice - List rewrittenIndices = new ArrayList<>(indices.length); + if (authorizedClusters.isEmpty()) { + logger.info("No remote clusters authorized, skipping..."); + return; + } + ResolvedIndices resolved = remoteClusterResolver.splitLocalAndRemoteIndexNames(indices); + // skip handling searches where there's both qualified and flat expressions to simplify POC + // in the real thing, we'd also rewrite these + if (resolved.getRemote().isEmpty() == false) { + return; + } + + List indexExpressions = new ArrayList<>(indices.length); for (var local : resolved.getLocal()) { - String rewritten = local; + List rewritten = new ArrayList<>(); + rewritten.add(local); for (var cluster : authorizedClusters) { - rewritten += "," + RemoteClusterAware.buildRemoteIndexName(cluster, local); - rewrittenIndices.add(rewritten); + rewritten.add(RemoteClusterAware.buildRemoteIndexName(cluster, local)); + indexExpressions.add(new FlatIndicesRequest.IndexExpression(local, rewritten)); } logger.info("Rewrote [{}] to [{}]", local, rewritten); } - if (resolved.getRemote().isEmpty() == false) { - rewrittenIndices.addAll(resolved.getRemote()); - } - request.indices(rewrittenIndices); - // skipping mixed expressions, _local expressions and all that jazz + + request.indexExpressions(indexExpressions); } /** diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java index 2ef0319e4729a..9c6b8c26ae312 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java @@ -999,8 +999,9 @@ static AuthorizedIndices resolveAuthorizedIndicesFromRole( } return predicate.test(indexAbstraction, selector); }, name -> { - // just some bogus predicate that lets us differentiate between roles - return Arrays.asList(role.names()).contains("remote_searcher"); + // just some bogus predicate that lets us differentiate between roles, not at all + // how this will work in the end + return Arrays.asList(role.names()).contains("_es_test_root"); }); } From 9d63d775476edcfde976118bb3bacc0a32f8a045 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 10 Jul 2025 11:07:21 +0200 Subject: [PATCH 05/28] More --- .../xpack/core/security/authz/AuthorizationEngine.java | 2 +- .../xpack/security/authz/IndicesAndAliasesResolver.java | 2 +- .../org/elasticsearch/xpack/security/authz/RBACEngine.java | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java index e2ffcf7480381..ef7446ae22ae5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java @@ -301,7 +301,7 @@ interface AuthorizedIndices { boolean check(String name, IndexComponentSelector selector); // Does not belong here - default boolean checkProject(String projectId) { + default boolean checkRemote(String remoteAlias) { return false; } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java index e895c8be3ce69..5db5b5c2b8cf4 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java @@ -152,7 +152,7 @@ void rewrite(FlatIndicesRequest request, AuthorizationEngine.AuthorizedIndices a var authorizedClusters = new HashSet(); for (var cluster : clusters) { - if (authorizedIndices.checkProject(cluster)) { + if (authorizedIndices.checkRemote(cluster)) { logger.info("Remote cluster [{}] authorized", cluster); authorizedClusters.add(cluster); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java index 9c6b8c26ae312..2e9a20d994e77 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java @@ -1158,9 +1158,9 @@ public boolean check(String name, IndexComponentSelector selector) { } @Override - public boolean checkProject(String name) { - Objects.requireNonNull(name, "must specify a project name for authorization check"); - return projectPredicate.test(name); + public boolean checkRemote(String remoteAlias) { + Objects.requireNonNull(remoteAlias, "must specify a project name for authorization check"); + return projectPredicate.test(remoteAlias); } } } From 8eec08ea1ee2bfb03545dc8977e8ba7254053d16 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 10 Jul 2025 12:55:15 +0200 Subject: [PATCH 06/28] Moar --- .../main/groovy/elasticsearch.run-ccs.gradle | 1 + .../org/elasticsearch/FlatIndicesRequest.java | 3 ++ .../action/search/SearchRequest.java | 22 +++++++++++++++ .../common/settings/ClusterSettings.java | 2 ++ .../rest/action/search/RestSearchAction.java | 15 ++++++++++ .../elasticsearch/search/SearchService.java | 2 ++ .../transport/RemoteClusterAware.java | 4 +++ .../transport/RemoteClusterService.java | 28 +++++++++++++++++++ .../transport/RemoteConnectionStrategy.java | 4 +++ 9 files changed, 81 insertions(+) diff --git a/build-tools-internal/src/main/groovy/elasticsearch.run-ccs.gradle b/build-tools-internal/src/main/groovy/elasticsearch.run-ccs.gradle index 587c97d3476ea..07abe9a8e7633 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.run-ccs.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.run-ccs.gradle @@ -48,6 +48,7 @@ tasks.register("run-ccs", RunTask) { useCluster queryingCluster doFirst { queryingCluster.get().getNodes().each { node -> + node.setting('cluster.remote.my_remote_cluster.tags', 'env-dev') if (proxyMode) { node.setting('cluster.remote.my_remote_cluster.mode', 'proxy') if (basicSecurityMode) { diff --git a/server/src/main/java/org/elasticsearch/FlatIndicesRequest.java b/server/src/main/java/org/elasticsearch/FlatIndicesRequest.java index 2d4594c05c6f1..b4ce4dbceaffc 100644 --- a/server/src/main/java/org/elasticsearch/FlatIndicesRequest.java +++ b/server/src/main/java/org/elasticsearch/FlatIndicesRequest.java @@ -10,6 +10,7 @@ package org.elasticsearch; import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.transport.RemoteClusterService; import java.util.List; @@ -18,5 +19,7 @@ public interface FlatIndicesRequest extends IndicesRequest { void indexExpressions(List indexExpressions); + boolean checkRemote(String remote, List tags); + record IndexExpression(String original, List rewritten) {} } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index cd5c0ad466bc7..9a21d2a05d77b 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -32,6 +32,7 @@ import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.transport.RemoteClusterService; import org.elasticsearch.xcontent.ToXContent; import java.io.IOException; @@ -74,6 +75,8 @@ public class SearchRequest extends LegacyActionRequest private SearchType searchType = SearchType.DEFAULT; private String[] indices = Strings.EMPTY_ARRAY; + // This will be a more complex thing in the real implementation -- a lucene expression instead of just a list of literals + private List routingTags = List.of(); @Nullable private List indexExpressions; @@ -408,6 +411,11 @@ public SearchRequest indices(String... indices) { return this; } + public SearchRequest routingTags(List routingTags) { + this.routingTags = routingTags; + return this; + } + private static void validateIndices(String... indices) { Objects.requireNonNull(indices, "indices must not be null"); for (String index : indices) { @@ -873,4 +881,18 @@ public void indexExpressions(List indexExpressions) { this.indexExpressions = indexExpressions; indices(indexExpressions.stream().flatMap(indexExpression -> indexExpression.rewritten().stream()).toArray(String[]::new)); } + + @Override + public boolean checkRemote(String remote, List tags) { + if (routingTags.isEmpty()) { + return true; // no routing requested, so no constraints + } + // if any tag in routingTags matches one in tags, return true + for (RemoteClusterService.RemoteTag tag : routingTags) { + if (tags.contains(tag)) { + return true; + } + } + return false; + } } diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index 1fbc8993cc5aa..f21f1231c76d0 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -367,6 +367,7 @@ public void apply(Settings value, Settings current, Settings previous) { TransportSearchAction.SHARD_COUNT_LIMIT_SETTING, TransportSearchAction.DEFAULT_PRE_FILTER_SHARD_SIZE, RemoteClusterService.REMOTE_CLUSTER_SKIP_UNAVAILABLE, + RemoteClusterService.REMOTE_CLUSTER_TAGS, SniffConnectionStrategy.REMOTE_CONNECTIONS_PER_CLUSTER, RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING, RemoteClusterService.REMOTE_NODE_ATTRIBUTE, @@ -483,6 +484,7 @@ public void apply(Settings value, Settings current, Settings previous) { SearchService.ALLOW_EXPENSIVE_QUERIES, SearchService.CCS_VERSION_CHECK_SETTING, SearchService.CCS_COLLECT_TELEMETRY, + SearchService.FLAT_WORLD_ENABLED, SearchService.BATCHED_QUERY_PHASE, SearchService.PREWARMING_THRESHOLD_THREADPOOL_SIZE_FACTOR_POOL_SIZE, MultiBucketConsumerService.MAX_BUCKET_SETTING, diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index 5eda47bc32354..ee67432e6050f 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -9,6 +9,8 @@ package org.elasticsearch.rest.action.search; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.search.SearchRequest; @@ -35,6 +37,7 @@ import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.search.suggest.SuggestBuilder; import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; +import org.elasticsearch.transport.RemoteClusterService; import org.elasticsearch.usage.SearchUsageHolder; import org.elasticsearch.xcontent.XContentParser; @@ -63,6 +66,7 @@ public class RestSearchAction extends BaseRestHandler { public static final String TYPED_KEYS_PARAM = "typed_keys"; public static final String INCLUDE_NAMED_QUERIES_SCORE_PARAM = "include_named_queries_score"; public static final Set RESPONSE_PARAMS = Set.of(TYPED_KEYS_PARAM, TOTAL_HITS_AS_INT_PARAM, INCLUDE_NAMED_QUERIES_SCORE_PARAM); + private static final Logger log = LogManager.getLogger(RestSearchAction.class); private final SearchUsageHolder searchUsageHolder; private final Predicate clusterSupportsFeature; @@ -98,6 +102,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC client.threadPool().getThreadContext().setErrorTraceTransportHeader(request); } SearchRequest searchRequest = new SearchRequest(); + // access the BwC param, but just drop it // this might be set by old clients request.param("min_compatible_shard_node"); @@ -167,6 +172,16 @@ public static void parseSearchRequest( searchRequest.source(new SearchSourceBuilder()); } searchRequest.indices(Strings.splitStringByCommaToArray(request.param("index"))); + + var routingTags = request.param("routing_tags", null); + if (routingTags != null) { + searchRequest.routingTags( + Arrays.stream(Strings.splitStringByCommaToArray(routingTags)).map(RemoteClusterService.RemoteTag::fromString).toList() + ); + } else { + log.info("No routing tags"); + } + if (requestContentParser != null) { if (searchUsageHolder == null) { searchRequest.source().parseXContent(requestContentParser, true, clusterSupportsFeature); diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index af568c7b5d2cb..ddd39676b35cd 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -297,6 +297,8 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv Setting.Property.NodeScope ); + public static final Setting FLAT_WORLD_ENABLED = Setting.boolSetting("search.flat_world.enabled", false, Property.NodeScope); + private static final boolean BATCHED_QUERY_PHASE_FEATURE_FLAG = new FeatureFlag("batched_query_phase").isEnabled(); /** diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java index 95e507f70d7a9..763faed028b10 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java @@ -56,6 +56,10 @@ protected static Set getEnabledRemoteClusters(final Settings settings) { return RemoteConnectionStrategy.getRemoteClusters(settings); } + protected static Map> getEnabledRemoteClustersWithTags(final Settings settings) { + return RemoteConnectionStrategy.getRemoteTags(settings); + } + /** * Check whether the index expression represents remote index or not. * The index name is assumed to be individual index (no commas) but can contain `-`, wildcards, diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java index fdb597b47c137..0ac19dff6971c 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java @@ -41,6 +41,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -97,6 +98,33 @@ public final class RemoteClusterService extends RemoteClusterAware (ns, key) -> boolSetting(key, true, new RemoteConnectionEnabled<>(ns, key), Setting.Property.Dynamic, Setting.Property.NodeScope) ); + public record RemoteTag(String key, String value) { + public static RemoteTag fromString(String tag) { + if (tag == null || tag.isEmpty()) { + throw new IllegalArgumentException("Remote tag must not be null or empty"); + } + // - as a separator to simplify search path param parsing; won't be like this in the real implementation + int idx = tag.indexOf('-'); + if (idx < 0) { + return new RemoteTag(tag, ""); + } else { + return new RemoteTag(tag.substring(0, idx), tag.substring(idx + 1)); + } + } + } + + public static final Setting.AffixSetting> REMOTE_CLUSTER_TAGS = Setting.affixKeySetting( + "cluster.remote.", + "tags", + (ns, key) -> Setting.listSetting( + key, + Collections.emptyList(), + RemoteTag::fromString, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ) + ); + public static final Setting.AffixSetting REMOTE_CLUSTER_PING_SCHEDULE = Setting.affixKeySetting( "cluster.remote.", "transport.ping_schedule", diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java index a715797b97977..5b1925b0a20fa 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java @@ -189,6 +189,10 @@ static Set getRemoteClusters(Settings settings) { return enablementSettings.flatMap(s -> getClusterAlias(settings, s)).collect(Collectors.toSet()); } + static Map> getRemoteTags(Settings settings) { + return RemoteClusterService.REMOTE_CLUSTER_TAGS.getAsMap(settings); + } + public static boolean isConnectionEnabled(String clusterAlias, Settings settings) { ConnectionStrategy mode = REMOTE_CONNECTION_MODE.getConcreteSettingForNamespace(clusterAlias).get(settings); if (mode.equals(ConnectionStrategy.SNIFF)) { From fe6b6967e1584f79c7f23f5d45207167512fedaa Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 10 Jul 2025 12:55:40 +0200 Subject: [PATCH 07/28] Also resolver --- .../authz/IndicesAndAliasesResolver.java | 48 ++++++++++++------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java index 5db5b5c2b8cf4..7b49a2dd4a0a9 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java @@ -34,8 +34,10 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.search.SearchService; import org.elasticsearch.transport.NoSuchRemoteClusterException; import org.elasticsearch.transport.RemoteClusterAware; +import org.elasticsearch.transport.RemoteClusterService; import org.elasticsearch.transport.RemoteConnectionStrategy; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine; @@ -63,11 +65,13 @@ class IndicesAndAliasesResolver { private final IndexNameExpressionResolver nameExpressionResolver; private final IndexAbstractionResolver indexAbstractionResolver; private final RemoteClusterResolver remoteClusterResolver; + private final boolean flatWorldEnabled; IndicesAndAliasesResolver(Settings settings, ClusterService clusterService, IndexNameExpressionResolver resolver) { this.nameExpressionResolver = resolver; this.indexAbstractionResolver = new IndexAbstractionResolver(resolver); this.remoteClusterResolver = new RemoteClusterResolver(settings, clusterService.getClusterSettings()); + this.flatWorldEnabled = SearchService.FLAT_WORLD_ENABLED.get(settings); } /** @@ -129,37 +133,42 @@ ResolvedIndices resolve( throw new IllegalStateException("Request [" + request + "] is not an Indices request, but should be."); } - if (request instanceof FlatIndicesRequest flatIndicesRequest && flatIndicesRequest.requiresRewrite()) { - rewrite(flatIndicesRequest, authorizedIndices); + if (flatWorldEnabled && request instanceof FlatIndicesRequest flatIndicesRequest && flatIndicesRequest.requiresRewrite()) { + rewriteFlatIndexExpression(flatIndicesRequest, authorizedIndices); } return resolveIndicesAndAliases(action, (IndicesRequest) request, projectMetadata, authorizedIndices); } - void rewrite(FlatIndicesRequest request, AuthorizationEngine.AuthorizedIndices authorizedIndices) { - assert request.requiresRewrite(); + void rewriteFlatIndexExpression(FlatIndicesRequest request, AuthorizationEngine.AuthorizedIndices authorizedIndices) { + assert flatWorldEnabled && request.requiresRewrite(); - var clusters = remoteClusterResolver.clusters(); - logger.info("Clusters available for remote indices: {}", clusters); + Set remotes = remoteClusterResolver.clusters(); + Map> tags = remoteClusterResolver.tags(); + + logger.info("Remote available: {} with tags {}", remotes, tags); // no remotes, nothing to rewrite... - if (clusters.isEmpty()) { - logger.info("No remote clusters linked, skipping..."); + if (remotes.isEmpty()) { + logger.info("No remotes, skipping..."); return; } var indices = request.indices(); // empty indices actually means search everything so would need to also rewrite that - var authorizedClusters = new HashSet(); - for (var cluster : clusters) { - if (authorizedIndices.checkRemote(cluster)) { - logger.info("Remote cluster [{}] authorized", cluster); - authorizedClusters.add(cluster); + var targetRemotes = new HashSet(); + for (var remote : remotes) { + List tagsForRemote = tags.get(remote); + logger.info("Remote [{}] has tags [{}]", remote, tagsForRemote); + // TODO routing also needs to apply to local + if (authorizedIndices.checkRemote(remote) && request.checkRemote(remote, tagsForRemote)) { + logger.info("Remote [{}] authorized and matches routing", remote); + targetRemotes.add(remote); } } - if (authorizedClusters.isEmpty()) { - logger.info("No remote clusters authorized, skipping..."); + if (targetRemotes.isEmpty()) { + logger.info("No target remotes, skipping..."); return; } @@ -174,7 +183,7 @@ void rewrite(FlatIndicesRequest request, AuthorizationEngine.AuthorizedIndices a for (var local : resolved.getLocal()) { List rewritten = new ArrayList<>(); rewritten.add(local); - for (var cluster : authorizedClusters) { + for (var cluster : targetRemotes) { rewritten.add(RemoteClusterAware.buildRemoteIndexName(cluster, local)); indexExpressions.add(new FlatIndicesRequest.IndexExpression(local, rewritten)); } @@ -601,10 +610,13 @@ private static List indicesList(String[] list) { private static class RemoteClusterResolver extends RemoteClusterAware { private final CopyOnWriteArraySet clusters; + // TODO consolidate + private final Map> tags; private RemoteClusterResolver(Settings settings, ClusterSettings clusterSettings) { super(settings); clusters = new CopyOnWriteArraySet<>(getEnabledRemoteClusters(settings)); + tags = RemoteClusterService.getEnabledRemoteClustersWithTags(settings); listenForUpdates(clusterSettings); } @@ -630,5 +642,9 @@ ResolvedIndices splitLocalAndRemoteIndexNames(String... indices) { Set clusters() { return Collections.unmodifiableSet(clusters); } + + Map> tags() { + return Collections.unmodifiableMap(tags); + } } } From 8dee748c1dbe3b7ad451266220387c92aae917c7 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 10 Jul 2025 13:44:01 +0200 Subject: [PATCH 08/28] field caps --- .../org/elasticsearch/FlatIndicesRequest.java | 2 +- .../fieldcaps/FieldCapabilitiesRequest.java | 29 ++++++++++++++++++- .../action/search/SearchRequest.java | 2 +- .../authz/IndicesAndAliasesResolver.java | 2 +- 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/FlatIndicesRequest.java b/server/src/main/java/org/elasticsearch/FlatIndicesRequest.java index b4ce4dbceaffc..a3ca1528bd7e1 100644 --- a/server/src/main/java/org/elasticsearch/FlatIndicesRequest.java +++ b/server/src/main/java/org/elasticsearch/FlatIndicesRequest.java @@ -19,7 +19,7 @@ public interface FlatIndicesRequest extends IndicesRequest { void indexExpressions(List indexExpressions); - boolean checkRemote(String remote, List tags); + boolean checkRemote(List tags); record IndexExpression(String original, List rewritten) {} } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java index 2e24858d9781f..ec04bb9bd67c5 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.fieldcaps; +import org.elasticsearch.FlatIndicesRequest; import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; @@ -18,11 +19,13 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.transport.RemoteClusterAware; +import org.elasticsearch.transport.RemoteClusterService; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -30,11 +33,16 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -public final class FieldCapabilitiesRequest extends LegacyActionRequest implements IndicesRequest.Replaceable, ToXContentObject { +public final class FieldCapabilitiesRequest extends LegacyActionRequest + implements + FlatIndicesRequest, + IndicesRequest.Replaceable, + ToXContentObject { public static final String NAME = "field_caps_request"; public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosed(); @@ -52,6 +60,8 @@ public final class FieldCapabilitiesRequest extends LegacyActionRequest implemen private QueryBuilder indexFilter; private Map runtimeFields = Collections.emptyMap(); private Long nowInMillis; + @Nullable + private List indexExpressions; public FieldCapabilitiesRequest(StreamInput in) throws IOException { super(in); @@ -323,4 +333,21 @@ public String getDescription() { } }; } + + @Override + public boolean requiresRewrite() { + return indexExpressions == null; + } + + @Override + public void indexExpressions(List indexExpressions) { + assert requiresRewrite(); + this.indexExpressions = indexExpressions; + indices(indexExpressions.stream().flatMap(indexExpression -> indexExpression.rewritten().stream()).toArray(String[]::new)); + } + + @Override + public boolean checkRemote(List tags) { + return true; + } } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 9a21d2a05d77b..7d890cd71140e 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -883,7 +883,7 @@ public void indexExpressions(List indexExpressions) { } @Override - public boolean checkRemote(String remote, List tags) { + public boolean checkRemote(List tags) { if (routingTags.isEmpty()) { return true; // no routing requested, so no constraints } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java index 7b49a2dd4a0a9..6057010c641cf 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java @@ -161,7 +161,7 @@ void rewriteFlatIndexExpression(FlatIndicesRequest request, AuthorizationEngine. List tagsForRemote = tags.get(remote); logger.info("Remote [{}] has tags [{}]", remote, tagsForRemote); // TODO routing also needs to apply to local - if (authorizedIndices.checkRemote(remote) && request.checkRemote(remote, tagsForRemote)) { + if (authorizedIndices.checkRemote(remote) && request.checkRemote(tagsForRemote)) { logger.info("Remote [{}] authorized and matches routing", remote); targetRemotes.add(remote); } From be2ab99d962f579f603fba5dba05e6401af1c79f Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 11 Jul 2025 12:04:05 +0200 Subject: [PATCH 09/28] SPI --- ...est.java => RewritableIndicesRequest.java} | 8 +- .../fieldcaps/FieldCapabilitiesRequest.java | 14 +-- .../action/search/SearchRequest.java | 14 +-- .../core/security/SecurityExtension.java | 5 ++ .../authz/CustomIndicesRequestRewriter.java | 22 +++++ .../xpack/security/Security.java | 45 +++++++++- .../security/authz/AuthorizationService.java | 6 +- .../authz/IndicesAndAliasesResolver.java | 89 ++++--------------- .../authz/AuthorizationServiceTests.java | 16 ++-- .../authz/IndicesAndAliasesResolverTests.java | 8 +- 10 files changed, 130 insertions(+), 97 deletions(-) rename server/src/main/java/org/elasticsearch/{FlatIndicesRequest.java => RewritableIndicesRequest.java} (74%) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CustomIndicesRequestRewriter.java diff --git a/server/src/main/java/org/elasticsearch/FlatIndicesRequest.java b/server/src/main/java/org/elasticsearch/RewritableIndicesRequest.java similarity index 74% rename from server/src/main/java/org/elasticsearch/FlatIndicesRequest.java rename to server/src/main/java/org/elasticsearch/RewritableIndicesRequest.java index a3ca1528bd7e1..c5185e449a3eb 100644 --- a/server/src/main/java/org/elasticsearch/FlatIndicesRequest.java +++ b/server/src/main/java/org/elasticsearch/RewritableIndicesRequest.java @@ -14,12 +14,12 @@ import java.util.List; -public interface FlatIndicesRequest extends IndicesRequest { - boolean requiresRewrite(); +public interface RewritableIndicesRequest extends IndicesRequest { + boolean rewritten(); - void indexExpressions(List indexExpressions); + void rewritten(List indexExpressions); boolean checkRemote(List tags); - record IndexExpression(String original, List rewritten) {} + record RewrittenIndexExpression(String original, List rewritten) {} } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java index ec04bb9bd67c5..451c3ad0c3a78 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java @@ -9,7 +9,7 @@ package org.elasticsearch.action.fieldcaps; -import org.elasticsearch.FlatIndicesRequest; +import org.elasticsearch.RewritableIndicesRequest; import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; @@ -40,7 +40,7 @@ public final class FieldCapabilitiesRequest extends LegacyActionRequest implements - FlatIndicesRequest, + RewritableIndicesRequest, IndicesRequest.Replaceable, ToXContentObject { public static final String NAME = "field_caps_request"; @@ -61,7 +61,7 @@ public final class FieldCapabilitiesRequest extends LegacyActionRequest private Map runtimeFields = Collections.emptyMap(); private Long nowInMillis; @Nullable - private List indexExpressions; + private List indexExpressions; public FieldCapabilitiesRequest(StreamInput in) throws IOException { super(in); @@ -335,13 +335,13 @@ public String getDescription() { } @Override - public boolean requiresRewrite() { - return indexExpressions == null; + public boolean rewritten() { + return indexExpressions != null; } @Override - public void indexExpressions(List indexExpressions) { - assert requiresRewrite(); + public void rewritten(List indexExpressions) { + assert false == rewritten(); this.indexExpressions = indexExpressions; indices(indexExpressions.stream().flatMap(indexExpression -> indexExpression.rewritten().stream()).toArray(String[]::new)); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 7d890cd71140e..0b9a44764bab5 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -9,7 +9,7 @@ package org.elasticsearch.action.search; -import org.elasticsearch.FlatIndicesRequest; +import org.elasticsearch.RewritableIndicesRequest; import org.elasticsearch.TransportVersions; import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; @@ -57,7 +57,7 @@ */ public class SearchRequest extends LegacyActionRequest implements - FlatIndicesRequest, + RewritableIndicesRequest, IndicesRequest.Replaceable, Rewriteable { @@ -79,7 +79,7 @@ public class SearchRequest extends LegacyActionRequest private List routingTags = List.of(); @Nullable - private List indexExpressions; + private List indexExpressions; @Nullable private String routing; @@ -871,13 +871,13 @@ public String toString() { } @Override - public boolean requiresRewrite() { - return indexExpressions == null; + public boolean rewritten() { + return indexExpressions != null; } @Override - public void indexExpressions(List indexExpressions) { - assert requiresRewrite(); + public void rewritten(List indexExpressions) { + assert false == rewritten(); this.indexExpressions = indexExpressions; indices(indexExpressions.stream().flatMap(indexExpression -> indexExpression.rewritten().stream()).toArray(String[]::new)); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java index f41b19de95272..463f7481720f5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.core.security.authc.service.ServiceAccountTokenStore; import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine; +import org.elasticsearch.xpack.core.security.authz.CustomIndicesRequestRewriter; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult; @@ -133,6 +134,10 @@ default CustomApiKeyAuthenticator getCustomApiKeyAuthenticator(SecurityComponent return null; } + default CustomIndicesRequestRewriter getCustomIndicesRequestRewriter(SecurityComponents components) { + return null; + } + /** * Returns a authorization engine for authorizing requests, or null to use the default authorization mechanism. * diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CustomIndicesRequestRewriter.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CustomIndicesRequestRewriter.java new file mode 100644 index 0000000000000..95847e51e3711 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CustomIndicesRequestRewriter.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.security.authz; + +import org.elasticsearch.RewritableIndicesRequest; + +public interface CustomIndicesRequestRewriter { + void rewrite(RewritableIndicesRequest request); + + class Default implements CustomIndicesRequestRewriter { + @Override + public void rewrite(RewritableIndicesRequest request) { + // No rewriting by default + // This is a no-op implementation + } + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index b1e30490990ff..70834cc936e62 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -213,6 +213,7 @@ import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine; import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; +import org.elasticsearch.xpack.core.security.authz.CustomIndicesRequestRewriter; import org.elasticsearch.xpack.core.security.authz.RestrictedIndices; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.accesscontrol.DocumentSubsetBitsetCache; @@ -1146,6 +1147,7 @@ Collection createComponents( if (authorizationDenialMessages.get() == null) { authorizationDenialMessages.set(new AuthorizationDenialMessages.Default()); } + final AuthorizationService authzService = new AuthorizationService( settings, allRolesStore, @@ -1162,7 +1164,8 @@ Collection createComponents( operatorPrivilegesService.get(), restrictedIndices, authorizationDenialMessages.get(), - projectResolver + projectResolver, + createCustomIndicesRequestRewriter(extensionComponents) ); components.add(nativeRolesStore); // used by roles actions @@ -1252,6 +1255,46 @@ Collection createComponents( return components; } + private CustomIndicesRequestRewriter createCustomIndicesRequestRewriter(SecurityExtension.SecurityComponents extensionComponents) { + final Map customByExtension = new HashMap<>(); + for (final SecurityExtension extension : securityExtensions) { + final CustomIndicesRequestRewriter custom = extension.getCustomIndicesRequestRewriter(extensionComponents); + if (custom != null) { + if (false == isInternalExtension(extension)) { + throw new IllegalStateException( + "The [" + + extension.extensionName() + + "] extension tried to install a custom CustomIndicesRequestRewriter. " + + "This functionality is not available to external extensions." + ); + } + customByExtension.put(extension.extensionName(), custom); + } + } + + if (customByExtension.isEmpty()) { + logger.debug( + "No custom implementation for [{}]. Falling-back to default implementation.", + CustomIndicesRequestRewriter.class.getCanonicalName() + ); + return new CustomIndicesRequestRewriter.Default(); + } else if (customByExtension.size() > 1) { + throw new IllegalStateException( + "Multiple extensions tried to install a custom CustomIndicesRequestRewriter: " + customByExtension.keySet() + ); + } else { + final var byExtensionEntry = customByExtension.entrySet().iterator().next(); + final CustomIndicesRequestRewriter custom = byExtensionEntry.getValue(); + final String extensionName = byExtensionEntry.getKey(); + logger.debug( + "CustomIndicesRequestRewriter implementation [{}] provided by extension [{}]", + custom.getClass().getCanonicalName(), + extensionName + ); + return custom; + } + } + private CustomApiKeyAuthenticator createCustomApiKeyAuthenticator(SecurityExtension.SecurityComponents extensionComponents) { final Map customApiKeyAuthenticatorByExtension = new HashMap<>(); for (final SecurityExtension extension : securityExtensions) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java index f7f0f48f1c0fe..1d951b6129f67 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java @@ -69,6 +69,7 @@ import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.ParentActionAuthorization; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.RequestInfo; import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; +import org.elasticsearch.xpack.core.security.authz.CustomIndicesRequestRewriter; import org.elasticsearch.xpack.core.security.authz.ResolvedIndices; import org.elasticsearch.xpack.core.security.authz.RestrictedIndices; import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection; @@ -164,12 +165,13 @@ public AuthorizationService( OperatorPrivilegesService operatorPrivilegesService, RestrictedIndices restrictedIndices, AuthorizationDenialMessages authorizationDenialMessages, - ProjectResolver projectResolver + ProjectResolver projectResolver, + CustomIndicesRequestRewriter customIndicesRequestRewriter ) { this.clusterService = clusterService; this.auditTrailService = auditTrailService; this.restrictedIndices = restrictedIndices; - this.indicesAndAliasesResolver = new IndicesAndAliasesResolver(settings, clusterService, resolver); + this.indicesAndAliasesResolver = new IndicesAndAliasesResolver(settings, clusterService, resolver, customIndicesRequestRewriter); this.authcFailureHandler = authcFailureHandler; this.threadContext = threadPool.getThreadContext(); this.securityContext = new SecurityContext(settings, this.threadContext); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java index 6057010c641cf..7e6417f37d5d0 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java @@ -6,9 +6,7 @@ */ package org.elasticsearch.xpack.security.authz; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.FlatIndicesRequest; +import org.elasticsearch.RewritableIndicesRequest; import org.elasticsearch.action.AliasesRequest; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; @@ -34,13 +32,13 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; -import org.elasticsearch.search.SearchService; import org.elasticsearch.transport.NoSuchRemoteClusterException; import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.transport.RemoteClusterService; import org.elasticsearch.transport.RemoteConnectionStrategy; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine; +import org.elasticsearch.xpack.core.security.authz.CustomIndicesRequestRewriter; import org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField; import org.elasticsearch.xpack.core.security.authz.ResolvedIndices; @@ -58,20 +56,23 @@ import static org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField.NO_INDEX_PLACEHOLDER; -class IndicesAndAliasesResolver { - - private static final Logger logger = LogManager.getLogger(IndicesAndAliasesResolver.class); +public class IndicesAndAliasesResolver { private final IndexNameExpressionResolver nameExpressionResolver; private final IndexAbstractionResolver indexAbstractionResolver; private final RemoteClusterResolver remoteClusterResolver; - private final boolean flatWorldEnabled; + private final CustomIndicesRequestRewriter customIndicesRequestRewriter; - IndicesAndAliasesResolver(Settings settings, ClusterService clusterService, IndexNameExpressionResolver resolver) { + IndicesAndAliasesResolver( + Settings settings, + ClusterService clusterService, + IndexNameExpressionResolver resolver, + CustomIndicesRequestRewriter customIndicesRequestRewriter + ) { this.nameExpressionResolver = resolver; this.indexAbstractionResolver = new IndexAbstractionResolver(resolver); this.remoteClusterResolver = new RemoteClusterResolver(settings, clusterService.getClusterSettings()); - this.flatWorldEnabled = SearchService.FLAT_WORLD_ENABLED.get(settings); + this.customIndicesRequestRewriter = customIndicesRequestRewriter; } /** @@ -133,66 +134,13 @@ ResolvedIndices resolve( throw new IllegalStateException("Request [" + request + "] is not an Indices request, but should be."); } - if (flatWorldEnabled && request instanceof FlatIndicesRequest flatIndicesRequest && flatIndicesRequest.requiresRewrite()) { - rewriteFlatIndexExpression(flatIndicesRequest, authorizedIndices); + if (request instanceof RewritableIndicesRequest rewritableIndicesRequest) { + customIndicesRequestRewriter.rewrite(rewritableIndicesRequest); } return resolveIndicesAndAliases(action, (IndicesRequest) request, projectMetadata, authorizedIndices); } - void rewriteFlatIndexExpression(FlatIndicesRequest request, AuthorizationEngine.AuthorizedIndices authorizedIndices) { - assert flatWorldEnabled && request.requiresRewrite(); - - Set remotes = remoteClusterResolver.clusters(); - Map> tags = remoteClusterResolver.tags(); - - logger.info("Remote available: {} with tags {}", remotes, tags); - // no remotes, nothing to rewrite... - if (remotes.isEmpty()) { - logger.info("No remotes, skipping..."); - return; - } - - var indices = request.indices(); - // empty indices actually means search everything so would need to also rewrite that - - var targetRemotes = new HashSet(); - for (var remote : remotes) { - List tagsForRemote = tags.get(remote); - logger.info("Remote [{}] has tags [{}]", remote, tagsForRemote); - // TODO routing also needs to apply to local - if (authorizedIndices.checkRemote(remote) && request.checkRemote(tagsForRemote)) { - logger.info("Remote [{}] authorized and matches routing", remote); - targetRemotes.add(remote); - } - } - - if (targetRemotes.isEmpty()) { - logger.info("No target remotes, skipping..."); - return; - } - - ResolvedIndices resolved = remoteClusterResolver.splitLocalAndRemoteIndexNames(indices); - // skip handling searches where there's both qualified and flat expressions to simplify POC - // in the real thing, we'd also rewrite these - if (resolved.getRemote().isEmpty() == false) { - return; - } - - List indexExpressions = new ArrayList<>(indices.length); - for (var local : resolved.getLocal()) { - List rewritten = new ArrayList<>(); - rewritten.add(local); - for (var cluster : targetRemotes) { - rewritten.add(RemoteClusterAware.buildRemoteIndexName(cluster, local)); - indexExpressions.add(new FlatIndicesRequest.IndexExpression(local, rewritten)); - } - logger.info("Rewrote [{}] to [{}]", local, rewritten); - } - - request.indexExpressions(indexExpressions); - } - /** * Attempt to resolve requested indices without expanding any wildcards. * @return The {@link ResolvedIndices} or null if wildcard expansion must be performed. @@ -607,13 +555,14 @@ private static List indicesList(String[] list) { return (list == null) ? null : Arrays.asList(list); } - private static class RemoteClusterResolver extends RemoteClusterAware { + public static class RemoteClusterResolver extends RemoteClusterAware { private final CopyOnWriteArraySet clusters; // TODO consolidate private final Map> tags; - private RemoteClusterResolver(Settings settings, ClusterSettings clusterSettings) { + @SuppressWarnings("this-escape") + public RemoteClusterResolver(Settings settings, ClusterSettings clusterSettings) { super(settings); clusters = new CopyOnWriteArraySet<>(getEnabledRemoteClusters(settings)); tags = RemoteClusterService.getEnabledRemoteClustersWithTags(settings); @@ -629,7 +578,7 @@ protected void updateRemoteCluster(String clusterAlias, Settings settings) { } } - ResolvedIndices splitLocalAndRemoteIndexNames(String... indices) { + public ResolvedIndices splitLocalAndRemoteIndexNames(String... indices) { final Map> map = super.groupClusterIndices(clusters, indices); final List local = map.remove(LOCAL_CLUSTER_GROUP_KEY); final List remote = map.entrySet() @@ -639,11 +588,11 @@ ResolvedIndices splitLocalAndRemoteIndexNames(String... indices) { return new ResolvedIndices(local == null ? List.of() : local, remote); } - Set clusters() { + public Set clusters() { return Collections.unmodifiableSet(clusters); } - Map> tags() { + public Map> tags() { return Collections.unmodifiableMap(tags); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java index e4bb33c66d983..2d9e21d545192 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java @@ -155,6 +155,7 @@ import org.elasticsearch.xpack.core.security.authc.Subject; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo; +import org.elasticsearch.xpack.core.security.authz.CustomIndicesRequestRewriter; import org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField; import org.elasticsearch.xpack.core.security.authz.ResolvedIndices; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; @@ -336,7 +337,8 @@ public void setup() { operatorPrivilegesService, RESTRICTED_INDICES, new AuthorizationDenialMessages.Default(), - projectResolver + projectResolver, + new CustomIndicesRequestRewriter.Default() ); } @@ -1769,7 +1771,8 @@ public void testDenialForAnonymousUser() { operatorPrivilegesService, RESTRICTED_INDICES, new AuthorizationDenialMessages.Default(), - projectResolver + projectResolver, + new CustomIndicesRequestRewriter.Default() ); RoleDescriptor role = new RoleDescriptor( @@ -1819,7 +1822,8 @@ public void testDenialForAnonymousUserAuthorizationExceptionDisabled() { operatorPrivilegesService, RESTRICTED_INDICES, new AuthorizationDenialMessages.Default(), - projectResolver + projectResolver, + new CustomIndicesRequestRewriter.Default() ); RoleDescriptor role = new RoleDescriptor( @@ -3357,7 +3361,8 @@ public void testAuthorizationEngineSelectionForCheckPrivileges() throws Exceptio operatorPrivilegesService, RESTRICTED_INDICES, new AuthorizationDenialMessages.Default(), - projectResolver + projectResolver, + new CustomIndicesRequestRewriter.Default() ); Subject subject = new Subject(new User("test", "a role"), mock(RealmRef.class)); @@ -3513,7 +3518,8 @@ public void getUserPrivileges(AuthorizationInfo authorizationInfo, ActionListene operatorPrivilegesService, RESTRICTED_INDICES, new AuthorizationDenialMessages.Default(), - projectResolver + projectResolver, + new CustomIndicesRequestRewriter.Default() ); Authentication authentication; try (StoredContext ignore = threadContext.stashContext()) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java index cb6db57ee5558..7e01409ed716a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java @@ -69,6 +69,7 @@ import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.authc.Subject; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizedIndices; +import org.elasticsearch.xpack.core.security.authz.CustomIndicesRequestRewriter; import org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField; import org.elasticsearch.xpack.core.security.authz.ResolvedIndices; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; @@ -417,7 +418,12 @@ public void setup() { ClusterService clusterService = mock(ClusterService.class); when(clusterService.getClusterSettings()).thenReturn(new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)); - defaultIndicesResolver = new IndicesAndAliasesResolver(settings, clusterService, indexNameExpressionResolver); + defaultIndicesResolver = new IndicesAndAliasesResolver( + settings, + clusterService, + indexNameExpressionResolver, + new CustomIndicesRequestRewriter.Default() + ); } public void testDashIndicesAreAllowedInShardLevelRequests() { From 710d789e04a92d718129a830abf04f7428b8effa Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 11 Jul 2025 14:16:59 +0200 Subject: [PATCH 10/28] SPI --- .../src/main/groovy/elasticsearch.run-ccs.gradle | 1 - .../action/search/SearchRequest.java | 1 - .../common/settings/ClusterSettings.java | 1 - .../org/elasticsearch/search/SearchService.java | 2 -- .../core/security/authz/AuthorizationEngine.java | 5 ----- .../xpack/security/authz/RBACEngine.java | 15 +-------------- 6 files changed, 1 insertion(+), 24 deletions(-) diff --git a/build-tools-internal/src/main/groovy/elasticsearch.run-ccs.gradle b/build-tools-internal/src/main/groovy/elasticsearch.run-ccs.gradle index 07abe9a8e7633..587c97d3476ea 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.run-ccs.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.run-ccs.gradle @@ -48,7 +48,6 @@ tasks.register("run-ccs", RunTask) { useCluster queryingCluster doFirst { queryingCluster.get().getNodes().each { node -> - node.setting('cluster.remote.my_remote_cluster.tags', 'env-dev') if (proxyMode) { node.setting('cluster.remote.my_remote_cluster.mode', 'proxy') if (basicSecurityMode) { diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 0b9a44764bab5..071dcc215ece3 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -75,7 +75,6 @@ public class SearchRequest extends LegacyActionRequest private SearchType searchType = SearchType.DEFAULT; private String[] indices = Strings.EMPTY_ARRAY; - // This will be a more complex thing in the real implementation -- a lucene expression instead of just a list of literals private List routingTags = List.of(); @Nullable diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index f21f1231c76d0..64575bac95f8d 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -484,7 +484,6 @@ public void apply(Settings value, Settings current, Settings previous) { SearchService.ALLOW_EXPENSIVE_QUERIES, SearchService.CCS_VERSION_CHECK_SETTING, SearchService.CCS_COLLECT_TELEMETRY, - SearchService.FLAT_WORLD_ENABLED, SearchService.BATCHED_QUERY_PHASE, SearchService.PREWARMING_THRESHOLD_THREADPOOL_SIZE_FACTOR_POOL_SIZE, MultiBucketConsumerService.MAX_BUCKET_SETTING, diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index ddd39676b35cd..af568c7b5d2cb 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -297,8 +297,6 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv Setting.Property.NodeScope ); - public static final Setting FLAT_WORLD_ENABLED = Setting.boolSetting("search.flat_world.enabled", false, Property.NodeScope); - private static final boolean BATCHED_QUERY_PHASE_FEATURE_FLAG = new FeatureFlag("batched_query_phase").isEnabled(); /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java index ef7446ae22ae5..2c831645d0e69 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java @@ -299,11 +299,6 @@ interface AuthorizedIndices { * Checks if an index-like resource name is authorized, for an action by a user. The resource might or might not exist. */ boolean check(String name, IndexComponentSelector selector); - - // Does not belong here - default boolean checkRemote(String remoteAlias) { - return false; - } } /** diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java index 2e9a20d994e77..1b99bd6888c4f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java @@ -998,10 +998,6 @@ static AuthorizedIndices resolveAuthorizedIndicesFromRole( } // we don't support granting access to a backing index with a failure selector via the parent data stream } return predicate.test(indexAbstraction, selector); - }, name -> { - // just some bogus predicate that lets us differentiate between roles, not at all - // how this will work in the end - return Arrays.asList(role.names()).contains("_es_test_root"); }); } @@ -1129,18 +1125,15 @@ static final class AuthorizedIndices implements AuthorizationEngine.AuthorizedIn private final CachedSupplier> authorizedAndAvailableDataResources; private final CachedSupplier> authorizedAndAvailableFailuresResources; private final BiPredicate isAuthorizedPredicate; - private final Predicate projectPredicate; AuthorizedIndices( Supplier> authorizedAndAvailableDataResources, Supplier> authorizedAndAvailableFailuresResources, - BiPredicate isAuthorizedPredicate, - Predicate projectPredicate + BiPredicate isAuthorizedPredicate ) { this.authorizedAndAvailableDataResources = CachedSupplier.wrap(authorizedAndAvailableDataResources); this.authorizedAndAvailableFailuresResources = CachedSupplier.wrap(authorizedAndAvailableFailuresResources); this.isAuthorizedPredicate = Objects.requireNonNull(isAuthorizedPredicate); - this.projectPredicate = projectPredicate; } @Override @@ -1156,11 +1149,5 @@ public boolean check(String name, IndexComponentSelector selector) { Objects.requireNonNull(selector, "must specify a selector for authorization check"); return isAuthorizedPredicate.test(name, selector); } - - @Override - public boolean checkRemote(String remoteAlias) { - Objects.requireNonNull(remoteAlias, "must specify a project name for authorization check"); - return projectPredicate.test(remoteAlias); - } } } From 3587e6a7d8960a43603338ac0d90e4734f58576e Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 11 Jul 2025 16:40:55 +0200 Subject: [PATCH 11/28] Remote conns --- .../transport/RemoteConnectionManager.java | 5 ++ .../authz/CustomIndicesRequestRewriter.java | 36 +++++++++++ .../xpack/security/Security.java | 4 +- .../SecurityServerTransportInterceptor.java | 61 +++++++++++++++++++ 4 files changed, 105 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionManager.java b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionManager.java index c27d9cf69a905..f6aa8aecd8778 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionManager.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionManager.java @@ -239,6 +239,11 @@ public static Optional resolveRemoteClusterAl return Optional.empty(); } + public static boolean isRemoteConnection(Transport.Connection connection) { + Transport.Connection unwrapped = TransportService.unwrapConnection(connection); + return unwrapped instanceof InternalRemoteConnection; + } + private Transport.Connection getConnectionInternal(DiscoveryNode node) throws NodeNotConnectedException { Transport.Connection connection = delegate.getConnection(node); return wrapConnectionWithRemoteClusterInfo(connection, clusterAlias, credentialsManager); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CustomIndicesRequestRewriter.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CustomIndicesRequestRewriter.java index 95847e51e3711..ecc54bfe36e90 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CustomIndicesRequestRewriter.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CustomIndicesRequestRewriter.java @@ -8,15 +8,51 @@ package org.elasticsearch.xpack.core.security.authz; import org.elasticsearch.RewritableIndicesRequest; +import org.elasticsearch.transport.Transport; +import org.elasticsearch.transport.TransportInterceptor; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportRequestOptions; +import org.elasticsearch.transport.TransportResponse; +import org.elasticsearch.transport.TransportResponseHandler; public interface CustomIndicesRequestRewriter { void rewrite(RewritableIndicesRequest request); + // TODO doesn't belong here + boolean enabled(); + + // TODO also doesn't belong here nor is this the right signature + void sendRequest( + TransportInterceptor.AsyncSender sender, + Transport.Connection connection, + String action, + TransportRequest request, + TransportRequestOptions options, + TransportResponseHandler handler + ); + class Default implements CustomIndicesRequestRewriter { @Override public void rewrite(RewritableIndicesRequest request) { // No rewriting by default // This is a no-op implementation } + + @Override + public boolean enabled() { + return false; // Default implementation is not enabled + } + + @Override + public void sendRequest( + TransportInterceptor.AsyncSender sender, + Transport.Connection connection, + String action, + TransportRequest request, + TransportRequestOptions options, + TransportResponseHandler handler + ) { + + } } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 70834cc936e62..7649cfc32ac1f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -1148,6 +1148,7 @@ Collection createComponents( authorizationDenialMessages.set(new AuthorizationDenialMessages.Default()); } + CustomIndicesRequestRewriter customIndicesRequestRewriter = createCustomIndicesRequestRewriter(extensionComponents); final AuthorizationService authzService = new AuthorizationService( settings, allRolesStore, @@ -1165,7 +1166,7 @@ Collection createComponents( restrictedIndices, authorizationDenialMessages.get(), projectResolver, - createCustomIndicesRequestRewriter(extensionComponents) + customIndicesRequestRewriter ); components.add(nativeRolesStore); // used by roles actions @@ -1197,6 +1198,7 @@ Collection createComponents( securityContext.get(), destructiveOperations, crossClusterAccessAuthcService.get(), + customIndicesRequestRewriter, getLicenseState() ) ); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java index 268f9e6375f0e..142f5954e129b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java @@ -42,6 +42,7 @@ import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.CrossClusterAccessSubjectInfo; +import org.elasticsearch.xpack.core.security.authz.CustomIndicesRequestRewriter; import org.elasticsearch.xpack.core.security.transport.ProfileConfigurations; import org.elasticsearch.xpack.core.security.user.InternalUser; import org.elasticsearch.xpack.core.security.user.SystemUser; @@ -99,6 +100,7 @@ public class SecurityServerTransportInterceptor implements TransportInterceptor private final CrossClusterAccessAuthenticationService crossClusterAccessAuthcService; private final Function> remoteClusterCredentialsResolver; private final XPackLicenseState licenseState; + private final CustomIndicesRequestRewriter customIndicesRequestRewriter; public SecurityServerTransportInterceptor( Settings settings, @@ -120,6 +122,33 @@ public SecurityServerTransportInterceptor( securityContext, destructiveOperations, crossClusterAccessAuthcService, + new CustomIndicesRequestRewriter.Default(), + licenseState + ); + } + + public SecurityServerTransportInterceptor( + Settings settings, + ThreadPool threadPool, + AuthenticationService authcService, + AuthorizationService authzService, + SSLService sslService, + SecurityContext securityContext, + DestructiveOperations destructiveOperations, + CrossClusterAccessAuthenticationService crossClusterAccessAuthcService, + CustomIndicesRequestRewriter customIndicesRequestRewriter, + XPackLicenseState licenseState + ) { + this( + settings, + threadPool, + authcService, + authzService, + sslService, + securityContext, + destructiveOperations, + crossClusterAccessAuthcService, + customIndicesRequestRewriter, licenseState, RemoteConnectionManager::resolveRemoteClusterAliasWithCredentials ); @@ -137,6 +166,35 @@ public SecurityServerTransportInterceptor( XPackLicenseState licenseState, // Inject for simplified testing Function> remoteClusterCredentialsResolver + ) { + this( + settings, + threadPool, + authcService, + authzService, + sslService, + securityContext, + destructiveOperations, + crossClusterAccessAuthcService, + new CustomIndicesRequestRewriter.Default(), + licenseState, + remoteClusterCredentialsResolver + ); + } + + SecurityServerTransportInterceptor( + Settings settings, + ThreadPool threadPool, + AuthenticationService authcService, + AuthorizationService authzService, + SSLService sslService, + SecurityContext securityContext, + DestructiveOperations destructiveOperations, + CrossClusterAccessAuthenticationService crossClusterAccessAuthcService, + CustomIndicesRequestRewriter customIndicesRequestRewriter, + XPackLicenseState licenseState, + // Inject for simplified testing + Function> remoteClusterCredentialsResolver ) { this.settings = settings; this.threadPool = threadPool; @@ -147,6 +205,7 @@ public SecurityServerTransportInterceptor( this.crossClusterAccessAuthcService = crossClusterAccessAuthcService; this.licenseState = licenseState; this.remoteClusterCredentialsResolver = remoteClusterCredentialsResolver; + this.customIndicesRequestRewriter = customIndicesRequestRewriter; this.profileFilters = initializeProfileFilters(destructiveOperations); } @@ -272,6 +331,8 @@ public void sendRequest( final Optional remoteClusterCredentials = getRemoteClusterCredentials(connection); if (remoteClusterCredentials.isPresent()) { sendWithCrossClusterAccessHeaders(remoteClusterCredentials.get(), connection, action, request, options, handler); + } else if (customIndicesRequestRewriter.enabled() && RemoteConnectionManager.isRemoteConnection(connection)) { + customIndicesRequestRewriter.sendRequest(sender, connection, action, request, options, handler); } else { // Send regular request, without cross cluster access headers try { From 2a45f5b3a65696f31b4fec685aa5dece6bef678b Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Sat, 12 Jul 2025 16:25:41 +0200 Subject: [PATCH 12/28] Extract interface --- ...ustomRemoteServerTransportInterceptor.java | 49 +++++++++++++++++++ .../core/security/SecurityExtension.java | 4 ++ .../authz/CustomIndicesRequestRewriter.java | 36 -------------- .../xpack/security/Security.java | 45 ++++++++++++++++- .../SecurityServerTransportInterceptor.java | 20 ++++---- 5 files changed, 107 insertions(+), 47 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomRemoteServerTransportInterceptor.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomRemoteServerTransportInterceptor.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomRemoteServerTransportInterceptor.java new file mode 100644 index 0000000000000..57d061f61da38 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomRemoteServerTransportInterceptor.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.security; + +import org.elasticsearch.transport.Transport; +import org.elasticsearch.transport.TransportInterceptor; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportRequestOptions; +import org.elasticsearch.transport.TransportResponse; +import org.elasticsearch.transport.TransportResponseHandler; + +public interface CustomRemoteServerTransportInterceptor { + // TODO probably don't want this + boolean enabled(); + + // TODO not the signature we want + void sendRequest( + TransportInterceptor.AsyncSender sender, + Transport.Connection connection, + String action, + TransportRequest request, + TransportRequestOptions options, + TransportResponseHandler handler + ); + + class Default implements CustomRemoteServerTransportInterceptor { + @Override + public boolean enabled() { + return false; + } + + @Override + public void sendRequest( + TransportInterceptor.AsyncSender sender, + Transport.Connection connection, + String action, + TransportRequest request, + TransportRequestOptions options, + TransportResponseHandler handler + ) { + sender.sendRequest(connection, action, request, options, handler); + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java index 463f7481720f5..01703fa29f1e0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java @@ -138,6 +138,10 @@ default CustomIndicesRequestRewriter getCustomIndicesRequestRewriter(SecurityCom return null; } + default CustomRemoteServerTransportInterceptor getCustomRemoteServerTransportInterceptor(SecurityComponents components) { + return null; + } + /** * Returns a authorization engine for authorizing requests, or null to use the default authorization mechanism. * diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CustomIndicesRequestRewriter.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CustomIndicesRequestRewriter.java index bcf1e0b81b468..95847e51e3711 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CustomIndicesRequestRewriter.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CustomIndicesRequestRewriter.java @@ -8,51 +8,15 @@ package org.elasticsearch.xpack.core.security.authz; import org.elasticsearch.RewritableIndicesRequest; -import org.elasticsearch.transport.Transport; -import org.elasticsearch.transport.TransportInterceptor; -import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.transport.TransportRequestOptions; -import org.elasticsearch.transport.TransportResponse; -import org.elasticsearch.transport.TransportResponseHandler; public interface CustomIndicesRequestRewriter { void rewrite(RewritableIndicesRequest request); - // TODO doesn't belong here - boolean enabled(); - - // TODO also doesn't belong here nor is this the right signature - void sendRequest( - TransportInterceptor.AsyncSender sender, - Transport.Connection connection, - String action, - TransportRequest request, - TransportRequestOptions options, - TransportResponseHandler handler - ); - class Default implements CustomIndicesRequestRewriter { @Override public void rewrite(RewritableIndicesRequest request) { // No rewriting by default // This is a no-op implementation } - - @Override - public boolean enabled() { - return false; // Default implementation is not enabled - } - - @Override - public void sendRequest( - TransportInterceptor.AsyncSender sender, - Transport.Connection connection, - String action, - TransportRequest request, - TransportRequestOptions options, - TransportResponseHandler handler - ) { - sender.sendRequest(connection, action, request, options, handler); // Default implementation simply forwards the request - } } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 7649cfc32ac1f..e1ac1c6b09793 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -127,6 +127,7 @@ import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; +import org.elasticsearch.xpack.core.security.CustomRemoteServerTransportInterceptor; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.SecurityExtension; import org.elasticsearch.xpack.core.security.SecurityField; @@ -1198,7 +1199,7 @@ Collection createComponents( securityContext.get(), destructiveOperations, crossClusterAccessAuthcService.get(), - customIndicesRequestRewriter, + createCustomRemoteServerTransportInterceptor(extensionComponents), getLicenseState() ) ); @@ -1257,6 +1258,48 @@ Collection createComponents( return components; } + private CustomRemoteServerTransportInterceptor createCustomRemoteServerTransportInterceptor( + SecurityExtension.SecurityComponents extensionComponents + ) { + final Map customByExtension = new HashMap<>(); + for (final SecurityExtension extension : securityExtensions) { + final CustomRemoteServerTransportInterceptor custom = extension.getCustomRemoteServerTransportInterceptor(extensionComponents); + if (custom != null) { + if (false == isInternalExtension(extension)) { + throw new IllegalStateException( + "The [" + + extension.extensionName() + + "] extension tried to install a custom CustomIndicesRequestRewriter. " + + "This functionality is not available to external extensions." + ); + } + customByExtension.put(extension.extensionName(), custom); + } + } + + if (customByExtension.isEmpty()) { + logger.debug( + "No custom implementation for [{}]. Falling-back to default implementation.", + CustomRemoteServerTransportInterceptor.class.getCanonicalName() + ); + return new CustomRemoteServerTransportInterceptor.Default(); + } else if (customByExtension.size() > 1) { + throw new IllegalStateException( + "Multiple extensions tried to install a custom CustomRemoteServerTransportInterceptor: " + customByExtension.keySet() + ); + } else { + final var byExtensionEntry = customByExtension.entrySet().iterator().next(); + final CustomRemoteServerTransportInterceptor custom = byExtensionEntry.getValue(); + final String extensionName = byExtensionEntry.getKey(); + logger.debug( + "CustomRemoteServerTransportInterceptor implementation [{}] provided by extension [{}]", + custom.getClass().getCanonicalName(), + extensionName + ); + return custom; + } + } + private CustomIndicesRequestRewriter createCustomIndicesRequestRewriter(SecurityExtension.SecurityComponents extensionComponents) { final Map customByExtension = new HashMap<>(); for (final SecurityExtension extension : securityExtensions) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java index 828acac914608..92ca24d63e900 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java @@ -39,10 +39,10 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.TransportService.ContextRestoreResponseHandler; import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.security.CustomRemoteServerTransportInterceptor; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.CrossClusterAccessSubjectInfo; -import org.elasticsearch.xpack.core.security.authz.CustomIndicesRequestRewriter; import org.elasticsearch.xpack.core.security.transport.ProfileConfigurations; import org.elasticsearch.xpack.core.security.user.InternalUser; import org.elasticsearch.xpack.core.security.user.SystemUser; @@ -100,7 +100,7 @@ public class SecurityServerTransportInterceptor implements TransportInterceptor private final CrossClusterAccessAuthenticationService crossClusterAccessAuthcService; private final Function> remoteClusterCredentialsResolver; private final XPackLicenseState licenseState; - private final CustomIndicesRequestRewriter customIndicesRequestRewriter; + private final CustomRemoteServerTransportInterceptor customRemoteServerTransportInterceptor; public SecurityServerTransportInterceptor( Settings settings, @@ -122,7 +122,7 @@ public SecurityServerTransportInterceptor( securityContext, destructiveOperations, crossClusterAccessAuthcService, - new CustomIndicesRequestRewriter.Default(), + new CustomRemoteServerTransportInterceptor.Default(), licenseState ); } @@ -136,7 +136,7 @@ public SecurityServerTransportInterceptor( SecurityContext securityContext, DestructiveOperations destructiveOperations, CrossClusterAccessAuthenticationService crossClusterAccessAuthcService, - CustomIndicesRequestRewriter customIndicesRequestRewriter, + CustomRemoteServerTransportInterceptor customRemoteServerTransportInterceptor, XPackLicenseState licenseState ) { this( @@ -148,7 +148,7 @@ public SecurityServerTransportInterceptor( securityContext, destructiveOperations, crossClusterAccessAuthcService, - customIndicesRequestRewriter, + customRemoteServerTransportInterceptor, licenseState, RemoteConnectionManager::resolveRemoteClusterAliasWithCredentials ); @@ -176,7 +176,7 @@ public SecurityServerTransportInterceptor( securityContext, destructiveOperations, crossClusterAccessAuthcService, - new CustomIndicesRequestRewriter.Default(), + new CustomRemoteServerTransportInterceptor.Default(), licenseState, remoteClusterCredentialsResolver ); @@ -191,7 +191,7 @@ public SecurityServerTransportInterceptor( SecurityContext securityContext, DestructiveOperations destructiveOperations, CrossClusterAccessAuthenticationService crossClusterAccessAuthcService, - CustomIndicesRequestRewriter customIndicesRequestRewriter, + CustomRemoteServerTransportInterceptor customRemoteServerTransportInterceptor, XPackLicenseState licenseState, // Inject for simplified testing Function> remoteClusterCredentialsResolver @@ -205,7 +205,7 @@ public SecurityServerTransportInterceptor( this.crossClusterAccessAuthcService = crossClusterAccessAuthcService; this.licenseState = licenseState; this.remoteClusterCredentialsResolver = remoteClusterCredentialsResolver; - this.customIndicesRequestRewriter = customIndicesRequestRewriter; + this.customRemoteServerTransportInterceptor = customRemoteServerTransportInterceptor; this.profileFilters = initializeProfileFilters(destructiveOperations); } @@ -331,9 +331,9 @@ public void sendRequest( final Optional remoteClusterCredentials = getRemoteClusterCredentials(connection); if (remoteClusterCredentials.isPresent()) { sendWithCrossClusterAccessHeaders(remoteClusterCredentials.get(), connection, action, request, options, handler); - } else if (customIndicesRequestRewriter.enabled() + } else if (customRemoteServerTransportInterceptor.enabled() && RemoteConnectionManager.resolveRemoteClusterAliasWithCredentials(connection).isPresent()) { - customIndicesRequestRewriter.sendRequest(sender, connection, action, request, options, handler); + customRemoteServerTransportInterceptor.sendRequest(sender, connection, action, request, options, handler); } else { // Send regular request, without cross cluster access headers try { From 9d29d4f785caaeccbaf5746b8b8353fbb8582b09 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Sun, 13 Jul 2025 12:19:49 +0200 Subject: [PATCH 13/28] Inject authenticator --- ...ustomRemoteServerTransportInterceptor.java | 9 +++- ...tomServerTransportFilterAuthenticator.java | 26 +++++++++++ .../CustomRemoteServerTransportFilter.java | 45 +++++++++++++++++++ .../SecurityServerTransportInterceptor.java | 40 ++++++++++++----- .../transport/ServerTransportFilter.java | 2 +- 5 files changed, 108 insertions(+), 14 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomServerTransportFilterAuthenticator.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CustomRemoteServerTransportFilter.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomRemoteServerTransportInterceptor.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomRemoteServerTransportInterceptor.java index 57d061f61da38..b59f29bb629b3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomRemoteServerTransportInterceptor.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomRemoteServerTransportInterceptor.java @@ -18,7 +18,7 @@ public interface CustomRemoteServerTransportInterceptor { // TODO probably don't want this boolean enabled(); - // TODO not the signature we want + // TODO this should be a wrapper around TransportInterceptor.AsyncSender instead void sendRequest( TransportInterceptor.AsyncSender sender, Transport.Connection connection, @@ -28,6 +28,8 @@ void sendRequest( TransportResponseHandler handler ); + CustomServerTransportFilterAuthenticator getAuthenticator(); + class Default implements CustomRemoteServerTransportInterceptor { @Override public boolean enabled() { @@ -45,5 +47,10 @@ public void sendRequest( ) { sender.sendRequest(connection, action, request, options, handler); } + + @Override + public CustomServerTransportFilterAuthenticator getAuthenticator() { + return new CustomServerTransportFilterAuthenticator.Default(); + } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomServerTransportFilterAuthenticator.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomServerTransportFilterAuthenticator.java new file mode 100644 index 0000000000000..6fe8e60645c04 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomServerTransportFilterAuthenticator.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.security; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.xpack.core.security.authc.Authentication; + +public interface CustomServerTransportFilterAuthenticator { + void authenticate(String securityAction, TransportRequest request, ActionListener authenticationListener); + + class Default implements CustomServerTransportFilterAuthenticator { + @Override + public void authenticate(String securityAction, TransportRequest request, ActionListener authenticationListener) { + // should never be called + authenticationListener.onFailure( + new UnsupportedOperationException("CustomServerTransportFilterAuthenticator not implemented for action: " + securityAction) + ); + } + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CustomRemoteServerTransportFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CustomRemoteServerTransportFilter.java new file mode 100644 index 0000000000000..0823109f387be --- /dev/null +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CustomRemoteServerTransportFilter.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.security.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.DestructiveOperations; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.xpack.core.security.CustomServerTransportFilterAuthenticator; +import org.elasticsearch.xpack.core.security.SecurityContext; +import org.elasticsearch.xpack.core.security.authc.Authentication; +import org.elasticsearch.xpack.security.authc.AuthenticationService; +import org.elasticsearch.xpack.security.authz.AuthorizationService; + +final class CustomRemoteServerTransportFilter extends ServerTransportFilter { + private static final Logger logger = LogManager.getLogger(CustomRemoteServerTransportFilter.class); + + private final CustomServerTransportFilterAuthenticator authenticator; + + CustomRemoteServerTransportFilter( + CustomServerTransportFilterAuthenticator authenticator, + AuthenticationService authcService, + AuthorizationService authzService, + ThreadContext threadContext, + boolean extractClientCert, + DestructiveOperations destructiveOperations, + SecurityContext securityContext + ) { + super(authcService, authzService, threadContext, extractClientCert, destructiveOperations, securityContext); + this.authenticator = authenticator; + } + + @Override + public void authenticate(String securityAction, TransportRequest request, ActionListener authenticationListener) { + logger.info("Custom authenticator authenticating request for action: {}", securityAction); + authenticator.authenticate(securityAction, request, authenticationListener); + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java index 92ca24d63e900..0654141af140b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java @@ -557,18 +557,34 @@ private Map initializeProfileFilters(DestructiveO final String profileName = entry.getKey(); final boolean useRemoteClusterProfile = remoteClusterPortEnabled && profileName.equals(REMOTE_CLUSTER_PROFILE); if (useRemoteClusterProfile) { - profileFilters.put( - profileName, - new CrossClusterAccessServerTransportFilter( - crossClusterAccessAuthcService, - authzService, - threadPool.getThreadContext(), - remoteClusterServerSSLEnabled && SSLService.isSSLClientAuthEnabled(profileConfiguration), - destructiveOperations, - securityContext, - licenseState - ) - ); + // probably not how we want to do this but ballpark correct + if (customRemoteServerTransportInterceptor.enabled()) { + profileFilters.put( + profileName, + new CustomRemoteServerTransportFilter( + customRemoteServerTransportInterceptor.getAuthenticator(), + authcService, + authzService, + threadPool.getThreadContext(), + remoteClusterServerSSLEnabled && SSLService.isSSLClientAuthEnabled(profileConfiguration), + destructiveOperations, + securityContext + ) + ); + } else { + profileFilters.put( + profileName, + new CrossClusterAccessServerTransportFilter( + crossClusterAccessAuthcService, + authzService, + threadPool.getThreadContext(), + remoteClusterServerSSLEnabled && SSLService.isSSLClientAuthEnabled(profileConfiguration), + destructiveOperations, + securityContext, + licenseState + ) + ); + } } else { profileFilters.put( profileName, diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java index c4a17c4a13729..b92435b3520b4 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java @@ -67,7 +67,7 @@ class ServerTransportFilter { * thrown by this method will stop the request from being handled and the error will * be sent back to the sender. */ - void inbound(String action, TransportRequest request, TransportChannel transportChannel, ActionListener listener) { + public void inbound(String action, TransportRequest request, TransportChannel transportChannel, ActionListener listener) { if (TransportCloseIndexAction.NAME.equals(action) || OpenIndexAction.NAME.equals(action) || TransportDeleteIndexAction.TYPE.name().equals(action)) { From 0ef2258f4209c3c772395b1b8a11ed8c8385707d Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Sun, 13 Jul 2025 20:21:34 +0200 Subject: [PATCH 14/28] Simplify --- .../transport/RemoteClusterAware.java | 4 --- .../transport/RemoteConnectionStrategy.java | 4 --- ...ustomRemoteServerTransportInterceptor.java | 6 ++--- .../security/CustomServerTransportFilter.java | 22 ++++++++++++++++ ...tomServerTransportFilterAuthenticator.java | 26 ------------------- .../netty4/SecurityNetty4Transport.java | 2 +- .../xpack/security/Security.java | 14 ++++++++-- ...ossClusterAccessAuthenticationService.java | 10 ++++++- .../authz/IndicesAndAliasesResolver.java | 12 --------- .../CustomRemoteServerTransportFilter.java | 21 +++++++++++---- .../SecurityServerTransportInterceptor.java | 2 +- 11 files changed, 64 insertions(+), 59 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomServerTransportFilter.java delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomServerTransportFilterAuthenticator.java diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java index 763faed028b10..95e507f70d7a9 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java @@ -56,10 +56,6 @@ protected static Set getEnabledRemoteClusters(final Settings settings) { return RemoteConnectionStrategy.getRemoteClusters(settings); } - protected static Map> getEnabledRemoteClustersWithTags(final Settings settings) { - return RemoteConnectionStrategy.getRemoteTags(settings); - } - /** * Check whether the index expression represents remote index or not. * The index name is assumed to be individual index (no commas) but can contain `-`, wildcards, diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java index 5b1925b0a20fa..a715797b97977 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java @@ -189,10 +189,6 @@ static Set getRemoteClusters(Settings settings) { return enablementSettings.flatMap(s -> getClusterAlias(settings, s)).collect(Collectors.toSet()); } - static Map> getRemoteTags(Settings settings) { - return RemoteClusterService.REMOTE_CLUSTER_TAGS.getAsMap(settings); - } - public static boolean isConnectionEnabled(String clusterAlias, Settings settings) { ConnectionStrategy mode = REMOTE_CONNECTION_MODE.getConcreteSettingForNamespace(clusterAlias).get(settings); if (mode.equals(ConnectionStrategy.SNIFF)) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomRemoteServerTransportInterceptor.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomRemoteServerTransportInterceptor.java index b59f29bb629b3..78c0319e402c3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomRemoteServerTransportInterceptor.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomRemoteServerTransportInterceptor.java @@ -28,7 +28,7 @@ void sendRequest( TransportResponseHandler handler ); - CustomServerTransportFilterAuthenticator getAuthenticator(); + CustomServerTransportFilter getFilter(); class Default implements CustomRemoteServerTransportInterceptor { @Override @@ -49,8 +49,8 @@ public void sendRequest( } @Override - public CustomServerTransportFilterAuthenticator getAuthenticator() { - return new CustomServerTransportFilterAuthenticator.Default(); + public CustomServerTransportFilter getFilter() { + return new CustomServerTransportFilter.Default(); } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomServerTransportFilter.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomServerTransportFilter.java new file mode 100644 index 0000000000000..b40e7b4ab60c7 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomServerTransportFilter.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.security; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.transport.TransportRequest; + +public interface CustomServerTransportFilter { + void filter(String securityAction, TransportRequest request, ActionListener authenticationListener); + + class Default implements CustomServerTransportFilter { + @Override + public void filter(String securityAction, TransportRequest request, ActionListener listener) { + listener.onResponse(null); + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomServerTransportFilterAuthenticator.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomServerTransportFilterAuthenticator.java deleted file mode 100644 index 6fe8e60645c04..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomServerTransportFilterAuthenticator.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.core.security; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.xpack.core.security.authc.Authentication; - -public interface CustomServerTransportFilterAuthenticator { - void authenticate(String securityAction, TransportRequest request, ActionListener authenticationListener); - - class Default implements CustomServerTransportFilterAuthenticator { - @Override - public void authenticate(String securityAction, TransportRequest request, ActionListener authenticationListener) { - // should never be called - authenticationListener.onFailure( - new UnsupportedOperationException("CustomServerTransportFilterAuthenticator not implemented for action: " + securityAction) - ); - } - } -} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/netty4/SecurityNetty4Transport.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/netty4/SecurityNetty4Transport.java index fa16de22c865c..e1a4cebaae4e6 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/netty4/SecurityNetty4Transport.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/netty4/SecurityNetty4Transport.java @@ -161,7 +161,7 @@ protected ChannelHandler getClientChannelInitializer(DiscoveryNode node, Connect @Override protected InboundPipeline getInboundPipeline(Channel channel, boolean isRemoteClusterServerChannel) { - if (false == isRemoteClusterServerChannel) { + if (false == isRemoteClusterServerChannel || crossClusterAccessAuthenticationService.skipTransportCheck()) { return super.getInboundPipeline(channel, false); } else { return new InboundPipeline( diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index e1ac1c6b09793..a446a629a4aef 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -1186,8 +1186,18 @@ Collection createComponents( ipFilter.set(new IPFilter(settings, auditTrailService, clusterService.getClusterSettings(), getLicenseState())); components.add(ipFilter.get()); + CustomRemoteServerTransportInterceptor customRemoteServerTransportInterceptor = createCustomRemoteServerTransportInterceptor( + extensionComponents + ); DestructiveOperations destructiveOperations = new DestructiveOperations(settings, clusterService.getClusterSettings()); - crossClusterAccessAuthcService.set(new CrossClusterAccessAuthenticationService(clusterService, apiKeyService, authcService.get())); + crossClusterAccessAuthcService.set( + new CrossClusterAccessAuthenticationService( + clusterService, + apiKeyService, + authcService.get(), + customRemoteServerTransportInterceptor.enabled() + ) + ); components.add(crossClusterAccessAuthcService.get()); securityInterceptor.set( new SecurityServerTransportInterceptor( @@ -1199,7 +1209,7 @@ Collection createComponents( securityContext.get(), destructiveOperations, crossClusterAccessAuthcService.get(), - createCustomRemoteServerTransportInterceptor(extensionComponents), + customRemoteServerTransportInterceptor, getLicenseState() ) ); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationService.java index 16f67c9077311..cc2812bca70b9 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationService.java @@ -41,15 +41,23 @@ public class CrossClusterAccessAuthenticationService { private final ClusterService clusterService; private final ApiKeyService apiKeyService; private final AuthenticationService authenticationService; + // TODO hack hack hack + private final boolean skipTransportCheck; public CrossClusterAccessAuthenticationService( ClusterService clusterService, ApiKeyService apiKeyService, - AuthenticationService authenticationService + AuthenticationService authenticationService, + boolean skipTransportCheck ) { this.clusterService = clusterService; this.apiKeyService = apiKeyService; this.authenticationService = authenticationService; + this.skipTransportCheck = skipTransportCheck; + } + + public boolean skipTransportCheck() { + return skipTransportCheck; } public void authenticate(final String action, final TransportRequest request, final ActionListener listener) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java index 7e6417f37d5d0..5ae061b5149e0 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java @@ -34,7 +34,6 @@ import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.transport.NoSuchRemoteClusterException; import org.elasticsearch.transport.RemoteClusterAware; -import org.elasticsearch.transport.RemoteClusterService; import org.elasticsearch.transport.RemoteConnectionStrategy; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine; @@ -558,14 +557,11 @@ private static List indicesList(String[] list) { public static class RemoteClusterResolver extends RemoteClusterAware { private final CopyOnWriteArraySet clusters; - // TODO consolidate - private final Map> tags; @SuppressWarnings("this-escape") public RemoteClusterResolver(Settings settings, ClusterSettings clusterSettings) { super(settings); clusters = new CopyOnWriteArraySet<>(getEnabledRemoteClusters(settings)); - tags = RemoteClusterService.getEnabledRemoteClustersWithTags(settings); listenForUpdates(clusterSettings); } @@ -587,13 +583,5 @@ public ResolvedIndices splitLocalAndRemoteIndexNames(String... indices) { .toList(); return new ResolvedIndices(local == null ? List.of() : local, remote); } - - public Set clusters() { - return Collections.unmodifiableSet(clusters); - } - - public Map> tags() { - return Collections.unmodifiableMap(tags); - } } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CustomRemoteServerTransportFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CustomRemoteServerTransportFilter.java index 0823109f387be..a349264bad7e4 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CustomRemoteServerTransportFilter.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CustomRemoteServerTransportFilter.java @@ -13,7 +13,7 @@ import org.elasticsearch.action.support.DestructiveOperations; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.xpack.core.security.CustomServerTransportFilterAuthenticator; +import org.elasticsearch.xpack.core.security.CustomServerTransportFilter; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.security.authc.AuthenticationService; @@ -22,10 +22,10 @@ final class CustomRemoteServerTransportFilter extends ServerTransportFilter { private static final Logger logger = LogManager.getLogger(CustomRemoteServerTransportFilter.class); - private final CustomServerTransportFilterAuthenticator authenticator; + private final CustomServerTransportFilter authenticator; CustomRemoteServerTransportFilter( - CustomServerTransportFilterAuthenticator authenticator, + CustomServerTransportFilter filter, AuthenticationService authcService, AuthorizationService authzService, ThreadContext threadContext, @@ -34,12 +34,23 @@ final class CustomRemoteServerTransportFilter extends ServerTransportFilter { SecurityContext securityContext ) { super(authcService, authzService, threadContext, extractClientCert, destructiveOperations, securityContext); - this.authenticator = authenticator; + this.authenticator = filter; } @Override public void authenticate(String securityAction, TransportRequest request, ActionListener authenticationListener) { logger.info("Custom authenticator authenticating request for action: {}", securityAction); - authenticator.authenticate(securityAction, request, authenticationListener); + authenticator.filter(securityAction, request, new ActionListener<>() { + @Override + public void onResponse(Void unused) { + CustomRemoteServerTransportFilter.super.authenticate(securityAction, request, authenticationListener); + } + + @Override + public void onFailure(Exception e) { + // TODO wrap exception + authenticationListener.onFailure(e); + } + }); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java index 0654141af140b..df547dca2bda1 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java @@ -562,7 +562,7 @@ private Map initializeProfileFilters(DestructiveO profileFilters.put( profileName, new CustomRemoteServerTransportFilter( - customRemoteServerTransportInterceptor.getAuthenticator(), + customRemoteServerTransportInterceptor.getFilter(), authcService, authzService, threadPool.getThreadContext(), From 88104ce4e78ee37383b9ae0cc9a9e9bbfe4d6932 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Sun, 13 Jul 2025 20:28:33 +0200 Subject: [PATCH 15/28] Scope --- .../xpack/security/authz/IndicesAndAliasesResolver.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java index 5ae061b5149e0..876033a3a66c1 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java @@ -554,12 +554,12 @@ private static List indicesList(String[] list) { return (list == null) ? null : Arrays.asList(list); } - public static class RemoteClusterResolver extends RemoteClusterAware { + private static class RemoteClusterResolver extends RemoteClusterAware { private final CopyOnWriteArraySet clusters; @SuppressWarnings("this-escape") - public RemoteClusterResolver(Settings settings, ClusterSettings clusterSettings) { + private RemoteClusterResolver(Settings settings, ClusterSettings clusterSettings) { super(settings); clusters = new CopyOnWriteArraySet<>(getEnabledRemoteClusters(settings)); listenForUpdates(clusterSettings); From 035ffc12652745a9a764764f8a2c0422d494077e Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 14 Jul 2025 13:16:02 +0200 Subject: [PATCH 16/28] Compile --- .../xpack/security/authz/IndicesAndAliasesResolver.java | 2 +- .../authc/CrossClusterAccessAuthenticationServiceTests.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java index 876033a3a66c1..641911aca24e2 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java @@ -554,7 +554,7 @@ private static List indicesList(String[] list) { return (list == null) ? null : Arrays.asList(list); } - private static class RemoteClusterResolver extends RemoteClusterAware { + static class RemoteClusterResolver extends RemoteClusterAware { private final CopyOnWriteArraySet clusters; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java index 31c6d6f0c2341..ff681864e8864 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java @@ -68,7 +68,8 @@ public void init() throws Exception { crossClusterAccessAuthenticationService = new CrossClusterAccessAuthenticationService( clusterService, apiKeyService, - authenticationService + authenticationService, + false ); } From fe3dea05a3fe5e4b313ebf3e5033c33d59f70da4 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 14 Jul 2025 13:42:45 +0200 Subject: [PATCH 17/28] Query routing --- .../org/elasticsearch/rest/action/search/RestSearchAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index ee67432e6050f..2480649a134d3 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -173,7 +173,7 @@ public static void parseSearchRequest( } searchRequest.indices(Strings.splitStringByCommaToArray(request.param("index"))); - var routingTags = request.param("routing_tags", null); + var routingTags = request.param("query_routing", null); if (routingTags != null) { searchRequest.routingTags( Arrays.stream(Strings.splitStringByCommaToArray(routingTags)).map(RemoteClusterService.RemoteTag::fromString).toList() From 1153dfa040268de1e0a02fd69d5f202f2359d382 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 14 Jul 2025 21:51:24 +0200 Subject: [PATCH 18/28] Some clean up --- ...sRequest.java => CrossProjectRequest.java} | 14 ++++--- .../action/search/SearchRequest.java | 38 ++++++++----------- .../rest/action/search/RestSearchAction.java | 11 ++---- .../core/security/SecurityExtension.java | 4 +- ...r.java => CrossProjectRequestHandler.java} | 10 ++--- .../xpack/security/Security.java | 18 ++++----- .../security/authz/AuthorizationService.java | 6 +-- .../authz/IndicesAndAliasesResolver.java | 14 +++---- .../authz/AuthorizationServiceTests.java | 12 +++--- .../authz/IndicesAndAliasesResolverTests.java | 4 +- 10 files changed, 61 insertions(+), 70 deletions(-) rename server/src/main/java/org/elasticsearch/{RewritableIndicesRequest.java => CrossProjectRequest.java} (61%) rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/{CustomIndicesRequestRewriter.java => CrossProjectRequestHandler.java} (61%) diff --git a/server/src/main/java/org/elasticsearch/RewritableIndicesRequest.java b/server/src/main/java/org/elasticsearch/CrossProjectRequest.java similarity index 61% rename from server/src/main/java/org/elasticsearch/RewritableIndicesRequest.java rename to server/src/main/java/org/elasticsearch/CrossProjectRequest.java index c5185e449a3eb..0c5fe52cfd11a 100644 --- a/server/src/main/java/org/elasticsearch/RewritableIndicesRequest.java +++ b/server/src/main/java/org/elasticsearch/CrossProjectRequest.java @@ -10,16 +10,18 @@ package org.elasticsearch; import org.elasticsearch.action.IndicesRequest; -import org.elasticsearch.transport.RemoteClusterService; +import org.elasticsearch.core.Nullable; import java.util.List; -public interface RewritableIndicesRequest extends IndicesRequest { - boolean rewritten(); +public interface CrossProjectRequest extends IndicesRequest { + // TODO naming for all these methods... + boolean alreadyHandled(); - void rewritten(List indexExpressions); + void qualified(List qualifiedExpressions); - boolean checkRemote(List tags); + @Nullable + String queryRouting(); - record RewrittenIndexExpression(String original, List rewritten) {} + record QualifiedExpression(String original, List qualified) {} } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 071dcc215ece3..d00adec5f1fa0 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -9,7 +9,7 @@ package org.elasticsearch.action.search; -import org.elasticsearch.RewritableIndicesRequest; +import org.elasticsearch.CrossProjectRequest; import org.elasticsearch.TransportVersions; import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; @@ -32,7 +32,6 @@ import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.transport.RemoteClusterService; import org.elasticsearch.xcontent.ToXContent; import java.io.IOException; @@ -57,7 +56,7 @@ */ public class SearchRequest extends LegacyActionRequest implements - RewritableIndicesRequest, + CrossProjectRequest, IndicesRequest.Replaceable, Rewriteable { @@ -75,10 +74,12 @@ public class SearchRequest extends LegacyActionRequest private SearchType searchType = SearchType.DEFAULT; private String[] indices = Strings.EMPTY_ARRAY; - private List routingTags = List.of(); @Nullable - private List indexExpressions; + private String queryRouting = null; + + @Nullable + private List indexExpressions; @Nullable private String routing; @@ -410,8 +411,8 @@ public SearchRequest indices(String... indices) { return this; } - public SearchRequest routingTags(List routingTags) { - this.routingTags = routingTags; + public SearchRequest queryRouting(String queryRouting) { + this.queryRouting = queryRouting; return this; } @@ -870,28 +871,19 @@ public String toString() { } @Override - public boolean rewritten() { + public boolean alreadyHandled() { return indexExpressions != null; } @Override - public void rewritten(List indexExpressions) { - assert false == rewritten(); - this.indexExpressions = indexExpressions; - indices(indexExpressions.stream().flatMap(indexExpression -> indexExpression.rewritten().stream()).toArray(String[]::new)); + public void qualified(List qualifiedExpressions) { + assert false == alreadyHandled(); + this.indexExpressions = qualifiedExpressions; + indices(qualifiedExpressions.stream().flatMap(indexExpression -> indexExpression.rewritten().stream()).toArray(String[]::new)); } @Override - public boolean checkRemote(List tags) { - if (routingTags.isEmpty()) { - return true; // no routing requested, so no constraints - } - // if any tag in routingTags matches one in tags, return true - for (RemoteClusterService.RemoteTag tag : routingTags) { - if (tags.contains(tag)) { - return true; - } - } - return false; + public String queryRouting() { + return queryRouting; } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index 2480649a134d3..3cd3644f418dc 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -37,7 +37,6 @@ import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.search.suggest.SuggestBuilder; import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; -import org.elasticsearch.transport.RemoteClusterService; import org.elasticsearch.usage.SearchUsageHolder; import org.elasticsearch.xcontent.XContentParser; @@ -173,13 +172,11 @@ public static void parseSearchRequest( } searchRequest.indices(Strings.splitStringByCommaToArray(request.param("index"))); - var routingTags = request.param("query_routing", null); - if (routingTags != null) { - searchRequest.routingTags( - Arrays.stream(Strings.splitStringByCommaToArray(routingTags)).map(RemoteClusterService.RemoteTag::fromString).toList() - ); + String queryRouting = request.param("query_routing", null); + if (queryRouting != null) { + searchRequest.queryRouting(queryRouting); } else { - log.info("No routing tags"); + log.info("No query routing defined"); } if (requestContentParser != null) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java index 01703fa29f1e0..bc1d45abc2f94 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java @@ -21,7 +21,7 @@ import org.elasticsearch.xpack.core.security.authc.service.ServiceAccountTokenStore; import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine; -import org.elasticsearch.xpack.core.security.authz.CustomIndicesRequestRewriter; +import org.elasticsearch.xpack.core.security.authz.CrossProjectRequestHandler; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult; @@ -134,7 +134,7 @@ default CustomApiKeyAuthenticator getCustomApiKeyAuthenticator(SecurityComponent return null; } - default CustomIndicesRequestRewriter getCustomIndicesRequestRewriter(SecurityComponents components) { + default CrossProjectRequestHandler getCustomIndicesRequestRewriter(SecurityComponents components) { return null; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CustomIndicesRequestRewriter.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectRequestHandler.java similarity index 61% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CustomIndicesRequestRewriter.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectRequestHandler.java index 95847e51e3711..56974190edd93 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CustomIndicesRequestRewriter.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectRequestHandler.java @@ -7,14 +7,14 @@ package org.elasticsearch.xpack.core.security.authz; -import org.elasticsearch.RewritableIndicesRequest; +import org.elasticsearch.CrossProjectRequest; -public interface CustomIndicesRequestRewriter { - void rewrite(RewritableIndicesRequest request); +public interface CrossProjectRequestHandler { + void handle(CrossProjectRequest request); - class Default implements CustomIndicesRequestRewriter { + class Default implements CrossProjectRequestHandler { @Override - public void rewrite(RewritableIndicesRequest request) { + public void handle(CrossProjectRequest request) { // No rewriting by default // This is a no-op implementation } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index a446a629a4aef..a2131468c03f8 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -214,7 +214,7 @@ import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine; import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; -import org.elasticsearch.xpack.core.security.authz.CustomIndicesRequestRewriter; +import org.elasticsearch.xpack.core.security.authz.CrossProjectRequestHandler; import org.elasticsearch.xpack.core.security.authz.RestrictedIndices; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.accesscontrol.DocumentSubsetBitsetCache; @@ -1149,7 +1149,7 @@ Collection createComponents( authorizationDenialMessages.set(new AuthorizationDenialMessages.Default()); } - CustomIndicesRequestRewriter customIndicesRequestRewriter = createCustomIndicesRequestRewriter(extensionComponents); + CrossProjectRequestHandler crossProjectRequestHandler = createCustomIndicesRequestRewriter(extensionComponents); final AuthorizationService authzService = new AuthorizationService( settings, allRolesStore, @@ -1167,7 +1167,7 @@ Collection createComponents( restrictedIndices, authorizationDenialMessages.get(), projectResolver, - customIndicesRequestRewriter + crossProjectRequestHandler ); components.add(nativeRolesStore); // used by roles actions @@ -1310,10 +1310,10 @@ private CustomRemoteServerTransportInterceptor createCustomRemoteServerTransport } } - private CustomIndicesRequestRewriter createCustomIndicesRequestRewriter(SecurityExtension.SecurityComponents extensionComponents) { - final Map customByExtension = new HashMap<>(); + private CrossProjectRequestHandler createCustomIndicesRequestRewriter(SecurityExtension.SecurityComponents extensionComponents) { + final Map customByExtension = new HashMap<>(); for (final SecurityExtension extension : securityExtensions) { - final CustomIndicesRequestRewriter custom = extension.getCustomIndicesRequestRewriter(extensionComponents); + final CrossProjectRequestHandler custom = extension.getCustomIndicesRequestRewriter(extensionComponents); if (custom != null) { if (false == isInternalExtension(extension)) { throw new IllegalStateException( @@ -1330,16 +1330,16 @@ private CustomIndicesRequestRewriter createCustomIndicesRequestRewriter(Security if (customByExtension.isEmpty()) { logger.debug( "No custom implementation for [{}]. Falling-back to default implementation.", - CustomIndicesRequestRewriter.class.getCanonicalName() + CrossProjectRequestHandler.class.getCanonicalName() ); - return new CustomIndicesRequestRewriter.Default(); + return new CrossProjectRequestHandler.Default(); } else if (customByExtension.size() > 1) { throw new IllegalStateException( "Multiple extensions tried to install a custom CustomIndicesRequestRewriter: " + customByExtension.keySet() ); } else { final var byExtensionEntry = customByExtension.entrySet().iterator().next(); - final CustomIndicesRequestRewriter custom = byExtensionEntry.getValue(); + final CrossProjectRequestHandler custom = byExtensionEntry.getValue(); final String extensionName = byExtensionEntry.getKey(); logger.debug( "CustomIndicesRequestRewriter implementation [{}] provided by extension [{}]", diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java index 1d951b6129f67..ce0b30008fbbe 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java @@ -69,7 +69,7 @@ import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.ParentActionAuthorization; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.RequestInfo; import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; -import org.elasticsearch.xpack.core.security.authz.CustomIndicesRequestRewriter; +import org.elasticsearch.xpack.core.security.authz.CrossProjectRequestHandler; import org.elasticsearch.xpack.core.security.authz.ResolvedIndices; import org.elasticsearch.xpack.core.security.authz.RestrictedIndices; import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection; @@ -166,12 +166,12 @@ public AuthorizationService( RestrictedIndices restrictedIndices, AuthorizationDenialMessages authorizationDenialMessages, ProjectResolver projectResolver, - CustomIndicesRequestRewriter customIndicesRequestRewriter + CrossProjectRequestHandler crossProjectRequestHandler ) { this.clusterService = clusterService; this.auditTrailService = auditTrailService; this.restrictedIndices = restrictedIndices; - this.indicesAndAliasesResolver = new IndicesAndAliasesResolver(settings, clusterService, resolver, customIndicesRequestRewriter); + this.indicesAndAliasesResolver = new IndicesAndAliasesResolver(settings, clusterService, resolver, crossProjectRequestHandler); this.authcFailureHandler = authcFailureHandler; this.threadContext = threadPool.getThreadContext(); this.securityContext = new SecurityContext(settings, this.threadContext); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java index 641911aca24e2..9aa3d545242d6 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java @@ -6,7 +6,7 @@ */ package org.elasticsearch.xpack.security.authz; -import org.elasticsearch.RewritableIndicesRequest; +import org.elasticsearch.CrossProjectRequest; import org.elasticsearch.action.AliasesRequest; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; @@ -37,7 +37,7 @@ import org.elasticsearch.transport.RemoteConnectionStrategy; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine; -import org.elasticsearch.xpack.core.security.authz.CustomIndicesRequestRewriter; +import org.elasticsearch.xpack.core.security.authz.CrossProjectRequestHandler; import org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField; import org.elasticsearch.xpack.core.security.authz.ResolvedIndices; @@ -60,18 +60,18 @@ public class IndicesAndAliasesResolver { private final IndexNameExpressionResolver nameExpressionResolver; private final IndexAbstractionResolver indexAbstractionResolver; private final RemoteClusterResolver remoteClusterResolver; - private final CustomIndicesRequestRewriter customIndicesRequestRewriter; + private final CrossProjectRequestHandler crossProjectRequestHandler; IndicesAndAliasesResolver( Settings settings, ClusterService clusterService, IndexNameExpressionResolver resolver, - CustomIndicesRequestRewriter customIndicesRequestRewriter + CrossProjectRequestHandler crossProjectRequestHandler ) { this.nameExpressionResolver = resolver; this.indexAbstractionResolver = new IndexAbstractionResolver(resolver); this.remoteClusterResolver = new RemoteClusterResolver(settings, clusterService.getClusterSettings()); - this.customIndicesRequestRewriter = customIndicesRequestRewriter; + this.crossProjectRequestHandler = crossProjectRequestHandler; } /** @@ -133,8 +133,8 @@ ResolvedIndices resolve( throw new IllegalStateException("Request [" + request + "] is not an Indices request, but should be."); } - if (request instanceof RewritableIndicesRequest rewritableIndicesRequest) { - customIndicesRequestRewriter.rewrite(rewritableIndicesRequest); + if (request instanceof CrossProjectRequest rewritableIndicesRequest) { + crossProjectRequestHandler.handle(rewritableIndicesRequest); } return resolveIndicesAndAliases(action, (IndicesRequest) request, projectMetadata, authorizedIndices); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java index 2d9e21d545192..b85be9fc98c5f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java @@ -155,7 +155,7 @@ import org.elasticsearch.xpack.core.security.authc.Subject; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo; -import org.elasticsearch.xpack.core.security.authz.CustomIndicesRequestRewriter; +import org.elasticsearch.xpack.core.security.authz.CrossProjectRequestHandler; import org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField; import org.elasticsearch.xpack.core.security.authz.ResolvedIndices; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; @@ -338,7 +338,7 @@ public void setup() { RESTRICTED_INDICES, new AuthorizationDenialMessages.Default(), projectResolver, - new CustomIndicesRequestRewriter.Default() + new CrossProjectRequestHandler.Default() ); } @@ -1772,7 +1772,7 @@ public void testDenialForAnonymousUser() { RESTRICTED_INDICES, new AuthorizationDenialMessages.Default(), projectResolver, - new CustomIndicesRequestRewriter.Default() + new CrossProjectRequestHandler.Default() ); RoleDescriptor role = new RoleDescriptor( @@ -1823,7 +1823,7 @@ public void testDenialForAnonymousUserAuthorizationExceptionDisabled() { RESTRICTED_INDICES, new AuthorizationDenialMessages.Default(), projectResolver, - new CustomIndicesRequestRewriter.Default() + new CrossProjectRequestHandler.Default() ); RoleDescriptor role = new RoleDescriptor( @@ -3362,7 +3362,7 @@ public void testAuthorizationEngineSelectionForCheckPrivileges() throws Exceptio RESTRICTED_INDICES, new AuthorizationDenialMessages.Default(), projectResolver, - new CustomIndicesRequestRewriter.Default() + new CrossProjectRequestHandler.Default() ); Subject subject = new Subject(new User("test", "a role"), mock(RealmRef.class)); @@ -3519,7 +3519,7 @@ public void getUserPrivileges(AuthorizationInfo authorizationInfo, ActionListene RESTRICTED_INDICES, new AuthorizationDenialMessages.Default(), projectResolver, - new CustomIndicesRequestRewriter.Default() + new CrossProjectRequestHandler.Default() ); Authentication authentication; try (StoredContext ignore = threadContext.stashContext()) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java index 7e01409ed716a..679f1c7cb0d1b 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java @@ -69,7 +69,7 @@ import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.authc.Subject; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizedIndices; -import org.elasticsearch.xpack.core.security.authz.CustomIndicesRequestRewriter; +import org.elasticsearch.xpack.core.security.authz.CrossProjectRequestHandler; import org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField; import org.elasticsearch.xpack.core.security.authz.ResolvedIndices; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; @@ -422,7 +422,7 @@ public void setup() { settings, clusterService, indexNameExpressionResolver, - new CustomIndicesRequestRewriter.Default() + new CrossProjectRequestHandler.Default() ); } From 3fd532ecbdc4071814054b8dcd70b28e7a8c2d9c Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 15 Jul 2025 10:55:24 +0200 Subject: [PATCH 19/28] WIP esql --- .../elasticsearch/CrossProjectRequest.java | 10 ++++-- .../fieldcaps/FieldCapabilitiesRequest.java | 31 ++++++++++++++++++- .../action/search/SearchRequest.java | 16 ++++++---- .../xpack/esql/plugin/ComputeService.java | 6 ++++ 4 files changed, 54 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/CrossProjectRequest.java b/server/src/main/java/org/elasticsearch/CrossProjectRequest.java index 0c5fe52cfd11a..582c2c49e9e6f 100644 --- a/server/src/main/java/org/elasticsearch/CrossProjectRequest.java +++ b/server/src/main/java/org/elasticsearch/CrossProjectRequest.java @@ -16,12 +16,18 @@ public interface CrossProjectRequest extends IndicesRequest { // TODO naming for all these methods... - boolean alreadyHandled(); + boolean alreadyQualified(); void qualified(List qualifiedExpressions); + List qualified(); + @Nullable String queryRouting(); - record QualifiedExpression(String original, List qualified) {} + record QualifiedExpression(String original, List qualified) {} + + // * -> (*, _local), (my_remote:*, my_remote) + + record ExpressionWithProject(String expression, String project) {} } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java index ec30886b1acbf..eb015fd29d037 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.fieldcaps; +import org.elasticsearch.CrossProjectRequest; import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; @@ -36,11 +37,16 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -public final class FieldCapabilitiesRequest extends LegacyActionRequest implements IndicesRequest.Replaceable, ToXContentObject { +public final class FieldCapabilitiesRequest extends LegacyActionRequest + implements + CrossProjectRequest, + IndicesRequest.Replaceable, + ToXContentObject { public static final String NAME = "field_caps_request"; public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosed(); @@ -58,6 +64,7 @@ public final class FieldCapabilitiesRequest extends LegacyActionRequest implemen private QueryBuilder indexFilter; private Map runtimeFields = Collections.emptyMap(); private Long nowInMillis; + private List qualifiedExpressions; public FieldCapabilitiesRequest(StreamInput in) throws IOException { super(in); @@ -373,4 +380,26 @@ public String getDescription() { } }; } + + @Override + public boolean alreadyQualified() { + return qualifiedExpressions != null; + } + + @Override + public void qualified(List qualifiedExpressions) { + assert false == alreadyQualified(); + this.qualifiedExpressions = qualifiedExpressions; + indices( + qualifiedExpressions.stream() + .flatMap(indexExpression -> indexExpression.qualified().stream().map(ExpressionWithProject::expression)) + .toArray(String[]::new) + ); + } + + @Override + public String queryRouting() { + // TODO how would this look in ES|QL? + return null; + } } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index d00adec5f1fa0..4120096d604df 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -79,7 +79,7 @@ public class SearchRequest extends LegacyActionRequest private String queryRouting = null; @Nullable - private List indexExpressions; + private List qualifiedExpressions; @Nullable private String routing; @@ -871,15 +871,19 @@ public String toString() { } @Override - public boolean alreadyHandled() { - return indexExpressions != null; + public boolean alreadyQualified() { + return qualifiedExpressions != null; } @Override public void qualified(List qualifiedExpressions) { - assert false == alreadyHandled(); - this.indexExpressions = qualifiedExpressions; - indices(qualifiedExpressions.stream().flatMap(indexExpression -> indexExpression.rewritten().stream()).toArray(String[]::new)); + assert false == alreadyQualified(); + this.qualifiedExpressions = qualifiedExpressions; + indices( + qualifiedExpressions.stream() + .flatMap(indexExpression -> indexExpression.qualified().stream().map(ExpressionWithProject::expression)) + .toArray(String[]::new) + ); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index 39e3503b5fdd9..b0e51b17df73c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -355,6 +355,7 @@ public void executePlan( return; } } + Map clusterToOriginalIndices = transportService.getRemoteClusterService() .groupIndices(SearchRequest.DEFAULT_INDICES_OPTIONS, PlannerUtils.planOriginalIndices(physicalPlan)); var localOriginalIndices = clusterToOriginalIndices.remove(LOCAL_CLUSTER); @@ -466,6 +467,11 @@ public void executePlan( } } // starts computes on remote clusters + + // * -> (*, _local), (my_remote:*, my_remote) + + // clusterToOriginalIndices: [*] + // clusterToConcreteIndices: my_remote: [*] final var remoteClusters = clusterComputeHandler.getRemoteClusters(clusterToConcreteIndices, clusterToOriginalIndices); for (ClusterComputeHandler.RemoteCluster cluster : remoteClusters) { if (execInfo.getCluster(cluster.clusterAlias()).getStatus() != EsqlExecutionInfo.Cluster.Status.RUNNING) { From 512222edbfd61aafab65f7de88a77c05101465a0 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 15 Jul 2025 11:02:08 +0200 Subject: [PATCH 20/28] Fix interface --- server/src/main/java/org/elasticsearch/CrossProjectRequest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/CrossProjectRequest.java b/server/src/main/java/org/elasticsearch/CrossProjectRequest.java index 582c2c49e9e6f..f8f73c2cfc522 100644 --- a/server/src/main/java/org/elasticsearch/CrossProjectRequest.java +++ b/server/src/main/java/org/elasticsearch/CrossProjectRequest.java @@ -20,8 +20,6 @@ public interface CrossProjectRequest extends IndicesRequest { void qualified(List qualifiedExpressions); - List qualified(); - @Nullable String queryRouting(); From b867e69486296a238864ccb64041407af0e988ec Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 15 Jul 2025 15:22:58 +0200 Subject: [PATCH 21/28] WIP resolver --- .../transport/RemoteClusterService.java | 12 ++++----- ...jectRemoteServerTransportInterceptor.java} | 4 +-- .../core/security/SecurityExtension.java | 2 +- .../authz/CrossProjectTargetResolver.java | 27 +++++++++++++++++++ .../xpack/security/Security.java | 25 ++++++++--------- .../SecurityServerTransportInterceptor.java | 24 ++++++++--------- 6 files changed, 61 insertions(+), 33 deletions(-) rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/{CustomRemoteServerTransportInterceptor.java => CrossProjectRemoteServerTransportInterceptor.java} (92%) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectTargetResolver.java diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java index 0ac19dff6971c..e6951ec29565b 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java @@ -98,28 +98,28 @@ public final class RemoteClusterService extends RemoteClusterAware (ns, key) -> boolSetting(key, true, new RemoteConnectionEnabled<>(ns, key), Setting.Property.Dynamic, Setting.Property.NodeScope) ); - public record RemoteTag(String key, String value) { - public static RemoteTag fromString(String tag) { + public record Metadata(String key, String value) { + public static Metadata fromString(String tag) { if (tag == null || tag.isEmpty()) { throw new IllegalArgumentException("Remote tag must not be null or empty"); } // - as a separator to simplify search path param parsing; won't be like this in the real implementation int idx = tag.indexOf('-'); if (idx < 0) { - return new RemoteTag(tag, ""); + return new Metadata(tag, ""); } else { - return new RemoteTag(tag.substring(0, idx), tag.substring(idx + 1)); + return new Metadata(tag.substring(0, idx), tag.substring(idx + 1)); } } } - public static final Setting.AffixSetting> REMOTE_CLUSTER_TAGS = Setting.affixKeySetting( + public static final Setting.AffixSetting> REMOTE_CLUSTER_TAGS = Setting.affixKeySetting( "cluster.remote.", "tags", (ns, key) -> Setting.listSetting( key, Collections.emptyList(), - RemoteTag::fromString, + Metadata::fromString, Setting.Property.Dynamic, Setting.Property.NodeScope ) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomRemoteServerTransportInterceptor.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CrossProjectRemoteServerTransportInterceptor.java similarity index 92% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomRemoteServerTransportInterceptor.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CrossProjectRemoteServerTransportInterceptor.java index 78c0319e402c3..6ba0c92e3ce0e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CustomRemoteServerTransportInterceptor.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CrossProjectRemoteServerTransportInterceptor.java @@ -14,7 +14,7 @@ import org.elasticsearch.transport.TransportResponse; import org.elasticsearch.transport.TransportResponseHandler; -public interface CustomRemoteServerTransportInterceptor { +public interface CrossProjectRemoteServerTransportInterceptor { // TODO probably don't want this boolean enabled(); @@ -30,7 +30,7 @@ void sendRequest( CustomServerTransportFilter getFilter(); - class Default implements CustomRemoteServerTransportInterceptor { + class Default implements CrossProjectRemoteServerTransportInterceptor { @Override public boolean enabled() { return false; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java index bc1d45abc2f94..68211ba90486f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java @@ -138,7 +138,7 @@ default CrossProjectRequestHandler getCustomIndicesRequestRewriter(SecurityCompo return null; } - default CustomRemoteServerTransportInterceptor getCustomRemoteServerTransportInterceptor(SecurityComponents components) { + default CrossProjectRemoteServerTransportInterceptor getCustomRemoteServerTransportInterceptor(SecurityComponents components) { return null; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectTargetResolver.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectTargetResolver.java new file mode 100644 index 0000000000000..3b91458c4e882 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectTargetResolver.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.security.authz; + +import org.elasticsearch.xpack.core.security.SecurityContext; + +import java.util.List; + +public interface CrossProjectTargetResolver { + ResolvedProjects resolve(SecurityContext securityContext); + + record ResolvedProjects(List projects) { + public static ResolvedProjects SKIPPED = new ResolvedProjects(List.of()); + } + + class Default implements CrossProjectTargetResolver { + @Override + public ResolvedProjects resolve(SecurityContext securityContext) { + return ResolvedProjects.SKIPPED; + } + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index a2131468c03f8..dfb956e02a727 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -127,7 +127,7 @@ import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; -import org.elasticsearch.xpack.core.security.CustomRemoteServerTransportInterceptor; +import org.elasticsearch.xpack.core.security.CrossProjectRemoteServerTransportInterceptor; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.SecurityExtension; import org.elasticsearch.xpack.core.security.SecurityField; @@ -1186,16 +1186,15 @@ Collection createComponents( ipFilter.set(new IPFilter(settings, auditTrailService, clusterService.getClusterSettings(), getLicenseState())); components.add(ipFilter.get()); - CustomRemoteServerTransportInterceptor customRemoteServerTransportInterceptor = createCustomRemoteServerTransportInterceptor( - extensionComponents - ); + CrossProjectRemoteServerTransportInterceptor crossProjectRemoteServerTransportInterceptor = + createCustomRemoteServerTransportInterceptor(extensionComponents); DestructiveOperations destructiveOperations = new DestructiveOperations(settings, clusterService.getClusterSettings()); crossClusterAccessAuthcService.set( new CrossClusterAccessAuthenticationService( clusterService, apiKeyService, authcService.get(), - customRemoteServerTransportInterceptor.enabled() + crossProjectRemoteServerTransportInterceptor.enabled() ) ); components.add(crossClusterAccessAuthcService.get()); @@ -1209,7 +1208,7 @@ Collection createComponents( securityContext.get(), destructiveOperations, crossClusterAccessAuthcService.get(), - customRemoteServerTransportInterceptor, + crossProjectRemoteServerTransportInterceptor, getLicenseState() ) ); @@ -1268,12 +1267,14 @@ Collection createComponents( return components; } - private CustomRemoteServerTransportInterceptor createCustomRemoteServerTransportInterceptor( + private CrossProjectRemoteServerTransportInterceptor createCustomRemoteServerTransportInterceptor( SecurityExtension.SecurityComponents extensionComponents ) { - final Map customByExtension = new HashMap<>(); + final Map customByExtension = new HashMap<>(); for (final SecurityExtension extension : securityExtensions) { - final CustomRemoteServerTransportInterceptor custom = extension.getCustomRemoteServerTransportInterceptor(extensionComponents); + final CrossProjectRemoteServerTransportInterceptor custom = extension.getCustomRemoteServerTransportInterceptor( + extensionComponents + ); if (custom != null) { if (false == isInternalExtension(extension)) { throw new IllegalStateException( @@ -1290,16 +1291,16 @@ private CustomRemoteServerTransportInterceptor createCustomRemoteServerTransport if (customByExtension.isEmpty()) { logger.debug( "No custom implementation for [{}]. Falling-back to default implementation.", - CustomRemoteServerTransportInterceptor.class.getCanonicalName() + CrossProjectRemoteServerTransportInterceptor.class.getCanonicalName() ); - return new CustomRemoteServerTransportInterceptor.Default(); + return new CrossProjectRemoteServerTransportInterceptor.Default(); } else if (customByExtension.size() > 1) { throw new IllegalStateException( "Multiple extensions tried to install a custom CustomRemoteServerTransportInterceptor: " + customByExtension.keySet() ); } else { final var byExtensionEntry = customByExtension.entrySet().iterator().next(); - final CustomRemoteServerTransportInterceptor custom = byExtensionEntry.getValue(); + final CrossProjectRemoteServerTransportInterceptor custom = byExtensionEntry.getValue(); final String extensionName = byExtensionEntry.getKey(); logger.debug( "CustomRemoteServerTransportInterceptor implementation [{}] provided by extension [{}]", diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java index df547dca2bda1..88e4dc4357f01 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java @@ -39,7 +39,7 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.TransportService.ContextRestoreResponseHandler; import org.elasticsearch.xpack.core.XPackSettings; -import org.elasticsearch.xpack.core.security.CustomRemoteServerTransportInterceptor; +import org.elasticsearch.xpack.core.security.CrossProjectRemoteServerTransportInterceptor; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.CrossClusterAccessSubjectInfo; @@ -100,7 +100,7 @@ public class SecurityServerTransportInterceptor implements TransportInterceptor private final CrossClusterAccessAuthenticationService crossClusterAccessAuthcService; private final Function> remoteClusterCredentialsResolver; private final XPackLicenseState licenseState; - private final CustomRemoteServerTransportInterceptor customRemoteServerTransportInterceptor; + private final CrossProjectRemoteServerTransportInterceptor crossProjectRemoteServerTransportInterceptor; public SecurityServerTransportInterceptor( Settings settings, @@ -122,7 +122,7 @@ public SecurityServerTransportInterceptor( securityContext, destructiveOperations, crossClusterAccessAuthcService, - new CustomRemoteServerTransportInterceptor.Default(), + new CrossProjectRemoteServerTransportInterceptor.Default(), licenseState ); } @@ -136,7 +136,7 @@ public SecurityServerTransportInterceptor( SecurityContext securityContext, DestructiveOperations destructiveOperations, CrossClusterAccessAuthenticationService crossClusterAccessAuthcService, - CustomRemoteServerTransportInterceptor customRemoteServerTransportInterceptor, + CrossProjectRemoteServerTransportInterceptor crossProjectRemoteServerTransportInterceptor, XPackLicenseState licenseState ) { this( @@ -148,7 +148,7 @@ public SecurityServerTransportInterceptor( securityContext, destructiveOperations, crossClusterAccessAuthcService, - customRemoteServerTransportInterceptor, + crossProjectRemoteServerTransportInterceptor, licenseState, RemoteConnectionManager::resolveRemoteClusterAliasWithCredentials ); @@ -176,7 +176,7 @@ public SecurityServerTransportInterceptor( securityContext, destructiveOperations, crossClusterAccessAuthcService, - new CustomRemoteServerTransportInterceptor.Default(), + new CrossProjectRemoteServerTransportInterceptor.Default(), licenseState, remoteClusterCredentialsResolver ); @@ -191,7 +191,7 @@ public SecurityServerTransportInterceptor( SecurityContext securityContext, DestructiveOperations destructiveOperations, CrossClusterAccessAuthenticationService crossClusterAccessAuthcService, - CustomRemoteServerTransportInterceptor customRemoteServerTransportInterceptor, + CrossProjectRemoteServerTransportInterceptor crossProjectRemoteServerTransportInterceptor, XPackLicenseState licenseState, // Inject for simplified testing Function> remoteClusterCredentialsResolver @@ -205,7 +205,7 @@ public SecurityServerTransportInterceptor( this.crossClusterAccessAuthcService = crossClusterAccessAuthcService; this.licenseState = licenseState; this.remoteClusterCredentialsResolver = remoteClusterCredentialsResolver; - this.customRemoteServerTransportInterceptor = customRemoteServerTransportInterceptor; + this.crossProjectRemoteServerTransportInterceptor = crossProjectRemoteServerTransportInterceptor; this.profileFilters = initializeProfileFilters(destructiveOperations); } @@ -331,9 +331,9 @@ public void sendRequest( final Optional remoteClusterCredentials = getRemoteClusterCredentials(connection); if (remoteClusterCredentials.isPresent()) { sendWithCrossClusterAccessHeaders(remoteClusterCredentials.get(), connection, action, request, options, handler); - } else if (customRemoteServerTransportInterceptor.enabled() + } else if (crossProjectRemoteServerTransportInterceptor.enabled() && RemoteConnectionManager.resolveRemoteClusterAliasWithCredentials(connection).isPresent()) { - customRemoteServerTransportInterceptor.sendRequest(sender, connection, action, request, options, handler); + crossProjectRemoteServerTransportInterceptor.sendRequest(sender, connection, action, request, options, handler); } else { // Send regular request, without cross cluster access headers try { @@ -558,11 +558,11 @@ private Map initializeProfileFilters(DestructiveO final boolean useRemoteClusterProfile = remoteClusterPortEnabled && profileName.equals(REMOTE_CLUSTER_PROFILE); if (useRemoteClusterProfile) { // probably not how we want to do this but ballpark correct - if (customRemoteServerTransportInterceptor.enabled()) { + if (crossProjectRemoteServerTransportInterceptor.enabled()) { profileFilters.put( profileName, new CustomRemoteServerTransportFilter( - customRemoteServerTransportInterceptor.getFilter(), + crossProjectRemoteServerTransportInterceptor.getFilter(), authcService, authzService, threadPool.getThreadContext(), From 08d3e4e319e0113d3fdc038c1f520a88c7f11d11 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 15 Jul 2025 19:27:35 +0200 Subject: [PATCH 22/28] Only inject resolver --- .../elasticsearch/CrossProjectRequest.java | 1 + .../core/security/SecurityExtension.java | 4 +- .../authz/CrossProjectRequestHandler.java | 22 ------ .../authz/CrossProjectTargetResolver.java | 11 +-- .../xpack/esql/plugin/ComputeService.java | 4 - .../xpack/security/Security.java | 18 ++--- .../security/authz/AuthorizationService.java | 22 ++++-- .../authz/IndicesAndAliasesResolver.java | 78 ++++++++++++++++--- .../authz/AuthorizationServiceTests.java | 12 +-- .../authz/IndicesAndAliasesResolverTests.java | 12 ++- 10 files changed, 117 insertions(+), 67 deletions(-) delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectRequestHandler.java diff --git a/server/src/main/java/org/elasticsearch/CrossProjectRequest.java b/server/src/main/java/org/elasticsearch/CrossProjectRequest.java index f8f73c2cfc522..18d5148bc7205 100644 --- a/server/src/main/java/org/elasticsearch/CrossProjectRequest.java +++ b/server/src/main/java/org/elasticsearch/CrossProjectRequest.java @@ -23,6 +23,7 @@ public interface CrossProjectRequest extends IndicesRequest { @Nullable String queryRouting(); + // we need to track if the original resource is qualified or flat record QualifiedExpression(String original, List qualified) {} // * -> (*, _local), (my_remote:*, my_remote) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java index 68211ba90486f..ca1d8ff055923 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java @@ -21,7 +21,7 @@ import org.elasticsearch.xpack.core.security.authc.service.ServiceAccountTokenStore; import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine; -import org.elasticsearch.xpack.core.security.authz.CrossProjectRequestHandler; +import org.elasticsearch.xpack.core.security.authz.CrossProjectTargetResolver; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult; @@ -134,7 +134,7 @@ default CustomApiKeyAuthenticator getCustomApiKeyAuthenticator(SecurityComponent return null; } - default CrossProjectRequestHandler getCustomIndicesRequestRewriter(SecurityComponents components) { + default CrossProjectTargetResolver getCustomIndicesRequestRewriter(SecurityComponents components) { return null; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectRequestHandler.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectRequestHandler.java deleted file mode 100644 index 56974190edd93..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectRequestHandler.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.core.security.authz; - -import org.elasticsearch.CrossProjectRequest; - -public interface CrossProjectRequestHandler { - void handle(CrossProjectRequest request); - - class Default implements CrossProjectRequestHandler { - @Override - public void handle(CrossProjectRequest request) { - // No rewriting by default - // This is a no-op implementation - } - } -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectTargetResolver.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectTargetResolver.java index 3b91458c4e882..079d513fad102 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectTargetResolver.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectTargetResolver.java @@ -14,14 +14,15 @@ public interface CrossProjectTargetResolver { ResolvedProjects resolve(SecurityContext securityContext); - record ResolvedProjects(List projects) { - public static ResolvedProjects SKIPPED = new ResolvedProjects(List.of()); - } - class Default implements CrossProjectTargetResolver { @Override public ResolvedProjects resolve(SecurityContext securityContext) { - return ResolvedProjects.SKIPPED; + return ResolvedProjects.VOID; } } + + record ResolvedProjects(List projects) { + // I need a better name + public static ResolvedProjects VOID = new ResolvedProjects(List.of()); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index b0e51b17df73c..68a90b50046fb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -468,10 +468,6 @@ public void executePlan( } // starts computes on remote clusters - // * -> (*, _local), (my_remote:*, my_remote) - - // clusterToOriginalIndices: [*] - // clusterToConcreteIndices: my_remote: [*] final var remoteClusters = clusterComputeHandler.getRemoteClusters(clusterToConcreteIndices, clusterToOriginalIndices); for (ClusterComputeHandler.RemoteCluster cluster : remoteClusters) { if (execInfo.getCluster(cluster.clusterAlias()).getStatus() != EsqlExecutionInfo.Cluster.Status.RUNNING) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index dfb956e02a727..b0db0fb74c132 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -214,7 +214,7 @@ import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine; import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; -import org.elasticsearch.xpack.core.security.authz.CrossProjectRequestHandler; +import org.elasticsearch.xpack.core.security.authz.CrossProjectTargetResolver; import org.elasticsearch.xpack.core.security.authz.RestrictedIndices; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.accesscontrol.DocumentSubsetBitsetCache; @@ -1149,7 +1149,7 @@ Collection createComponents( authorizationDenialMessages.set(new AuthorizationDenialMessages.Default()); } - CrossProjectRequestHandler crossProjectRequestHandler = createCustomIndicesRequestRewriter(extensionComponents); + CrossProjectTargetResolver crossProjectTargetResolver = createCustomIndicesRequestRewriter(extensionComponents); final AuthorizationService authzService = new AuthorizationService( settings, allRolesStore, @@ -1167,7 +1167,7 @@ Collection createComponents( restrictedIndices, authorizationDenialMessages.get(), projectResolver, - crossProjectRequestHandler + crossProjectTargetResolver ); components.add(nativeRolesStore); // used by roles actions @@ -1311,10 +1311,10 @@ private CrossProjectRemoteServerTransportInterceptor createCustomRemoteServerTra } } - private CrossProjectRequestHandler createCustomIndicesRequestRewriter(SecurityExtension.SecurityComponents extensionComponents) { - final Map customByExtension = new HashMap<>(); + private CrossProjectTargetResolver createCustomIndicesRequestRewriter(SecurityExtension.SecurityComponents extensionComponents) { + final Map customByExtension = new HashMap<>(); for (final SecurityExtension extension : securityExtensions) { - final CrossProjectRequestHandler custom = extension.getCustomIndicesRequestRewriter(extensionComponents); + final CrossProjectTargetResolver custom = extension.getCustomIndicesRequestRewriter(extensionComponents); if (custom != null) { if (false == isInternalExtension(extension)) { throw new IllegalStateException( @@ -1331,16 +1331,16 @@ private CrossProjectRequestHandler createCustomIndicesRequestRewriter(SecurityEx if (customByExtension.isEmpty()) { logger.debug( "No custom implementation for [{}]. Falling-back to default implementation.", - CrossProjectRequestHandler.class.getCanonicalName() + CrossProjectTargetResolver.class.getCanonicalName() ); - return new CrossProjectRequestHandler.Default(); + return new CrossProjectTargetResolver.Default(); } else if (customByExtension.size() > 1) { throw new IllegalStateException( "Multiple extensions tried to install a custom CustomIndicesRequestRewriter: " + customByExtension.keySet() ); } else { final var byExtensionEntry = customByExtension.entrySet().iterator().next(); - final CrossProjectRequestHandler custom = byExtensionEntry.getValue(); + final CrossProjectTargetResolver custom = byExtensionEntry.getValue(); final String extensionName = byExtensionEntry.getKey(); logger.debug( "CustomIndicesRequestRewriter implementation [{}] provided by extension [{}]", diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java index ce0b30008fbbe..5ba8220f73be7 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java @@ -9,6 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.CrossProjectRequest; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchRoleRestrictionException; import org.elasticsearch.ElasticsearchSecurityException; @@ -69,7 +70,7 @@ import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.ParentActionAuthorization; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.RequestInfo; import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; -import org.elasticsearch.xpack.core.security.authz.CrossProjectRequestHandler; +import org.elasticsearch.xpack.core.security.authz.CrossProjectTargetResolver; import org.elasticsearch.xpack.core.security.authz.ResolvedIndices; import org.elasticsearch.xpack.core.security.authz.RestrictedIndices; import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection; @@ -144,6 +145,7 @@ public class AuthorizationService { private final RestrictedIndices restrictedIndices; private final AuthorizationDenialMessages authorizationDenialMessages; private final ProjectResolver projectResolver; + private final CrossProjectTargetResolver crossProjectTargetResolver; private final boolean isAnonymousEnabled; private final boolean anonymousAuthzExceptionEnabled; @@ -166,12 +168,12 @@ public AuthorizationService( RestrictedIndices restrictedIndices, AuthorizationDenialMessages authorizationDenialMessages, ProjectResolver projectResolver, - CrossProjectRequestHandler crossProjectRequestHandler + CrossProjectTargetResolver crossProjectTargetResolver ) { this.clusterService = clusterService; this.auditTrailService = auditTrailService; this.restrictedIndices = restrictedIndices; - this.indicesAndAliasesResolver = new IndicesAndAliasesResolver(settings, clusterService, resolver, crossProjectRequestHandler); + this.indicesAndAliasesResolver = new IndicesAndAliasesResolver(settings, clusterService, resolver, crossProjectTargetResolver); this.authcFailureHandler = authcFailureHandler; this.threadContext = threadPool.getThreadContext(); this.securityContext = new SecurityContext(settings, this.threadContext); @@ -192,6 +194,7 @@ public AuthorizationService( this.indicesAccessControlWrapper = new DlsFlsFeatureTrackingIndicesAccessControlWrapper(settings, licenseState); this.authorizationDenialMessages = authorizationDenialMessages; this.projectResolver = projectResolver; + this.crossProjectTargetResolver = crossProjectTargetResolver; } public void checkPrivileges( @@ -487,13 +490,20 @@ private void authorizeAction( })); } else if (isIndexAction(action)) { final ProjectMetadata projectMetadata = projectResolver.getProjectMetadata(clusterService.state()); + final CrossProjectTargetResolver.ResolvedProjects resolvedProjects = request instanceof CrossProjectRequest + ? crossProjectTargetResolver.resolve(securityContext) + : CrossProjectTargetResolver.ResolvedProjects.VOID; assert projectMetadata != null; final AsyncSupplier resolvedIndicesAsyncSupplier = new CachingAsyncSupplier<>(() -> { if (request instanceof SearchRequest searchRequest && searchRequest.pointInTimeBuilder() != null) { - var resolvedIndices = indicesAndAliasesResolver.resolvePITIndices(searchRequest); + var resolvedIndices = indicesAndAliasesResolver.resolvePITIndices(searchRequest, resolvedProjects); return SubscribableListener.newSucceeded(resolvedIndices); } - final ResolvedIndices resolvedIndices = indicesAndAliasesResolver.tryResolveWithoutWildcards(action, request); + final ResolvedIndices resolvedIndices = indicesAndAliasesResolver.tryResolveWithoutWildcards( + action, + request, + resolvedProjects + ); if (resolvedIndices != null) { return SubscribableListener.newSucceeded(resolvedIndices); } else { @@ -504,7 +514,7 @@ private void authorizeAction( projectMetadata.getIndicesLookup(), ActionListener.wrap( authorizedIndices -> resolvedIndicesListener.onResponse( - indicesAndAliasesResolver.resolve(action, request, projectMetadata, authorizedIndices) + indicesAndAliasesResolver.resolve(action, request, projectMetadata, authorizedIndices, resolvedProjects) ), e -> { if (e instanceof InvalidIndexNameException diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java index 9aa3d545242d6..83bc3506f0c43 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java @@ -32,12 +32,14 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.elasticsearch.transport.NoSuchRemoteClusterException; import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.transport.RemoteConnectionStrategy; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine; -import org.elasticsearch.xpack.core.security.authz.CrossProjectRequestHandler; +import org.elasticsearch.xpack.core.security.authz.CrossProjectTargetResolver; import org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField; import org.elasticsearch.xpack.core.security.authz.ResolvedIndices; @@ -57,21 +59,23 @@ public class IndicesAndAliasesResolver { + private static final Logger logger = LogManager.getLogger(IndicesAndAliasesResolver.class); + private final IndexNameExpressionResolver nameExpressionResolver; private final IndexAbstractionResolver indexAbstractionResolver; private final RemoteClusterResolver remoteClusterResolver; - private final CrossProjectRequestHandler crossProjectRequestHandler; + private final CrossProjectTargetResolver crossProjectTargetResolver; IndicesAndAliasesResolver( Settings settings, ClusterService clusterService, IndexNameExpressionResolver resolver, - CrossProjectRequestHandler crossProjectRequestHandler + CrossProjectTargetResolver crossProjectTargetResolver ) { this.nameExpressionResolver = resolver; this.indexAbstractionResolver = new IndexAbstractionResolver(resolver); this.remoteClusterResolver = new RemoteClusterResolver(settings, clusterService.getClusterSettings()); - this.crossProjectRequestHandler = crossProjectRequestHandler; + this.crossProjectTargetResolver = crossProjectTargetResolver; } /** @@ -116,7 +120,8 @@ ResolvedIndices resolve( String action, TransportRequest request, ProjectMetadata projectMetadata, - AuthorizationEngine.AuthorizedIndices authorizedIndices + AuthorizationEngine.AuthorizedIndices authorizedIndices, + CrossProjectTargetResolver.ResolvedProjects resolvedProjects ) { if (request instanceof IndicesAliasesRequest indicesAliasesRequest) { ResolvedIndices.Builder resolvedIndicesBuilder = new ResolvedIndices.Builder(); @@ -133,19 +138,64 @@ ResolvedIndices resolve( throw new IllegalStateException("Request [" + request + "] is not an Indices request, but should be."); } - if (request instanceof CrossProjectRequest rewritableIndicesRequest) { - crossProjectRequestHandler.handle(rewritableIndicesRequest); + if (request instanceof CrossProjectRequest crossProjectRequest) { + maybeRewriteCrossProjectRequest(resolvedProjects, crossProjectRequest); } return resolveIndicesAndAliases(action, (IndicesRequest) request, projectMetadata, authorizedIndices); } + void maybeRewriteCrossProjectRequest(CrossProjectTargetResolver.ResolvedProjects resolvedProjects, CrossProjectRequest request) { + if (resolvedProjects == CrossProjectTargetResolver.ResolvedProjects.VOID) { + logger.info("Cross-project search is disabled or not applicable, skipping request [{}]...", request); + return; + } + + if (resolvedProjects.projects().isEmpty()) { + // This is NOT correct + logger.info("No target projects available for cross-project search, skipping request [{}]...", request); + return; + } + + String[] indices = request.indices(); + ResolvedIndices resolved = remoteClusterResolver.splitLocalAndRemoteIndexNames(indices); + logger.info("Resolved indices: local [{}], remote [{}]", resolved.getLocal(), resolved.getRemote()); + + // skip handling searches where there's both qualified and flat expressions to simplify POC + // in the real thing, we'd also rewrite these + if (resolved.getRemote().isEmpty() == false) { + return; + } + + List qualifiedExpressions = new ArrayList<>(indices.length); + for (String local : resolved.getLocal()) { + List expressionWithProjects = new ArrayList<>(); + expressionWithProjects.add(new CrossProjectRequest.ExpressionWithProject(local, "_local")); + for (String targetProject : resolvedProjects.projects()) { + expressionWithProjects.add( + new CrossProjectRequest.ExpressionWithProject( + RemoteClusterAware.buildRemoteIndexName(targetProject, local), + targetProject + ) + ); + qualifiedExpressions.add(new CrossProjectRequest.QualifiedExpression(local, expressionWithProjects)); + } + logger.info("Rewrote [{}] to [{}]", local, expressionWithProjects); + } + + request.qualified(qualifiedExpressions); + } + /** * Attempt to resolve requested indices without expanding any wildcards. * @return The {@link ResolvedIndices} or null if wildcard expansion must be performed. */ @Nullable - ResolvedIndices tryResolveWithoutWildcards(String action, TransportRequest transportRequest) { + ResolvedIndices tryResolveWithoutWildcards( + String action, + TransportRequest transportRequest, + CrossProjectTargetResolver.ResolvedProjects resolvedProjects + ) { // We only take care of IndicesRequest if (false == transportRequest instanceof IndicesRequest) { return null; @@ -155,7 +205,7 @@ ResolvedIndices tryResolveWithoutWildcards(String action, TransportRequest trans return null; } // It's safe to cast IndicesRequest since the above test guarantees it - return resolveIndicesAndAliasesWithoutWildcards(action, indicesRequest); + return resolveIndicesAndAliasesWithoutWildcards(action, indicesRequest, resolvedProjects); } private static boolean requiresWildcardExpansion(IndicesRequest indicesRequest) { @@ -171,6 +221,14 @@ private static boolean requiresWildcardExpansion(IndicesRequest indicesRequest) } ResolvedIndices resolveIndicesAndAliasesWithoutWildcards(String action, IndicesRequest indicesRequest) { + return resolveIndicesAndAliasesWithoutWildcards(action, indicesRequest, CrossProjectTargetResolver.ResolvedProjects.VOID); + } + + ResolvedIndices resolveIndicesAndAliasesWithoutWildcards( + String action, + IndicesRequest indicesRequest, + CrossProjectTargetResolver.ResolvedProjects resolvedProjects + ) { assert false == requiresWildcardExpansion(indicesRequest) : "request must not require wildcard expansion"; final String[] indices = indicesRequest.indices(); if (indices == null || indices.length == 0) { @@ -243,7 +301,7 @@ ResolvedIndices resolveIndicesAndAliasesWithoutWildcards(String action, IndicesR /** * Returns the resolved indices from the {@link SearchContextId} within the provided {@link SearchRequest}. */ - ResolvedIndices resolvePITIndices(SearchRequest request) { + ResolvedIndices resolvePITIndices(SearchRequest request, CrossProjectTargetResolver.ResolvedProjects resolvedProjects) { assert request.pointInTimeBuilder() != null; var indices = SearchContextId.decodeIndices(request.pointInTimeBuilder().getEncodedId()); final ResolvedIndices split; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java index b85be9fc98c5f..f2bec1021d5e3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java @@ -155,7 +155,7 @@ import org.elasticsearch.xpack.core.security.authc.Subject; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo; -import org.elasticsearch.xpack.core.security.authz.CrossProjectRequestHandler; +import org.elasticsearch.xpack.core.security.authz.CrossProjectTargetResolver; import org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField; import org.elasticsearch.xpack.core.security.authz.ResolvedIndices; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; @@ -338,7 +338,7 @@ public void setup() { RESTRICTED_INDICES, new AuthorizationDenialMessages.Default(), projectResolver, - new CrossProjectRequestHandler.Default() + new CrossProjectTargetResolver.Default() ); } @@ -1772,7 +1772,7 @@ public void testDenialForAnonymousUser() { RESTRICTED_INDICES, new AuthorizationDenialMessages.Default(), projectResolver, - new CrossProjectRequestHandler.Default() + new CrossProjectTargetResolver.Default() ); RoleDescriptor role = new RoleDescriptor( @@ -1823,7 +1823,7 @@ public void testDenialForAnonymousUserAuthorizationExceptionDisabled() { RESTRICTED_INDICES, new AuthorizationDenialMessages.Default(), projectResolver, - new CrossProjectRequestHandler.Default() + new CrossProjectTargetResolver.Default() ); RoleDescriptor role = new RoleDescriptor( @@ -3362,7 +3362,7 @@ public void testAuthorizationEngineSelectionForCheckPrivileges() throws Exceptio RESTRICTED_INDICES, new AuthorizationDenialMessages.Default(), projectResolver, - new CrossProjectRequestHandler.Default() + new CrossProjectTargetResolver.Default() ); Subject subject = new Subject(new User("test", "a role"), mock(RealmRef.class)); @@ -3519,7 +3519,7 @@ public void getUserPrivileges(AuthorizationInfo authorizationInfo, ActionListene RESTRICTED_INDICES, new AuthorizationDenialMessages.Default(), projectResolver, - new CrossProjectRequestHandler.Default() + new CrossProjectTargetResolver.Default() ); Authentication authentication; try (StoredContext ignore = threadContext.stashContext()) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java index 679f1c7cb0d1b..c9f9b55126bf7 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java @@ -69,7 +69,7 @@ import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.authc.Subject; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizedIndices; -import org.elasticsearch.xpack.core.security.authz.CrossProjectRequestHandler; +import org.elasticsearch.xpack.core.security.authz.CrossProjectTargetResolver; import org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField; import org.elasticsearch.xpack.core.security.authz.ResolvedIndices; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; @@ -422,7 +422,7 @@ public void setup() { settings, clusterService, indexNameExpressionResolver, - new CrossProjectRequestHandler.Default() + new CrossProjectTargetResolver.Default() ); } @@ -2672,7 +2672,13 @@ private ResolvedIndices resolveIndices(TransportRequest request, AuthorizedIndic } private ResolvedIndices resolveIndices(String action, TransportRequest request, AuthorizedIndices authorizedIndices) { - return defaultIndicesResolver.resolve(action, request, this.projectMetadata, authorizedIndices); + return defaultIndicesResolver.resolve( + action, + request, + this.projectMetadata, + authorizedIndices, + CrossProjectTargetResolver.ResolvedProjects.VOID + ); } private static void assertNoIndices(IndicesRequest.Replaceable request, ResolvedIndices resolvedIndices) { From d36c3f6a062a73e2eb64660de30849fafa997033 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 15 Jul 2025 19:54:59 +0200 Subject: [PATCH 23/28] Clean up --- .../xpack/core/security/SecurityExtension.java | 2 +- .../org/elasticsearch/xpack/security/Security.java | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java index ca1d8ff055923..93dfb3d1baac6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java @@ -134,7 +134,7 @@ default CustomApiKeyAuthenticator getCustomApiKeyAuthenticator(SecurityComponent return null; } - default CrossProjectTargetResolver getCustomIndicesRequestRewriter(SecurityComponents components) { + default CrossProjectTargetResolver getCrossProjectTargetResolver(SecurityComponents components) { return null; } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index b0db0fb74c132..415164ac9145b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -1149,7 +1149,7 @@ Collection createComponents( authorizationDenialMessages.set(new AuthorizationDenialMessages.Default()); } - CrossProjectTargetResolver crossProjectTargetResolver = createCustomIndicesRequestRewriter(extensionComponents); + CrossProjectTargetResolver crossProjectTargetResolver = createCrossProjectTargetResolver(extensionComponents); final AuthorizationService authzService = new AuthorizationService( settings, allRolesStore, @@ -1311,16 +1311,16 @@ private CrossProjectRemoteServerTransportInterceptor createCustomRemoteServerTra } } - private CrossProjectTargetResolver createCustomIndicesRequestRewriter(SecurityExtension.SecurityComponents extensionComponents) { + private CrossProjectTargetResolver createCrossProjectTargetResolver(SecurityExtension.SecurityComponents extensionComponents) { final Map customByExtension = new HashMap<>(); for (final SecurityExtension extension : securityExtensions) { - final CrossProjectTargetResolver custom = extension.getCustomIndicesRequestRewriter(extensionComponents); + final CrossProjectTargetResolver custom = extension.getCrossProjectTargetResolver(extensionComponents); if (custom != null) { if (false == isInternalExtension(extension)) { throw new IllegalStateException( "The [" + extension.extensionName() - + "] extension tried to install a custom CustomIndicesRequestRewriter. " + + "] extension tried to install a custom CrossProjectTargetResolver. " + "This functionality is not available to external extensions." ); } @@ -1336,14 +1336,14 @@ private CrossProjectTargetResolver createCustomIndicesRequestRewriter(SecurityEx return new CrossProjectTargetResolver.Default(); } else if (customByExtension.size() > 1) { throw new IllegalStateException( - "Multiple extensions tried to install a custom CustomIndicesRequestRewriter: " + customByExtension.keySet() + "Multiple extensions tried to install a custom CrossProjectTargetResolver: " + customByExtension.keySet() ); } else { final var byExtensionEntry = customByExtension.entrySet().iterator().next(); final CrossProjectTargetResolver custom = byExtensionEntry.getValue(); final String extensionName = byExtensionEntry.getKey(); logger.debug( - "CustomIndicesRequestRewriter implementation [{}] provided by extension [{}]", + "CrossProjectTargetResolver implementation [{}] provided by extension [{}]", custom.getClass().getCanonicalName(), extensionName ); From d20a8c92c44335868c47cb0c44695d1df0b60a13 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 16 Jul 2025 11:44:06 +0200 Subject: [PATCH 24/28] esql patch by @idegtiarenko --- .../xpack/esql/planner/PlannerUtils.java | 20 +++++++++++++++++-- .../xpack/esql/session/EsqlSession.java | 2 +- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java index e9d8c93511106..95799282b0390 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java @@ -20,6 +20,7 @@ import org.elasticsearch.index.query.CoordinatorRewriteContext; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.AttributeSet; @@ -64,7 +65,6 @@ import java.util.function.Consumer; import java.util.function.Predicate; -import static java.util.Arrays.asList; import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.DOC_VALUES; import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS; import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.NONE; @@ -153,10 +153,26 @@ public static String[] planOriginalIndices(PhysicalPlan plan) { return Strings.EMPTY_ARRAY; } var indices = new LinkedHashSet(); - forEachRelation(plan, relation -> indices.addAll(asList(Strings.commaDelimitedListToStringArray(relation.indexPattern())))); + forEachRelation(plan, relation -> indices.addAll(expressions(relation.indexPattern()))); return indices.toArray(String[]::new); } + public static List expressions(String indexPattern) { + var result = new ArrayList(); + for (var expression : Strings.commaDelimitedListToStringArray(indexPattern)) { + var clusterAndExpression = RemoteClusterAware.splitIndexName(expression); + if (clusterAndExpression[0] == null) { + // unqualified, convert to `index`,`*:index` + result.add(expression); + result.add("*:" + expression); + } else { + // qualified expression, keep as is + result.add(expression); + } + } + return result; + } + private static void forEachRelation(PhysicalPlan plan, Consumer action) { plan.forEachDown(FragmentExec.class, f -> f.fragment().forEachDown(EsRelation.class, r -> { if (r.indexMode() != IndexMode.LOOKUP) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 40a859e3f5b58..a140315f8971e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -614,7 +614,7 @@ private void initializeClusterData(List indices, EsqlExecutionInfo Map clusterIndices = indicesExpressionGrouper.groupIndices( configuredClusters, IndicesOptions.DEFAULT, - indices.getFirst().indexPattern() + String.join(",", PlannerUtils.expressions(indices.getFirst().indexPattern())) ); for (Map.Entry entry : clusterIndices.entrySet()) { final String clusterAlias = entry.getKey(); From 32ed394e1ffdd06281f0e0fc874a82a17dc94691 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 17 Jul 2025 15:44:29 +0200 Subject: [PATCH 25/28] More context --- ...t.java => CrossProjectEnabledRequest.java} | 28 +++++++++++++------ .../fieldcaps/FieldCapabilitiesRequest.java | 7 ++--- .../action/search/SearchRequest.java | 7 ++--- 3 files changed, 26 insertions(+), 16 deletions(-) rename server/src/main/java/org/elasticsearch/{CrossProjectRequest.java => CrossProjectEnabledRequest.java} (53%) diff --git a/server/src/main/java/org/elasticsearch/CrossProjectRequest.java b/server/src/main/java/org/elasticsearch/CrossProjectEnabledRequest.java similarity index 53% rename from server/src/main/java/org/elasticsearch/CrossProjectRequest.java rename to server/src/main/java/org/elasticsearch/CrossProjectEnabledRequest.java index 18d5148bc7205..a91934cace43e 100644 --- a/server/src/main/java/org/elasticsearch/CrossProjectRequest.java +++ b/server/src/main/java/org/elasticsearch/CrossProjectEnabledRequest.java @@ -14,19 +14,31 @@ import java.util.List; -public interface CrossProjectRequest extends IndicesRequest { - // TODO naming for all these methods... - boolean alreadyQualified(); - +public interface CrossProjectEnabledRequest extends IndicesRequest { + /** + * Can be used to determine if this should be processed in cross-project mode vs. stateful CCS. + */ + boolean crossProjectModeEnabled(); + + /** + * Only called if cross-project rewriting (flat-world, linked project filtering) was applied + */ void qualified(List qualifiedExpressions); @Nullable String queryRouting(); - // we need to track if the original resource is qualified or flat - record QualifiedExpression(String original, List qualified) {} - - // * -> (*, _local), (my_remote:*, my_remote) + /** + * Used to track a mapping from original expression (potentially flat-world) to canonicalized CCS expressions. + * e.g. for an original index expression `logs-*`, this would be: + * original=logs-* + * qualified=[(logs-*, _local), (my-remote:logs-*, my-remote)] + */ + record QualifiedExpression(String original, List qualified) { + public boolean hasFlatOriginalExpression() { + return true; + } + } record ExpressionWithProject(String expression, String project) {} } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java index eb015fd29d037..91e9c8484f099 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java @@ -9,7 +9,7 @@ package org.elasticsearch.action.fieldcaps; -import org.elasticsearch.CrossProjectRequest; +import org.elasticsearch.CrossProjectEnabledRequest; import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; @@ -44,7 +44,7 @@ public final class FieldCapabilitiesRequest extends LegacyActionRequest implements - CrossProjectRequest, + CrossProjectEnabledRequest, IndicesRequest.Replaceable, ToXContentObject { public static final String NAME = "field_caps_request"; @@ -382,13 +382,12 @@ public String getDescription() { } @Override - public boolean alreadyQualified() { + public boolean crossProjectModeEnabled() { return qualifiedExpressions != null; } @Override public void qualified(List qualifiedExpressions) { - assert false == alreadyQualified(); this.qualifiedExpressions = qualifiedExpressions; indices( qualifiedExpressions.stream() diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 4120096d604df..86e590d8bd585 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -9,7 +9,7 @@ package org.elasticsearch.action.search; -import org.elasticsearch.CrossProjectRequest; +import org.elasticsearch.CrossProjectEnabledRequest; import org.elasticsearch.TransportVersions; import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; @@ -56,7 +56,7 @@ */ public class SearchRequest extends LegacyActionRequest implements - CrossProjectRequest, + CrossProjectEnabledRequest, IndicesRequest.Replaceable, Rewriteable { @@ -871,13 +871,12 @@ public String toString() { } @Override - public boolean alreadyQualified() { + public boolean crossProjectModeEnabled() { return qualifiedExpressions != null; } @Override public void qualified(List qualifiedExpressions) { - assert false == alreadyQualified(); this.qualifiedExpressions = qualifiedExpressions; indices( qualifiedExpressions.stream() From 67f5cdf970927d8be1ecf71e3b0834f3694eb6fc Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 17 Jul 2025 15:46:41 +0200 Subject: [PATCH 26/28] More still --- .../authz/CrossProjectTargetResolver.java | 6 +++++- .../security/authz/AuthorizationService.java | 4 ++-- .../authz/IndicesAndAliasesResolver.java | 18 +++++++++--------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectTargetResolver.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectTargetResolver.java index 079d513fad102..0498b01957159 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectTargetResolver.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectTargetResolver.java @@ -11,6 +11,11 @@ import java.util.List; +/** + * Resolves linked and authorized projects, if running in a cross-project environment. + * + * In non-cross-project environments, `resolve` returns `ResolvedProjects.VOID` + */ public interface CrossProjectTargetResolver { ResolvedProjects resolve(SecurityContext securityContext); @@ -22,7 +27,6 @@ public ResolvedProjects resolve(SecurityContext securityContext) { } record ResolvedProjects(List projects) { - // I need a better name public static ResolvedProjects VOID = new ResolvedProjects(List.of()); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java index 5ba8220f73be7..d891992046f53 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java @@ -9,7 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.CrossProjectRequest; +import org.elasticsearch.CrossProjectEnabledRequest; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchRoleRestrictionException; import org.elasticsearch.ElasticsearchSecurityException; @@ -490,7 +490,7 @@ private void authorizeAction( })); } else if (isIndexAction(action)) { final ProjectMetadata projectMetadata = projectResolver.getProjectMetadata(clusterService.state()); - final CrossProjectTargetResolver.ResolvedProjects resolvedProjects = request instanceof CrossProjectRequest + final CrossProjectTargetResolver.ResolvedProjects resolvedProjects = request instanceof CrossProjectEnabledRequest ? crossProjectTargetResolver.resolve(securityContext) : CrossProjectTargetResolver.ResolvedProjects.VOID; assert projectMetadata != null; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java index 83bc3506f0c43..0b98a10a94fc2 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java @@ -6,7 +6,7 @@ */ package org.elasticsearch.xpack.security.authz; -import org.elasticsearch.CrossProjectRequest; +import org.elasticsearch.CrossProjectEnabledRequest; import org.elasticsearch.action.AliasesRequest; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; @@ -138,14 +138,14 @@ ResolvedIndices resolve( throw new IllegalStateException("Request [" + request + "] is not an Indices request, but should be."); } - if (request instanceof CrossProjectRequest crossProjectRequest) { - maybeRewriteCrossProjectRequest(resolvedProjects, crossProjectRequest); + if (request instanceof CrossProjectEnabledRequest crossProjectEnabledRequest) { + maybeRewriteCrossProjectRequest(resolvedProjects, crossProjectEnabledRequest); } return resolveIndicesAndAliases(action, (IndicesRequest) request, projectMetadata, authorizedIndices); } - void maybeRewriteCrossProjectRequest(CrossProjectTargetResolver.ResolvedProjects resolvedProjects, CrossProjectRequest request) { + void maybeRewriteCrossProjectRequest(CrossProjectTargetResolver.ResolvedProjects resolvedProjects, CrossProjectEnabledRequest request) { if (resolvedProjects == CrossProjectTargetResolver.ResolvedProjects.VOID) { logger.info("Cross-project search is disabled or not applicable, skipping request [{}]...", request); return; @@ -167,18 +167,18 @@ void maybeRewriteCrossProjectRequest(CrossProjectTargetResolver.ResolvedProjects return; } - List qualifiedExpressions = new ArrayList<>(indices.length); + List qualifiedExpressions = new ArrayList<>(indices.length); for (String local : resolved.getLocal()) { - List expressionWithProjects = new ArrayList<>(); - expressionWithProjects.add(new CrossProjectRequest.ExpressionWithProject(local, "_local")); + List expressionWithProjects = new ArrayList<>(); + expressionWithProjects.add(new CrossProjectEnabledRequest.ExpressionWithProject(local, "_local")); for (String targetProject : resolvedProjects.projects()) { expressionWithProjects.add( - new CrossProjectRequest.ExpressionWithProject( + new CrossProjectEnabledRequest.ExpressionWithProject( RemoteClusterAware.buildRemoteIndexName(targetProject, local), targetProject ) ); - qualifiedExpressions.add(new CrossProjectRequest.QualifiedExpression(local, expressionWithProjects)); + qualifiedExpressions.add(new CrossProjectEnabledRequest.QualifiedExpression(local, expressionWithProjects)); } logger.info("Rewrote [{}] to [{}]", local, expressionWithProjects); } From e25d8b273aa66ac5c972565b3f5b6dfba6d9b473 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 17 Jul 2025 15:47:47 +0200 Subject: [PATCH 27/28] CrossProjectAware --- ...uest.java => CrossProjectAwareRequest.java} | 2 +- .../fieldcaps/FieldCapabilitiesRequest.java | 4 ++-- .../action/search/SearchRequest.java | 4 ++-- .../security/authz/AuthorizationService.java | 4 ++-- .../authz/IndicesAndAliasesResolver.java | 18 +++++++++--------- 5 files changed, 16 insertions(+), 16 deletions(-) rename server/src/main/java/org/elasticsearch/{CrossProjectEnabledRequest.java => CrossProjectAwareRequest.java} (95%) diff --git a/server/src/main/java/org/elasticsearch/CrossProjectEnabledRequest.java b/server/src/main/java/org/elasticsearch/CrossProjectAwareRequest.java similarity index 95% rename from server/src/main/java/org/elasticsearch/CrossProjectEnabledRequest.java rename to server/src/main/java/org/elasticsearch/CrossProjectAwareRequest.java index a91934cace43e..394f42a1f1aa7 100644 --- a/server/src/main/java/org/elasticsearch/CrossProjectEnabledRequest.java +++ b/server/src/main/java/org/elasticsearch/CrossProjectAwareRequest.java @@ -14,7 +14,7 @@ import java.util.List; -public interface CrossProjectEnabledRequest extends IndicesRequest { +public interface CrossProjectAwareRequest extends IndicesRequest { /** * Can be used to determine if this should be processed in cross-project mode vs. stateful CCS. */ diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java index 91e9c8484f099..cdb3a193f629b 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java @@ -9,7 +9,7 @@ package org.elasticsearch.action.fieldcaps; -import org.elasticsearch.CrossProjectEnabledRequest; +import org.elasticsearch.CrossProjectAwareRequest; import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; @@ -44,7 +44,7 @@ public final class FieldCapabilitiesRequest extends LegacyActionRequest implements - CrossProjectEnabledRequest, + CrossProjectAwareRequest, IndicesRequest.Replaceable, ToXContentObject { public static final String NAME = "field_caps_request"; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 86e590d8bd585..b06b9707d9a5f 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -9,7 +9,7 @@ package org.elasticsearch.action.search; -import org.elasticsearch.CrossProjectEnabledRequest; +import org.elasticsearch.CrossProjectAwareRequest; import org.elasticsearch.TransportVersions; import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; @@ -56,7 +56,7 @@ */ public class SearchRequest extends LegacyActionRequest implements - CrossProjectEnabledRequest, + CrossProjectAwareRequest, IndicesRequest.Replaceable, Rewriteable { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java index d891992046f53..a5471eb4a59f8 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java @@ -9,7 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.CrossProjectEnabledRequest; +import org.elasticsearch.CrossProjectAwareRequest; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchRoleRestrictionException; import org.elasticsearch.ElasticsearchSecurityException; @@ -490,7 +490,7 @@ private void authorizeAction( })); } else if (isIndexAction(action)) { final ProjectMetadata projectMetadata = projectResolver.getProjectMetadata(clusterService.state()); - final CrossProjectTargetResolver.ResolvedProjects resolvedProjects = request instanceof CrossProjectEnabledRequest + final CrossProjectTargetResolver.ResolvedProjects resolvedProjects = request instanceof CrossProjectAwareRequest ? crossProjectTargetResolver.resolve(securityContext) : CrossProjectTargetResolver.ResolvedProjects.VOID; assert projectMetadata != null; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java index 0b98a10a94fc2..c376a73afe3eb 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java @@ -6,7 +6,7 @@ */ package org.elasticsearch.xpack.security.authz; -import org.elasticsearch.CrossProjectEnabledRequest; +import org.elasticsearch.CrossProjectAwareRequest; import org.elasticsearch.action.AliasesRequest; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; @@ -138,14 +138,14 @@ ResolvedIndices resolve( throw new IllegalStateException("Request [" + request + "] is not an Indices request, but should be."); } - if (request instanceof CrossProjectEnabledRequest crossProjectEnabledRequest) { - maybeRewriteCrossProjectRequest(resolvedProjects, crossProjectEnabledRequest); + if (request instanceof CrossProjectAwareRequest crossProjectAwareRequest) { + maybeRewriteCrossProjectRequest(resolvedProjects, crossProjectAwareRequest); } return resolveIndicesAndAliases(action, (IndicesRequest) request, projectMetadata, authorizedIndices); } - void maybeRewriteCrossProjectRequest(CrossProjectTargetResolver.ResolvedProjects resolvedProjects, CrossProjectEnabledRequest request) { + void maybeRewriteCrossProjectRequest(CrossProjectTargetResolver.ResolvedProjects resolvedProjects, CrossProjectAwareRequest request) { if (resolvedProjects == CrossProjectTargetResolver.ResolvedProjects.VOID) { logger.info("Cross-project search is disabled or not applicable, skipping request [{}]...", request); return; @@ -167,18 +167,18 @@ void maybeRewriteCrossProjectRequest(CrossProjectTargetResolver.ResolvedProjects return; } - List qualifiedExpressions = new ArrayList<>(indices.length); + List qualifiedExpressions = new ArrayList<>(indices.length); for (String local : resolved.getLocal()) { - List expressionWithProjects = new ArrayList<>(); - expressionWithProjects.add(new CrossProjectEnabledRequest.ExpressionWithProject(local, "_local")); + List expressionWithProjects = new ArrayList<>(); + expressionWithProjects.add(new CrossProjectAwareRequest.ExpressionWithProject(local, "_local")); for (String targetProject : resolvedProjects.projects()) { expressionWithProjects.add( - new CrossProjectEnabledRequest.ExpressionWithProject( + new CrossProjectAwareRequest.ExpressionWithProject( RemoteClusterAware.buildRemoteIndexName(targetProject, local), targetProject ) ); - qualifiedExpressions.add(new CrossProjectEnabledRequest.QualifiedExpression(local, expressionWithProjects)); + qualifiedExpressions.add(new CrossProjectAwareRequest.QualifiedExpression(local, expressionWithProjects)); } logger.info("Rewrote [{}] to [{}]", local, expressionWithProjects); } From 137195fa4c6c7a74090549a5b33c9f4e612c7371 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 17 Jul 2025 15:55:15 +0200 Subject: [PATCH 28/28] Javadoc --- .../xpack/core/security/authz/CrossProjectTargetResolver.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectTargetResolver.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectTargetResolver.java index 0498b01957159..40d969783f873 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectTargetResolver.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/CrossProjectTargetResolver.java @@ -13,7 +13,6 @@ /** * Resolves linked and authorized projects, if running in a cross-project environment. - * * In non-cross-project environments, `resolve` returns `ResolvedProjects.VOID` */ public interface CrossProjectTargetResolver {