Skip to content

Commit 6d49122

Browse files
moschemosche
authored
Support entitlements in internal cluster tests (#130710) (#131456)
* To prevent an implicit grant-all if storing node homes inside the Java temp dir, the temporary folder of ESTestCase is configured separately from the Java temp dir in internalClusterTests (by means of the system property tempDir, see TestRuleTemporaryFilesCleanup) * Move ReloadingDatabasesWhilePerformingGeoLookupsIT from internalClusterTest to test, file permissions in internalClusterTest are stricter on the lucene tempDir
1 parent e9e44ae commit 6d49122

File tree

12 files changed

+148
-55
lines changed

12 files changed

+148
-55
lines changed

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

Lines changed: 41 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import java.io.File;
3535
import java.util.List;
3636
import java.util.Map;
37+
import java.util.Set;
3738
import java.util.stream.Stream;
3839

3940
import javax.inject.Inject;
@@ -50,6 +51,8 @@ public abstract class ElasticsearchTestBasePlugin implements Plugin<Project> {
5051

5152
public static final String DUMP_OUTPUT_ON_FAILURE_PROP_NAME = "dumpOutputOnFailure";
5253

54+
public static final Set<String> TEST_TASKS_WITH_ENTITLEMENTS = Set.of("test", "internalClusterTest");
55+
5356
@Inject
5457
protected abstract ProviderFactory getProviderFactory();
5558

@@ -173,14 +176,23 @@ public void execute(Task t) {
173176
nonInputProperties.systemProperty("workspace.dir", Util.locateElasticsearchWorkspace(project.getGradle()));
174177
// we use 'temp' relative to CWD since this is per JVM and tests are forbidden from writing to CWD
175178
nonInputProperties.systemProperty("java.io.tmpdir", test.getWorkingDir().toPath().resolve("temp"));
179+
if (test.getName().equals("internalClusterTest")) {
180+
// configure a node home directory independent of the Java temp dir so that entitlements can be properly enforced
181+
nonInputProperties.systemProperty("tempDir", test.getWorkingDir().toPath().resolve("nodesTemp"));
182+
}
176183

177184
SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
178185
SourceSet mainSourceSet = sourceSets.findByName(SourceSet.MAIN_SOURCE_SET_NAME);
179186
SourceSet testSourceSet = sourceSets.findByName(SourceSet.TEST_SOURCE_SET_NAME);
180-
if ("test".equals(test.getName()) && mainSourceSet != null && testSourceSet != null) {
187+
SourceSet internalClusterTestSourceSet = sourceSets.findByName("internalClusterTest");
188+
189+
if (TEST_TASKS_WITH_ENTITLEMENTS.contains(test.getName()) && mainSourceSet != null && testSourceSet != null) {
181190
FileCollection mainRuntime = mainSourceSet.getRuntimeClasspath();
182191
FileCollection testRuntime = testSourceSet.getRuntimeClasspath();
183-
FileCollection testOnlyFiles = testRuntime.minus(mainRuntime);
192+
FileCollection internalClusterTestRuntime = ("internalClusterTest".equals(test.getName())
193+
&& internalClusterTestSourceSet != null) ? internalClusterTestSourceSet.getRuntimeClasspath() : project.files();
194+
FileCollection testOnlyFiles = testRuntime.plus(internalClusterTestRuntime).minus(mainRuntime);
195+
184196
test.doFirst(task -> test.environment("es.entitlement.testOnlyPath", testOnlyFiles.getAsPath()));
185197
}
186198

@@ -240,14 +252,15 @@ public void execute(Task t) {
240252
* Computes and sets the {@code --patch-module=java.base} and {@code --add-opens=java.base} JVM command line options.
241253
*/
242254
private void configureJavaBaseModuleOptions(Project project) {
243-
project.getTasks().withType(Test.class).matching(task -> task.getName().equals("test")).configureEach(test -> {
244-
FileCollection patchedImmutableCollections = patchedImmutableCollections(project);
255+
project.getTasks().withType(Test.class).configureEach(test -> {
256+
// patch immutable collections only for "test" task
257+
FileCollection patchedImmutableCollections = test.getName().equals("test") ? patchedImmutableCollections(project) : null;
245258
if (patchedImmutableCollections != null) {
246259
test.getInputs().files(patchedImmutableCollections);
247260
test.systemProperty("tests.hackImmutableCollections", "true");
248261
}
249262

250-
FileCollection entitlementBridge = entitlementBridge(project);
263+
FileCollection entitlementBridge = TEST_TASKS_WITH_ENTITLEMENTS.contains(test.getName()) ? entitlementBridge(project) : null;
251264
if (entitlementBridge != null) {
252265
test.getInputs().files(entitlementBridge);
253266
}
@@ -311,27 +324,30 @@ private static void configureEntitlements(Project project) {
311324
}
312325
FileCollection bridgeFiles = bridgeConfig;
313326

314-
project.getTasks().withType(Test.class).configureEach(test -> {
315-
// See also SystemJvmOptions.maybeAttachEntitlementAgent.
316-
317-
// Agent
318-
if (agentFiles.isEmpty() == false) {
319-
test.getInputs().files(agentFiles);
320-
test.systemProperty("es.entitlement.agentJar", agentFiles.getAsPath());
321-
test.systemProperty("jdk.attach.allowAttachSelf", true);
322-
}
327+
project.getTasks()
328+
.withType(Test.class)
329+
.matching(test -> TEST_TASKS_WITH_ENTITLEMENTS.contains(test.getName()))
330+
.configureEach(test -> {
331+
// See also SystemJvmOptions.maybeAttachEntitlementAgent.
332+
333+
// Agent
334+
if (agentFiles.isEmpty() == false) {
335+
test.getInputs().files(agentFiles);
336+
test.systemProperty("es.entitlement.agentJar", agentFiles.getAsPath());
337+
test.systemProperty("jdk.attach.allowAttachSelf", true);
338+
}
323339

324-
// Bridge
325-
if (bridgeFiles.isEmpty() == false) {
326-
String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
327-
test.getInputs().files(bridgeFiles);
328-
// Tests may not be modular, but the JDK still is
329-
test.jvmArgs(
330-
"--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
331-
+ modulesContainingEntitlementInstrumentation
332-
);
333-
}
334-
});
340+
// Bridge
341+
if (bridgeFiles.isEmpty() == false) {
342+
String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
343+
test.getInputs().files(bridgeFiles);
344+
// Tests may not be modular, but the JDK still is
345+
test.jvmArgs(
346+
"--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
347+
+ modulesContainingEntitlementInstrumentation
348+
);
349+
}
350+
});
335351
}
336352

337353
}

‎build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,12 @@ public void apply(Project project) {
5858
});
5959

6060
if (project.getRootProject().getName().equals("elasticsearch")) {
61-
project.getTasks().withType(Test.class).matching(test -> List.of("test").contains(test.getName())).configureEach(test -> {
62-
test.systemProperty("es.entitlement.enableForTests", "true");
63-
});
61+
project.getTasks()
62+
.withType(Test.class)
63+
.matching(test -> List.of("test", "internalClusterTest").contains(test.getName()))
64+
.configureEach(test -> {
65+
test.systemProperty("es.entitlement.enableForTests", "true");
66+
});
6467
}
6568
}
6669
}

‎libs/build.gradle

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ configure(childProjects.values()) {
5050
// Omit oddball libraries that aren't in server.
5151
def nonServerLibs = ['plugin-scanner']
5252
if (false == nonServerLibs.contains(project.name)) {
53-
project.getTasks().withType(Test.class).matching(test -> ['test'].contains(test.name)).configureEach(test -> {
53+
project.getTasks().withType(Test.class).matching(test -> ['test', 'internalClusterTest'].contains(test.name)).configureEach(test -> {
5454
test.systemProperty('es.entitlement.enableForTests', 'true')
5555
})
5656
}

‎modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java renamed to ‎modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@
5252
// 'WindowsFS.checkDeleteAccess(...)').
5353
}
5454
)
55-
public class ReloadingDatabasesWhilePerformingGeoLookupsIT extends ESTestCase {
55+
public class ReloadingDatabasesWhilePerformingGeoLookupsTests extends ESTestCase {
5656

5757
/**
5858
* This tests essentially verifies that a Maxmind database reader doesn't fail with:

‎server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.elasticsearch.gateway.PersistedClusterStateService;
2525
import org.elasticsearch.node.Node;
2626
import org.elasticsearch.test.ESIntegTestCase;
27+
import org.elasticsearch.test.ESTestCase;
2728
import org.elasticsearch.test.InternalTestCluster;
2829

2930
import java.io.IOException;
@@ -39,6 +40,7 @@
3940
import static org.hamcrest.Matchers.notNullValue;
4041

4142
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
43+
@ESTestCase.WithoutEntitlements // CLI tools don't run with entitlements enforced
4244
public class UnsafeBootstrapAndDetachCommandIT extends ESIntegTestCase {
4345

4446
private MockTerminal executeCommand(ElasticsearchNodeCommand command, Environment environment, boolean abort) throws Exception {

‎test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
import org.elasticsearch.common.network.IfConfig;
1515
import org.elasticsearch.common.settings.Settings;
1616
import org.elasticsearch.core.Booleans;
17-
import org.elasticsearch.core.Nullable;
1817
import org.elasticsearch.core.PathUtils;
1918
import org.elasticsearch.entitlement.bootstrap.TestEntitlementBootstrap;
2019
import org.elasticsearch.jdk.JarHell;
@@ -76,20 +75,12 @@ public class BootstrapForTesting {
7675

7776
// Fire up entitlements
7877
try {
79-
TestEntitlementBootstrap.bootstrap(javaTmpDir, maybePath(System.getProperty("tests.config")));
78+
TestEntitlementBootstrap.bootstrap(javaTmpDir);
8079
} catch (IOException e) {
8180
throw new IllegalStateException(e.getClass().getSimpleName() + " while initializing entitlements for tests", e);
8281
}
8382
}
8483

85-
private static @Nullable Path maybePath(String str) {
86-
if (str == null) {
87-
return null;
88-
} else {
89-
return PathUtils.get(str);
90-
}
91-
}
92-
9384
// does nothing, just easy way to make sure the class is loaded.
9485
public static void ensureInitialized() {}
9586
}

‎test/framework/src/main/java/org/elasticsearch/entitlement/bootstrap/TestEntitlementBootstrap.java

Lines changed: 72 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,15 @@
1212
import org.elasticsearch.bootstrap.TestBuildInfo;
1313
import org.elasticsearch.bootstrap.TestBuildInfoParser;
1414
import org.elasticsearch.bootstrap.TestScopeResolver;
15+
import org.elasticsearch.common.settings.Settings;
1516
import org.elasticsearch.core.Booleans;
1617
import org.elasticsearch.core.Nullable;
1718
import org.elasticsearch.core.PathUtils;
1819
import org.elasticsearch.core.Strings;
1920
import org.elasticsearch.core.SuppressForbidden;
2021
import org.elasticsearch.entitlement.initialization.EntitlementInitialization;
2122
import org.elasticsearch.entitlement.runtime.policy.PathLookup;
23+
import org.elasticsearch.entitlement.runtime.policy.PathLookup.BaseDir;
2224
import org.elasticsearch.entitlement.runtime.policy.Policy;
2325
import org.elasticsearch.entitlement.runtime.policy.PolicyParser;
2426
import org.elasticsearch.entitlement.runtime.policy.TestPathLookup;
@@ -32,39 +34,106 @@
3234
import java.net.URI;
3335
import java.net.URL;
3436
import java.nio.file.Path;
37+
import java.nio.file.Paths;
3538
import java.util.ArrayList;
3639
import java.util.Arrays;
40+
import java.util.Collection;
3741
import java.util.HashMap;
42+
import java.util.HashSet;
3843
import java.util.List;
3944
import java.util.Map;
4045
import java.util.Set;
4146
import java.util.TreeSet;
47+
import java.util.concurrent.ConcurrentHashMap;
48+
import java.util.function.BiFunction;
49+
import java.util.function.Consumer;
4250

4351
import static java.util.stream.Collectors.toCollection;
4452
import static java.util.stream.Collectors.toSet;
45-
import static org.elasticsearch.entitlement.runtime.policy.PathLookup.BaseDir.CONFIG;
4653
import static org.elasticsearch.entitlement.runtime.policy.PathLookup.BaseDir.TEMP;
54+
import static org.elasticsearch.env.Environment.PATH_DATA_SETTING;
55+
import static org.elasticsearch.env.Environment.PATH_HOME_SETTING;
56+
import static org.elasticsearch.env.Environment.PATH_REPO_SETTING;
4757

4858
public class TestEntitlementBootstrap {
4959

5060
private static final Logger logger = LogManager.getLogger(TestEntitlementBootstrap.class);
5161

62+
private static Map<BaseDir, Collection<Path>> baseDirPaths = new ConcurrentHashMap<>();
5263
private static TestPolicyManager policyManager;
5364

5465
/**
5566
* Activates entitlement checking in tests.
5667
*/
57-
public static void bootstrap(@Nullable Path tempDir, @Nullable Path configDir) throws IOException {
68+
public static void bootstrap(@Nullable Path tempDir) throws IOException {
5869
if (isEnabledForTest() == false) {
5970
return;
6071
}
61-
TestPathLookup pathLookup = new TestPathLookup(Map.of(TEMP, zeroOrOne(tempDir), CONFIG, zeroOrOne(configDir)));
72+
var previousTempDir = baseDirPaths.put(TEMP, zeroOrOne(tempDir));
73+
assert previousTempDir == null : "Test entitlement bootstrap called multiple times";
74+
TestPathLookup pathLookup = new TestPathLookup(baseDirPaths);
6275
policyManager = createPolicyManager(pathLookup);
6376
EntitlementInitialization.initializeArgs = new EntitlementInitialization.InitializeArgs(pathLookup, Set.of(), policyManager);
6477
logger.debug("Loading entitlement agent");
6578
EntitlementBootstrap.loadAgent(EntitlementBootstrap.findAgentJar(), EntitlementInitialization.class.getName());
6679
}
6780

81+
public static void registerNodeBaseDirs(Settings settings, Path configPath) {
82+
if (policyManager == null) {
83+
return;
84+
}
85+
Path homeDir = absolutePath(PATH_HOME_SETTING.get(settings));
86+
Path configDir = configPath != null ? configPath : homeDir.resolve("config");
87+
Collection<Path> dataDirs = dataDirs(settings, homeDir);
88+
Collection<Path> repoDirs = repoDirs(settings);
89+
logger.debug("Registering node dirs: config [{}], dataDirs [{}], repoDirs [{}]", configDir, dataDirs, repoDirs);
90+
baseDirPaths.compute(BaseDir.CONFIG, baseDirModifier(paths -> paths.add(configDir)));
91+
baseDirPaths.compute(BaseDir.DATA, baseDirModifier(paths -> paths.addAll(dataDirs)));
92+
baseDirPaths.compute(BaseDir.SHARED_REPO, baseDirModifier(paths -> paths.addAll(repoDirs)));
93+
policyManager.reset();
94+
}
95+
96+
public static void unregisterNodeBaseDirs(Settings settings, Path configPath) {
97+
if (policyManager == null) {
98+
return;
99+
}
100+
Path homeDir = absolutePath(PATH_HOME_SETTING.get(settings));
101+
Path configDir = configPath != null ? configPath : homeDir.resolve("config");
102+
Collection<Path> dataDirs = dataDirs(settings, homeDir);
103+
Collection<Path> repoDirs = repoDirs(settings);
104+
logger.debug("Unregistering node dirs: config [{}], dataDirs [{}], repoDirs [{}]", configDir, dataDirs, repoDirs);
105+
baseDirPaths.compute(BaseDir.CONFIG, baseDirModifier(paths -> paths.remove(configDir)));
106+
baseDirPaths.compute(BaseDir.DATA, baseDirModifier(paths -> paths.removeAll(dataDirs)));
107+
baseDirPaths.compute(BaseDir.SHARED_REPO, baseDirModifier(paths -> paths.removeAll(repoDirs)));
108+
policyManager.reset();
109+
}
110+
111+
private static Collection<Path> dataDirs(Settings settings, Path homeDir) {
112+
List<String> dataDirs = PATH_DATA_SETTING.get(settings);
113+
return dataDirs.isEmpty()
114+
? List.of(homeDir.resolve("data"))
115+
: dataDirs.stream().map(TestEntitlementBootstrap::absolutePath).toList();
116+
}
117+
118+
private static Collection<Path> repoDirs(Settings settings) {
119+
return PATH_REPO_SETTING.get(settings).stream().map(TestEntitlementBootstrap::absolutePath).toList();
120+
}
121+
122+
private static BiFunction<BaseDir, Collection<Path>, Collection<Path>> baseDirModifier(Consumer<Collection<Path>> consumer) {
123+
return (BaseDir baseDir, Collection<Path> paths) -> {
124+
if (paths == null) {
125+
paths = new HashSet<>();
126+
}
127+
consumer.accept(paths);
128+
return paths;
129+
};
130+
}
131+
132+
@SuppressForbidden(reason = "must be resolved using the default file system, rather then the mocked test file system")
133+
private static Path absolutePath(String path) {
134+
return Paths.get(path).toAbsolutePath().normalize();
135+
}
136+
68137
private static <T> List<T> zeroOrOne(T item) {
69138
if (item == null) {
70139
return List.of();

‎test/framework/src/main/java/org/elasticsearch/node/MockNode.java

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.elasticsearch.common.util.MockBigArrays;
2424
import org.elasticsearch.common.util.MockPageCacheRecycler;
2525
import org.elasticsearch.common.util.PageCacheRecycler;
26+
import org.elasticsearch.entitlement.bootstrap.TestEntitlementBootstrap;
2627
import org.elasticsearch.env.Environment;
2728
import org.elasticsearch.http.HttpServerTransport;
2829
import org.elasticsearch.indices.ExecutorSelector;
@@ -53,6 +54,7 @@
5354
import org.elasticsearch.transport.TransportService;
5455
import org.elasticsearch.transport.TransportSettings;
5556

57+
import java.io.IOException;
5658
import java.nio.file.Path;
5759
import java.util.Collection;
5860
import java.util.Collections;
@@ -252,16 +254,7 @@ public MockNode(
252254
final Path configPath,
253255
final boolean forbidPrivateIndexSettings
254256
) {
255-
this(
256-
InternalSettingsPreparer.prepareEnvironment(
257-
Settings.builder().put(TransportSettings.PORT.getKey(), ESTestCase.getPortRange()).put(settings).build(),
258-
Collections.emptyMap(),
259-
configPath,
260-
() -> "mock_ node"
261-
),
262-
classpathPlugins,
263-
forbidPrivateIndexSettings
264-
);
257+
this(prepareEnvironment(settings, configPath), classpathPlugins, forbidPrivateIndexSettings);
265258
}
266259

267260
private MockNode(
@@ -280,6 +273,25 @@ PluginsService newPluginService(Environment environment, PluginsLoader pluginsLo
280273
this.classpathPlugins = classpathPlugins;
281274
}
282275

276+
private static Environment prepareEnvironment(final Settings settings, final Path configPath) {
277+
TestEntitlementBootstrap.registerNodeBaseDirs(settings, configPath);
278+
return InternalSettingsPreparer.prepareEnvironment(
279+
Settings.builder().put(TransportSettings.PORT.getKey(), ESTestCase.getPortRange()).put(settings).build(),
280+
Collections.emptyMap(),
281+
configPath,
282+
() -> "mock_ node"
283+
);
284+
}
285+
286+
@Override
287+
public synchronized void close() throws IOException {
288+
try {
289+
super.close();
290+
} finally {
291+
TestEntitlementBootstrap.unregisterNodeBaseDirs(getEnvironment().settings(), getEnvironment().configDir());
292+
}
293+
}
294+
283295
/**
284296
* The classpath plugins this node was constructed with.
285297
*/

‎test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -266,7 +266,6 @@
266266
* </ul>
267267
*/
268268
@LuceneTestCase.SuppressFileSystems("ExtrasFS") // doesn't work with potential multi data path from test cluster yet
269-
@ESTestCase.WithoutEntitlements // ES-12042
270269
public abstract class ESIntegTestCase extends ESTestCase {
271270

272271
/** node names of the corresponding clusters will start with these prefixes */

‎test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,6 @@
8585
* A test that keep a singleton node started for all tests that can be used to get
8686
* references to Guice injectors in unit tests.
8787
*/
88-
@ESTestCase.WithoutEntitlements // ES-12042
8988
public abstract class ESSingleNodeTestCase extends ESTestCase {
9089

9190
private static Node NODE = null;

0 commit comments

Comments
 (0)