Skip to content

Commit c583951

Browse files
authored
acc: Support ENVFILTER for filtering tests by Env/EnvMatrix setting (#2944)
## Why Useful for selecting only subtests for tests with a lot of combinations, e.g default-python/combinations. Usiong it in #2926 to select single deployment backend on CI. ## Tests Manually. ``` ~/work/cli-main/acceptance/selftest/envmatrix % ENVFILTER=FIRST=one111 go test ../.. -run ^TestAccept$/^selftest$/^envmatrix$ -v | grep SKIP --- SKIP: TestAccept/selftest/envmatrix (0.01s) --- SKIP: TestAccept/selftest/envmatrix/inner/B_REGULAR_VAR=hello/FIRST=two222/SECOND=variantB/THIRD=three_$FIRST (0.01s) --- SKIP: TestAccept/selftest/envmatrix/inner/B_REGULAR_VAR=hello/FIRST=two222/SECOND=variantA/THIRD=three_$FIRST (0.01s) ```
1 parent 5fbad90 commit c583951

File tree

1 file changed

+46
-2
lines changed

1 file changed

+46
-2
lines changed

acceptance/acceptance_test.go

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,11 @@ const (
7575
MaxFileSize = 100_000
7676
// Filename to save replacements to (used by diff.py)
7777
ReplsFile = "repls.json"
78+
79+
// ENVFILTER allows filtering subtests matching certain env var.
80+
// e.g. ENVFILTER=SERVERLESS=yes will run all tests that run SERVERLESS to "yes"
81+
// The tests the don't set SERVERLESS variable or set to empty string will also be run.
82+
EnvFilterVar = "ENVFILTER"
7883
)
7984

8085
var Scripts = map[string]bool{
@@ -214,6 +219,8 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
214219
totalDirs := 0
215220
selectedDirs := 0
216221

222+
envFilters := getEnvFilters(t)
223+
217224
for _, dir := range testDirs {
218225
totalDirs += 1
219226

@@ -245,15 +252,15 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
245252
if len(expanded[0]) > 0 {
246253
t.Logf("Running test with env %v", expanded[0])
247254
}
248-
runTest(t, dir, 0, coverDir, repls.Clone(), config, configPath, expanded[0], inprocessMode)
255+
runTest(t, dir, 0, coverDir, repls.Clone(), config, configPath, expanded[0], inprocessMode, envFilters)
249256
} else {
250257
for ind, envset := range expanded {
251258
envname := strings.Join(envset, "/")
252259
t.Run(envname, func(t *testing.T) {
253260
if !inprocessMode {
254261
t.Parallel()
255262
}
256-
runTest(t, dir, ind, coverDir, repls.Clone(), config, configPath, envset, inprocessMode)
263+
runTest(t, dir, ind, coverDir, repls.Clone(), config, configPath, envset, inprocessMode, envFilters)
257264
})
258265
}
259266
}
@@ -265,6 +272,27 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
265272
return len(testDirs)
266273
}
267274

275+
func getEnvFilters(t *testing.T) []string {
276+
envFilterValue := os.Getenv(EnvFilterVar)
277+
if envFilterValue == "" {
278+
return nil
279+
}
280+
281+
filters := strings.Split(envFilterValue, ",")
282+
283+
for _, filter := range filters {
284+
items := strings.Split(filter, "=")
285+
if len(items) != 2 || len(items[0]) == 0 {
286+
t.Fatalf("Invalid filter %q in %s=%q", filter, EnvFilterVar, envFilterValue)
287+
}
288+
key := items[0]
289+
// Clear it just to be sure, since it's going to be part of os.Environ() and we're going to add different value based on settings.
290+
os.Unsetenv(key)
291+
}
292+
293+
return filters
294+
}
295+
268296
func getTests(t *testing.T) []string {
269297
testDirs := make([]string, 0, 128)
270298

@@ -354,6 +382,7 @@ func runTest(t *testing.T,
354382
configPath string,
355383
customEnv []string,
356384
inprocessMode bool,
385+
envFilters []string,
357386
) {
358387
if LogConfig {
359388
configBytes, err := json.MarshalIndent(config, "", " ")
@@ -487,6 +516,21 @@ func runTest(t *testing.T,
487516
cmd.Env = addEnvVar(t, cmd.Env, &repls, key, value, config.EnvRepl, len(config.EnvMatrix[key]) > 1 && len(value) >= 4)
488517
}
489518

519+
for filterInd, filterEnv := range envFilters {
520+
filterEnvKey := strings.Split(filterEnv, "=")[0]
521+
for ind := range cmd.Env {
522+
// Search backwards, because the latest settings is what is actually applicable.
523+
envPair := cmd.Env[len(cmd.Env)-1-ind]
524+
if strings.Split(envPair, "=")[0] == filterEnvKey {
525+
if envPair == filterEnv {
526+
break
527+
} else {
528+
t.Skipf("Skipping because test environment %s does not match ENVFILTER#%d: %s", envPair, filterInd, filterEnv)
529+
}
530+
}
531+
}
532+
}
533+
490534
absDir, err := filepath.Abs(dir)
491535
require.NoError(t, err)
492536
cmd.Env = append(cmd.Env, "TESTDIR="+absDir)

0 commit comments

Comments
 (0)