Skip to content

Commit 2961ab0

Browse files
velovelo
authored
Merge pull request #2 from DataSQRL/validate_env_vars
Validate if all environment variables are present
2 parents a884825 + 8cbbdda commit 2961ab0

File tree

6 files changed

+211
-11
lines changed

6 files changed

+211
-11
lines changed

‎fink-dockerfile-example/build.gradle

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ ext {
2727
flinkVersion = "1.19.1"
2828
jdbcVersion = "3.2.0-1.19"
2929
kafkaVersion = "3.2.0-1.19"
30-
sqrlVersion = "0.5.6"
30+
sqrlVersion = "0.5.7"
3131
icebergVersion = "1.6.0"
3232
}
3333

‎src/main/java/com/datasqrl/EnvironmentVariablesUtils.java

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/*
2+
* Copyright © 2024 DataSQRL (contact@datasqrl.com)
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.datasqrl;
17+
18+
import java.util.Collections;
19+
import java.util.Map;
20+
import java.util.Set;
21+
import java.util.TreeSet;
22+
import java.util.regex.Matcher;
23+
import java.util.regex.Pattern;
24+
import lombok.experimental.UtilityClass;
25+
import lombok.extern.slf4j.Slf4j;
26+
27+
@Slf4j
28+
@UtilityClass
29+
public class EnvironmentVariablesUtils {
30+
31+
private static final Pattern ENVIRONMENT_VARIABLE_PATTERN = Pattern.compile("\\$\\{(.*?)\\}");
32+
33+
public static String replaceWithEnv(String command) {
34+
return replaceWithEnv(command, System.getenv());
35+
}
36+
37+
public static String replaceWithEnv(String command, Map<String, String> envVariables) {
38+
String substitutedStr = command;
39+
StringBuffer result = new StringBuffer();
40+
// First pass to replace environment variables
41+
Matcher matcher = ENVIRONMENT_VARIABLE_PATTERN.matcher(substitutedStr);
42+
while (matcher.find()) {
43+
String key = matcher.group(1);
44+
String envValue = envVariables.get(key);
45+
if (envValue == null) {
46+
throw new IllegalStateException(String.format("Missing environment variable: %s", key));
47+
}
48+
matcher.appendReplacement(result, Matcher.quoteReplacement(envValue));
49+
}
50+
matcher.appendTail(result);
51+
52+
return result.toString();
53+
}
54+
55+
public Set<String> validateEnvironmentVariables(String script) {
56+
return validateEnvironmentVariables(System.getenv(), script);
57+
}
58+
59+
public Set<String> validateEnvironmentVariables(Map<String, String> envVariables, String script) {
60+
Matcher matcher = ENVIRONMENT_VARIABLE_PATTERN.matcher(script);
61+
62+
Set<String> scriptEnvironmentVariables = new TreeSet<>();
63+
while (matcher.find()) {
64+
scriptEnvironmentVariables.add(matcher.group(1));
65+
}
66+
67+
if (envVariables.keySet().containsAll(scriptEnvironmentVariables)) {
68+
log.info("All environment variables are available: {}", scriptEnvironmentVariables);
69+
return Collections.emptySet();
70+
}
71+
72+
scriptEnvironmentVariables.removeAll(envVariables.keySet());
73+
return Collections.unmodifiableSet(scriptEnvironmentVariables);
74+
}
75+
}

‎src/main/java/com/datasqrl/SqlExecutor.java

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -74,15 +74,6 @@ public TableResult executeScript(String script) throws Exception {
7474
for (String statement : statements) {
7575
tableResult = executeStatement(statement);
7676
}
77-
//
78-
// TableEnvironmentImpl tEnv1 = (TableEnvironmentImpl) tableEnv;
79-
//
80-
// StatementSetOperation parse = (StatementSetOperation)tEnv1.getParser()
81-
// .parse(statements.get(statements.size()-1)).get(0);
82-
//
83-
// CompiledPlan plan = tEnv1.compilePlan(parse.getOperations());
84-
//
85-
// plan.writeToFile("/Users/henneberger/flink-jar-runner/src/test/resources/sql/compiled-plan-udf.json");
8677

8778
return tableResult;
8879
}
@@ -107,7 +98,7 @@ private TableResult executeStatement(String statement) {
10798
} else {
10899
System.out.println(statement);
109100
log.info("Executing statement:\n{}", statement);
110-
tableResult = tableEnv.executeSql(replaceWithEnv(statement));
101+
tableResult = tableEnv.executeSql(EnvironmentVariablesUtils.replaceWithEnv(statement));
111102
}
112103
} catch (Exception e) {
113104
log.error("Failed to execute statement: {}", statement, e);

‎src/main/java/com/datasqrl/SqlRunner.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,16 @@ public Integer call() throws Exception {
8989
if (sqlFile != null) {
9090
// Single SQL file mode
9191
String script = FileUtils.readFileUtf8(sqlFile);
92+
93+
Set<String> missingEnvironmentVariables =
94+
EnvironmentVariablesUtils.validateEnvironmentVariables(script);
95+
if (!missingEnvironmentVariables.isEmpty()) {
96+
throw new IllegalStateException(
97+
String.format(
98+
"Could not find the following environment variables: %s",
99+
missingEnvironmentVariables));
100+
}
101+
92102
tableResult = sqlExecutor.executeScript(script);
93103
} else if (planFile != null) {
94104
// Compiled plan JSON file

‎src/main/java/com/datasqrl/SqlUtils.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,10 @@
1717

1818
import java.util.ArrayList;
1919
import java.util.List;
20+
import lombok.experimental.UtilityClass;
2021

2122
/** Utility class for parsing SQL scripts. */
23+
@UtilityClass
2224
class SqlUtils {
2325

2426
private static final String STATEMENT_DELIMITER = ";"; // a statement should end with `;`

‎src/test/java/com/datasqrl/EnvironmentVariablesUtilsTest.java

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
/*
2+
* Copyright © 2024 DataSQRL (contact@datasqrl.com)
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.datasqrl;
17+
18+
import static org.assertj.core.api.Assertions.assertThat;
19+
import static org.assertj.core.api.Assertions.assertThatThrownBy;
20+
21+
import java.util.Map;
22+
import java.util.Set;
23+
import org.junit.jupiter.params.ParameterizedTest;
24+
import org.junit.jupiter.params.provider.CsvSource;
25+
26+
class EnvironmentVariablesUtilsTest {
27+
28+
@ParameterizedTest
29+
@CsvSource({
30+
"'Hello, ${USER}', 'Hello, John'", // Positive case: USER variable exists
31+
"'Path: ${PATH}', 'Path: /usr/bin'", // Positive case: PATH variable exists
32+
"'No match here', 'No match here'", // Case with no placeholders
33+
"'Partial ${USER_HOME', 'Partial ${USER_HOME'" // Case to ensure partial match is not replaced
34+
})
35+
void givenEnvVariables_whenReplaceWithEnv_thenReplaceCorrectly(String command, String expected) {
36+
Map<String, String> envVariables =
37+
Map.of(
38+
"USER", "John",
39+
"PATH", "/usr/bin");
40+
String result = EnvironmentVariablesUtils.replaceWithEnv(command, envVariables);
41+
assertThat(result).isEqualTo(expected);
42+
}
43+
44+
@ParameterizedTest
45+
@CsvSource({
46+
"'Hello, ${USER}'", // USER variable missing
47+
"'Path: ${UNKNOWN}'", // UNKNOWN variable missing
48+
"'Combined ${VAR1} and ${VAR2}'" // Multiple missing variables
49+
})
50+
void givenMissingEnvVariables_whenReplaceWithEnv_thenThrowException(String command) {
51+
Map<String, String> envVariables = Map.of(); // Empty map to simulate missing variables
52+
assertThatThrownBy(() -> EnvironmentVariablesUtils.replaceWithEnv(command, envVariables))
53+
.isInstanceOf(IllegalStateException.class)
54+
.hasMessageContaining("Missing environment variable");
55+
}
56+
57+
@ParameterizedTest
58+
@CsvSource({
59+
"'${USER} is different from ${USER_NAME}'",
60+
"'${NAME} is different from ${USER_NAME}'"
61+
})
62+
void givenSimilarEnvVariableNames_whenReplaceWithEnv_thenPartialMatchDoesNotOccur(
63+
String command) {
64+
Map<String, String> envVariables =
65+
Map.of(
66+
"USER", "John",
67+
"NAME", "exists");
68+
assertThatThrownBy(() -> EnvironmentVariablesUtils.replaceWithEnv(command, envVariables))
69+
.isInstanceOf(IllegalStateException.class)
70+
.hasMessageContaining("Missing environment variable: USER_NAME");
71+
}
72+
73+
@ParameterizedTest
74+
@CsvSource({"'Run with ${VAR1} and ${VAR2}'", "'Execute ${USER} ${HOME}'", "'Simple command'"})
75+
void givenAllEnvVariablesInScript_whenValidateEnvironmentVariables_thenReturnEmptySet(
76+
String script) {
77+
Map<String, String> envVariables =
78+
Map.of(
79+
"VAR1", "1",
80+
"VAR2", "2",
81+
"USER", "admin",
82+
"HOME", "/home/admin");
83+
Set<String> missingVars =
84+
EnvironmentVariablesUtils.validateEnvironmentVariables(envVariables, script);
85+
assertThat(missingVars).isEmpty();
86+
}
87+
88+
@ParameterizedTest
89+
@CsvSource({
90+
"'Run with ${VAR1} and ${VAR2}', 'VAR2'",
91+
"'Execute ${USER} ${HOME} ${PATH}', 'PATH'",
92+
"'${MISSING_VAR}', 'MISSING_VAR'"
93+
})
94+
void givenSomeEnvVariablesInScript_whenValidateEnvironmentVariables_thenReturnMissingVariables(
95+
String script, String expectedMissing) {
96+
Map<String, String> envVariables =
97+
Map.of(
98+
"VAR1", "1",
99+
"USER", "admin",
100+
"HOME", "/home/admin");
101+
Set<String> missingVars =
102+
EnvironmentVariablesUtils.validateEnvironmentVariables(envVariables, script);
103+
assertThat(missingVars).containsExactlyInAnyOrder(expectedMissing.split(","));
104+
}
105+
106+
@ParameterizedTest
107+
@CsvSource({
108+
"'${USER} is different from ${USER_NAME}'",
109+
"'${NAME} is different from ${USER_NAME}'"
110+
})
111+
void givenSimilarVariableNames_whenValidateEnvironmentVariables_thenNoPartialMatch(
112+
String script) {
113+
Map<String, String> envVariables =
114+
Map.of(
115+
"USER", "admin",
116+
"NAME", "admin");
117+
Set<String> missingVars =
118+
EnvironmentVariablesUtils.validateEnvironmentVariables(envVariables, script);
119+
assertThat(missingVars)
120+
.containsExactly("USER_NAME"); // Ensure only USERNAME is reported missing
121+
}
122+
}

0 commit comments

Comments
 (0)