Skip to content

Commit f5faeb3

Browse files
cgivrecgivre
committed
WIP
1 parent f9b5036 commit f5faeb3

File tree

5 files changed

+172
-173
lines changed

5 files changed

+172
-173
lines changed

‎exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java

Lines changed: 24 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -23,99 +23,27 @@
2323
import org.apache.drill.exec.expr.DrillSimpleFunc;
2424
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
2525
import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
26+
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
2627
import org.apache.drill.exec.expr.annotations.Output;
2728
import org.apache.drill.exec.expr.annotations.Param;
2829
import org.apache.drill.exec.expr.annotations.Workspace;
2930
import org.apache.drill.exec.expr.holders.NullableVarBinaryHolder;
30-
import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
31-
import org.apache.drill.exec.expr.holders.VarBinaryHolder;
3231
import org.apache.drill.exec.expr.holders.VarCharHolder;
3332
import org.apache.drill.exec.physical.resultSet.ResultSetLoader;
3433
import org.apache.drill.exec.server.options.OptionManager;
34+
import org.apache.drill.exec.vector.complex.writer.BaseWriter;
3535
import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
3636

3737
public class JsonConvertFrom {
3838

39-
private JsonConvertFrom() {
40-
}
41-
42-
@FunctionTemplate(name = "convert_fromJSON", scope = FunctionScope.SIMPLE, isRandom = true)
43-
public static class ConvertFromJson implements DrillSimpleFunc {
44-
45-
@Param VarBinaryHolder in;
46-
@Inject
47-
ResultSetLoader loader;
48-
@Workspace
49-
org.apache.drill.exec.store.easy.json.loader.JsonLoaderImpl.JsonLoaderBuilder jsonLoaderBuilder;
50-
51-
@Inject
52-
OptionManager options;
53-
54-
@Output ComplexWriter writer;
55-
56-
@Override
57-
public void setup() {
58-
jsonLoaderBuilder = new org.apache.drill.exec.store.easy.json.loader.JsonLoaderImpl.JsonLoaderBuilder()
59-
.resultSetLoader(loader)
60-
.standardOptions(options);
61-
}
62-
63-
@Override
64-
public void eval() {
65-
try {
66-
jsonLoaderBuilder.fromStream(in.start, in.end, in.buffer);
67-
org.apache.drill.exec.store.easy.json.loader.JsonLoader jsonLoader = jsonLoaderBuilder.build();
68-
loader.startBatch();
69-
jsonLoader.readBatch();
70-
loader.close();
71-
72-
} catch (Exception e) {
73-
throw new org.apache.drill.common.exceptions.DrillRuntimeException("Error while converting from JSON. ", e);
74-
}
75-
}
76-
}
77-
78-
@FunctionTemplate(name = "convert_fromJSON", scope = FunctionScope.SIMPLE, isRandom = true)
79-
public static class ConvertFromJsonVarchar implements DrillSimpleFunc {
80-
81-
@Param VarCharHolder in;
82-
@Workspace
83-
org.apache.drill.exec.store.easy.json.loader.JsonLoaderImpl.JsonLoaderBuilder jsonLoaderBuilder;
84-
85-
@Inject
86-
OptionManager options;
87-
88-
@Inject
89-
ResultSetLoader loader;
90-
91-
@Output ComplexWriter writer;
92-
93-
@Override
94-
public void setup() {
95-
jsonLoaderBuilder = new org.apache.drill.exec.store.easy.json.loader.JsonLoaderImpl.JsonLoaderBuilder()
96-
.resultSetLoader(loader)
97-
.standardOptions(options);
98-
}
99-
100-
@Override
101-
public void eval() {
102-
try {
103-
jsonLoaderBuilder.fromStream(in.start, in.end, in.buffer);
104-
org.apache.drill.exec.store.easy.json.loader.JsonLoader jsonLoader = jsonLoaderBuilder.build();
105-
loader.startBatch();
106-
jsonLoader.readBatch();
107-
loader.close();
108-
109-
} catch (Exception e) {
110-
throw new org.apache.drill.common.exceptions.DrillRuntimeException("Error while converting from JSON. ", e);
111-
}
112-
}
113-
}
39+
private JsonConvertFrom() {}
11440

115-
@FunctionTemplate(name = "convert_fromJSON", scope = FunctionScope.SIMPLE, isRandom = true)
41+
@FunctionTemplate(names = {"convert_fromJSON", "convertFromJson", "convert_from_json"},
42+
scope = FunctionScope.SIMPLE, nulls = NullHandling.INTERNAL)
11643
public static class ConvertFromJsonNullableInput implements DrillSimpleFunc {
11744

118-
@Param NullableVarBinaryHolder in;
45+
@Param
46+
NullableVarBinaryHolder in;
11947

12048
@Workspace
12149
org.apache.drill.exec.store.easy.json.loader.JsonLoaderImpl.JsonLoaderBuilder jsonLoaderBuilder;
@@ -126,7 +54,8 @@ public static class ConvertFromJsonNullableInput implements DrillSimpleFunc {
12654
@Inject
12755
ResultSetLoader loader;
12856

129-
@Output ComplexWriter writer;
57+
@Output
58+
BaseWriter.ComplexWriter writer;
13059

13160
@Override
13261
public void setup() {
@@ -137,7 +66,7 @@ public void setup() {
13766

13867
@Override
13968
public void eval() {
140-
if (in.isSet == 0) {
69+
if (in.end == 0) {
14170
// Return empty map
14271
org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter mapWriter = writer.rootAsMap();
14372
mapWriter.start();
@@ -157,10 +86,15 @@ public void eval() {
15786
}
15887
}
15988

160-
@FunctionTemplate(name = "convert_fromJSON", scope = FunctionScope.SIMPLE, isRandom = true)
161-
public static class ConvertFromJsonVarcharNullableInput implements DrillSimpleFunc {
89+
@FunctionTemplate(names = {"convert_fromJSON", "convertFromJson", "convert_from_json"},
90+
scope = FunctionScope.SIMPLE)
91+
public static class ConvertFromJsonVarcharInput implements DrillSimpleFunc {
16292

163-
@Param NullableVarCharHolder in;
93+
@Param
94+
VarCharHolder in;
95+
96+
@Output
97+
ComplexWriter writer;
16498

16599
@Workspace
166100
org.apache.drill.exec.store.easy.json.loader.JsonLoaderImpl.JsonLoaderBuilder jsonLoaderBuilder;
@@ -171,27 +105,27 @@ public static class ConvertFromJsonVarcharNullableInput implements DrillSimpleFu
171105
@Inject
172106
ResultSetLoader loader;
173107

174-
@Output ComplexWriter writer;
175-
176108
@Override
177109
public void setup() {
178-
jsonLoaderBuilder = new org.apache.drill.exec.store.easy.json.loader.JsonLoaderImpl.JsonLoaderBuilder()
110+
jsonLoaderBuilder = new org.apache.drill.exec.store.easy.json.loader.JsonLoaderImpl.JsonLoaderBuilder()
179111
.resultSetLoader(loader)
180112
.standardOptions(options);
181113
}
182114

183115
@Override
184116
public void eval() {
185-
if (in.isSet == 0) {
186-
// Return empty map
117+
String jsonString = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(in.start, in.end, in.buffer);
118+
119+
// If the input is null or empty, return an empty map
120+
if (jsonString.length() == 0) {
187121
org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter mapWriter = writer.rootAsMap();
188122
mapWriter.start();
189123
mapWriter.end();
190124
return;
191125
}
192126

193127
try {
194-
jsonLoaderBuilder.fromStream(in.start, in.end, in.buffer);
128+
jsonLoaderBuilder.fromString(jsonString);
195129
org.apache.drill.exec.store.easy.json.loader.JsonLoader jsonLoader = jsonLoaderBuilder.build();
196130
loader.startBatch();
197131
jsonLoader.readBatch();

‎exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonConversionUDF.java

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.drill.exec.store.json;
20+
21+
22+
import ch.qos.logback.classic.Level;
23+
import org.apache.drill.exec.ExecConstants;
24+
import org.apache.drill.exec.physical.impl.project.ProjectRecordBatch;
25+
import org.apache.drill.exec.physical.impl.validate.IteratorValidatorBatchIterator;
26+
import org.apache.drill.exec.physical.rowSet.RowSet;
27+
import org.apache.drill.exec.store.easy.json.loader.JsonLoaderImpl;
28+
import org.apache.drill.test.ClusterFixture;
29+
import org.apache.drill.test.ClusterTest;
30+
import org.apache.drill.test.LogFixture;
31+
import org.junit.BeforeClass;
32+
import org.junit.Test;
33+
34+
import static org.junit.Assert.assertEquals;
35+
36+
37+
public class TestJsonConversionUDF extends ClusterTest {
38+
39+
protected static LogFixture logFixture;
40+
private final static Level CURRENT_LOG_LEVEL = Level.DEBUG;
41+
@BeforeClass
42+
public static void setup() throws Exception {
43+
logFixture = LogFixture.builder()
44+
.toConsole()
45+
.logger(ProjectRecordBatch.class, CURRENT_LOG_LEVEL)
46+
.logger(JsonLoaderImpl.class, CURRENT_LOG_LEVEL)
47+
.logger(IteratorValidatorBatchIterator.class, CURRENT_LOG_LEVEL)
48+
.build();
49+
50+
startCluster(ClusterFixture.builder(dirTestWatcher));
51+
}
52+
53+
@Test
54+
public void testConvertFromJsonFunctionWithBinaryInput() throws Exception {
55+
client.alterSession(ExecConstants.JSON_READER_NAN_INF_NUMBERS, true);
56+
String sql = "SELECT string_binary(convert_toJSON(convert_fromJSON(columns[1]))) as col FROM cp.`jsoninput/nan_test.csv`";
57+
RowSet results = client.queryBuilder().sql(sql).rowSet();
58+
assertEquals("Query result must contain 1 row", 1, results.rowCount());
59+
60+
results.print();
61+
}
62+
63+
@Test
64+
public void testConvertFromJSONWithStringInput() throws Exception {
65+
// String sql = "SELECT *, convert_FromJSON('{\"foo\":\"bar\"}') FROM cp.`jsoninput/allTypes.csv`";
66+
String sql = "SELECT convert_FromJSON('{\"foo\":\"bar\"}') FROM (VALUES(1))";
67+
RowSet results = client.queryBuilder().sql(sql).rowSet();
68+
results.print();
69+
}
70+
71+
/*
72+
private void doTestConvertToJsonFunction() throws Exception {
73+
String table = "nan_test.csv";
74+
File file = new File(dirTestWatcher.getRootDir(), table);
75+
String csv = "col_0, {\"nan_col\":NaN}";
76+
String query = String.format("select string_binary(convert_toJSON(convert_fromJSON(columns[1]))) as col " +
77+
"from dfs.`%s` where columns[0]='col_0'", table);
78+
try {
79+
FileUtils.writeStringToFile(file, csv, Charset.defaultCharset());
80+
List<QueryDataBatch> results = testSqlWithResults(query);
81+
RecordBatchLoader batchLoader = new RecordBatchLoader(getAllocator());
82+
assertEquals("Query result must contain 1 row", 1, results.size());
83+
QueryDataBatch batch = results.get(0);
84+
85+
batchLoader.load(batch.getHeader().getDef(), batch.getData());
86+
VectorWrapper<?> vw = batchLoader.getValueAccessorById(VarCharVector.class, batchLoader.getValueVectorId(SchemaPath.getCompoundPath("col")).getFieldIds());
87+
// ensuring that `NaN` token ARE NOT enclosed with double quotes
88+
String resultJson = vw.getValueVector().getAccessor().getObject(0).toString();
89+
int nanIndex = resultJson.indexOf("NaN");
90+
assertNotEquals("`NaN` must not be enclosed with \"\" ", '"', resultJson.charAt(nanIndex - 1));
91+
assertNotEquals("`NaN` must not be enclosed with \"\" ", '"', resultJson.charAt(nanIndex + "NaN".length()));
92+
batch.release();
93+
batchLoader.clear();
94+
} finally {
95+
FileUtils.deleteQuietly(file);
96+
}
97+
}
98+
99+
@Test
100+
public void testConvertFromJsonFunction() throws Exception {
101+
//runBoth(this::doTestConvertFromJsonFunction);
102+
}
103+
104+
private void doTestConvertFromJsonFunction() throws Exception {
105+
String table = "nan_test.csv";
106+
File file = new File(dirTestWatcher.getRootDir(), table);
107+
String csv = "col_0, {\"nan_col\":NaN}";
108+
try {
109+
FileUtils.writeStringToFile(file, csv);
110+
testBuilder()
111+
.sqlQuery(String.format("select convert_fromJSON(columns[1]) as col from dfs.`%s`", table))
112+
.unOrdered()
113+
.baselineColumns("col")
114+
.baselineValues(mapOf("nan_col", Double.NaN))
115+
.go();
116+
} finally {
117+
FileUtils.deleteQuietly(file);
118+
}
119+
}
120+
*/
121+
122+
}

0 commit comments

Comments
 (0)