This is an automated email from the ASF dual-hosted git repository.
jamesbognar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/juneau.git
The following commit(s) were added to refs/heads/master by this push:
new adc877845b Parquet fix
adc877845b is described below
commit adc877845bc54df345e279d9533308f317fe8644
Author: James Bognar <[email protected]>
AuthorDate: Wed Mar 18 20:30:39 2026 -0400
Parquet fix
---
.../juneau/markdown/MarkdownParserSession.java | 8 +++--
.../org/apache/juneau/parquet/ParquetParser.java | 5 ++++
.../juneau/parquet/ParquetParserSession.java | 33 ++++++++++++--------
.../apache/juneau/parquet/ParquetSerializer.java | 5 ++++
.../juneau/parquet/ParquetSerializerSession.java | 6 ++--
.../a/rttests/TrimStrings_RoundTripTest.java | 35 ++++++++++------------
6 files changed, 56 insertions(+), 36 deletions(-)
diff --git
a/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/markdown/MarkdownParserSession.java
b/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/markdown/MarkdownParserSession.java
index bc974776d3..a8ceb87b17 100644
---
a/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/markdown/MarkdownParserSession.java
+++
b/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/markdown/MarkdownParserSession.java
@@ -593,8 +593,12 @@ public class MarkdownParserSession extends
ReaderParserSession {
* Returns a Json5Parser that shares this session's bean context
(swaps, implClasses, dictionary).
*/
private JsonParser getJson5Parser() {
- if (json5Parser == null)
- json5Parser =
Json5Parser.create().beanContext((BeanContext) getContext()).build();
+ if (json5Parser == null) {
+ var b = Json5Parser.create().beanContext((BeanContext)
getContext());
+ if (isTrimStrings())
+ b = b.trimStrings();
+ json5Parser = b.build();
+ }
return json5Parser;
}
diff --git
a/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/parquet/ParquetParser.java
b/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/parquet/ParquetParser.java
index 346f4dec75..cbf3399173 100644
---
a/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/parquet/ParquetParser.java
+++
b/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/parquet/ParquetParser.java
@@ -128,6 +128,11 @@ public class ParquetParser extends InputStreamParser
implements ParquetMetaProvi
nullKeyString = builder.nullKeyString;
}
+ @Override /* Overridden from Context */
+ public Builder copy() {
+ return new Builder(this);
+ }
+
@Override
public ParquetParserSession.Builder createSession() {
return ParquetParserSession.create(this);
diff --git
a/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/parquet/ParquetParserSession.java
b/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/parquet/ParquetParserSession.java
index c282ad3e9a..995a648d58 100644
---
a/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/parquet/ParquetParserSession.java
+++
b/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/parquet/ParquetParserSession.java
@@ -446,12 +446,13 @@ public class ParquetParserSession extends
InputStreamParserSession {
for (var cc : firstGroup.columns()) {
var path = String.join(".", cc.pathInSchema());
List<Object> values;
+ var trim = isTrimStrings();
if (isListColumnPath(path))
- values = readListColumnChunk(fileBytes, cc,
numRows);
+ values = readListColumnChunk(fileBytes, cc,
numRows, trim);
else if (isMapKeyValueColumnPath(path))
- values = readMapKeyValueColumnChunk(fileBytes,
cc, numRows);
+ values = readMapKeyValueColumnChunk(fileBytes,
cc, numRows, trim);
else
- values = readColumnChunk(fileBytes, cc,
numRows, schemaRepetition);
+ values = readColumnChunk(fileBytes, cc,
numRows, schemaRepetition, trim);
columnData.put(path, values);
}
if (parquetDebug()) {
@@ -507,7 +508,7 @@ public class ParquetParserSession extends
InputStreamParserSession {
return idx < 0 ? listColumnPath : listColumnPath.substring(0,
idx);
}
- private static List<Object> readListColumnChunk(byte[] fileBytes,
ColumnChunkMeta cc, int numRows) throws ParseException {
+ private static List<Object> readListColumnChunk(byte[] fileBytes,
ColumnChunkMeta cc, int numRows, boolean trimStrings) throws ParseException {
try {
var path = String.join(".", cc.pathInSchema());
var rowRelPath = rowRelativePath(path);
@@ -584,7 +585,7 @@ public class ParquetParserSession extends
InputStreamParserSession {
int def = defDecoder.readInt();
Object val;
if (def >= maxDef) {
- val = readValue(valueReader, cc.type());
+ val = readValue(valueReader, cc.type(),
trimStrings);
} else {
val = null;
}
@@ -710,7 +711,7 @@ public class ParquetParserSession extends
InputStreamParserSession {
private static final int MAP_MAX_DEF = 2;
private static final int MAP_MAX_REP = 1;
- private static List<Object> readMapKeyValueColumnChunk(byte[]
fileBytes, ColumnChunkMeta cc, int numRows) throws ParseException {
+ private static List<Object> readMapKeyValueColumnChunk(byte[]
fileBytes, ColumnChunkMeta cc, int numRows, boolean trimStrings) throws
ParseException {
try {
int maxDef = MAP_MAX_DEF;
int maxRep = MAP_MAX_REP;
@@ -783,7 +784,7 @@ public class ParquetParserSession extends
InputStreamParserSession {
int def = defDecoder.readInt();
Object val;
if (def >= maxDef) {
- val = readValue(valueReader, cc.type());
+ val = readValue(valueReader, cc.type(),
trimStrings);
} else {
val = null;
}
@@ -827,7 +828,7 @@ public class ParquetParserSession extends
InputStreamParserSession {
return result;
}
- private static List<Object> readColumnChunk(byte[] fileBytes,
ColumnChunkMeta cc, int numRows, Map<String, Integer> schemaRepetition) throws
ParseException {
+ private static List<Object> readColumnChunk(byte[] fileBytes,
ColumnChunkMeta cc, int numRows, Map<String, Integer> schemaRepetition, boolean
trimStrings) throws ParseException {
try {
int off = (int)cc.dataPageOffset();
var bais = new ByteArrayInputStream(fileBytes, off,
fileBytes.length - off);
@@ -876,7 +877,7 @@ public class ParquetParserSession extends
InputStreamParserSession {
var reader = new ParquetColumnReader(decompressed,
valuesToRead, maxDefLevel);
var values = new ArrayList<>();
while (reader.hasNext()) {
- values.add(readValue(reader, cc.type()));
+ values.add(readValue(reader, cc.type(),
trimStrings));
}
if (parquetDebug())
parquetDebugLog("readColumnChunk values: path="
+ path + " values=" + values);
@@ -886,7 +887,7 @@ public class ParquetParserSession extends
InputStreamParserSession {
}
}
- private static Object readValue(ParquetColumnReader reader, int type)
throws IOException {
+ private static Object readValue(ParquetColumnReader reader, int type,
boolean trimStrings) throws IOException {
reader.advance();
if (reader.isNull())
return null;
@@ -896,9 +897,15 @@ public class ParquetParserSession extends
InputStreamParserSession {
case TYPE_INT64 -> reader.readInt64();
case TYPE_FLOAT -> reader.readFloat();
case TYPE_DOUBLE -> reader.readDouble();
- case TYPE_BYTE_ARRAY -> reader.readByteArrayAsString();
+ case TYPE_BYTE_ARRAY -> {
+ var s = reader.readByteArrayAsString();
+ yield trimStrings && s != null ? s.trim() : s;
+ }
case TYPE_FIXED_LEN_BYTE_ARRAY ->
reader.readFixedLenByteArray(16);
- default -> reader.readByteArrayAsString();
+ default -> {
+ var s = reader.readByteArrayAsString();
+ yield trimStrings && s != null ? s.trim() : s;
+ }
};
}
@@ -917,6 +924,8 @@ public class ParquetParserSession extends
InputStreamParserSession {
var values = e.getValue();
var v = i < values.size() ? values.get(i) :
null;
var path = isListColumnPath(fullPath) ?
listProp : rowRelPath;
+ if (isTrimStrings())
+ path = path.trim();
setByPath(row, path, v);
}
for (var e : listBeanColumns.entrySet()) {
diff --git
a/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/parquet/ParquetSerializer.java
b/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/parquet/ParquetSerializer.java
index 305650d66e..254a613acb 100644
---
a/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/parquet/ParquetSerializer.java
+++
b/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/parquet/ParquetSerializer.java
@@ -239,6 +239,11 @@ public class ParquetSerializer extends
OutputStreamSerializer implements Parquet
nullKeyString = builder.nullKeyString;
}
+ @Override /* Overridden from Context */
+ public Builder copy() {
+ return new Builder(this);
+ }
+
@Override
public ParquetSerializerSession.Builder createSession() {
return ParquetSerializerSession.create(this);
diff --git
a/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/parquet/ParquetSerializerSession.java
b/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/parquet/ParquetSerializerSession.java
index 5cb13867e9..66db8fa731 100644
---
a/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/parquet/ParquetSerializerSession.java
+++
b/juneau-core/juneau-marshall/src/main/java/org/apache/juneau/parquet/ParquetSerializerSession.java
@@ -194,7 +194,7 @@ public class ParquetSerializerSession extends
OutputStreamSerializerSession {
// Replace null keys with the null-key sentinel
so they survive as a named column in the schema.
var swappedMap = new LinkedHashMap<>();
for (var e : m.entrySet()) {
- var key = e.getKey() == null ?
ctx.nullKeyString : e.getKey();
+ var key = e.getKey() == null ?
ctx.nullKeyString : trim(e.getKey());
var val = e.getValue();
if (val != null)
val = applySwap(val,
getClassMetaForObject(val));
@@ -444,7 +444,7 @@ public class ParquetSerializerSession extends
OutputStreamSerializerSession {
}
if (key instanceof Enum<?> e)
return ctx.getBeanContext().isUseEnumNames() ? e.name()
: e.toString();
- return String.valueOf(generalize(key,
getClassMetaForObject(key)));
+ return trim(String.valueOf(generalize(key,
getClassMetaForObject(key))));
}
private List<FlattenedEntry> extractFlattenedListValues(List<?> rows,
String path) throws SerializeException {
@@ -735,7 +735,7 @@ public class ParquetSerializerSession extends
OutputStreamSerializerSession {
if (v instanceof byte[] b)
return b;
if (v instanceof String s)
- return s.getBytes(StandardCharsets.UTF_8);
+ return trim(s).getBytes(StandardCharsets.UTF_8);
if (v instanceof Enum<?> e)
return (ctx.getBeanContext().isUseEnumNames() ?
e.name() : e.toString()).getBytes(StandardCharsets.UTF_8);
if (v instanceof Duration d)
diff --git
a/juneau-utest/src/test/java/org/apache/juneau/a/rttests/TrimStrings_RoundTripTest.java
b/juneau-utest/src/test/java/org/apache/juneau/a/rttests/TrimStrings_RoundTripTest.java
index da81f4307e..63070cda83 100755
---
a/juneau-utest/src/test/java/org/apache/juneau/a/rttests/TrimStrings_RoundTripTest.java
+++
b/juneau-utest/src/test/java/org/apache/juneau/a/rttests/TrimStrings_RoundTripTest.java
@@ -21,7 +21,6 @@ import static
org.apache.juneau.commons.utils.CollectionUtils.*;
import static org.junit.jupiter.api.Assertions.*;
import org.apache.juneau.collections.*;
-import org.apache.juneau.markdown.*;
import org.apache.juneau.parquet.*;
import org.apache.juneau.parser.*;
import org.apache.juneau.serializer.*;
@@ -42,12 +41,6 @@ class TrimStrings_RoundTripTest extends RoundTripTest_Base {
void a01_basic(RoundTrip_Tester t) throws Exception {
if (t.isValidationOnly())
return;
- // TODO - Skip Markdown - inline JSON5 trimStrings behavior
needs investigation
- if (t.getSerializer() instanceof MarkdownSerializer)
- return;
- // TODO - Skip Parquet - copy() returns Serializer.Builder
incompatible with ParquetSerializer constructor
- if (t.getSerializer() instanceof ParquetSerializer)
- return;
var s = t.getSerializer();
var p = t.getParser();
@@ -68,19 +61,23 @@ class TrimStrings_RoundTripTest extends RoundTripTest_Base {
a = p2.parse(s.serialize(in), JsonMap.class);
assertEquals(json(a), json(e));
- in = new JsonList("[' foo ', {' foo ': ' bar '}]");
- e = new JsonList("['foo',{foo:'bar'}]");
- a = p.parse(s2.serialize(in), JsonList.class);
- assertEquals(json(a), json(e));
- a = p2.parse(s.serialize(in), JsonList.class);
- assertEquals(json(a), json(e));
+ // Skip Parquet for mixed-type JsonList and bean with
JsonList/JsonMap properties.
+ // Parquet's static schema cannot represent mixed-type
collections (string + map in same list).
+ if (!(s instanceof ParquetSerializer)) {
+ in = new JsonList("[' foo ', {' foo ': ' bar '}]");
+ e = new JsonList("['foo',{foo:'bar'}]");
+ a = p.parse(s2.serialize(in), JsonList.class);
+ assertEquals(json(a), json(e));
+ a = p2.parse(s.serialize(in), JsonList.class);
+ assertEquals(json(a), json(e));
- in = new A().init1();
- e = new A().init2();
- a = p.parse(s2.serialize(in), A.class);
- assertEquals(json(a), json(e));
- a = p2.parse(s.serialize(in), A.class);
- assertEquals(json(a), json(e));
+ in = new A().init1();
+ e = new A().init2();
+ a = p.parse(s2.serialize(in), A.class);
+ assertEquals(json(a), json(e));
+ a = p2.parse(s.serialize(in), A.class);
+ assertEquals(json(a), json(e));
+ }
}
public static class A {
