gene-db commented on code in PR #3202: URL: https://github.com/apache/parquet-java/pull/3202#discussion_r2072179064
########## parquet-variant/src/main/java/org/apache/parquet/variant/VariantArrayBuilder.java: ########## @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.parquet.variant; + +import java.util.ArrayList; + +/** + * Builder for creating Variant arrays, used by VariantBuilder. + */ +public class VariantArrayBuilder extends VariantBuilder { + /** The parent VariantBuilder. */ + private final VariantBuilder parent; + /** The offsets of the elements in this array. */ + private final ArrayList<Integer> offsets; + /** The number of values appended to this array. */ + protected long numValues = 0; + + VariantArrayBuilder(VariantBuilder parent) { + this.parent = parent; + this.offsets = new ArrayList<>(); + } + + /** + * @return the list of element offsets in this array + */ + ArrayList<Integer> validateAndGetOffsets() { + if (offsets.size() != numValues) { + throw new IllegalStateException(String.format( + "Number of offsets (%d) do not match the number of values (%d).", offsets.size(), numValues)); + } + return offsets; + } + + @Override + protected void onAppend() { + checkAppendWhileNested(); + offsets.add(writePos); + } + + @Override + protected void onStartNested() { + checkMultipleNested(); + offsets.add(writePos); + } + + @Override + protected void incrementNumValues() { Review Comment: Moved to `onAppend`. ########## parquet-variant/src/main/java/org/apache/parquet/variant/VariantObjectBuilder.java: ########## @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.parquet.variant; + +import java.util.ArrayList; + +/** + * Builder for creating Variant object, used by VariantBuilder. + */ +public class VariantObjectBuilder extends VariantBuilder { + /** The parent VariantBuilder. */ + private final VariantBuilder parent; + /** The FieldEntry list for the fields of this object. */ + private final ArrayList<VariantBuilder.FieldEntry> fields; + /** The number of values appended to this object. */ + protected long numValues = 0; + + VariantObjectBuilder(VariantBuilder parent) { + this.parent = parent; + this.fields = new ArrayList<>(); + } + + /** + * Appends an object key to this object. This method must be called before appending any value. + * @param key the key to append + */ + void appendKey(String key) { + if (fields.size() > numValues) { + throw new IllegalStateException("Cannot call appendKey() before appending an object value."); Review Comment: Yes, updated. ########## parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java: ########## @@ -0,0 +1,590 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.parquet.variant; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; + +/** + * Builder for creating Variant value and metadata. + */ +public class VariantBuilder { + /** + * Creates a VariantBuilder. + * @param allowDuplicateKeys if true, only the last occurrence of a duplicate key will be kept. + * Otherwise, an exception will be thrown. + */ + public VariantBuilder(boolean allowDuplicateKeys) { + this.allowDuplicateKeys = allowDuplicateKeys; + } + + /** + * @return the Variant value + */ + public Variant build() { + int numKeys = dictionaryKeys.size(); + // Use long to avoid overflow in accumulating lengths. + long dictionaryStringSize = 0; + for (byte[] key : dictionaryKeys) { + dictionaryStringSize += key.length; + } + // Determine the number of bytes required per offset entry. + // The largest offset is the one-past-the-end value, which is total string size. It's very + // unlikely that the number of keys could be larger, but incorporate that into the calculation + // in case of pathological data. + long maxSize = Math.max(dictionaryStringSize, numKeys); + int offsetSize = getMinIntegerSize((int) maxSize); + + int offsetStart = 1 + offsetSize; + int stringStart = offsetStart + (numKeys + 1) * offsetSize; + long metadataSize = stringStart + dictionaryStringSize; + + byte[] metadata = new byte[(int) metadataSize]; + int headerByte = VariantUtil.VERSION | ((offsetSize - 1) << 6); + VariantUtil.writeLong(metadata, 0, headerByte, 1); + VariantUtil.writeLong(metadata, 1, numKeys, offsetSize); + int currentOffset = 0; + for (int i = 0; i < numKeys; ++i) { + VariantUtil.writeLong(metadata, offsetStart + i * offsetSize, currentOffset, offsetSize); + byte[] key = dictionaryKeys.get(i); + System.arraycopy(key, 0, metadata, stringStart + currentOffset, key.length); + currentOffset += key.length; + } + VariantUtil.writeLong(metadata, offsetStart + numKeys * offsetSize, currentOffset, offsetSize); + return new Variant(Arrays.copyOfRange(writeBuffer, 0, writePos), metadata); Review Comment: Ok. Added a TODO to mention this. ########## parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java: ########## @@ -0,0 +1,658 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.parquet.variant; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; + +/** + * Builder for creating Variant value and metadata. + */ +public class VariantBuilder { + /** The buffer for building the Variant value. The first `writePos` bytes have been written. */ + protected byte[] writeBuffer = new byte[1024]; + + protected int writePos = 0; + /** The dictionary for mapping keys to monotonically increasing ids. */ + private final HashMap<String, Integer> dictionary = new HashMap<>(); + /** The keys in the dictionary, in id order. */ + private final ArrayList<byte[]> dictionaryKeys = new ArrayList<>(); + + /** + * These are used to build nested objects and arrays, via startObject() and startArray(). + * Only one of these can be non-null at a time. If one of these is non-null, then no append() + * methods can be called on this builder, until endObject() or endArray() is called. + */ + protected VariantObjectBuilder objectBuilder = null; + + protected VariantArrayBuilder arrayBuilder = null; + + /** + * Creates a VariantBuilder. + */ + public VariantBuilder() {} + + /** + * @return the Variant value + */ + public Variant build() { + if (objectBuilder != null) { + throw new IllegalStateException( + "Cannot call build() while an object is being built. Must call endObject() first."); + } + if (arrayBuilder != null) { + throw new IllegalStateException( + "Cannot call build() while an array is being built. Must call endArray() first."); + } + int numKeys = dictionaryKeys.size(); + // Use long to avoid overflow in accumulating lengths. + long dictionaryTotalDataSize = 0; + for (byte[] key : dictionaryKeys) { + dictionaryTotalDataSize += key.length; + } + // Determine the number of bytes required per offset entry. + // The largest offset is the one-past-the-end value, which is total data size. It's very + // unlikely that the number of keys could be larger, but incorporate that into the calculation + // in case of pathological data. + long maxSize = Math.max(dictionaryTotalDataSize, numKeys); + int offsetSize = getMinIntegerSize((int) maxSize); + + int offsetListOffset = 1 + offsetSize; + int dataOffset = offsetListOffset + (numKeys + 1) * offsetSize; + long metadataSize = dataOffset + dictionaryTotalDataSize; + + byte[] metadata = new byte[(int) metadataSize]; + // Only unsorted dictionary keys are supported. + // TODO: Support sorted dictionary keys. + int headerByte = VariantUtil.VERSION | ((offsetSize - 1) << 6); + VariantUtil.writeLong(metadata, 0, headerByte, 1); + VariantUtil.writeLong(metadata, 1, numKeys, offsetSize); + int currentOffset = 0; + for (int i = 0; i < numKeys; ++i) { + VariantUtil.writeLong(metadata, offsetListOffset + i * offsetSize, currentOffset, offsetSize); + byte[] key = dictionaryKeys.get(i); + System.arraycopy(key, 0, metadata, dataOffset + currentOffset, key.length); + currentOffset += key.length; + } + VariantUtil.writeLong(metadata, offsetListOffset + numKeys * offsetSize, currentOffset, offsetSize); + // Copying the data to a new buffer, to retain only the required data length, not the capacity. + return new Variant(Arrays.copyOfRange(writeBuffer, 0, writePos), metadata); + } + + /** + * Appends a string value to the Variant builder. + * @param str the string value to append + */ + public void appendString(String str) { + onAppend(); + byte[] data = str.getBytes(StandardCharsets.UTF_8); + boolean longStr = data.length > VariantUtil.MAX_SHORT_STR_SIZE; + checkCapacity((longStr ? 1 + VariantUtil.U32_SIZE : 1) + data.length); + if (longStr) { + writeBuffer[writePos] = VariantUtil.HEADER_LONG_STRING; + writePos += 1; + VariantUtil.writeLong(writeBuffer, writePos, data.length, VariantUtil.U32_SIZE); + writePos += VariantUtil.U32_SIZE; + } else { + writeBuffer[writePos] = VariantUtil.shortStrHeader(data.length); + writePos += 1; + } + System.arraycopy(data, 0, writeBuffer, writePos, data.length); + writePos += data.length; + incrementNumValues(); Review Comment: Added that additional validation for both the array builder and object builder, and added tests for those scenarios. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
