This is an automated email from the ASF dual-hosted git repository.
curth pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-dotnet.git
The following commit(s) were added to refs/heads/main by this push:
new d1c0a72 Implement largelistview support (#303)
d1c0a72 is described below
commit d1c0a72cda966037fb56daf7eac981b39e6e01fd
Author: Curt Hagenlocher <[email protected]>
AuthorDate: Tue Mar 31 06:16:44 2026 -0700
Implement largelistview support (#303)
## What's Changed
Support for largelistview has been implemented. This still doesn't
support lists which are actually more than 2GB, but at least smaller
lists of this type can be loaded.
---
src/Apache.Arrow/Arrays/ArrayDataConcatenator.cs | 51 ++++++
.../Arrays/ArrowArrayBuilderFactory.cs | 2 +
src/Apache.Arrow/Arrays/ArrowArrayFactory.cs | 2 +
.../{ListViewArray.cs => LargeListViewArray.cs} | 44 +++---
src/Apache.Arrow/Arrays/ListViewArray.cs | 1 +
src/Apache.Arrow/C/CArrowArrayImporter.cs | 30 ++++
src/Apache.Arrow/C/CArrowSchemaExporter.cs | 1 +
src/Apache.Arrow/C/CArrowSchemaImporter.cs | 11 +-
src/Apache.Arrow/Ipc/ArrowReaderImplementation.cs | 1 +
src/Apache.Arrow/Ipc/ArrowStreamWriter.cs | 69 ++++++++
src/Apache.Arrow/Ipc/ArrowTypeFlatbufferBuilder.cs | 9 ++
src/Apache.Arrow/Ipc/MessageSerializer.cs | 6 +
src/Apache.Arrow/Types/IArrowType.cs | 1 +
src/Apache.Arrow/Types/LargeListViewType.cs | 36 +++++
test/Apache.Arrow.IntegrationTest/JsonFile.cs | 36 +++++
test/Apache.Arrow.Tests/ArrayBuilderTests.cs | 29 ++++
.../ArrowArrayConcatenatorTests.cs | 13 ++
test/Apache.Arrow.Tests/ArrowReaderVerifier.cs | 41 +++++
.../LargeListViewArrayBuilderTests.cs | 156 +++++++++++++++++++
test/Apache.Arrow.Tests/LargeListViewArrayTests.cs | 173 +++++++++++++++++++++
test/Apache.Arrow.Tests/TableTests.cs | 4 +-
test/Apache.Arrow.Tests/TestData.cs | 34 ++++
22 files changed, 721 insertions(+), 29 deletions(-)
diff --git a/src/Apache.Arrow/Arrays/ArrayDataConcatenator.cs
b/src/Apache.Arrow/Arrays/ArrayDataConcatenator.cs
index a835435..68f9fff 100644
--- a/src/Apache.Arrow/Arrays/ArrayDataConcatenator.cs
+++ b/src/Apache.Arrow/Arrays/ArrayDataConcatenator.cs
@@ -58,6 +58,7 @@ namespace Apache.Arrow
IArrowTypeVisitor<LargeBinaryType>,
IArrowTypeVisitor<LargeStringType>,
IArrowTypeVisitor<LargeListType>,
+ IArrowTypeVisitor<LargeListViewType>,
IArrowTypeVisitor<MapType>
{
public ArrayData Result { get; private set; }
@@ -110,6 +111,56 @@ namespace Apache.Arrow
public void Visit(LargeListType type) =>
ConcatenateLargeLists(type);
+ public void Visit(LargeListViewType type)
+ {
+ CheckData(type, 3);
+ ArrowBuffer validityBuffer = ConcatenateValidityBuffer();
+ ArrowBuffer sizesBuffer =
ConcatenateFixedWidthTypeValueBuffer(2, Int64Type.Default);
+
+ var children = new List<ArrayData>(_arrayDataList.Count);
+ var offsetsBuilder = new
ArrowBuffer.Builder<long>(_totalLength);
+ long baseOffset = 0;
+
+ foreach (ArrayData arrayData in _arrayDataList)
+ {
+ if (arrayData.Length == 0)
+ {
+ continue;
+ }
+
+ var child = arrayData.Children[0];
+ ReadOnlySpan<long> offsets =
arrayData.Buffers[1].Span.CastTo<long>().Slice(arrayData.Offset,
arrayData.Length);
+ ReadOnlySpan<long> sizes =
arrayData.Buffers[2].Span.CastTo<long>().Slice(arrayData.Offset,
arrayData.Length);
+ var minOffset = offsets[0];
+ long maxEnd = 0;
+
+ for (int i = 0; i < arrayData.Length; ++i)
+ {
+ minOffset = Math.Min(minOffset, offsets[i]);
+ maxEnd = Math.Max(maxEnd, offsets[i] + sizes[i]);
+ }
+
+ foreach (long offset in offsets)
+ {
+ offsetsBuilder.Append(baseOffset + offset - minOffset);
+ }
+
+ var childLength = maxEnd - minOffset;
+ if (minOffset != 0 || childLength != child.Length)
+ {
+ child = child.Slice(checked((int)minOffset),
checked((int)childLength));
+ }
+
+ baseOffset += childLength;
+ children.Add(child);
+ }
+
+ ArrowBuffer offsetBuffer = offsetsBuilder.Build(_allocator);
+ ArrayData combinedChild = Concatenate(children, _allocator);
+
+ Result = new ArrayData(type, _totalLength, _totalNullCount, 0,
new ArrowBuffer[] { validityBuffer, offsetBuffer, sizesBuffer }, new[] {
combinedChild });
+ }
+
public void Visit(ListType type) => ConcatenateLists(type);
public void Visit(ListViewType type)
diff --git a/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs
b/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs
index f4f551c..ec0206a 100644
--- a/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs
+++ b/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs
@@ -101,6 +101,8 @@ namespace Apache.Arrow
return new LargeStringArray.Builder();
case ArrowTypeId.LargeList:
return new LargeListArray.Builder(dataType as
LargeListType);
+ case ArrowTypeId.LargeListView:
+ return new LargeListViewArray.Builder(dataType as
LargeListViewType);
case ArrowTypeId.Map:
return new MapArray.Builder(dataType as MapType);
case ArrowTypeId.Struct:
diff --git a/src/Apache.Arrow/Arrays/ArrowArrayFactory.cs
b/src/Apache.Arrow/Arrays/ArrowArrayFactory.cs
index ae5d7be..0f81336 100644
--- a/src/Apache.Arrow/Arrays/ArrowArrayFactory.cs
+++ b/src/Apache.Arrow/Arrays/ArrowArrayFactory.cs
@@ -71,6 +71,8 @@ namespace Apache.Arrow
return new ListViewArray(data);
case ArrowTypeId.LargeList:
return new LargeListArray(data);
+ case ArrowTypeId.LargeListView:
+ return new LargeListViewArray(data);
case ArrowTypeId.Map:
return new MapArray(data);
case ArrowTypeId.Struct:
diff --git a/src/Apache.Arrow/Arrays/ListViewArray.cs
b/src/Apache.Arrow/Arrays/LargeListViewArray.cs
similarity index 80%
copy from src/Apache.Arrow/Arrays/ListViewArray.cs
copy to src/Apache.Arrow/Arrays/LargeListViewArray.cs
index 7bbb292..240b731 100644
--- a/src/Apache.Arrow/Arrays/ListViewArray.cs
+++ b/src/Apache.Arrow/Arrays/LargeListViewArray.cs
@@ -1,4 +1,4 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
+// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
@@ -19,17 +19,17 @@ using Apache.Arrow.Types;
namespace Apache.Arrow
{
- public class ListViewArray : Array
+ public class LargeListViewArray : Array
{
- public class Builder : IArrowArrayBuilder<ListViewArray, Builder>
+ public class Builder : IArrowArrayBuilder<LargeListViewArray, Builder>
{
public IArrowArrayBuilder<IArrowArray,
IArrowArrayBuilder<IArrowArray>> ValueBuilder { get; }
public int Length => ValueOffsetsBufferBuilder.Length;
- private ArrowBuffer.Builder<int> ValueOffsetsBufferBuilder { get; }
+ private ArrowBuffer.Builder<long> ValueOffsetsBufferBuilder { get;
}
- private ArrowBuffer.Builder<int> SizesBufferBuilder { get; }
+ private ArrowBuffer.Builder<long> SizesBufferBuilder { get; }
private ArrowBuffer.BitmapBuilder ValidityBufferBuilder { get; }
@@ -39,19 +39,19 @@ namespace Apache.Arrow
private int Start { get; set; }
- public Builder(IArrowType valueDataType) : this(new
ListViewType(valueDataType))
+ public Builder(IArrowType valueDataType) : this(new
LargeListViewType(valueDataType))
{
}
- public Builder(Field valueField) : this(new
ListViewType(valueField))
+ public Builder(Field valueField) : this(new
LargeListViewType(valueField))
{
}
- internal Builder(ListViewType dataType)
+ internal Builder(LargeListViewType dataType)
{
ValueBuilder =
ArrowArrayBuilderFactory.Build(dataType.ValueDataType);
- ValueOffsetsBufferBuilder = new ArrowBuffer.Builder<int>();
- SizesBufferBuilder = new ArrowBuffer.Builder<int>();
+ ValueOffsetsBufferBuilder = new ArrowBuffer.Builder<long>();
+ SizesBufferBuilder = new ArrowBuffer.Builder<long>();
ValidityBufferBuilder = new ArrowBuffer.BitmapBuilder();
DataType = dataType;
Start = -1;
@@ -61,8 +61,7 @@ namespace Apache.Arrow
/// Start a new variable-length list slot
///
/// This function should be called before beginning to append
elements to the
- /// value builder. TODO: Consider adding builder APIs to support
construction
- /// of overlapping lists.
+ /// value builder.
/// </summary>
public Builder Append()
{
@@ -96,7 +95,7 @@ namespace Apache.Arrow
Start = ValueBuilder.Length;
}
- public ListViewArray Build(MemoryAllocator allocator = default)
+ public LargeListViewArray Build(MemoryAllocator allocator =
default)
{
AppendPrevious();
@@ -104,7 +103,7 @@ namespace Apache.Arrow
?
ValidityBufferBuilder.Build(allocator)
: ArrowBuffer.Empty;
- return new ListViewArray(DataType, Length,
+ return new LargeListViewArray(DataType, Length,
ValueOffsetsBufferBuilder.Build(allocator),
SizesBufferBuilder.Build(allocator),
ValueBuilder.Build(allocator),
validityBuffer, NullCount, 0);
@@ -133,6 +132,7 @@ namespace Apache.Arrow
ValueBuilder.Clear();
ValidityBufferBuilder.Clear();
NullCount = 0;
+ Start = -1;
return this;
}
}
@@ -141,13 +141,13 @@ namespace Apache.Arrow
public ArrowBuffer ValueOffsetsBuffer => Data.Buffers[1];
- public ReadOnlySpan<int> ValueOffsets =>
ValueOffsetsBuffer.Span.CastTo<int>().Slice(Offset, Length);
+ public ReadOnlySpan<long> ValueOffsets =>
ValueOffsetsBuffer.Span.CastTo<long>().Slice(Offset, Length);
public ArrowBuffer SizesBuffer => Data.Buffers[2];
- public ReadOnlySpan<int> Sizes =>
SizesBuffer.Span.CastTo<int>().Slice(Offset, Length);
+ public ReadOnlySpan<long> Sizes =>
SizesBuffer.Span.CastTo<long>().Slice(Offset, Length);
- public ListViewArray(IArrowType dataType, int length,
+ public LargeListViewArray(IArrowType dataType, int length,
ArrowBuffer valueOffsetsBuffer, ArrowBuffer sizesBuffer,
IArrowArray values,
ArrowBuffer nullBitmapBuffer, int nullCount = 0, int offset = 0)
: this(new ArrayData(dataType, length, nullCount, offset,
@@ -156,15 +156,15 @@ namespace Apache.Arrow
{
}
- public ListViewArray(ArrayData data)
+ public LargeListViewArray(ArrayData data)
: this(data, ArrowArrayFactory.BuildArray(data.Children[0]))
{
}
- private ListViewArray(ArrayData data, IArrowArray values) : base(data)
+ private LargeListViewArray(ArrayData data, IArrowArray values) :
base(data)
{
data.EnsureBufferCount(3);
- data.EnsureDataType(ArrowTypeId.ListView);
+ data.EnsureDataType(ArrowTypeId.LargeListView);
Values = values;
}
@@ -182,7 +182,7 @@ namespace Apache.Arrow
return 0;
}
- return Sizes[index];
+ return checked((int)Sizes[index]);
}
public IArrowArray GetSlicedValues(int index)
@@ -202,7 +202,7 @@ namespace Apache.Arrow
return default;
}
- return array.Slice(ValueOffsets[index], GetValueLength(index));
+ return array.Slice(checked((int)ValueOffsets[index]),
GetValueLength(index));
}
protected override void Dispose(bool disposing)
diff --git a/src/Apache.Arrow/Arrays/ListViewArray.cs
b/src/Apache.Arrow/Arrays/ListViewArray.cs
index 7bbb292..ab8eaa9 100644
--- a/src/Apache.Arrow/Arrays/ListViewArray.cs
+++ b/src/Apache.Arrow/Arrays/ListViewArray.cs
@@ -133,6 +133,7 @@ namespace Apache.Arrow
ValueBuilder.Clear();
ValidityBufferBuilder.Clear();
NullCount = 0;
+ Start = -1;
return this;
}
}
diff --git a/src/Apache.Arrow/C/CArrowArrayImporter.cs
b/src/Apache.Arrow/C/CArrowArrayImporter.cs
index ed2462b..e87f3ca 100644
--- a/src/Apache.Arrow/C/CArrowArrayImporter.cs
+++ b/src/Apache.Arrow/C/CArrowArrayImporter.cs
@@ -179,6 +179,10 @@ namespace Apache.Arrow.C
children = ProcessListChildren(cArray,
((LargeListType)storageType).ValueDataType);
buffers = ImportLargeListBuffers(cArray);
break;
+ case ArrowTypeId.LargeListView:
+ children = ProcessListChildren(cArray,
((LargeListViewType)storageType).ValueDataType);
+ buffers = ImportLargeListViewBuffers(cArray);
+ break;
case ArrowTypeId.FixedSizeList:
children = ProcessListChildren(cArray,
((FixedSizeListType)storageType).ValueDataType);
buffers = ImportFixedSizeListBuffers(cArray);
@@ -393,6 +397,32 @@ namespace Apache.Arrow.C
return buffers;
}
+ private ArrowBuffer[] ImportLargeListViewBuffers(CArrowArray*
cArray)
+ {
+ if (cArray->n_buffers != 3)
+ {
+ throw new InvalidOperationException("Large list view
arrays are expected to have exactly three buffers");
+ }
+
+ const int maxLength = int.MaxValue / 8;
+ if (cArray->length > maxLength)
+ {
+ throw new OverflowException(
+ $"Cannot import large list view array. Array length
{cArray->length} " +
+ $"is greater than the maximum supported large list
view array length ({maxLength})");
+ }
+
+ int length = checked((int)cArray->offset +
(int)cArray->length);
+ int bufferLength = checked(length * 8);
+
+ ArrowBuffer[] buffers = new ArrowBuffer[3];
+ buffers[0] = ImportValidityBuffer(cArray);
+ buffers[1] = ImportCArrayBuffer(cArray, 1, bufferLength);
+ buffers[2] = ImportCArrayBuffer(cArray, 2, bufferLength);
+
+ return buffers;
+ }
+
private ArrowBuffer[] ImportLargeListBuffers(CArrowArray* cArray)
{
if (cArray->n_buffers != 2)
diff --git a/src/Apache.Arrow/C/CArrowSchemaExporter.cs
b/src/Apache.Arrow/C/CArrowSchemaExporter.cs
index 8865c18..3206642 100644
--- a/src/Apache.Arrow/C/CArrowSchemaExporter.cs
+++ b/src/Apache.Arrow/C/CArrowSchemaExporter.cs
@@ -231,6 +231,7 @@ namespace Apache.Arrow.C
case ListType _: return "+l";
case ListViewType _: return "+vl";
case LargeListType _: return "+L";
+ case LargeListViewType _: return "+vL";
case FixedSizeListType fixedListType:
return $"+w:{fixedListType.ListSize}";
case StructType _: return "+s";
diff --git a/src/Apache.Arrow/C/CArrowSchemaImporter.cs
b/src/Apache.Arrow/C/CArrowSchemaImporter.cs
index fd06793..706890b 100644
--- a/src/Apache.Arrow/C/CArrowSchemaImporter.cs
+++ b/src/Apache.Arrow/C/CArrowSchemaImporter.cs
@@ -185,7 +185,7 @@ namespace Apache.Arrow.C
}
// Special handling for nested types
- if (format == "+l" || format == "+vl" || format == "+L")
+ if (format == "+l" || format == "+vl" || format == "+L" ||
format == "+vL")
{
if (_cSchema->n_children != 1)
{
@@ -200,11 +200,12 @@ namespace Apache.Arrow.C
Field childField = childSchema.GetAsField();
- return format[1] switch
+ return format switch
{
- 'l' => new ListType(childField),
- 'v' => new ListViewType(childField),
- 'L' => new LargeListType(childField),
+ "+l" => new ListType(childField),
+ "+vl" => new ListViewType(childField),
+ "+L" => new LargeListType(childField),
+ "+vL" => new LargeListViewType(childField),
_ => throw new InvalidDataException($"Invalid format
for list: '{format}'"),
};
}
diff --git a/src/Apache.Arrow/Ipc/ArrowReaderImplementation.cs
b/src/Apache.Arrow/Ipc/ArrowReaderImplementation.cs
index 1ed8d8a..28757b9 100644
--- a/src/Apache.Arrow/Ipc/ArrowReaderImplementation.cs
+++ b/src/Apache.Arrow/Ipc/ArrowReaderImplementation.cs
@@ -301,6 +301,7 @@ namespace Apache.Arrow.Ipc
case ArrowTypeId.LargeString:
case ArrowTypeId.LargeBinary:
case ArrowTypeId.ListView:
+ case ArrowTypeId.LargeListView:
buffers = 3;
break;
case ArrowTypeId.StringView:
diff --git a/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs
b/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs
index c5eed3b..18cc538 100644
--- a/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs
+++ b/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs
@@ -58,6 +58,7 @@ namespace Apache.Arrow.Ipc
IArrowArrayVisitor<ListArray>,
IArrowArrayVisitor<ListViewArray>,
IArrowArrayVisitor<LargeListArray>,
+ IArrowArrayVisitor<LargeListViewArray>,
IArrowArrayVisitor<FixedSizeListArray>,
IArrowArrayVisitor<StringArray>,
IArrowArrayVisitor<StringViewArray>,
@@ -228,6 +229,23 @@ namespace Apache.Arrow.Ipc
VisitArray(values);
}
+ public void Visit(LargeListViewArray array)
+ {
+ var (valueOffsetsBuffer, minOffset, maxEnd) =
GetZeroBasedLargeListViewOffsets(array);
+
+ _buffers.Add(CreateBitmapBuffer(array.NullBitmapBuffer,
array.Offset, array.Length, false));
+ _buffers.Add(CreateBuffer(valueOffsetsBuffer, true));
+ _buffers.Add(CreateSlicedBuffer<long>(array.SizesBuffer,
array.Offset, array.Length, false));
+
+ IArrowArray values = array.Values;
+ if (minOffset != 0 || values.Length != maxEnd)
+ {
+ values = ArrowArrayFactory.Slice(values,
checked((int)minOffset), checked((int)(maxEnd - minOffset)));
+ }
+
+ VisitArray(values);
+ }
+
public void Visit(FixedSizeListArray array)
{
_buffers.Add(CreateBitmapBuffer(array.NullBitmapBuffer,
array.Offset, array.Length, false));
@@ -467,6 +485,57 @@ namespace Apache.Arrow.Ipc
return (new ArrowBuffer(newOffsetsBuffer), minOffset, maxEnd);
}
+ private (ArrowBuffer Buffer, long minOffset, long maxEnd)
GetZeroBasedLargeListViewOffsets(LargeListViewArray array)
+ {
+ if (array.Length == 0)
+ {
+ return (ArrowBuffer.Empty, 0, 0);
+ }
+
+ var offsets = array.ValueOffsets;
+ var sizes = array.Sizes;
+
+ long minOffset = offsets[0];
+ long maxEnd = offsets[array.Length - 1] + sizes[array.Length -
1];
+
+ if (minOffset != 0 || maxEnd != array.Values.Length)
+ {
+ for (int i = 0; i < array.Length; ++i)
+ {
+ minOffset = Math.Min(minOffset, offsets[i]);
+ maxEnd = Math.Max(maxEnd, offsets[i] + sizes[i]);
+ }
+ }
+
+ var requiredBytes = CalculatePaddedBufferLength(sizeof(long) *
array.Length);
+
+ if (minOffset == 0)
+ {
+ ArrowBuffer buffer;
+ if (array.Offset != 0 || array.ValueOffsetsBuffer.Length >
requiredBytes)
+ {
+ var byteOffset = sizeof(long) * array.Offset;
+ var sliceLength = Math.Min(requiredBytes,
array.ValueOffsetsBuffer.Length - byteOffset);
+ buffer = new
ArrowBuffer(array.ValueOffsetsBuffer.Memory.Slice(byteOffset, sliceLength));
+ }
+ else
+ {
+ buffer = new
ArrowBuffer(array.ValueOffsetsBuffer.Memory);
+ }
+
+ return (buffer, minOffset, maxEnd);
+ }
+
+ var newOffsetsBuffer = _allocator.Allocate(requiredBytes);
+ var newOffsets = newOffsetsBuffer.Memory.Span.CastTo<long>();
+ for (int i = 0; i < array.Length; ++i)
+ {
+ newOffsets[i] = offsets[i] - minOffset;
+ }
+
+ return (new ArrowBuffer(newOffsetsBuffer), minOffset, maxEnd);
+ }
+
private Buffer CreateBitmapBuffer(ArrowBuffer buffer, int offset,
int length, bool locallyOwned)
{
if (buffer.IsEmpty)
diff --git a/src/Apache.Arrow/Ipc/ArrowTypeFlatbufferBuilder.cs
b/src/Apache.Arrow/Ipc/ArrowTypeFlatbufferBuilder.cs
index 503680a..893ae63 100644
--- a/src/Apache.Arrow/Ipc/ArrowTypeFlatbufferBuilder.cs
+++ b/src/Apache.Arrow/Ipc/ArrowTypeFlatbufferBuilder.cs
@@ -71,6 +71,7 @@ namespace Apache.Arrow.Ipc
IArrowTypeVisitor<ListType>,
IArrowTypeVisitor<ListViewType>,
IArrowTypeVisitor<LargeListType>,
+ IArrowTypeVisitor<LargeListViewType>,
IArrowTypeVisitor<FixedSizeListType>,
IArrowTypeVisitor<UnionType>,
IArrowTypeVisitor<StructType>,
@@ -157,6 +158,14 @@ namespace Apache.Arrow.Ipc
Flatbuf.LargeList.EndLargeList(Builder));
}
+ public void Visit(LargeListViewType type)
+ {
+ Flatbuf.LargeListView.StartLargeListView(Builder);
+ Result = FieldType.Build(
+ Flatbuf.Type.LargeListView,
+ Flatbuf.LargeListView.EndLargeListView(Builder));
+ }
+
public void Visit(FixedSizeListType type)
{
Result = FieldType.Build(
diff --git a/src/Apache.Arrow/Ipc/MessageSerializer.cs
b/src/Apache.Arrow/Ipc/MessageSerializer.cs
index 0984faf..645e6c8 100644
--- a/src/Apache.Arrow/Ipc/MessageSerializer.cs
+++ b/src/Apache.Arrow/Ipc/MessageSerializer.cs
@@ -237,6 +237,12 @@ namespace Apache.Arrow.Ipc
throw new InvalidDataException($"Large list type must
have exactly one child.");
}
return new Types.LargeListType(childFields[0]);
+ case Flatbuf.Type.LargeListView:
+ if (childFields == null || childFields.Length != 1)
+ {
+ throw new InvalidDataException($"Large list view type
must have exactly one child.");
+ }
+ return new Types.LargeListViewType(childFields[0]);
case Flatbuf.Type.FixedSizeList:
if (childFields == null || childFields.Length != 1)
{
diff --git a/src/Apache.Arrow/Types/IArrowType.cs
b/src/Apache.Arrow/Types/IArrowType.cs
index bf16368..ef1ec10 100644
--- a/src/Apache.Arrow/Types/IArrowType.cs
+++ b/src/Apache.Arrow/Types/IArrowType.cs
@@ -59,6 +59,7 @@ namespace Apache.Arrow.Types
Decimal32,
Decimal64,
Extension,
+ LargeListView,
}
public interface IArrowType
diff --git a/src/Apache.Arrow/Types/LargeListViewType.cs
b/src/Apache.Arrow/Types/LargeListViewType.cs
new file mode 100644
index 0000000..e28e067
--- /dev/null
+++ b/src/Apache.Arrow/Types/LargeListViewType.cs
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+namespace Apache.Arrow.Types
+{
+ public sealed class LargeListViewType : NestedType
+ {
+ public override ArrowTypeId TypeId => ArrowTypeId.LargeListView;
+
+ public override string Name => "large_listview";
+
+ public Field ValueField => Fields[0];
+
+ public IArrowType ValueDataType => Fields[0].DataType;
+
+ public LargeListViewType(Field valueField)
+ : base(valueField) { }
+
+ public LargeListViewType(IArrowType valueDataType)
+ : this(new Field("item", valueDataType, true)) { }
+
+ public override void Accept(IArrowTypeVisitor visitor) => Accept(this,
visitor);
+ }
+}
diff --git a/test/Apache.Arrow.IntegrationTest/JsonFile.cs
b/test/Apache.Arrow.IntegrationTest/JsonFile.cs
index a75c419..71831a7 100644
--- a/test/Apache.Arrow.IntegrationTest/JsonFile.cs
+++ b/test/Apache.Arrow.IntegrationTest/JsonFile.cs
@@ -191,6 +191,7 @@ namespace Apache.Arrow.IntegrationTest
"list" => ToListArrowType(type, children),
"listview" => ToListViewArrowType(type, children),
"largelist" => ToLargeListArrowType(type, children),
+ "largelistview" => ToLargeListViewArrowType(type, children),
"fixedsizelist" => ToFixedSizeListArrowType(type, children),
"struct" => ToStructArrowType(type, children),
"union" => ToUnionArrowType(type, children),
@@ -315,6 +316,11 @@ namespace Apache.Arrow.IntegrationTest
return new LargeListType(children[0]);
}
+ private static IArrowType ToLargeListViewArrowType(JsonArrowType type,
Field[] children)
+ {
+ return new LargeListViewType(children[0]);
+ }
+
private static IArrowType ToFixedSizeListArrowType(JsonArrowType type,
Field[] children)
{
return new FixedSizeListType(children[0], type.ListSize);
@@ -483,6 +489,7 @@ namespace Apache.Arrow.IntegrationTest
IArrowTypeVisitor<ListType>,
IArrowTypeVisitor<ListViewType>,
IArrowTypeVisitor<LargeListType>,
+ IArrowTypeVisitor<LargeListViewType>,
IArrowTypeVisitor<FixedSizeListType>,
IArrowTypeVisitor<StructType>,
IArrowTypeVisitor<UnionType>,
@@ -875,6 +882,22 @@ namespace Apache.Arrow.IntegrationTest
Array = new LargeListArray(arrayData);
}
+ public void Visit(LargeListViewType type)
+ {
+ ArrowBuffer validityBuffer = GetValidityBuffer(out int
nullCount);
+ ArrowBuffer offsetBuffer = GetLargeOffsetBuffer();
+ ArrowBuffer sizeBuffer = GetLargeSizeBuffer();
+
+ var data = JsonFieldData;
+ JsonFieldData = data.Children[0];
+ type.ValueDataType.Accept(this);
+ JsonFieldData = data;
+
+ ArrayData arrayData = new ArrayData(type, JsonFieldData.Count,
nullCount, 0,
+ new[] { validityBuffer, offsetBuffer, sizeBuffer }, new[]
{ Array.Data });
+ Array = new LargeListViewArray(arrayData);
+ }
+
public void Visit(FixedSizeListType type)
{
ArrowBuffer validityBuffer = GetValidityBuffer(out int
nullCount);
@@ -1068,6 +1091,13 @@ namespace Apache.Arrow.IntegrationTest
return valueSizes.Build(default);
}
+ private ArrowBuffer GetLargeSizeBuffer()
+ {
+ ArrowBuffer.Builder<long> valueSizes = new
ArrowBuffer.Builder<long>(JsonFieldData.Size.Count);
+ valueSizes.AppendRange(JsonFieldData.LongSize);
+ return valueSizes.Build(default);
+ }
+
private ArrowBuffer GetTypeIdBuffer()
{
ArrowBuffer.Builder<byte> typeIds = new
ArrowBuffer.Builder<byte>(JsonFieldData.TypeId.Length);
@@ -1137,6 +1167,12 @@ namespace Apache.Arrow.IntegrationTest
get { return Size.Select(GetInt); }
}
+ [JsonIgnore]
+ public IEnumerable<long> LongSize
+ {
+ get { return Size.Select(GetLong); }
+ }
+
static int GetInt(JsonNode node)
{
try
diff --git a/test/Apache.Arrow.Tests/ArrayBuilderTests.cs
b/test/Apache.Arrow.Tests/ArrayBuilderTests.cs
index 2db71b2..ddf4dc1 100644
--- a/test/Apache.Arrow.Tests/ArrayBuilderTests.cs
+++ b/test/Apache.Arrow.Tests/ArrayBuilderTests.cs
@@ -349,7 +349,36 @@ namespace Apache.Arrow.Tests
valueBuilder.Append(10);
var clearedList = builder.Build();
+ Assert.Equal(1, clearedList.Length);
Assert.Equal(0, clearedList.NullCount);
+ var slice = (Int64Array)clearedList.GetSlicedValues(0);
+ Assert.Equal(1, slice.Length);
+ Assert.Equal(10, slice.GetValue(0));
+ }
+
+ [Fact]
+ public void ListViewArrayBuilderClearWithoutBuildResetsCorrectly()
+ {
+ var builder = new ListViewArray.Builder(Int64Type.Default);
+ var valueBuilder = (Int64Array.Builder)builder.ValueBuilder;
+
+ builder.Append();
+ valueBuilder.Append(1);
+ valueBuilder.Append(2);
+
+ // Clear without calling Build first
+ builder.Clear();
+
+ builder.Append();
+ valueBuilder.Append(10);
+
+ var array = builder.Build();
+
+ Assert.Equal(1, array.Length);
+ Assert.Equal(0, array.NullCount);
+ var slice = (Int64Array)array.GetSlicedValues(0);
+ Assert.Equal(1, slice.Length);
+ Assert.Equal(10, slice.GetValue(0));
}
[Fact]
diff --git a/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs
b/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs
index f9589ef..e15e8e0 100644
--- a/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs
+++ b/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs
@@ -89,6 +89,7 @@ namespace Apache.Arrow.Tests
new LargeListType(Int64Type.Default),
new ListType(Int64Type.Default),
new ListViewType(Int64Type.Default),
+ new LargeListViewType(Int64Type.Default),
new StructType(new List<Field>{
new
Field.Builder().Name("Strings").DataType(StringType.Default).Nullable(true).Build(),
new
Field.Builder().Name("Ints").DataType(Int32Type.Default).Nullable(true).Build()
@@ -160,6 +161,7 @@ namespace Apache.Arrow.Tests
IArrowTypeVisitor<LargeBinaryType>,
IArrowTypeVisitor<LargeStringType>,
IArrowTypeVisitor<LargeListType>,
+ IArrowTypeVisitor<LargeListViewType>,
IArrowTypeVisitor<MapType>
{
private readonly List<List<int?>> _baseData;
@@ -529,6 +531,17 @@ namespace Apache.Arrow.Tests
builder.ValueBuilder.Reserve(length);
});
+ public void Visit(LargeListViewType type) =>
+ GenerateTestData<LargeListViewArray,
LargeListViewArray.Builder>(type, (builder, x) =>
+ {
+ builder.Append();
+ ((Int64Array.Builder)builder.ValueBuilder).Append(x);
+ }, initAction: (builder, length) =>
+ {
+ builder.Reserve(length);
+ builder.ValueBuilder.Reserve(length);
+ });
+
public void Visit(MapType type) =>
GenerateTestData<MapArray, MapArray.Builder>(type, (builder,
x) =>
{
diff --git a/test/Apache.Arrow.Tests/ArrowReaderVerifier.cs
b/test/Apache.Arrow.Tests/ArrowReaderVerifier.cs
index bdbe4bb..2f647ae 100644
--- a/test/Apache.Arrow.Tests/ArrowReaderVerifier.cs
+++ b/test/Apache.Arrow.Tests/ArrowReaderVerifier.cs
@@ -96,6 +96,7 @@ namespace Apache.Arrow.Tests
IArrowArrayVisitor<ListArray>,
IArrowArrayVisitor<ListViewArray>,
IArrowArrayVisitor<LargeListArray>,
+ IArrowArrayVisitor<LargeListViewArray>,
IArrowArrayVisitor<FixedSizeListArray>,
IArrowArrayVisitor<StringArray>,
IArrowArrayVisitor<StringViewArray>,
@@ -150,6 +151,7 @@ namespace Apache.Arrow.Tests
public void Visit(ListArray array) => CompareArrays(array);
public void Visit(ListViewArray array) => CompareArrays(array);
public void Visit(LargeListArray array) => CompareArrays(array);
+ public void Visit(LargeListViewArray array) =>
CompareArrays(array);
public void Visit(FixedSizeListArray array) =>
CompareArrays(array);
public void Visit(FixedSizeBinaryArray array) =>
CompareArrays(array);
public void Visit(Decimal32Array array) => CompareArrays(array);
@@ -551,6 +553,45 @@ namespace Apache.Arrow.Tests
}
}
+ private void CompareArrays(LargeListViewArray actualArray)
+ {
+ Assert.IsAssignableFrom<LargeListViewArray>(_expectedArray);
+ LargeListViewArray expectedArray =
(LargeListViewArray)_expectedArray;
+
+ actualArray.Data.DataType.Accept(_arrayTypeComparer);
+
+ Assert.Equal(expectedArray.Length, actualArray.Length);
+ Assert.Equal(expectedArray.NullCount, actualArray.NullCount);
+
+ CompareValidityBuffer(
+ expectedArray.NullCount, _expectedArray.Length,
expectedArray.NullBitmapBuffer,
+ expectedArray.Offset, actualArray.NullBitmapBuffer,
actualArray.Offset);
+
+ if (_strictCompare)
+ {
+ Assert.Equal(expectedArray.Offset, actualArray.Offset);
+
Assert.True(expectedArray.ValueOffsetsBuffer.Span.SequenceEqual(actualArray.ValueOffsetsBuffer.Span));
+
Assert.True(expectedArray.SizesBuffer.Span.SequenceEqual(actualArray.SizesBuffer.Span));
+ actualArray.Values.Accept(new
ArrayComparer(expectedArray.Values, _strictCompare));
+ }
+ else
+ {
+ for (int i = 0; i < actualArray.Length; ++i)
+ {
+ if (expectedArray.IsNull(i))
+ {
+ Assert.True(actualArray.IsNull(i));
+ }
+ else
+ {
+ var expectedList =
expectedArray.GetSlicedValues(i);
+ var actualList = actualArray.GetSlicedValues(i);
+ actualList.Accept(new ArrayComparer(expectedList,
_strictCompare));
+ }
+ }
+ }
+ }
+
private void CompareArrays(FixedSizeListArray actualArray)
{
Assert.IsAssignableFrom<FixedSizeListArray>(_expectedArray);
diff --git a/test/Apache.Arrow.Tests/LargeListViewArrayBuilderTests.cs
b/test/Apache.Arrow.Tests/LargeListViewArrayBuilderTests.cs
new file mode 100644
index 0000000..ad172fe
--- /dev/null
+++ b/test/Apache.Arrow.Tests/LargeListViewArrayBuilderTests.cs
@@ -0,0 +1,156 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using Apache.Arrow.Types;
+using Xunit;
+
+namespace Apache.Arrow.Tests
+{
+ public class LargeListViewArrayBuilderTests
+ {
+ [Fact]
+ public void AppendBuildsCorrectArray()
+ {
+ var builder = new LargeListViewArray.Builder(Int32Type.Default);
+ var valueBuilder = (Int32Array.Builder)builder.ValueBuilder;
+
+ builder.Append();
+ valueBuilder.Append(1);
+ valueBuilder.Append(2);
+
+ builder.Append();
+ valueBuilder.Append(3);
+
+ builder.AppendNull();
+
+ builder.Append();
+
+ var array = builder.Build();
+
+ Assert.Equal(4, array.Length);
+
+ var slice0 = (Int32Array)array.GetSlicedValues(0);
+ Assert.Equal(2, slice0.Length);
+ Assert.Equal(1, slice0.GetValue(0));
+ Assert.Equal(2, slice0.GetValue(1));
+
+ var slice1 = (Int32Array)array.GetSlicedValues(1);
+ Assert.Equal(1, slice1.Length);
+ Assert.Equal(3, slice1.GetValue(0));
+
+ Assert.Null(array.GetSlicedValues(2));
+
+ var slice3 = (Int32Array)array.GetSlicedValues(3);
+ Assert.Equal(0, slice3.Length);
+ }
+
+ [Fact]
+ public void NullCountIsCorrect()
+ {
+ var builder = new LargeListViewArray.Builder(Int32Type.Default);
+ var valueBuilder = (Int32Array.Builder)builder.ValueBuilder;
+
+ builder.Append();
+ valueBuilder.Append(1);
+
+ builder.AppendNull();
+ builder.AppendNull();
+
+ builder.Append();
+ valueBuilder.Append(2);
+
+ var array = builder.Build();
+
+ Assert.Equal(4, array.Length);
+ Assert.Equal(2, array.NullCount);
+ }
+
+ [Fact]
+ public void ClearResetsBuilder()
+ {
+ var builder = new LargeListViewArray.Builder(Int32Type.Default);
+ var valueBuilder = (Int32Array.Builder)builder.ValueBuilder;
+
+ builder.Append();
+ valueBuilder.Append(1);
+ builder.AppendNull();
+
+ builder.Clear();
+
+ builder.Append();
+ valueBuilder.Append(99);
+
+ var array = builder.Build();
+
+ Assert.Equal(1, array.Length);
+ Assert.Equal(0, array.NullCount);
+ var slice = (Int32Array)array.GetSlicedValues(0);
+ Assert.Equal(1, slice.Length);
+ Assert.Equal(99, slice.GetValue(0));
+ }
+
+ [Fact]
+ public void ClearWithoutBuildResetsBuilder()
+ {
+ var builder = new LargeListViewArray.Builder(Int32Type.Default);
+ var valueBuilder = (Int32Array.Builder)builder.ValueBuilder;
+
+ builder.Append();
+ valueBuilder.Append(1);
+ valueBuilder.Append(2);
+
+ // Clear without calling Build first
+ builder.Clear();
+
+ builder.Append();
+ valueBuilder.Append(99);
+
+ var array = builder.Build();
+
+ Assert.Equal(1, array.Length);
+ Assert.Equal(0, array.NullCount);
+ var slice = (Int32Array)array.GetSlicedValues(0);
+ Assert.Equal(1, slice.Length);
+ Assert.Equal(99, slice.GetValue(0));
+ }
+
+ [Fact]
+ public void NestedStringListBuildsCorrectly()
+ {
+ var builder = new LargeListViewArray.Builder(StringType.Default);
+ var valueBuilder = (StringArray.Builder)builder.ValueBuilder;
+
+ builder.Append();
+ valueBuilder.Append("hello");
+ valueBuilder.Append("world");
+
+ builder.Append();
+ valueBuilder.Append("foo");
+
+ var array = builder.Build();
+
+ Assert.Equal(2, array.Length);
+
+ var slice0 = (StringArray)array.GetSlicedValues(0);
+ Assert.Equal(2, slice0.Length);
+ Assert.Equal("hello", slice0.GetString(0));
+ Assert.Equal("world", slice0.GetString(1));
+
+ var slice1 = (StringArray)array.GetSlicedValues(1);
+ Assert.Equal(1, slice1.Length);
+ Assert.Equal("foo", slice1.GetString(0));
+ }
+ }
+}
diff --git a/test/Apache.Arrow.Tests/LargeListViewArrayTests.cs
b/test/Apache.Arrow.Tests/LargeListViewArrayTests.cs
new file mode 100644
index 0000000..6e0286a
--- /dev/null
+++ b/test/Apache.Arrow.Tests/LargeListViewArrayTests.cs
@@ -0,0 +1,173 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.Linq;
+using Apache.Arrow.Types;
+using Xunit;
+
+namespace Apache.Arrow.Tests;
+
+public class LargeListViewArrayTests
+{
+ [Fact]
+ public void GetSlicedValuesReturnsCorrectValues()
+ {
+ var values = new int?[][]
+ {
+ new int?[] { 0, 1, 2 },
+ System.Array.Empty<int?>(),
+ null,
+ new int?[] { 3, 4, null, 6 },
+ };
+
+ var array = BuildArray(values);
+
+ Assert.Equal(values.Length, array.Length);
+ for (int i = 0; i < values.Length; ++i)
+ {
+ Assert.Equal(values[i] == null, array.IsNull(i));
+ var arrayItem = (Int32Array)array.GetSlicedValues(i);
+ if (values[i] == null)
+ {
+ Assert.Null(arrayItem);
+ }
+ else
+ {
+ Assert.Equal(values[i], arrayItem.ToArray());
+ }
+ }
+ }
+
+ [Fact]
+ public void GetSlicedValuesChecksForOffsetOverflow()
+ {
+ var valuesArray = new Int32Array.Builder().Build();
+ var offsetBuffer = new ArrowBuffer.Builder<long>();
+ var sizesBuffer = new ArrowBuffer.Builder<long>();
+ var validityBuffer = new ArrowBuffer.BitmapBuilder();
+
+ offsetBuffer.Append((long)int.MaxValue + 1);
+ sizesBuffer.Append(1);
+ validityBuffer.Append(true);
+
+ var array = new LargeListViewArray(
+ new LargeListViewType(new Int32Type()), length: 1,
+ offsetBuffer.Build(), sizesBuffer.Build(), valuesArray,
validityBuffer.Build(),
+ validityBuffer.UnsetBitCount);
+
+ Assert.Throws<OverflowException>(() => array.GetSlicedValues(0));
+ }
+
+ [Fact]
+ public void GetSlicedValuesChecksForSizeOverflow()
+ {
+ var valuesArray = new Int32Array.Builder().Build();
+ var offsetBuffer = new ArrowBuffer.Builder<long>();
+ var sizesBuffer = new ArrowBuffer.Builder<long>();
+ var validityBuffer = new ArrowBuffer.BitmapBuilder();
+
+ offsetBuffer.Append(0L);
+ sizesBuffer.Append((long)int.MaxValue + 1);
+ validityBuffer.Append(true);
+
+ var array = new LargeListViewArray(
+ new LargeListViewType(new Int32Type()), length: 1,
+ offsetBuffer.Build(), sizesBuffer.Build(), valuesArray,
validityBuffer.Build(),
+ validityBuffer.UnsetBitCount);
+
+ Assert.Throws<OverflowException>(() => array.GetValueLength(0));
+ }
+
+ [Fact]
+ public void SliceReturnsCorrectValues()
+ {
+ var values = new int?[][]
+ {
+ new int?[] { 10, 20 },
+ new int?[] { 30 },
+ new int?[] { 40, 50, 60 },
+ new int?[] { 70 },
+ };
+
+ var array = BuildArray(values);
+ var sliced = (LargeListViewArray)array.Slice(1, 2);
+
+ Assert.Equal(2, sliced.Length);
+
+ var list0 = (Int32Array)sliced.GetSlicedValues(0);
+ Assert.Equal(1, list0.Length);
+ Assert.Equal(30, list0.GetValue(0));
+
+ var list1 = (Int32Array)sliced.GetSlicedValues(1);
+ Assert.Equal(3, list1.Length);
+ Assert.Equal(40, list1.GetValue(0));
+ Assert.Equal(60, list1.GetValue(2));
+ }
+
+ [Fact]
+ public void PropertiesReturnCorrectValues()
+ {
+ var values = new int?[][]
+ {
+ new int?[] { 1, 2, 3 },
+ null,
+ new int?[] { 4, 5 },
+ };
+
+ var array = BuildArray(values);
+
+ Assert.Equal(3, array.Length);
+ Assert.Equal(1, array.NullCount);
+ Assert.Equal(ArrowTypeId.LargeListView, array.Data.DataType.TypeId);
+
+ Assert.Equal(3, array.GetValueLength(0));
+ Assert.Equal(0, array.GetValueLength(1));
+ Assert.Equal(2, array.GetValueLength(2));
+ }
+
+ private static LargeListViewArray BuildArray(int?[][] values)
+ {
+ var valuesBuilder = new Int32Array.Builder();
+ var offsetBuffer = new ArrowBuffer.Builder<long>();
+ var sizesBuffer = new ArrowBuffer.Builder<long>();
+ var validityBuffer = new ArrowBuffer.BitmapBuilder();
+
+ foreach (var listValue in values)
+ {
+ if (listValue == null)
+ {
+ offsetBuffer.Append(valuesBuilder.Length);
+ sizesBuffer.Append(0);
+ validityBuffer.Append(false);
+ }
+ else
+ {
+ offsetBuffer.Append(valuesBuilder.Length);
+ sizesBuffer.Append(listValue.Length);
+ foreach (var value in listValue)
+ {
+ valuesBuilder.Append(value);
+ }
+ validityBuffer.Append(true);
+ }
+ }
+
+ return new LargeListViewArray(
+ new LargeListViewType(new Int32Type()), values.Length,
+ offsetBuffer.Build(), sizesBuffer.Build(), valuesBuilder.Build(),
validityBuffer.Build(),
+ validityBuffer.UnsetBitCount);
+ }
+}
diff --git a/test/Apache.Arrow.Tests/TableTests.cs
b/test/Apache.Arrow.Tests/TableTests.cs
index a12e7b9..309ef7b 100644
--- a/test/Apache.Arrow.Tests/TableTests.cs
+++ b/test/Apache.Arrow.Tests/TableTests.cs
@@ -63,9 +63,9 @@ namespace Apache.Arrow.Tests
Table table1 = Table.TableFromRecordBatches(recordBatch1.Schema,
recordBatches);
Assert.Equal(20, table1.RowCount);
#if NET5_0_OR_GREATER
- Assert.Equal(40, table1.ColumnCount);
+ Assert.Equal(41, table1.ColumnCount);
#else
- Assert.Equal(39, table1.ColumnCount);
+ Assert.Equal(40, table1.ColumnCount);
#endif
Assert.Equal("ChunkedArray: Length=20, DataType=list",
table1.Column(0).Data.ToString());
diff --git a/test/Apache.Arrow.Tests/TestData.cs
b/test/Apache.Arrow.Tests/TestData.cs
index 3991bcd..08205cb 100644
--- a/test/Apache.Arrow.Tests/TestData.cs
+++ b/test/Apache.Arrow.Tests/TestData.cs
@@ -51,6 +51,7 @@ namespace Apache.Arrow.Tests
AddField(CreateField(new ListType(Int64Type.Default), i));
AddField(CreateField(new ListViewType(Int64Type.Default), i));
AddField(CreateField(new LargeListType(Int64Type.Default), i));
+ AddField(CreateField(new LargeListViewType(Int64Type.Default),
i));
AddField(CreateField(BooleanType.Default, i));
AddField(CreateField(UInt8Type.Default, i));
AddField(CreateField(Int8Type.Default, i));
@@ -154,6 +155,7 @@ namespace Apache.Arrow.Tests
IArrowTypeVisitor<ListType>,
IArrowTypeVisitor<ListViewType>,
IArrowTypeVisitor<LargeListType>,
+ IArrowTypeVisitor<LargeListViewType>,
IArrowTypeVisitor<FixedSizeListType>,
IArrowTypeVisitor<StructType>,
IArrowTypeVisitor<UnionType>,
@@ -489,6 +491,38 @@ namespace Apache.Arrow.Tests
validityBuffer.UnsetBitCount);
}
+ public void Visit(LargeListViewType type)
+ {
+ var valueBuilder = new Int64Array.Builder().Reserve(Length * 3
/ 2);
+ var offsetBuffer = new ArrowBuffer.Builder<long>();
+ var sizesBuffer = new ArrowBuffer.Builder<long>();
+ var validityBuffer = new ArrowBuffer.BitmapBuilder();
+
+ for (var i = 0; i < Length; i++)
+ {
+ if (i % 10 == 2)
+ {
+ offsetBuffer.Append(valueBuilder.Length);
+ sizesBuffer.Append(0);
+ validityBuffer.Append(false);
+ }
+ else
+ {
+ var listLength = i % 4;
+ offsetBuffer.Append(valueBuilder.Length);
+ sizesBuffer.Append(listLength);
+ valueBuilder.AppendRange(Enumerable.Range(i,
listLength).Select(x => (long)x));
+ validityBuffer.Append(true);
+ }
+ }
+
+ var validity = validityBuffer.UnsetBitCount > 0 ?
validityBuffer.Build() : ArrowBuffer.Empty;
+ Array = new LargeListViewArray(
+ new LargeListViewType(new Int64Type()), Length,
+ offsetBuffer.Build(), sizesBuffer.Build(),
valueBuilder.Build(), validity,
+ validityBuffer.UnsetBitCount);
+ }
+
public void Visit(FixedSizeListType type)
{
var builder = new FixedSizeListArray.Builder(type.ValueField,
type.ListSize).Reserve(Length);
