Thanks Wes! So I create it this way, but I still don't know how to populate
and
auto element = PrimitiveNode::Make("element", Repetition::OPTIONAL,
Type::INT32);
auto list = GroupNode::Make("list", Repetition::REPEATED, {element});
auto my_array = GroupNode::Make("my_array", Repetition::REQUIRED, {list},
LogicalType::LIST);
fields.push_back(PrimitiveNode::Make("id", Repetition::REQUIRED,
Type::INT32, LogicalType::NONE));
fields.push_back(my_array);
auto my_schema = GroupNode::Make("schema", Repetition::REQUIRED, fields);
I tried populating it this way:
parquet::Int32Writer* int32_writer1 =
static_cast<parquet::Int32Writer*>(rg_writer->NextColumn());
for (int i = 0; i < NROWS_GROUP; i++) {
int32_t value = i;
int16_t definition_level = 1;
int16_t repetition_level = 0;
if ((i+1)%2 == 0) {
repetition_level = 1; // start of a new record
}
int32_writer1->WriteBatch(1, &definition_level, &repetition_level,
&value);
}
That seems to work, but I can't use the generated file on Athena and using
the parquet_reader from parquet_cpp returns NULLs on the elements. Is it
that I have to get a handle to the list element? Thanks again for the help!
