jecsand838 commented on PR #17861:
URL: https://github.com/apache/datafusion/pull/17861#issuecomment-3577323411
> @nathaniel-d-ef No not yet. I'm still looking for a solution. Whether or
not projection is applied, I couldn't retrieve the schema metadata when reading
the file with ReaderBuilder. I don't know the `arrow-avro` internal
implementation very well, so I'm investigating. I'll share it once I find a
solution.
>
> ```rust
> let avro_reader = ReaderBuilder::new().build(BufReader::new(reader))?;
> println!("AVRO READER METADATA : {:?}", avro_reader.schema().metadata); //
{}
> ```
@getChan Try using `avro_header` instead to get the OCF `Header`:
```rust
let avro_reader =
ReaderBuilder::new().build(BufReader::new(reader))?;
let header = avro_reader.avro_header();
println!("\nAVRO HEADER METADATA BYTES: {:?}",
header.metadata().collect::<Vec<_>>()); // {}
let writer_avro = AvroSchema::new(
std::str::from_utf8(
header
.get(SCHEMA_METADATA_KEY.as_bytes())
.expect("missing avro.schema metadata"),
)
.unwrap()
.to_string(),
);
println!("AVRO HEADER SCHEMA METADATA : {:?}", writer_avro);
```
You should see an output like this:
```
AVRO HEADER METADATA BYTES: [([97, 118, 114, 111, 46, 115, 99, 104, 101,
109, 97], [123, 34, 116, 121, 112, 101, 34, 58, 34, 114, 101, 99, 111, 114,
100, 34, 44, 34, 110, 97, 109, 101, 34, 58, 34, 116, 111, 112, 76, 101, 118,
101, 108, 82, 101, 99, 111, 114, 100, 34, 44, 34, 102, 105, 101, 108, 100, 115,
34, 58, 91, 123, 34, 110, 97, 109, 101, 34, 58, 34, 105, 100, 34, 44, 34, 116,
121, 112, 101, 34, 58, 91, 34, 105, 110, 116, 34, 44, 34, 110, 117, 108, 108,
34, 93, 125, 44, 123, 34, 110, 97, 109, 101, 34, 58, 34, 98, 111, 111, 108, 95,
99, 111, 108, 34, 44, 34, 116, 121, 112, 101, 34, 58, 91, 34, 98, 111, 111,
108, 101, 97, 110, 34, 44, 34, 110, 117, 108, 108, 34, 93, 125, 44, 123, 34,
110, 97, 109, 101, 34, 58, 34, 116, 105, 110, 121, 105, 110, 116, 95, 99, 111,
108, 34, 44, 34, 116, 121, 112, 101, 34, 58, 91, 34, 105, 110, 116, 34, 44, 34,
110, 117, 108, 108, 34, 93, 125, 44, 123, 34, 110, 97, 109, 101, 34, 58, 34,
115, 109, 97, 108, 108, 105, 110, 116, 95, 99, 111, 108, 34, 44
, 34, 116, 121, 112, 101, 34, 58, 91, 34, 105, 110, 116, 34, 44, 34, 110, 117,
108, 108, 34, 93, 125, 44, 123, 34, 110, 97, 109, 101, 34, 58, 34, 105, 110,
116, 95, 99, 111, 108, 34, 44, 34, 116, 121, 112, 101, 34, 58, 91, 34, 105,
110, 116, 34, 44, 34, 110, 117, 108, 108, 34, 93, 125, 44, 123, 34, 110, 97,
109, 101, 34, 58, 34, 98, 105, 103, 105, 110, 116, 95, 99, 111, 108, 34, 44,
34, 116, 121, 112, 101, 34, 58, 91, 34, 108, 111, 110, 103, 34, 44, 34, 110,
117, 108, 108, 34, 93, 125, 44, 123, 34, 110, 97, 109, 101, 34, 58, 34, 102,
108, 111, 97, 116, 95, 99, 111, 108, 34, 44, 34, 116, 121, 112, 101, 34, 58,
91, 34, 102, 108, 111, 97, 116, 34, 44, 34, 110, 117, 108, 108, 34, 93, 125,
44, 123, 34, 110, 97, 109, 101, 34, 58, 34, 100, 111, 117, 98, 108, 101, 95,
99, 111, 108, 34, 44, 34, 116, 121, 112, 101, 34, 58, 91, 34, 100, 111, 117,
98, 108, 101, 34, 44, 34, 110, 117, 108, 108, 34, 93, 125, 44, 123, 34, 110,
97, 109, 101, 34, 58, 34, 100, 97, 116, 101, 95, 115, 116, 114, 105, 110
, 103, 95, 99, 111, 108, 34, 44, 34, 116, 121, 112, 101, 34, 58, 91, 34, 98,
121, 116, 101, 115, 34, 44, 34, 110, 117, 108, 108, 34, 93, 125, 44, 123, 34,
110, 97, 109, 101, 34, 58, 34, 115, 116, 114, 105, 110, 103, 95, 99, 111, 108,
34, 44, 34, 116, 121, 112, 101, 34, 58, 91, 34, 98, 121, 116, 101, 115, 34, 44,
34, 110, 117, 108, 108, 34, 93, 125, 44, 123, 34, 110, 97, 109, 101, 34, 58,
34, 116, 105, 109, 101, 115, 116, 97, 109, 112, 95, 99, 111, 108, 34, 44, 34,
116, 121, 112, 101, 34, 58, 91, 123, 34, 116, 121, 112, 101, 34, 58, 34, 108,
111, 110, 103, 34, 44, 34, 108, 111, 103, 105, 99, 97, 108, 84, 121, 112, 101,
34, 58, 34, 116, 105, 109, 101, 115, 116, 97, 109, 112, 45, 109, 105, 99, 114,
111, 115, 34, 125, 44, 34, 110, 117, 108, 108, 34, 93, 125, 93, 125]), ([111,
114, 103, 46, 97, 112, 97, 99, 104, 101, 46, 115, 112, 97, 114, 107, 46, 118,
101, 114, 115, 105, 111, 110], [51, 46, 49, 46, 50]), ([97, 118, 114, 111, 46,
99, 111, 100, 101, 99], [115, 110, 97, 112, 112, 121])]
AVRO HEADER SCHEMA METADATA : AvroSchema { json_string:
"{\"type\":\"record\",\"name\":\"topLevelRecord\",\"fields\":[{\"name\":\"id\",\"type\":[\"int\",\"null\"]},{\"name\":\"bool_col\",\"type\":[\"boolean\",\"null\"]},{\"name\":\"tinyint_col\",\"type\":[\"int\",\"null\"]},{\"name\":\"smallint_col\",\"type\":[\"int\",\"null\"]},{\"name\":\"int_col\",\"type\":[\"int\",\"null\"]},{\"name\":\"bigint_col\",\"type\":[\"long\",\"null\"]},{\"name\":\"float_col\",\"type\":[\"float\",\"null\"]},{\"name\":\"double_col\",\"type\":[\"double\",\"null\"]},{\"name\":\"date_string_col\",\"type\":[\"bytes\",\"null\"]},{\"name\":\"string_col\",\"type\":[\"bytes\",\"null\"]},{\"name\":\"timestamp_col\",\"type\":[{\"type\":\"long\",\"logicalType\":\"timestamp-micros\"},\"null\"]}]}"
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
