paleolimbot commented on code in PR #471: URL: https://github.com/apache/sedona-db/pull/471#discussion_r2776812561
########## c/sedona-s2geography/s2geography: ########## Review Comment: Does this submodule need to be reverted or updated? ########## examples/sedonadb-rust-pointcloud/src/main.rs: ########## Review Comment: Thanks! We can add this here to run in CI: https://github.com/apache/sedona-db/blob/d17d7f08dd65fe42d9bf64126ae2a383666c72f6/.github/workflows/examples.yml#L76-L90 ########## rust/sedona-pointcloud/src/laz/builder.rs: ########## @@ -0,0 +1,571 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::{fmt::Debug, sync::Arc}; + +use arrow_array::{ + builder::{ + ArrayBuilder, BinaryBuilder, BooleanBuilder, FixedSizeBinaryBuilder, Float32Builder, + Float64Builder, Int16Builder, Int32Builder, Int64Builder, Int8Builder, UInt16Builder, + UInt32Builder, UInt64Builder, UInt8Builder, + }, + Array, ArrayRef, BooleanArray, FixedSizeBinaryArray, Float32Array, Float64Array, StructArray, + UInt16Array, UInt8Array, +}; +use arrow_buffer::ScalarBuffer; +use arrow_schema::{ArrowError, DataType}; +use geoarrow_array::{ + array::{CoordBuffer, PointArray, SeparatedCoordBuffer}, + GeoArrowArray, +}; +use geoarrow_schema::Dimension; +use las::{Header, Point}; + +use crate::laz::{ + metadata::ExtraAttribute, + options::{LasExtraBytes, LasPointEncoding}, + schema::try_schema_from_header, +}; + +#[derive(Debug)] +pub struct RowBuilder { + x: Float64Builder, + y: Float64Builder, + z: Float64Builder, + intensity: UInt16Builder, + return_number: UInt8Builder, + number_of_returns: UInt8Builder, + is_synthetic: BooleanBuilder, + is_key_point: BooleanBuilder, + is_withheld: BooleanBuilder, + is_overlap: BooleanBuilder, + scanner_channel: UInt8Builder, + scan_direction: UInt8Builder, + is_edge_of_flight_line: BooleanBuilder, + classification: UInt8Builder, + user_data: UInt8Builder, + scan_angle: Float32Builder, + point_source_id: UInt16Builder, + gps_time: Float64Builder, + red: UInt16Builder, + green: UInt16Builder, + blue: UInt16Builder, + nir: UInt16Builder, + extra: FixedSizeBinaryBuilder, + header: Arc<Header>, + point_encoding: LasPointEncoding, + extra_bytes: LasExtraBytes, + extra_attributes: Arc<Vec<ExtraAttribute>>, +} + +impl RowBuilder { + pub fn new(capacity: usize, header: Arc<Header>) -> Self { + Self { + x: Float64Array::builder(capacity), + y: Float64Array::builder(capacity), + z: Float64Array::builder(capacity), + intensity: UInt16Array::builder(capacity), + return_number: UInt8Array::builder(capacity), + number_of_returns: UInt8Array::builder(capacity), + is_synthetic: BooleanArray::builder(capacity), + is_key_point: BooleanArray::builder(capacity), + is_withheld: BooleanArray::builder(capacity), + is_overlap: BooleanArray::builder(capacity), + scanner_channel: UInt8Array::builder(capacity), + scan_direction: UInt8Array::builder(capacity), + is_edge_of_flight_line: BooleanArray::builder(capacity), + classification: UInt8Array::builder(capacity), + user_data: UInt8Array::builder(capacity), + scan_angle: Float32Array::builder(capacity), + point_source_id: UInt16Array::builder(capacity), + gps_time: Float64Array::builder(capacity), + red: UInt16Array::builder(capacity), + green: UInt16Array::builder(capacity), + blue: UInt16Array::builder(capacity), + nir: UInt16Array::builder(capacity), + extra: FixedSizeBinaryBuilder::with_capacity( + capacity, + header.point_format().extra_bytes as i32, + ), + + header, + point_encoding: Default::default(), + extra_bytes: Default::default(), + extra_attributes: Arc::new(Vec::new()), + } + } + + pub fn with_point_encoding(mut self, point_encoding: LasPointEncoding) -> Self { + self.point_encoding = point_encoding; + self + } + + pub fn with_extra_attributes( + mut self, + attributes: Arc<Vec<ExtraAttribute>>, + extra_bytes: LasExtraBytes, + ) -> Self { + self.extra_attributes = attributes; + self.extra_bytes = extra_bytes; + self + } + + pub fn append(&mut self, p: Point) { + self.x.append_value(p.x); + self.y.append_value(p.y); + self.z.append_value(p.z); + self.intensity.append_option(Some(p.intensity)); + self.return_number.append_value(p.return_number); + self.number_of_returns.append_value(p.number_of_returns); + self.is_synthetic.append_value(p.is_synthetic); + self.is_key_point.append_value(p.is_key_point); + self.is_withheld.append_value(p.is_withheld); + self.is_overlap.append_value(p.is_overlap); + self.scanner_channel.append_value(p.scanner_channel); + self.scan_direction.append_value(p.scan_direction as u8); + self.is_edge_of_flight_line + .append_value(p.is_edge_of_flight_line); + self.classification.append_value(u8::from(p.classification)); + self.user_data.append_value(p.user_data); + self.scan_angle.append_value(p.scan_angle); + self.point_source_id.append_value(p.point_source_id); + if self.header.point_format().has_gps_time { + self.gps_time.append_value(p.gps_time.unwrap()); + } + if self.header.point_format().has_color { + let color = p.color.unwrap(); + self.red.append_value(color.red); + self.green.append_value(color.green); + self.blue.append_value(color.blue); + } + if self.header.point_format().has_nir { + self.nir.append_value(p.nir.unwrap()); + } + if self.header.point_format().extra_bytes > 0 { + self.extra.append_value(p.extra_bytes).unwrap(); + } + } + + /// Note: returns StructArray to allow nesting within another array if desired + pub fn finish(&mut self) -> Result<StructArray, ArrowError> { + let schema = try_schema_from_header(&self.header, self.point_encoding, self.extra_bytes)?; + + let mut columns = match self.point_encoding { + LasPointEncoding::Plain => vec![ + Arc::new(self.x.finish()) as ArrayRef, + Arc::new(self.y.finish()) as ArrayRef, + Arc::new(self.z.finish()) as ArrayRef, + ], + LasPointEncoding::Wkb => { + const POINT_SIZE: usize = 29; + + let n: usize = self.x.len(); + + let mut builder = BinaryBuilder::with_capacity(n, n * POINT_SIZE); + + let x = self.x.finish(); + let y = self.y.finish(); + let z = self.z.finish(); + + let mut wkb_bytes = [0_u8; POINT_SIZE]; + wkb_bytes[0] = 0x01; // Little-endian + wkb_bytes[1..5].copy_from_slice(&[0xE9, 0x03, 0x00, 0x00]); // Point Z type (1001) + + for i in 0..n { + let x = unsafe { x.value_unchecked(i) }; + let y = unsafe { y.value_unchecked(i) }; + let z = unsafe { z.value_unchecked(i) }; + + wkb_bytes[5..13].copy_from_slice(x.to_le_bytes().as_slice()); + wkb_bytes[13..21].copy_from_slice(y.to_le_bytes().as_slice()); + wkb_bytes[21..29].copy_from_slice(z.to_le_bytes().as_slice()); + + builder.append_value(wkb_bytes); + } + + vec![Arc::new(builder.finish()) as ArrayRef] + } + LasPointEncoding::Native => { + let buffers = [ + self.x.finish().into_parts().1, + self.y.finish().into_parts().1, + self.z.finish().into_parts().1, + ScalarBuffer::from(vec![]), + ]; + let coords = CoordBuffer::Separated(SeparatedCoordBuffer::from_array( + buffers, + Dimension::XYZ, + )?); + let points = PointArray::new(coords, None, Default::default()); + vec![points.to_array_ref()] + } + }; + + columns.extend([ + Arc::new(self.intensity.finish()) as ArrayRef, + Arc::new(self.return_number.finish()) as ArrayRef, + Arc::new(self.number_of_returns.finish()) as ArrayRef, + Arc::new(self.is_synthetic.finish()) as ArrayRef, + Arc::new(self.is_key_point.finish()) as ArrayRef, + Arc::new(self.is_withheld.finish()) as ArrayRef, + Arc::new(self.is_overlap.finish()) as ArrayRef, + Arc::new(self.scanner_channel.finish()) as ArrayRef, + Arc::new(self.scan_direction.finish()) as ArrayRef, + Arc::new(self.is_edge_of_flight_line.finish()) as ArrayRef, + Arc::new(self.classification.finish()) as ArrayRef, + Arc::new(self.user_data.finish()) as ArrayRef, + Arc::new(self.scan_angle.finish()) as ArrayRef, + Arc::new(self.point_source_id.finish()) as ArrayRef, + ]); + if self.header.point_format().has_gps_time { + columns.push(Arc::new(self.gps_time.finish()) as ArrayRef); + } + if self.header.point_format().has_color { + columns.extend([ + Arc::new(self.red.finish()) as ArrayRef, + Arc::new(self.green.finish()) as ArrayRef, + Arc::new(self.blue.finish()) as ArrayRef, + ]); + } + if self.header.point_format().has_nir { + columns.push(Arc::new(self.nir.finish()) as ArrayRef); + } + + // extra bytes + let num_extra_bytes = self.header.point_format().extra_bytes as usize; + if num_extra_bytes > 0 { + match self.extra_bytes { + LasExtraBytes::Typed => { + let extra = self.extra.finish(); + + let mut pos = 0; + + for attribute in self.extra_attributes.iter() { + pos += build_attribute(attribute, pos, &extra, &mut columns)?; + } + } + LasExtraBytes::Blob => columns.push(Arc::new(self.extra.finish())), + LasExtraBytes::Ignore => (), + } + } + + Ok(StructArray::new(schema.fields.to_owned(), columns, None)) + } +} + +fn build_attribute( + attribute: &ExtraAttribute, + pos: usize, + extra: &FixedSizeBinaryArray, + columns: &mut Vec<ArrayRef>, +) -> Result<usize, ArrowError> { + let scale = attribute.scale.unwrap_or(1.0); + let offset = attribute.offset.unwrap_or(0.0); + + let width = if let DataType::FixedSizeBinary(width) = attribute.data_type { + width as usize + } else { + attribute.data_type.primitive_width().unwrap() + }; + + let iter = extra.iter().map(|b| &b.unwrap()[pos..pos + width]); + + match &attribute.data_type { + DataType::FixedSizeBinary(_) => { + let data = FixedSizeBinaryArray::try_from_iter(iter)?; + columns.push(Arc::new(data) as ArrayRef) + } + DataType::Int8 => { + let no_data = attribute.no_data.map(i64::from_le_bytes); + + let iter = iter.map(|d| { + let v = d[0] as i8; + if let Some(no_data) = no_data { + if no_data == v as i64 { + return None; + } + } + Some(v) + }); + + if attribute.scale.is_some() || attribute.offset.is_some() { + let mut builder = Float64Builder::with_capacity(extra.len()); + for v in iter { + builder.append_option(v.map(|v| v as f64 * scale + offset)); + } + columns.push(Arc::new(builder.finish()) as ArrayRef) + } else { + let mut builder = Int8Builder::with_capacity(extra.len()); + for v in iter { + builder.append_option(v); + } + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + } + DataType::Int16 => { + let no_data = attribute.no_data.map(i64::from_le_bytes); + + let iter = iter.map(|d| { + let v = i16::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v as i64 { + return None; + } + } + Some(v) + }); + + if attribute.scale.is_some() || attribute.offset.is_some() { + let mut builder = Float64Builder::with_capacity(extra.len()); + for v in iter { + builder.append_option(v.map(|v| v as f64 * scale + offset)); + } + columns.push(Arc::new(builder.finish()) as ArrayRef) + } else { + let mut builder = Int16Builder::with_capacity(extra.len()); + for v in iter { + builder.append_option(v); + } + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + } + DataType::Int32 => { + let no_data = attribute.no_data.map(i64::from_le_bytes); + + let iter = iter.map(|d| { + let v = i32::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v as i64 { + return None; + } + } + Some(v) + }); + + if attribute.scale.is_some() || attribute.offset.is_some() { + let mut builder = Float64Builder::with_capacity(extra.len()); + for v in iter { + builder.append_option(v.map(|v| v as f64 * scale + offset)); + } + columns.push(Arc::new(builder.finish()) as ArrayRef) + } else { + let mut builder = Int32Builder::with_capacity(extra.len()); + for v in iter { + builder.append_option(v); + } + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + } + DataType::Int64 => { + let no_data = attribute.no_data.map(i64::from_le_bytes); + + let iter = iter.map(|d| { + let v = i64::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v { + return None; + } + } + Some(v) + }); + + if attribute.scale.is_some() || attribute.offset.is_some() { + let mut builder = Float64Builder::with_capacity(extra.len()); + for v in iter { + builder.append_option(v.map(|v| v as f64 * scale + offset)); + } + columns.push(Arc::new(builder.finish()) as ArrayRef) + } else { + let mut builder = Int64Builder::with_capacity(extra.len()); + for v in iter { + builder.append_option(v); + } + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + } + DataType::UInt8 => { + let no_data = attribute.no_data.map(u64::from_le_bytes); + + let iter = iter.map(|d| { + let v = d[0]; + if let Some(no_data) = no_data { + if no_data == v as u64 { + return None; + } + } + Some(v) + }); + + if attribute.scale.is_some() || attribute.offset.is_some() { + let mut builder = Float64Builder::with_capacity(extra.len()); + for v in iter { + builder.append_option(v.map(|v| v as f64 * scale + offset)); + } + columns.push(Arc::new(builder.finish()) as ArrayRef) + } else { + let mut builder = UInt8Builder::with_capacity(extra.len()); + for v in iter { + builder.append_option(v); + } + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + } + DataType::UInt16 => { + let no_data = attribute.no_data.map(u64::from_le_bytes); + + let iter = iter.map(|d| { + let v = u16::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v as u64 { + return None; + } + } + Some(v) + }); + + if attribute.scale.is_some() || attribute.offset.is_some() { + let mut builder = Float64Builder::with_capacity(extra.len()); + for v in iter { + builder.append_option(v.map(|v| v as f64 * scale + offset)); + } + columns.push(Arc::new(builder.finish()) as ArrayRef) + } else { + let mut builder = UInt16Builder::with_capacity(extra.len()); + for v in iter { + builder.append_option(v); + } + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + } + DataType::UInt32 => { + let no_data = attribute.no_data.map(u64::from_le_bytes); + + let iter = iter.map(|d| { + let v = u32::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v as u64 { + return None; + } + } + Some(v) + }); + + if attribute.scale.is_some() || attribute.offset.is_some() { + let mut builder = Float64Builder::with_capacity(extra.len()); + for v in iter { + builder.append_option(v.map(|v| v as f64 * scale + offset)); + } + columns.push(Arc::new(builder.finish()) as ArrayRef) + } else { + let mut builder = UInt32Builder::with_capacity(extra.len()); + for v in iter { + builder.append_option(v); + } + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + } + DataType::UInt64 => { + let no_data = attribute.no_data.map(u64::from_le_bytes); + + let iter = iter.map(|d| { + let v = u64::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v { + return None; + } + } + Some(v) + }); + + if attribute.scale.is_some() || attribute.offset.is_some() { + let mut builder = Float64Builder::with_capacity(extra.len()); + for v in iter { + builder.append_option(v.map(|v| v as f64 * scale + offset)); + } + columns.push(Arc::new(builder.finish()) as ArrayRef) + } else { + let mut builder = UInt64Builder::with_capacity(extra.len()); + for v in iter { + builder.append_option(v); + } + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + } + DataType::Float32 => { + let no_data = attribute.no_data.map(f64::from_le_bytes); + + let iter = iter.map(|d| { + let v = f32::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v as f64 { + return None; + } + } + Some(v) + }); + + if attribute.scale.is_some() || attribute.offset.is_some() { + let mut builder = Float64Builder::with_capacity(extra.len()); + for v in iter { + builder.append_option(v.map(|v| v as f64 * scale + offset)); + } + columns.push(Arc::new(builder.finish()) as ArrayRef) + } else { + let mut builder = Float32Builder::with_capacity(extra.len()); + for v in iter { + builder.append_option(v); + } + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + } + DataType::Float64 => { + let no_data = attribute.no_data.map(f64::from_le_bytes); + + let iter = iter.map(|d| { + let v = f64::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v { + return None; + } + } + Some(v) + }); + + if attribute.scale.is_some() || attribute.offset.is_some() { + let mut builder = Float64Builder::with_capacity(extra.len()); + for v in iter { + builder.append_option(v.map(|v| v * scale + offset)); + } + columns.push(Arc::new(builder.finish()) as ArrayRef) + } else { + let mut builder = Float64Builder::with_capacity(extra.len()); + for v in iter { + builder.append_option(v); + } + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + } + + dt => { + return Err(ArrowError::ExternalError( + format!("Unsupported data type for extra bytes: `{dt}`").into(), + )) + } + } + + Ok(width) +} Review Comment: I left a suggestion inline...anywhere in rust/sedona-pointcloud that makes sense for the purposes of this PR! ########## rust/sedona-pointcloud/src/laz/builder.rs: ########## @@ -0,0 +1,926 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::{fmt::Debug, sync::Arc}; + +use arrow_array::{ + builder::{ + ArrayBuilder, BinaryBuilder, BooleanBuilder, FixedSizeBinaryBuilder, Float32Builder, + Float64Builder, Int16Builder, Int32Builder, Int64Builder, Int8Builder, UInt16Builder, + UInt32Builder, UInt64Builder, UInt8Builder, + }, + Array, ArrayRef, BooleanArray, FixedSizeBinaryArray, Float32Array, Float64Array, StructArray, + UInt16Array, UInt8Array, +}; +use arrow_buffer::ScalarBuffer; +use arrow_schema::{ArrowError, DataType}; +use geoarrow_array::{ + array::{CoordBuffer, PointArray, SeparatedCoordBuffer}, + GeoArrowArray, +}; +use geoarrow_schema::Dimension; +use las::{Header, Point}; + +use crate::laz::{ + metadata::ExtraAttribute, + options::{LasExtraBytes, LasPointEncoding}, + schema::try_schema_from_header, +}; + +#[derive(Debug)] +pub struct RowBuilder { + x: Float64Builder, + y: Float64Builder, + z: Float64Builder, + intensity: UInt16Builder, + return_number: UInt8Builder, + number_of_returns: UInt8Builder, + is_synthetic: BooleanBuilder, + is_key_point: BooleanBuilder, + is_withheld: BooleanBuilder, + is_overlap: BooleanBuilder, + scanner_channel: UInt8Builder, + scan_direction: UInt8Builder, + is_edge_of_flight_line: BooleanBuilder, + classification: UInt8Builder, + user_data: UInt8Builder, + scan_angle: Float32Builder, + point_source_id: UInt16Builder, + gps_time: Float64Builder, + red: UInt16Builder, + green: UInt16Builder, + blue: UInt16Builder, + nir: UInt16Builder, + extra: FixedSizeBinaryBuilder, + header: Arc<Header>, + point_encoding: LasPointEncoding, + extra_bytes: LasExtraBytes, + extra_attributes: Arc<Vec<ExtraAttribute>>, +} + +impl RowBuilder { + pub fn new(capacity: usize, header: Arc<Header>) -> Self { + Self { + x: Float64Array::builder(capacity), + y: Float64Array::builder(capacity), + z: Float64Array::builder(capacity), + intensity: UInt16Array::builder(capacity), + return_number: UInt8Array::builder(capacity), + number_of_returns: UInt8Array::builder(capacity), + is_synthetic: BooleanArray::builder(capacity), + is_key_point: BooleanArray::builder(capacity), + is_withheld: BooleanArray::builder(capacity), + is_overlap: BooleanArray::builder(capacity), + scanner_channel: UInt8Array::builder(capacity), + scan_direction: UInt8Array::builder(capacity), + is_edge_of_flight_line: BooleanArray::builder(capacity), + classification: UInt8Array::builder(capacity), + user_data: UInt8Array::builder(capacity), + scan_angle: Float32Array::builder(capacity), + point_source_id: UInt16Array::builder(capacity), + gps_time: Float64Array::builder(capacity), + red: UInt16Array::builder(capacity), + green: UInt16Array::builder(capacity), + blue: UInt16Array::builder(capacity), + nir: UInt16Array::builder(capacity), + extra: FixedSizeBinaryBuilder::with_capacity( + capacity, + header.point_format().extra_bytes as i32, + ), + + header, + point_encoding: Default::default(), + extra_bytes: Default::default(), + extra_attributes: Arc::new(Vec::new()), + } + } + + pub fn with_point_encoding(mut self, point_encoding: LasPointEncoding) -> Self { + self.point_encoding = point_encoding; + self + } + + pub fn with_extra_attributes( + mut self, + attributes: Arc<Vec<ExtraAttribute>>, + extra_bytes: LasExtraBytes, + ) -> Self { + self.extra_attributes = attributes; + self.extra_bytes = extra_bytes; + self + } + + pub fn append(&mut self, p: Point) { + self.x.append_value(p.x); + self.y.append_value(p.y); + self.z.append_value(p.z); + self.intensity.append_option(Some(p.intensity)); + self.return_number.append_value(p.return_number); + self.number_of_returns.append_value(p.number_of_returns); + self.is_synthetic.append_value(p.is_synthetic); + self.is_key_point.append_value(p.is_key_point); + self.is_withheld.append_value(p.is_withheld); + self.is_overlap.append_value(p.is_overlap); + self.scanner_channel.append_value(p.scanner_channel); + self.scan_direction.append_value(p.scan_direction as u8); + self.is_edge_of_flight_line + .append_value(p.is_edge_of_flight_line); + self.classification.append_value(u8::from(p.classification)); + self.user_data.append_value(p.user_data); + self.scan_angle.append_value(p.scan_angle); + self.point_source_id.append_value(p.point_source_id); + if self.header.point_format().has_gps_time { + self.gps_time.append_value(p.gps_time.unwrap()); + } + if self.header.point_format().has_color { + let color = p.color.unwrap(); + self.red.append_value(color.red); + self.green.append_value(color.green); + self.blue.append_value(color.blue); + } + if self.header.point_format().has_nir { + self.nir.append_value(p.nir.unwrap()); + } + if self.header.point_format().extra_bytes > 0 { + self.extra.append_value(p.extra_bytes).unwrap(); + } + } + + /// Note: returns StructArray to allow nesting within another array if desired + pub fn finish(&mut self) -> Result<StructArray, ArrowError> { + let schema = try_schema_from_header(&self.header, self.point_encoding, self.extra_bytes)?; + + let mut columns = match self.point_encoding { + LasPointEncoding::Plain => vec![ + Arc::new(self.x.finish()) as ArrayRef, + Arc::new(self.y.finish()) as ArrayRef, + Arc::new(self.z.finish()) as ArrayRef, + ], + LasPointEncoding::Wkb => { + const POINT_SIZE: usize = 29; + + let n: usize = self.x.len(); + + let mut builder = BinaryBuilder::with_capacity(n, n * POINT_SIZE); + + let x = self.x.finish(); + let y = self.y.finish(); + let z = self.z.finish(); + + let mut wkb_bytes = [0_u8; POINT_SIZE]; + wkb_bytes[0] = 0x01; // Little-endian + wkb_bytes[1..5].copy_from_slice(&[0xE9, 0x03, 0x00, 0x00]); // Point Z type (1001) + + for i in 0..n { + let x = unsafe { x.value_unchecked(i) }; + let y = unsafe { y.value_unchecked(i) }; + let z = unsafe { z.value_unchecked(i) }; + + wkb_bytes[5..13].copy_from_slice(x.to_le_bytes().as_slice()); + wkb_bytes[13..21].copy_from_slice(y.to_le_bytes().as_slice()); + wkb_bytes[21..29].copy_from_slice(z.to_le_bytes().as_slice()); + + builder.append_value(wkb_bytes); + } + + vec![Arc::new(builder.finish()) as ArrayRef] + } + LasPointEncoding::Native => { + let buffers = [ + self.x.finish().into_parts().1, + self.y.finish().into_parts().1, + self.z.finish().into_parts().1, + ScalarBuffer::from(vec![]), + ]; + let coords = CoordBuffer::Separated(SeparatedCoordBuffer::from_array( + buffers, + Dimension::XYZ, + )?); + let points = PointArray::new(coords, None, Default::default()); + vec![points.to_array_ref()] + } + }; + + columns.extend([ + Arc::new(self.intensity.finish()) as ArrayRef, + Arc::new(self.return_number.finish()) as ArrayRef, + Arc::new(self.number_of_returns.finish()) as ArrayRef, + Arc::new(self.is_synthetic.finish()) as ArrayRef, + Arc::new(self.is_key_point.finish()) as ArrayRef, + Arc::new(self.is_withheld.finish()) as ArrayRef, + Arc::new(self.is_overlap.finish()) as ArrayRef, + Arc::new(self.scanner_channel.finish()) as ArrayRef, + Arc::new(self.scan_direction.finish()) as ArrayRef, + Arc::new(self.is_edge_of_flight_line.finish()) as ArrayRef, + Arc::new(self.classification.finish()) as ArrayRef, + Arc::new(self.user_data.finish()) as ArrayRef, + Arc::new(self.scan_angle.finish()) as ArrayRef, + Arc::new(self.point_source_id.finish()) as ArrayRef, + ]); + if self.header.point_format().has_gps_time { + columns.push(Arc::new(self.gps_time.finish()) as ArrayRef); + } + if self.header.point_format().has_color { + columns.extend([ + Arc::new(self.red.finish()) as ArrayRef, + Arc::new(self.green.finish()) as ArrayRef, + Arc::new(self.blue.finish()) as ArrayRef, + ]); + } + if self.header.point_format().has_nir { + columns.push(Arc::new(self.nir.finish()) as ArrayRef); + } + + // extra bytes + let num_extra_bytes = self.header.point_format().extra_bytes as usize; + if num_extra_bytes > 0 { + match self.extra_bytes { + LasExtraBytes::Typed => { + let extra = self.extra.finish(); + + let mut pos = 0; + + for attribute in self.extra_attributes.iter() { + pos += build_attribute(attribute, pos, &extra, &mut columns)?; + } + } + LasExtraBytes::Blob => columns.push(Arc::new(self.extra.finish())), + LasExtraBytes::Ignore => (), + } + } + + Ok(StructArray::new(schema.fields.to_owned(), columns, None)) + } +} + +fn build_attribute( + attribute: &ExtraAttribute, + pos: usize, + extra: &FixedSizeBinaryArray, + columns: &mut Vec<ArrayRef>, +) -> Result<usize, ArrowError> { + let scale = attribute.scale.unwrap_or(1.0); + let offset = attribute.offset.unwrap_or(0.0); + + let width = if let DataType::FixedSizeBinary(width) = attribute.data_type { + width as usize + } else { + attribute.data_type.primitive_width().unwrap() + }; + + let iter = extra.iter().map(|b| &b.unwrap()[pos..pos + width]); + + match &attribute.data_type { + DataType::FixedSizeBinary(_) => { + let data = FixedSizeBinaryArray::try_from_iter(iter)?; + columns.push(Arc::new(data) as ArrayRef) + } + DataType::Int8 => { + let mut builder = Int8Builder::with_capacity(extra.len()); + let no_data = attribute.no_data.map(i64::from_le_bytes); + + for d in iter { + let mut v = i8::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v as i64 { + builder.append_null(); + continue; + } + } + if attribute.scale.is_some() || attribute.offset.is_some() { + v = (v as f64 * scale + offset) as i8; + } + builder.append_value(v) + } + + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + DataType::Int16 => { + let mut builder = Int16Builder::with_capacity(extra.len()); + let no_data = attribute.no_data.map(i64::from_le_bytes); + + for d in iter { + let mut v = i16::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v as i64 { + builder.append_null(); + continue; + } + } + if attribute.scale.is_some() || attribute.offset.is_some() { + v = (v as f64 * scale + offset) as i16; + } + builder.append_value(v) + } + + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + DataType::Int32 => { + let mut builder = Int32Builder::with_capacity(extra.len()); + let no_data = attribute.no_data.map(i64::from_le_bytes); + + for d in iter { + let mut v = i32::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v as i64 { + builder.append_null(); + continue; + } + } + if attribute.scale.is_some() || attribute.offset.is_some() { + v = (v as f64 * scale + offset) as i32; + } + builder.append_value(v) + } + + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + DataType::Int64 => { + let mut builder = Int64Builder::with_capacity(extra.len()); + let no_data = attribute.no_data.map(i64::from_le_bytes); + + for d in iter { + let mut v = i64::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v { + builder.append_null(); + continue; + } + } + if attribute.scale.is_some() || attribute.offset.is_some() { + v = (v as f64 * scale + offset) as i64; + } + builder.append_value(v) + } + + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + DataType::UInt8 => { + let mut builder = UInt8Builder::with_capacity(extra.len()); + let no_data = attribute.no_data.map(u64::from_le_bytes); + + for d in iter { + let mut v = u8::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v as u64 { + builder.append_null(); + continue; + } + } + if attribute.scale.is_some() || attribute.offset.is_some() { + v = (v as f64 * scale + offset) as u8; + } + builder.append_value(v) + } + + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + DataType::UInt16 => { + let mut builder = UInt16Builder::with_capacity(extra.len()); + let no_data = attribute.no_data.map(u64::from_le_bytes); + + for d in iter { + let mut v = u16::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v as u64 { + builder.append_null(); + continue; + } + } + if attribute.scale.is_some() || attribute.offset.is_some() { + v = (v as f64 * scale + offset) as u16; + } + builder.append_value(v) + } + + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + DataType::UInt32 => { + let mut builder = UInt32Builder::with_capacity(extra.len()); + let no_data = attribute.no_data.map(u64::from_le_bytes); + + for d in iter { + let mut v = u32::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v as u64 { + builder.append_null(); + continue; + } + } + if attribute.scale.is_some() || attribute.offset.is_some() { + v = (v as f64 * scale + offset) as u32; + } + builder.append_value(v) + } + + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + DataType::UInt64 => { + let mut builder = UInt64Builder::with_capacity(extra.len()); + let no_data = attribute.no_data.map(u64::from_le_bytes); + + for d in iter { + let mut v = u64::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v { + builder.append_null(); + continue; + } + } + if attribute.scale.is_some() || attribute.offset.is_some() { + v = (v as f64 * scale + offset) as u64; + } + builder.append_value(v) + } + + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + DataType::Float32 => { + let mut builder = Float32Builder::with_capacity(extra.len()); + let no_data = attribute.no_data.map(f64::from_le_bytes); + + for d in iter { + let mut v = f32::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v as f64 { + builder.append_null(); + continue; + } + } + if attribute.scale.is_some() || attribute.offset.is_some() { + v = (v as f64 * scale + offset) as f32; + } + builder.append_value(v) + } + + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + DataType::Float64 => { + let mut builder = Float64Builder::with_capacity(extra.len()); + let no_data = attribute.no_data.map(f64::from_le_bytes); + + for d in iter { + let mut v = f64::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v { + builder.append_null(); + continue; + } + } + if attribute.scale.is_some() || attribute.offset.is_some() { + v = v * scale + offset; + } + builder.append_value(v); + } + + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + + dt => { + return Err(ArrowError::ExternalError( + format!("Unsupported data type for extra bytes: `{dt}`").into(), + )) + } + } + + Ok(width) +} + +#[cfg(test)] +mod tests { + use std::{fs::File, sync::Arc}; + + use arrow_array::{ + cast::AsArray, + types::{ + Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, UInt16Type, + UInt32Type, UInt64Type, UInt8Type, + }, + }; + use datafusion_datasource::PartitionedFile; + use las::{point::Format, Builder, Writer}; + use object_store::{local::LocalFileSystem, path::Path, ObjectStore}; + + use crate::laz::{ + options::{LasExtraBytes, LazTableOptions}, + reader::LazFileReaderFactory, + }; + + #[tokio::test] + async fn point_formats() { + let tmpdir = tempfile::tempdir().unwrap(); + + for format in 0..=10 { + let tmp_path = tmpdir.path().join("tmp.laz"); + let tmp_file = File::create(&tmp_path).unwrap(); + + // create laz file + let mut builder = Builder::from((1, 4)); + builder.point_format = Format::new(format).unwrap(); + builder.point_format.is_compressed = true; + let header = builder.into_header().unwrap(); + let mut writer = Writer::new(tmp_file, header).unwrap(); + writer.close().unwrap(); + + // read batch with `LazFileReader` + let store = LocalFileSystem::new(); + let location = Path::from_filesystem_path(tmp_path).unwrap(); + let object = store.head(&location).await.unwrap(); + + let laz_file_reader = LazFileReaderFactory::new(Arc::new(store), None) + .create_reader( + PartitionedFile::new(location, object.size), + LazTableOptions::default(), + ) + .unwrap(); + let metadata = laz_file_reader.get_metadata().await.unwrap(); + + let batch = laz_file_reader + .get_batch(&metadata.chunk_table[0]) + .await + .unwrap(); + + match format { + 0 => assert_eq!(batch.num_columns(), 17), + 1 | 4 | 6 | 9 => assert_eq!(batch.num_columns(), 18), + 2 => assert_eq!(batch.num_columns(), 20), + 3 | 5 | 7 => assert_eq!(batch.num_columns(), 21), + 8 | 10 => assert_eq!(batch.num_columns(), 22), + _ => unreachable!(), + } + } + } + + #[tokio::test] + async fn extra_attributes() { + // create extra file + let extra = [ + 76, 65, 83, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 79, + 84, 72, 69, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 108, 97, 115, 112, 121, 32, 50, 46, 55, 46, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 234, 7, 119, 1, 145, 24, 0, 0, 2, 0, 0, 0, 134, + 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, + 153, 153, 153, 153, 153, 185, 63, 154, 153, 153, 153, 153, 153, 185, 63, 154, 153, 153, + 153, 153, 153, 185, 63, 0, 0, 0, 0, 0, 0, 240, 63, 0, 0, 0, 0, 0, 0, 240, 63, 0, 0, 0, + 0, 0, 0, 240, 63, 0, 0, 0, 0, 0, 0, 224, 63, 0, 0, 0, 0, 0, 0, 224, 63, 0, 0, 0, 0, 0, + 0, 224, 63, 0, 0, 0, 0, 0, 0, 224, 63, 0, 0, 0, 0, 0, 0, 224, 63, 0, 0, 0, 0, 0, 0, + 224, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 76, 65, 83, 70, 95, 83, 112, 101, 99, 0, 0, 0, 0, 0, 0, 0, 4, + 0, 128, 22, 69, 120, 116, 114, 97, 32, 66, 121, 116, 101, 115, 32, 82, 101, 99, 111, + 114, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 117, 105, 110, 116, 56, + 95, 112, 108, 97, 105, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 30, 117, 105, 110, 116, 56, 95, 115, + 99, 97, 108, 101, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 153, 153, 153, 153, 153, + 185, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 64, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 117, 105, 110, 116, + 56, 95, 110, 111, 100, 97, 116, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 255, 255, 255, 255, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 105, 110, 116, 56, + 95, 112, 108, 97, 105, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 30, 105, 110, 116, 56, 95, 115, 99, + 97, 108, 101, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 153, 153, 153, 153, 153, + 185, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 64, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7, 105, 110, 116, 56, + 95, 110, 111, 100, 97, 116, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 255, 255, 255, 255, 255, 255, 255, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6, 117, 105, 110, + 116, 49, 54, 95, 112, 108, 97, 105, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 30, 117, 105, 110, 116, 49, + 54, 95, 115, 99, 97, 108, 101, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 153, 153, 153, 153, + 153, 185, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 64, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7, 117, 105, 110, + 116, 49, 54, 95, 110, 111, 100, 97, 116, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 255, 255, 255, 255, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 105, 110, + 116, 49, 54, 95, 112, 108, 97, 105, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 30, 105, 110, 116, 49, 54, + 95, 115, 99, 97, 108, 101, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 153, 153, 153, 153, + 153, 185, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 64, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7, 105, 110, 116, 49, + 54, 95, 110, 111, 100, 97, 116, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 255, 255, 255, 255, 255, 255, 255, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 117, 105, 110, + 116, 51, 50, 95, 112, 108, 97, 105, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 30, 117, 105, 110, 116, 51, + 50, 95, 115, 99, 97, 108, 101, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 153, 153, 153, 153, + 153, 185, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 64, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7, 117, 105, 110, + 116, 51, 50, 95, 110, 111, 100, 97, 116, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 255, 255, 255, 255, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 105, 110, + 116, 51, 50, 95, 112, 108, 97, 105, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 30, 105, 110, 116, 51, 50, + 95, 115, 99, 97, 108, 101, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 153, 153, 153, 153, + 153, 185, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 64, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 105, 110, 116, 51, + 50, 95, 110, 111, 100, 97, 116, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 255, 255, 255, 255, 255, 255, 255, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 117, 105, 110, + 116, 54, 52, 95, 112, 108, 97, 105, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 30, 117, 105, 110, 116, 54, + 52, 95, 115, 99, 97, 108, 101, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 153, 153, 153, 153, + 153, 185, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 64, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 117, 105, 110, + 116, 54, 52, 95, 110, 111, 100, 97, 116, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 255, 255, 255, 255, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 105, 110, + 116, 54, 52, 95, 112, 108, 97, 105, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 30, 105, 110, 116, 54, 52, + 95, 115, 99, 97, 108, 101, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 153, 153, 153, 153, + 153, 185, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 64, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 7, 105, 110, 116, 54, + 52, 95, 110, 111, 100, 97, 116, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 255, 255, 255, 255, 255, 255, 255, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 6, 102, 108, 111, + 97, 116, 51, 50, 95, 112, 108, 97, 105, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 53, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 30, 102, 108, 111, 97, + 116, 51, 50, 95, 115, 99, 97, 108, 101, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 128, 91, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 128, 91, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 153, 153, + 153, 153, 153, 185, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 36, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 7, 102, + 108, 111, 97, 116, 51, 50, 95, 110, 111, 100, 97, 116, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 64, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 255, 255, 255, 239, 127, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 255, 255, 255, 239, 255, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 10, 6, 102, 108, 111, 97, 116, 54, 52, 95, 112, 108, 97, 105, 110, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 64, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 10, 30, 102, 108, 111, 97, 116, 54, 52, 95, 115, 99, 97, 108, 101, 100, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 91, 64, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 91, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 154, 153, 153, 153, 153, 153, 185, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 10, 7, 102, 108, 111, 97, 116, 54, 52, 95, 110, 111, 100, 97, 116, + 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 69, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 255, 255, 255, + 239, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 255, 255, 255, + 239, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 97, 115, 122, 105, 112, 32, 101, 110, + 99, 111, 100, 101, 100, 0, 0, 188, 86, 46, 0, 104, 116, 116, 112, 58, 47, 47, 108, 97, + 115, 122, 105, 112, 46, 111, 114, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, + 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 80, 195, 0, 0, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 2, 0, 10, 0, 30, 0, 3, 0, 14, 0, 126, 0, 3, 0, + 93, 27, 0, 0, 0, 0, 0, 0, 251, 255, 255, 255, 251, 255, 255, 255, 251, 255, 255, 255, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 110, 42, 21, 110, 42, 21, 0, + 110, 0, 42, 0, 21, 0, 110, 0, 42, 0, 21, 0, 0, 0, 110, 0, 0, 0, 42, 0, 0, 0, 21, 0, 0, + 0, 110, 0, 0, 0, 42, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 42, 0, + 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 168, 65, 0, 0, 220, 66, 0, 0, 40, 66, 0, 0, 0, 0, 0, 0, 53, 64, 0, 0, 0, 0, + 0, 128, 91, 64, 0, 0, 0, 0, 0, 0, 69, 64, 1, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 82, 237, 50, 0, 0, + ]; + + let tmpdir = tempfile::tempdir().unwrap(); + let tmp_path = tmpdir.path().join("extra.laz"); + std::fs::write(&tmp_path, extra).unwrap(); + + // TODO: use sedona-testing or similar + // let path = "../../submodules/sedona-testing/data/pointclouds/extra.laz"; + Review Comment: (This is a good idea but for now I think the test bytes work) ```suggestion ``` ########## rust/sedona-pointcloud/src/laz/format.rs: ########## @@ -0,0 +1,209 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::{any::Any, collections::HashMap, fmt, sync::Arc}; + +use arrow_schema::{Schema, SchemaRef}; +use datafusion_catalog::{memory::DataSourceExec, Session}; +use datafusion_common::{ + config::ExtensionOptions, error::DataFusionError, internal_err, + parsers::CompressionTypeVariant, GetExt, Statistics, +}; +use datafusion_datasource::{ + file::FileSource, + file_compression_type::FileCompressionType, + file_format::{FileFormat, FileFormatFactory}, + file_scan_config::FileScanConfig, +}; +use datafusion_physical_plan::ExecutionPlan; +use futures::{StreamExt, TryStreamExt}; +use object_store::{ObjectMeta, ObjectStore}; + +use crate::laz::{metadata::LazMetadataReader, options::LazTableOptions, source::LazSource}; + +const DEFAULT_LAZ_EXTENSION: &str = ".laz"; + +/// Factory struct used to create [LazFormat] +#[derive(Default)] +pub struct LazFormatFactory { + // inner options for LAZ + pub options: Option<LazTableOptions>, +} + +impl LazFormatFactory { + /// Creates an instance of [LazFormatFactory] + pub fn new() -> Self { + Self { options: None } + } + + /// Creates an instance of [LazFormatFactory] with customized default options + pub fn new_with(options: LazTableOptions) -> Self { + Self { + options: Some(options), + } + } +} + +impl FileFormatFactory for LazFormatFactory { + fn create( + &self, + state: &dyn Session, + format_options: &HashMap<String, String>, + ) -> Result<Arc<dyn FileFormat>, DataFusionError> { + let mut laz_options = state + .config_options() + .extensions + .get::<LazTableOptions>() + .or_else(|| state.table_options().extensions.get::<LazTableOptions>()) + .cloned() + .or(self.options.clone()) + .unwrap_or_default(); + + for (k, v) in format_options { + laz_options.set(k, v)?; + } + + Ok(Arc::new(LazFormat::default().with_options(laz_options))) + } + + fn default(&self) -> Arc<dyn FileFormat> { + Arc::new(LazFormat::default()) + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +impl GetExt for LazFormatFactory { + fn get_ext(&self) -> String { + // Removes the dot, i.e. ".laz" -> "laz" + DEFAULT_LAZ_EXTENSION[1..].to_string() + } +} + +impl fmt::Debug for LazFormatFactory { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LazFormatFactory") + .field("LazFormatFactory", &self.options) + .finish() + } +} + +/// The LAZ `FileFormat` implementation +#[derive(Debug, Default)] +pub struct LazFormat { + pub options: LazTableOptions, +} + +impl LazFormat { + pub fn with_options(mut self, options: LazTableOptions) -> Self { + self.options = options; + self + } +} + +#[async_trait::async_trait] +impl FileFormat for LazFormat { + fn as_any(&self) -> &dyn Any { + self + } + + fn get_ext(&self) -> String { + LazFormatFactory::new().get_ext() + } + + fn get_ext_with_compression( + &self, + file_compression_type: &FileCompressionType, + ) -> Result<String, DataFusionError> { + let ext = self.get_ext(); + match file_compression_type.get_variant() { + CompressionTypeVariant::UNCOMPRESSED => Ok(ext), + _ => internal_err!("Laz FileFormat does not support compression."), + } + } + + fn compression_type(&self) -> Option<FileCompressionType> { + Some(FileCompressionType::UNCOMPRESSED) + } + + async fn infer_schema( + &self, + state: &dyn Session, + store: &Arc<dyn ObjectStore>, + objects: &[ObjectMeta], + ) -> Result<SchemaRef, DataFusionError> { + let file_metadata_cache = state.runtime_env().cache_manager.get_file_metadata_cache(); + + let mut schemas: Vec<_> = futures::stream::iter(objects) + .map(|object_meta| async { + let loc_path = object_meta.location.clone(); + + let schema = LazMetadataReader::new(store, object_meta) + .with_file_metadata_cache(Some(Arc::clone(&file_metadata_cache))) + .with_options(self.options.clone()) + .fetch_schema() + .await?; + + Ok::<_, DataFusionError>((loc_path, schema)) + }) + .boxed() // Workaround https://github.com/rust-lang/rust/issues/64552 + // fetch schemas concurrently, if requested + .buffered(state.config_options().execution.meta_fetch_concurrency) + .try_collect() + .await?; + + schemas.sort_by(|(location1, _), (location2, _)| location1.cmp(location2)); + + let schemas = schemas + .into_iter() + .map(|(_, schema)| schema) + .collect::<Vec<_>>(); + + let schema = Schema::try_merge(schemas)?; + + Ok(Arc::new(schema)) + } + + async fn infer_stats( + &self, + state: &dyn Session, + store: &Arc<dyn ObjectStore>, + table_schema: SchemaRef, + object: &ObjectMeta, + ) -> Result<Statistics, DataFusionError> { + let file_metadata_cache = state.runtime_env().cache_manager.get_file_metadata_cache(); + LazMetadataReader::new(store, object) + .with_options(self.options.clone()) + .with_file_metadata_cache(Some(Arc::clone(&file_metadata_cache))) + .fetch_statistics(&table_schema) + .await + } + + async fn create_physical_plan( + &self, + _state: &dyn Session, + conf: FileScanConfig, + ) -> Result<Arc<dyn ExecutionPlan>, DataFusionError> { + Ok(DataSourceExec::from_data_source(conf)) + } + + fn file_source(&self) -> Arc<dyn FileSource> { + Arc::new(LazSource::default().with_options(self.options.clone())) + } +} Review Comment: In `sedona-geoparquet/src/format.rs` you'll see a few tests for the Parquet format that verify a few basic properties of the format in the spirit of "is it plugged in". I think this file would benefit from having those (both to make sure everything here is plugged in and to ensure future contributors to this file have a place to test bug fixes). ########## rust/sedona-pointcloud/src/laz/builder.rs: ########## @@ -0,0 +1,926 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::{fmt::Debug, sync::Arc}; + +use arrow_array::{ + builder::{ + ArrayBuilder, BinaryBuilder, BooleanBuilder, FixedSizeBinaryBuilder, Float32Builder, + Float64Builder, Int16Builder, Int32Builder, Int64Builder, Int8Builder, UInt16Builder, + UInt32Builder, UInt64Builder, UInt8Builder, + }, + Array, ArrayRef, BooleanArray, FixedSizeBinaryArray, Float32Array, Float64Array, StructArray, + UInt16Array, UInt8Array, +}; +use arrow_buffer::ScalarBuffer; +use arrow_schema::{ArrowError, DataType}; +use geoarrow_array::{ + array::{CoordBuffer, PointArray, SeparatedCoordBuffer}, + GeoArrowArray, +}; +use geoarrow_schema::Dimension; +use las::{Header, Point}; + +use crate::laz::{ + metadata::ExtraAttribute, + options::{LasExtraBytes, LasPointEncoding}, + schema::try_schema_from_header, +}; + +#[derive(Debug)] +pub struct RowBuilder { + x: Float64Builder, + y: Float64Builder, + z: Float64Builder, + intensity: UInt16Builder, + return_number: UInt8Builder, + number_of_returns: UInt8Builder, + is_synthetic: BooleanBuilder, + is_key_point: BooleanBuilder, + is_withheld: BooleanBuilder, + is_overlap: BooleanBuilder, + scanner_channel: UInt8Builder, + scan_direction: UInt8Builder, + is_edge_of_flight_line: BooleanBuilder, + classification: UInt8Builder, + user_data: UInt8Builder, + scan_angle: Float32Builder, + point_source_id: UInt16Builder, + gps_time: Float64Builder, + red: UInt16Builder, + green: UInt16Builder, + blue: UInt16Builder, + nir: UInt16Builder, + extra: FixedSizeBinaryBuilder, + header: Arc<Header>, + point_encoding: LasPointEncoding, + extra_bytes: LasExtraBytes, + extra_attributes: Arc<Vec<ExtraAttribute>>, +} + +impl RowBuilder { + pub fn new(capacity: usize, header: Arc<Header>) -> Self { + Self { + x: Float64Array::builder(capacity), + y: Float64Array::builder(capacity), + z: Float64Array::builder(capacity), + intensity: UInt16Array::builder(capacity), + return_number: UInt8Array::builder(capacity), + number_of_returns: UInt8Array::builder(capacity), + is_synthetic: BooleanArray::builder(capacity), + is_key_point: BooleanArray::builder(capacity), + is_withheld: BooleanArray::builder(capacity), + is_overlap: BooleanArray::builder(capacity), + scanner_channel: UInt8Array::builder(capacity), + scan_direction: UInt8Array::builder(capacity), + is_edge_of_flight_line: BooleanArray::builder(capacity), + classification: UInt8Array::builder(capacity), + user_data: UInt8Array::builder(capacity), + scan_angle: Float32Array::builder(capacity), + point_source_id: UInt16Array::builder(capacity), + gps_time: Float64Array::builder(capacity), + red: UInt16Array::builder(capacity), + green: UInt16Array::builder(capacity), + blue: UInt16Array::builder(capacity), + nir: UInt16Array::builder(capacity), + extra: FixedSizeBinaryBuilder::with_capacity( + capacity, + header.point_format().extra_bytes as i32, + ), + + header, + point_encoding: Default::default(), + extra_bytes: Default::default(), + extra_attributes: Arc::new(Vec::new()), + } + } + + pub fn with_point_encoding(mut self, point_encoding: LasPointEncoding) -> Self { + self.point_encoding = point_encoding; + self + } + + pub fn with_extra_attributes( + mut self, + attributes: Arc<Vec<ExtraAttribute>>, + extra_bytes: LasExtraBytes, + ) -> Self { + self.extra_attributes = attributes; + self.extra_bytes = extra_bytes; + self + } + + pub fn append(&mut self, p: Point) { + self.x.append_value(p.x); + self.y.append_value(p.y); + self.z.append_value(p.z); + self.intensity.append_option(Some(p.intensity)); + self.return_number.append_value(p.return_number); + self.number_of_returns.append_value(p.number_of_returns); + self.is_synthetic.append_value(p.is_synthetic); + self.is_key_point.append_value(p.is_key_point); + self.is_withheld.append_value(p.is_withheld); + self.is_overlap.append_value(p.is_overlap); + self.scanner_channel.append_value(p.scanner_channel); + self.scan_direction.append_value(p.scan_direction as u8); + self.is_edge_of_flight_line + .append_value(p.is_edge_of_flight_line); + self.classification.append_value(u8::from(p.classification)); + self.user_data.append_value(p.user_data); + self.scan_angle.append_value(p.scan_angle); + self.point_source_id.append_value(p.point_source_id); + if self.header.point_format().has_gps_time { + self.gps_time.append_value(p.gps_time.unwrap()); + } + if self.header.point_format().has_color { + let color = p.color.unwrap(); + self.red.append_value(color.red); + self.green.append_value(color.green); + self.blue.append_value(color.blue); + } + if self.header.point_format().has_nir { + self.nir.append_value(p.nir.unwrap()); + } + if self.header.point_format().extra_bytes > 0 { + self.extra.append_value(p.extra_bytes).unwrap(); + } + } + + /// Note: returns StructArray to allow nesting within another array if desired + pub fn finish(&mut self) -> Result<StructArray, ArrowError> { + let schema = try_schema_from_header(&self.header, self.point_encoding, self.extra_bytes)?; + + let mut columns = match self.point_encoding { + LasPointEncoding::Plain => vec![ + Arc::new(self.x.finish()) as ArrayRef, + Arc::new(self.y.finish()) as ArrayRef, + Arc::new(self.z.finish()) as ArrayRef, + ], + LasPointEncoding::Wkb => { + const POINT_SIZE: usize = 29; + + let n: usize = self.x.len(); + + let mut builder = BinaryBuilder::with_capacity(n, n * POINT_SIZE); + + let x = self.x.finish(); + let y = self.y.finish(); + let z = self.z.finish(); + + let mut wkb_bytes = [0_u8; POINT_SIZE]; + wkb_bytes[0] = 0x01; // Little-endian + wkb_bytes[1..5].copy_from_slice(&[0xE9, 0x03, 0x00, 0x00]); // Point Z type (1001) + + for i in 0..n { + let x = unsafe { x.value_unchecked(i) }; + let y = unsafe { y.value_unchecked(i) }; + let z = unsafe { z.value_unchecked(i) }; + + wkb_bytes[5..13].copy_from_slice(x.to_le_bytes().as_slice()); + wkb_bytes[13..21].copy_from_slice(y.to_le_bytes().as_slice()); + wkb_bytes[21..29].copy_from_slice(z.to_le_bytes().as_slice()); + + builder.append_value(wkb_bytes); + } + + vec![Arc::new(builder.finish()) as ArrayRef] + } + LasPointEncoding::Native => { + let buffers = [ + self.x.finish().into_parts().1, + self.y.finish().into_parts().1, + self.z.finish().into_parts().1, + ScalarBuffer::from(vec![]), + ]; + let coords = CoordBuffer::Separated(SeparatedCoordBuffer::from_array( + buffers, + Dimension::XYZ, + )?); + let points = PointArray::new(coords, None, Default::default()); + vec![points.to_array_ref()] + } + }; + + columns.extend([ + Arc::new(self.intensity.finish()) as ArrayRef, + Arc::new(self.return_number.finish()) as ArrayRef, + Arc::new(self.number_of_returns.finish()) as ArrayRef, + Arc::new(self.is_synthetic.finish()) as ArrayRef, + Arc::new(self.is_key_point.finish()) as ArrayRef, + Arc::new(self.is_withheld.finish()) as ArrayRef, + Arc::new(self.is_overlap.finish()) as ArrayRef, + Arc::new(self.scanner_channel.finish()) as ArrayRef, + Arc::new(self.scan_direction.finish()) as ArrayRef, + Arc::new(self.is_edge_of_flight_line.finish()) as ArrayRef, + Arc::new(self.classification.finish()) as ArrayRef, + Arc::new(self.user_data.finish()) as ArrayRef, + Arc::new(self.scan_angle.finish()) as ArrayRef, + Arc::new(self.point_source_id.finish()) as ArrayRef, + ]); + if self.header.point_format().has_gps_time { + columns.push(Arc::new(self.gps_time.finish()) as ArrayRef); + } + if self.header.point_format().has_color { + columns.extend([ + Arc::new(self.red.finish()) as ArrayRef, + Arc::new(self.green.finish()) as ArrayRef, + Arc::new(self.blue.finish()) as ArrayRef, + ]); + } + if self.header.point_format().has_nir { + columns.push(Arc::new(self.nir.finish()) as ArrayRef); + } + + // extra bytes + let num_extra_bytes = self.header.point_format().extra_bytes as usize; + if num_extra_bytes > 0 { + match self.extra_bytes { + LasExtraBytes::Typed => { + let extra = self.extra.finish(); + + let mut pos = 0; + + for attribute in self.extra_attributes.iter() { + pos += build_attribute(attribute, pos, &extra, &mut columns)?; + } + } + LasExtraBytes::Blob => columns.push(Arc::new(self.extra.finish())), + LasExtraBytes::Ignore => (), + } + } + + Ok(StructArray::new(schema.fields.to_owned(), columns, None)) + } +} + +fn build_attribute( + attribute: &ExtraAttribute, + pos: usize, + extra: &FixedSizeBinaryArray, + columns: &mut Vec<ArrayRef>, +) -> Result<usize, ArrowError> { + let scale = attribute.scale.unwrap_or(1.0); + let offset = attribute.offset.unwrap_or(0.0); + + let width = if let DataType::FixedSizeBinary(width) = attribute.data_type { + width as usize + } else { + attribute.data_type.primitive_width().unwrap() + }; + + let iter = extra.iter().map(|b| &b.unwrap()[pos..pos + width]); + + match &attribute.data_type { + DataType::FixedSizeBinary(_) => { + let data = FixedSizeBinaryArray::try_from_iter(iter)?; + columns.push(Arc::new(data) as ArrayRef) + } + DataType::Int8 => { + let mut builder = Int8Builder::with_capacity(extra.len()); + let no_data = attribute.no_data.map(i64::from_le_bytes); + + for d in iter { + let mut v = i8::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v as i64 { + builder.append_null(); + continue; + } + } + if attribute.scale.is_some() || attribute.offset.is_some() { + v = (v as f64 * scale + offset) as i8; + } + builder.append_value(v) + } + + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + DataType::Int16 => { + let mut builder = Int16Builder::with_capacity(extra.len()); + let no_data = attribute.no_data.map(i64::from_le_bytes); + + for d in iter { + let mut v = i16::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v as i64 { + builder.append_null(); + continue; + } + } + if attribute.scale.is_some() || attribute.offset.is_some() { + v = (v as f64 * scale + offset) as i16; + } + builder.append_value(v) + } + + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + DataType::Int32 => { + let mut builder = Int32Builder::with_capacity(extra.len()); + let no_data = attribute.no_data.map(i64::from_le_bytes); + + for d in iter { + let mut v = i32::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v as i64 { + builder.append_null(); + continue; + } + } + if attribute.scale.is_some() || attribute.offset.is_some() { + v = (v as f64 * scale + offset) as i32; + } + builder.append_value(v) + } + + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + DataType::Int64 => { + let mut builder = Int64Builder::with_capacity(extra.len()); + let no_data = attribute.no_data.map(i64::from_le_bytes); + + for d in iter { + let mut v = i64::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v { + builder.append_null(); + continue; + } + } + if attribute.scale.is_some() || attribute.offset.is_some() { + v = (v as f64 * scale + offset) as i64; + } + builder.append_value(v) + } + + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + DataType::UInt8 => { + let mut builder = UInt8Builder::with_capacity(extra.len()); + let no_data = attribute.no_data.map(u64::from_le_bytes); + + for d in iter { + let mut v = u8::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v as u64 { + builder.append_null(); + continue; + } + } + if attribute.scale.is_some() || attribute.offset.is_some() { + v = (v as f64 * scale + offset) as u8; + } + builder.append_value(v) + } + + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + DataType::UInt16 => { + let mut builder = UInt16Builder::with_capacity(extra.len()); + let no_data = attribute.no_data.map(u64::from_le_bytes); + + for d in iter { + let mut v = u16::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v as u64 { + builder.append_null(); + continue; + } + } + if attribute.scale.is_some() || attribute.offset.is_some() { + v = (v as f64 * scale + offset) as u16; + } + builder.append_value(v) + } + + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + DataType::UInt32 => { + let mut builder = UInt32Builder::with_capacity(extra.len()); + let no_data = attribute.no_data.map(u64::from_le_bytes); + + for d in iter { + let mut v = u32::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v as u64 { + builder.append_null(); + continue; + } + } + if attribute.scale.is_some() || attribute.offset.is_some() { + v = (v as f64 * scale + offset) as u32; + } + builder.append_value(v) + } + + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + DataType::UInt64 => { + let mut builder = UInt64Builder::with_capacity(extra.len()); + let no_data = attribute.no_data.map(u64::from_le_bytes); + + for d in iter { + let mut v = u64::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v { + builder.append_null(); + continue; + } + } + if attribute.scale.is_some() || attribute.offset.is_some() { + v = (v as f64 * scale + offset) as u64; + } + builder.append_value(v) + } + + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + DataType::Float32 => { + let mut builder = Float32Builder::with_capacity(extra.len()); + let no_data = attribute.no_data.map(f64::from_le_bytes); + + for d in iter { + let mut v = f32::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v as f64 { + builder.append_null(); + continue; + } + } + if attribute.scale.is_some() || attribute.offset.is_some() { + v = (v as f64 * scale + offset) as f32; + } + builder.append_value(v) + } + + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + DataType::Float64 => { + let mut builder = Float64Builder::with_capacity(extra.len()); + let no_data = attribute.no_data.map(f64::from_le_bytes); + + for d in iter { + let mut v = f64::from_le_bytes(d.try_into().unwrap()); + if let Some(no_data) = no_data { + if no_data == v { + builder.append_null(); + continue; + } + } + if attribute.scale.is_some() || attribute.offset.is_some() { + v = v * scale + offset; + } + builder.append_value(v); + } + + columns.push(Arc::new(builder.finish()) as ArrayRef) + } + + dt => { + return Err(ArrowError::ExternalError( + format!("Unsupported data type for extra bytes: `{dt}`").into(), + )) + } + } + + Ok(width) +} + +#[cfg(test)] +mod tests { + use std::{fs::File, sync::Arc}; + + use arrow_array::{ + cast::AsArray, + types::{ + Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, UInt16Type, + UInt32Type, UInt64Type, UInt8Type, + }, + }; + use datafusion_datasource::PartitionedFile; + use las::{point::Format, Builder, Writer}; + use object_store::{local::LocalFileSystem, path::Path, ObjectStore}; + + use crate::laz::{ + options::{LasExtraBytes, LazTableOptions}, + reader::LazFileReaderFactory, + }; + + #[tokio::test] + async fn point_formats() { + let tmpdir = tempfile::tempdir().unwrap(); + + for format in 0..=10 { + let tmp_path = tmpdir.path().join("tmp.laz"); + let tmp_file = File::create(&tmp_path).unwrap(); + + // create laz file + let mut builder = Builder::from((1, 4)); + builder.point_format = Format::new(format).unwrap(); + builder.point_format.is_compressed = true; + let header = builder.into_header().unwrap(); + let mut writer = Writer::new(tmp_file, header).unwrap(); + writer.close().unwrap(); + + // read batch with `LazFileReader` + let store = LocalFileSystem::new(); + let location = Path::from_filesystem_path(tmp_path).unwrap(); + let object = store.head(&location).await.unwrap(); + + let laz_file_reader = LazFileReaderFactory::new(Arc::new(store), None) + .create_reader( + PartitionedFile::new(location, object.size), + LazTableOptions::default(), + ) + .unwrap(); + let metadata = laz_file_reader.get_metadata().await.unwrap(); + + let batch = laz_file_reader + .get_batch(&metadata.chunk_table[0]) + .await + .unwrap(); + + match format { + 0 => assert_eq!(batch.num_columns(), 17), + 1 | 4 | 6 | 9 => assert_eq!(batch.num_columns(), 18), + 2 => assert_eq!(batch.num_columns(), 20), + 3 | 5 | 7 => assert_eq!(batch.num_columns(), 21), + 8 | 10 => assert_eq!(batch.num_columns(), 22), + _ => unreachable!(), + } + } + } + + #[tokio::test] + async fn extra_attributes() { + // create extra file Review Comment: Can you add a quick description of what this file contains? Probably you can just include this as a file and perhaps some other files in the rust/sedona-pointcloud directory with the Python script that generates it (and we can follow up with adding them to sedona-testing without blocking this PR). I believe cargo test sets the working directory such that this is reasonable to do. ########## examples/sedonadb-rust-pointcloud/src/main.rs: ########## @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// Because a number of methods only return Err() for not implemented, +// the compiler doesn't know how to guess which impl RecordBatchReader +// will be returned. When we implement the methods, we can remove this. + +use std::error::Error; + +use sedona::context::{SedonaContext, SedonaDataFrame}; + +#[tokio::main] +async fn main() -> Result<(), Box<dyn Error>> { Review Comment: Optional nit (we use `datafusion_common::Result` pretty much everywhere else) ```suggestion async fn main() -> Result<()> { ``` ########## rust/sedona-pointcloud/src/laz/options.rs: ########## @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::{fmt::Display, str::FromStr}; + +use datafusion_common::{ + config::{ConfigExtension, ConfigField, Visit}, + error::DataFusionError, + extensions_options, +}; + +/// Geometry representation +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub enum LasPointEncoding { + /// Use plain coordinates as three fields `x`, `y`, `z` with datatype Float64 encoding. + #[default] + Plain, + /// Resolves the coordinates to a fields `geometry` with WKB encoding. + Wkb, + /// Resolves the coordinates to a fields `geometry` with separated GeoArrow encoding. + Native, +} + +impl Display for LasPointEncoding { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + LasPointEncoding::Plain => f.write_str("plain"), + LasPointEncoding::Wkb => f.write_str("wkb"), + LasPointEncoding::Native => f.write_str("nativ"), + } + } +} + +impl FromStr for LasPointEncoding { + type Err = String; + + fn from_str(s: &str) -> Result<Self, Self::Err> { + match s.to_lowercase().as_str() { + "plain" => Ok(Self::Plain), + "wkb" => Ok(Self::Wkb), + "native" => Ok(Self::Native), + s => Err(format!("Unable to parse from `{s}`")), + } + } +} + +impl ConfigField for LasPointEncoding { + fn visit<V: Visit>(&self, v: &mut V, key: &str, _description: &'static str) { + v.some( + &format!("{key}.point_encoding"), + self, + "Specify point encoding", + ); + } + + fn set(&mut self, _key: &str, value: &str) -> Result<(), DataFusionError> { + *self = value.parse().map_err(DataFusionError::Configuration)?; + Ok(()) + } +} + +/// LAS extra bytes handling +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub enum LasExtraBytes { + /// Resolve to typed and named attributes + Typed, + /// Keep as binary blob + Blob, + /// Drop/ignore extrabytes + #[default] + Ignore, +} + +impl Display for LasExtraBytes { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + LasExtraBytes::Typed => f.write_str("typed"), + LasExtraBytes::Blob => f.write_str("blob"), + LasExtraBytes::Ignore => f.write_str("ignore"), + } + } +} + +impl FromStr for LasExtraBytes { + type Err = String; + + fn from_str(s: &str) -> Result<Self, Self::Err> { + match s.to_lowercase().as_str() { + "typed" => Ok(Self::Typed), + "blob" => Ok(Self::Blob), + "ignore" => Ok(Self::Ignore), + s => Err(format!("Unable to parse from `{s}`")), + } + } +} + +impl ConfigField for LasExtraBytes { + fn visit<V: Visit>(&self, v: &mut V, key: &str, _description: &'static str) { + v.some( + &format!("{key}.extra_bytes"), + self, + "Specify extra bytes handling", + ); + } + + fn set(&mut self, _key: &str, value: &str) -> Result<(), DataFusionError> { + *self = value.parse().map_err(DataFusionError::Configuration)?; + Ok(()) + } +} + +// Define a new configuration struct using the `extensions_options` macro +extensions_options! { + /// The LAZ config options + pub struct LazTableOptions { + pub point_encoding: LasPointEncoding, default = LasPointEncoding::default() + pub extra_bytes: LasExtraBytes, default = LasExtraBytes::default() + } + +} + +impl ConfigExtension for LazTableOptions { + const PREFIX: &'static str = "laz"; +} Review Comment: Should this be `pointcloud`? (e.g., will these apply to las files as well as laz files?) ########## rust/sedona-pointcloud/src/laz/options.rs: ########## @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::{fmt::Display, str::FromStr}; + +use datafusion_common::{ + config::{ConfigExtension, ConfigField, Visit}, + error::DataFusionError, + extensions_options, +}; + +/// Geometry representation +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub enum LasPointEncoding { + /// Use plain coordinates as three fields `x`, `y`, `z` with datatype Float64 encoding. + #[default] + Plain, + /// Resolves the coordinates to a fields `geometry` with WKB encoding. + Wkb, + /// Resolves the coordinates to a fields `geometry` with separated GeoArrow encoding. + Native, +} + +impl Display for LasPointEncoding { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + LasPointEncoding::Plain => f.write_str("plain"), + LasPointEncoding::Wkb => f.write_str("wkb"), + LasPointEncoding::Native => f.write_str("nativ"), Review Comment: Optional nit: ```suggestion LasPointEncoding::Native => f.write_str("native"), ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
