qzyu999 commented on code in PR #23656:
URL: https://github.com/apache/datafusion/pull/23656#discussion_r3679389935
##########
datafusion/datasource-parquet/src/sink.rs:
##########
@@ -752,3 +777,278 @@ async fn output_single_parquet_file_parallelized(
.map_err(|e| DataFusionError::ExecutionJoin(Box::new(e)))??;
Ok(parquet_meta_data)
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use arrow::array::{ArrayRef, StringArray};
+ use arrow::datatypes::{DataType, Field, Schema};
+ use datafusion_common::config::TableParquetOptions;
+ use datafusion_datasource::PartitionedFile;
+ use datafusion_datasource::file_groups::FileGroup;
+ use datafusion_datasource::file_sink_config::{FileOutputMode,
FileSinkConfig};
+ use datafusion_datasource::sink::{DataSink, DataSinkExec};
+ use datafusion_datasource::url::ListingTableUrl;
+ use datafusion_execution::config::SessionConfig;
+ use datafusion_execution::object_store::ObjectStoreUrl;
+ use datafusion_execution::runtime_env::RuntimeEnv;
+ use datafusion_expr::dml::InsertOp;
+ use datafusion_physical_plan::stream::RecordBatchStreamAdapter;
+ use object_store::local::LocalFileSystem;
+
+ fn build_test_ctx(store_url: &ObjectStoreUrl) -> Arc<TaskContext> {
+ let tmp_dir = tempfile::TempDir::new().unwrap();
+ let local = Arc::new(
+ LocalFileSystem::new_with_prefix(&tmp_dir)
+ .expect("should create object store"),
+ );
+
+ let session = SessionConfig::default();
+ let runtime = RuntimeEnv::default();
+ runtime
+ .object_store_registry
+ .register_store(store_url.as_ref(), local);
+
+ Arc::new(
+ TaskContext::default()
+ .with_session_config(session)
+ .with_runtime(Arc::new(runtime)),
+ )
+ }
+
+ fn create_test_sink() -> (Arc<ParquetSink>, SchemaRef, ObjectStoreUrl) {
+ let field_a = Field::new("a", DataType::Utf8, false);
+ let field_b = Field::new("b", DataType::Utf8, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+ let object_store_url = ObjectStoreUrl::local_filesystem();
+
+ let file_sink_config = FileSinkConfig {
+ original_url: String::default(),
+ object_store_url: object_store_url.clone(),
+ file_group:
FileGroup::new(vec![PartitionedFile::new("/tmp".to_string(), 1)]),
+ table_paths:
vec![ListingTableUrl::parse("file:///tmp/test/").unwrap()],
+ output_schema: Arc::clone(&schema),
+ table_partition_cols: vec![],
+ insert_op: InsertOp::Overwrite,
+ keep_partition_by_columns: false,
+ file_extension: "parquet".into(),
+ file_output_mode: FileOutputMode::Automatic,
+ };
+
+ let parquet_sink = Arc::new(ParquetSink::new(
+ file_sink_config,
+ TableParquetOptions::default(),
+ ));
+
+ (parquet_sink, schema, object_store_url)
+ }
+
+ fn make_test_batch(schema: &SchemaRef) -> RecordBatch {
+ let col_a: ArrayRef = Arc::new(StringArray::from(vec!["foo", "bar",
"baz"]));
+ let col_b: ArrayRef = Arc::new(StringArray::from(vec!["one", "two",
"three"]));
+ RecordBatch::try_new(Arc::clone(schema), vec![col_a, col_b]).unwrap()
Review Comment:
Replaced manual `StringArray` + `RecordBatch::try_new` with the macro:
```rust
fn make_test_batch(_schema: &SchemaRef) -> RecordBatch {
record_batch!(
("a", Utf8, ["foo", "bar", "baz"]),
("b", Utf8, ["one", "two", "three"])
)
.unwrap()
}
```
Same for the partitioned test:
```rust
let batch = record_batch!(
("a", Utf8, ["x", "y", "x"]),
("b", Utf8, ["one", "two", "three"])
)
.unwrap();
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
