jayzhan211 commented on code in PR #14362:
URL: https://github.com/apache/datafusion/pull/14362#discussion_r1946315799
##########
datafusion/common/src/dfschema.rs:
##########
@@ -1056,6 +1079,107 @@ pub fn qualified_name(qualifier:
Option<&TableReference>, name: &str) -> String
}
}
+/// Extension trait to manage DataFusion specific metadata on Arrow fields.
+pub trait FieldExt {
+ /// Check if this field is a system columns.
+ ///
+ /// System columns are columns which meant to be semi-public stores of the
internal details of the table.
+ /// For example, `ctid` in Postgres would be considered a metadata column
+ /// (Postgres calls these "system columns", see [the Postgres
docs](https://www.postgresql.org/docs/current/ddl-system-columns.html) for more
information and examples.
+ /// Spark has a `_metadata` column that it uses to include details about
each file read in a query (see [Spark's
docs](https://docs.databricks.com/en/ingestion/file-metadata-column.html)).
+ ///
+ /// DataFusion allows fields to be declared as metadata columns by setting
the `datafusion.system_column` key in the field's metadata
+ /// to `true`.
+ ///
+ /// As an example of how this works in practice, if you have the following
Postgres table:
+ ///
+ /// ```sql
+ /// CREATE TABLE t (x int);
+ /// INSERT INTO t VALUES (1);
+ /// ```
+ ///
+ /// And you do a `SELECT * FROM t`, you would get the following schema:
+ ///
+ /// ```text
+ /// +---+
+ /// | x |
+ /// +---+
+ /// | 1 |
+ /// +---+
+ /// ```
+ ///
+ /// But if you do `SELECT ctid, * FROM t`, you would get the following
schema (ignore the meaning of the value of `ctid`, this is just an example):
+ ///
+ /// ```text
+ /// +-----+---+
+ /// | ctid| x |
+ /// +-----+---+
+ /// | 0 | 1 |
+ /// +-----+---+
+ /// ```
+ fn is_system_column(&self) -> bool;
+
+ /// Mark this field as a system column.
+ ///
+ /// See [`FieldExt::is_system_column`] for more information on what a
system column is.
+ fn to_system_column(self) -> Self;
+
+ /// Mark this field as a non system column by removing the
`datafusion.system_column` key from the field's metadata.
+ ///
+ /// See [`FieldExt::is_system_column`] for more information on what a
system column is.
+ fn to_non_system_column(self) -> Self;
+}
+
+/// See [`FieldExt`].
+impl FieldExt for Field {
+ /// Check if this field is a system column.
+ /// See [`FieldExt::is_system_column`] for more information on what a
system column is.
+ fn is_system_column(&self) -> bool {
+ self.metadata()
+ .get("datafusion.system_column")
Review Comment:
Make it a const?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For additional commands, e-mail: github-h...@datafusion.apache.org
