ericm-db commented on code in PR #49277:
URL: https://github.com/apache/spark/pull/49277#discussion_r1913906096
##########
sql/core/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala:
##########
@@ -372,6 +373,178 @@ object SchemaConverters extends Logging {
schema
}
}
+
+ /**
+ * Creates default values for Spark SQL data types when converting to Avro.
+ * This ensures fields have appropriate defaults during schema evolution.
+ *
+ * This method recursively processes Spark SQL data types and generates
corresponding
+ * default values that are compatible with Avro schema specifications. It
handles
+ * both primitive types (like Boolean, Integer) and complex types (Arrays,
Maps, Structs).
+ *
+ * @param dataType The Spark SQL DataType to create a default value for
+ * @return A default value appropriate for the given data type that's
compatible with Avro
+ */
+ private def getDefaultValue(dataType: DataType): Any = {
+ def createNestedDefault(st: StructType): java.util.HashMap[String, Any] = {
+ val defaultMap = new java.util.HashMap[String, Any]()
+ st.fields.foreach { field =>
+ defaultMap.put(field.name, getDefaultValue(field.dataType))
+ }
+ defaultMap
+ }
+
+ dataType match {
+ // Basic types
+ case BooleanType => false
+ case ByteType | ShortType | IntegerType => 0
+ case LongType => 0L
+ case FloatType => 0.0f
+ case DoubleType => 0.0
+ case StringType => ""
+ case BinaryType => java.nio.ByteBuffer.allocate(0)
+
+ // Complex types
+ case ArrayType(elementType, _) =>
+ new java.util.ArrayList[Any]()
+ case MapType(StringType, valueType, _) =>
+ new java.util.HashMap[String, Any]()
+ case st: StructType => createNestedDefault(st)
+
+ // Special types
+ case _: DecimalType => java.nio.ByteBuffer.allocate(0)
+ case DateType => 0
+ case TimestampType => 0L
+ case TimestampNTZType => 0L
+ case NullType => null
+ case _ => null
+ }
+ }
+
+ /**
+ * Converts a Spark SQL schema to a corresponding Avro schema.
+ * This method provides comprehensive support for schema evolution and
handles
+ * complex nested types while maintaining type safety and compatibility.
+ *
+ * The conversion process includes:
+ * - Converting primitive Spark SQL types to Avro types
+ * - Handling complex types (arrays, maps, structs) with proper nesting
+ * - Supporting nullable fields through Avro unions
+ * - Managing logical types for dates, timestamps, and decimals
+ * - Generating unique names for nested records
+ * - Preserving namespace hierarchy for nested structures
+ *
+ * @param catalystType The Spark SQL DataType to convert
+ * @param nullable Whether the field can contain null values
+ * @param recordName The name to use for the record in the Avro schema
+ * @param namespace The namespace for the Avro schema
+ * @param nestingLevel Current nesting level for generating unique names
+ * @return An Avro Schema corresponding to the input Spark SQL type
+ * @throws IncompatibleSchemaException if the input type cannot be converted
to Avro
+ */
+ def toAvroTypeWithDefaults(
+ catalystType: DataType,
+ nullable: Boolean = false,
Review Comment:
Added the assertion
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
