anishshri-db commented on code in PR #49277:
URL: https://github.com/apache/spark/pull/49277#discussion_r1901417065
##########
sql/core/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala:
##########
@@ -372,6 +373,158 @@ object SchemaConverters extends Logging {
schema
}
}
+
+ /**
+ * Creates default values for Spark SQL data types when converting to Avro.
+ * This ensures fields have appropriate defaults during schema evolution.
+ */
+ private def getDefaultValue(dataType: DataType): Any = {
+ def createNestedDefault(st: StructType): java.util.HashMap[String, Any] = {
+ val defaultMap = new java.util.HashMap[String, Any]()
+ st.fields.foreach { field =>
+ defaultMap.put(field.name, getDefaultValue(field.dataType))
+ }
+ defaultMap
+ }
+
+ dataType match {
+ // Basic types
+ case BooleanType => false
+ case ByteType | ShortType | IntegerType => 0
+ case LongType => 0L
+ case FloatType => 0.0f
+ case DoubleType => 0.0
+ case StringType => ""
+ case BinaryType => java.nio.ByteBuffer.allocate(0)
+
+ // Complex types
+ case ArrayType(elementType, _) =>
+ val defaultArray = new java.util.ArrayList[Any]()
+ defaultArray.add(getDefaultValue(elementType))
+ defaultArray
+ case MapType(StringType, valueType, _) =>
+ val defaultMap = new java.util.HashMap[String, Any]()
+ defaultMap.put("defaultKey", getDefaultValue(valueType))
+ defaultMap
+ case st: StructType => createNestedDefault(st)
+
+ // Special types
+ case _: DecimalType => java.nio.ByteBuffer.allocate(0)
+ case DateType => 0
+ case TimestampType => 0L
+ case TimestampNTZType => 0L
+ case NullType => null
+ case _ => null
+ }
+ }
+
+ /**
+ * Converts a Spark SQL schema to a corresponding Avro schema.
+ * Handles nested types and adds support for schema evolution.
Review Comment:
Could we add more details here ? Also maybe add comments for all the
function args ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
