xiaonanyang-db commented on code in PR #50300:
URL: https://github.com/apache/spark/pull/50300#discussion_r2031794919
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/StaxXmlParser.scala:
##########
@@ -897,4 +914,220 @@ object StaxXmlParser {
curRecord
}
}
+
+ /**
+ * Parse the input XML string as a Varaint value
+ */
+ def parseVariant(xml: String, options: XmlOptions): VariantVal = {
+ val parser = StaxXmlParserUtils.filteredReader(xml)
+ val rootEvent =
+ StaxXmlParserUtils.skipUntil(parser, XMLStreamConstants.START_ELEMENT)
+ val rootAttributes = rootEvent.asStartElement.getAttributes.asScala.toArray
+ val variant = convertVariant(parser, rootAttributes, options)
+ val v = new VariantVal(variant.getValue, variant.getMetadata)
+ parser.close()
+ v
+ }
+
+ /**
+ * Parse an XML element from the XML event stream into a Variant.
+ * This method transforms the XML element along with its attributes and
child elements
+ * into a hierarchical Variant data structure that preserves the XML
structure.
+ *
+ * @param parser The XML event stream reader positioned after the start
element
+ * @param attributes The attributes of the current XML element to be
included in the Variant
+ * @param options Configuration options that control how XML is parsed into
Variants
+ * @return A Variant representing the XML element with its attributes and
child content
+ */
+ def convertVariant(
+ parser: XMLEventReader,
+ attributes: Array[Attribute],
+ options: XmlOptions): Variant = {
+ // The variant builder for the root startElement
+ val rootBuilder = new VariantBuilder(false)
+ val start = rootBuilder.getWritePos
+
+ // Map to store the variant values of all child fields
+ // Each field could have multiple entries, which means it's an array
+ val fieldToVariants = collection.mutable.TreeMap.empty[String,
java.util.ArrayList[Variant]]
+
+ // Handle attributes first
+ StaxXmlParserUtils.convertAttributesToValuesMap(attributes,
options).foreach {
+ case (f, v) =>
+ val builder = new VariantBuilder(false)
+ appendXMLCharacterToVariant(builder, v, options)
+ addOrUpdateVariantFields(fieldToVariants, f, builder.result())
+ }
+
+ var shouldStop = false
+ while (!shouldStop) {
+ parser.nextEvent() match {
+ case s: StartElement =>
+ // For each child element, convert it to a variant and keep track of
it in
+ // fieldsToVariants
+ val attributes =
s.getAttributes.asScala.map(_.asInstanceOf[Attribute]).toArray
+ val field = StaxXmlParserUtils.getName(s.asStartElement.getName,
options)
+ addOrUpdateVariantFields(
+ fieldToVariants = fieldToVariants,
+ field = field,
+ variant = convertVariant(parser, attributes, options)
+ )
+
+ case c: Characters if !c.isWhiteSpace =>
+ // Treat the character as a value tag field, where we use the
[[XMLOptions.valueTag]] as
+ // the field key
+ val builder = new VariantBuilder(false)
+ appendXMLCharacterToVariant(builder, c.getData, options)
+ addOrUpdateVariantFields(
+ fieldToVariants = fieldToVariants,
+ field = options.valueTag,
+ variant = builder.result()
+ )
+
+ case _: EndElement =>
+ if (fieldToVariants.nonEmpty) {
+ val onlyValueTagField = fieldToVariants.keySet.forall(_ ==
options.valueTag)
+ if (onlyValueTagField) {
+ // If the element only has value tag field, parse the element as
a variant primitive
+
rootBuilder.appendVariant(fieldToVariants(options.valueTag).get(0))
+ } else {
+ // Otherwise, build the element as an object with all the fields
in fieldToVariants
+ val rootFieldEntries = new java.util.ArrayList[FieldEntry]()
+
+ fieldToVariants.foreach {
+ case (field, variants) =>
+ // Add the field key to the variant metadata
+ val fieldId = rootBuilder.addKey(field)
+ rootFieldEntries.add(
+ new FieldEntry(field, fieldId, rootBuilder.getWritePos -
start)
+ )
+
+ val fieldValue = if (variants.size() > 1) {
+ // If the field has more than one entry, it's an array
field. Build a Variant
+ // array as the field value
+ val arrayBuilder = new VariantBuilder(false)
+ val start = arrayBuilder.getWritePos
+ val offsets = new util.ArrayList[Integer]()
+ variants.asScala.foreach { v =>
+ offsets.add(arrayBuilder.getWritePos - start)
+ arrayBuilder.appendVariant(v)
+ }
+ arrayBuilder.finishWritingArray(start, offsets)
+ arrayBuilder.result()
+ } else {
+ // Otherwise, just use the first variant as the field value
+ variants.get(0)
+ }
+
+ // Append the field value to the variant builder
+ rootBuilder.appendVariant(fieldValue)
+ }
+
+ // Finish writing the root element as an object if it has more
than one child element
+ // or attribute
+ rootBuilder.finishWritingObject(start, rootFieldEntries)
+ }
+ }
+ shouldStop = true
+
+ case _: EndDocument => shouldStop = true
+
+ case _ => // do nothing
+ }
+ }
+
+ // If the element is empty, we treat it as a Variant null
+ if (rootBuilder.getWritePos == start) {
+ rootBuilder.appendNull()
+ }
+
+ rootBuilder.result()
+ }
+
+ /**
+ * Add or update the given field and its corresponding variant values in the
fieldToVariants map.
+ * If a field has multiple variant values, it will be parsed as a Variant
array.
+ *
+ * This method handles the case sensitivity of the field names based on the
SQLConf setting.
+ */
+ private def addOrUpdateVariantFields(
+ fieldToVariants: collection.mutable.TreeMap[String,
java.util.ArrayList[Variant]],
+ field: String,
+ variant: Variant): Unit = {
+ val variants = if (SQLConf.get.caseSensitiveAnalysis) {
+ // If case-sensitive analysis is enabled, we need to use the original
field name
+ // to avoid case-insensitive key collision
+ fieldToVariants.getOrElseUpdate(field, new
java.util.ArrayList[Variant]())
+ } else {
+ // Otherwise, we can use the lower-case field name for case-insensitive
key collision
+ fieldToVariants.get(field.toLowerCase(Locale.ROOT)) match {
+ case Some(variantList) => variantList
+ case _ =>
+ // If the field doesn't exist, create the entry with the original
field name
+ fieldToVariants.getOrElseUpdate(field, new
java.util.ArrayList[Variant]())
+ }
+ }
+ variants.add(variant)
+ }
+
+ /**
+ * Convert an XML Character value `s` into a variant value and append the
result to `builder`.
+ * The result can only be one of a variant boolean/long/decimal/string.
Anything other than
+ * the supported types will be appended to the Variant builder as a string.
+ *
+ * Floating point types (double, float) are not considered to avoid
precision loss.
+ */
+ private def appendXMLCharacterToVariant(
+ builder: VariantBuilder,
+ s: String,
+ options: XmlOptions): Unit = {
+ if (s == null || s == options.nullValue) {
+ builder.appendNull()
+ return
+ }
+
+ val value = if (options.ignoreSurroundingSpaces) s.trim() else s
+
+ // Exit early for empty strings
+ if (s.isEmpty) {
+ builder.appendString(s)
+ return
+ }
+
+ // Try parsing the value as boolean first
+ if (value.toLowerCase(Locale.ROOT) == "true") {
+ builder.appendBoolean(true)
+ return
+ }
+ if (value.toLowerCase(Locale.ROOT) == "false") {
+ builder.appendBoolean(false)
+ return
+ }
+
+ // Try parsing the value as a long
+ allCatch opt value.toLong match {
+ case Some(l) =>
+ builder.appendLong(l)
+ return
+ case _ =>
+ }
+
+ // Try parsing the value as decimal
+ val decimalParser = ExprUtils.getDecimalParser(options.locale)
+ allCatch opt decimalParser(value) match {
+ case Some(decimalValue) =>
+ var d = decimalValue
+ if (d.scale() < 0) {
+ d = d.setScale(0)
+ }
+ if (d.scale <= MAX_DECIMAL16_PRECISION && d.precision <=
MAX_DECIMAL16_PRECISION) {
+ builder.appendDecimal(d)
+ }
+ return
Review Comment:
ah, good catch. Will add a test case for it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
