sandip-db commented on code in PR #50300:
URL: https://github.com/apache/spark/pull/50300#discussion_r2034327293
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/StaxXmlParser.scala:
##########
@@ -897,4 +914,219 @@ object StaxXmlParser {
curRecord
}
}
+
+ /**
+ * Parse the input XML string as a Variant value
+ */
+ def parseVariant(xml: String, options: XmlOptions): VariantVal = {
+ val parser = StaxXmlParserUtils.filteredReader(xml)
+ val rootEvent =
+ StaxXmlParserUtils.skipUntil(parser, XMLStreamConstants.START_ELEMENT)
+ val rootAttributes = rootEvent.asStartElement.getAttributes.asScala.toArray
+ val variant = convertVariant(parser, rootAttributes, options)
+ val v = new VariantVal(variant.getValue, variant.getMetadata)
+ parser.close()
+ v
+ }
+
+ /**
+ * Parse an XML element from the XML event stream into a Variant.
+ * This method transforms the XML element along with its attributes and
child elements
+ * into a hierarchical Variant data structure that preserves the XML
structure.
+ *
+ * @param parser The XML event stream reader positioned after the start
element
+ * @param attributes The attributes of the current XML element to be
included in the Variant
+ * @param options Configuration options that control how XML is parsed into
Variants
+ * @return A Variant representing the XML element with its attributes and
child content
+ */
+ def convertVariant(
+ parser: XMLEventReader,
+ attributes: Array[Attribute],
+ options: XmlOptions): Variant = {
+ // The variant builder for the root startElement
+ val rootBuilder = new VariantBuilder(false)
+ val start = rootBuilder.getWritePos
+
+ // Map to store the variant values of all child fields
+ // Each field could have multiple entries, which means it's an array
+ val fieldToVariants = collection.mutable.TreeMap.empty[String,
java.util.ArrayList[Variant]]
+
+ // Handle attributes first
+ StaxXmlParserUtils.convertAttributesToValuesMap(attributes,
options).foreach {
+ case (f, v) =>
+ val builder = new VariantBuilder(false)
+ appendXMLCharacterToVariant(builder, v, options)
+ val variants = fieldToVariants.getOrElseUpdate(f, new
java.util.ArrayList[Variant]())
+ variants.add(builder.result())
+ }
+
+ var shouldStop = false
+ while (!shouldStop) {
+ parser.nextEvent() match {
+ case s: StartElement =>
+ // For each child element, convert it to a variant and keep track of
it in
+ // fieldsToVariants
+ val attributes =
s.getAttributes.asScala.map(_.asInstanceOf[Attribute]).toArray
+ val field = StaxXmlParserUtils.getName(s.asStartElement.getName,
options)
+ val variants = fieldToVariants.getOrElseUpdate(field, new
java.util.ArrayList[Variant]())
+ variants.add(convertVariant(parser, attributes, options))
+
+ case c: Characters if !c.isWhiteSpace =>
+ // Treat the character as a value tag field, where we use the
[[XMLOptions.valueTag]] as
+ // the field key
+ val builder = new VariantBuilder(false)
+ appendXMLCharacterToVariant(builder, c.getData, options)
+ val variants = fieldToVariants.getOrElseUpdate(
+ options.valueTag,
+ new java.util.ArrayList[Variant]()
+ )
+ variants.add(builder.result())
+
+ case _: EndElement =>
+ if (fieldToVariants.nonEmpty) {
+ val onlyValueTagField = fieldToVariants.keySet.forall(_ ==
options.valueTag)
+ if (onlyValueTagField) {
+ // If the element only has value tag field, parse the element as
a variant primitive
+
rootBuilder.appendVariant(fieldToVariants(options.valueTag).get(0))
+ } else {
+ writeVariantObject(rootBuilder, start, fieldToVariants)
+ }
+ }
+ shouldStop = true
Review Comment:
validate `startElementName`
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/StaxXmlParser.scala:
##########
@@ -897,4 +914,219 @@ object StaxXmlParser {
curRecord
}
}
+
+ /**
+ * Parse the input XML string as a Variant value
+ */
+ def parseVariant(xml: String, options: XmlOptions): VariantVal = {
+ val parser = StaxXmlParserUtils.filteredReader(xml)
+ val rootEvent =
+ StaxXmlParserUtils.skipUntil(parser, XMLStreamConstants.START_ELEMENT)
+ val rootAttributes = rootEvent.asStartElement.getAttributes.asScala.toArray
+ val variant = convertVariant(parser, rootAttributes, options)
+ val v = new VariantVal(variant.getValue, variant.getMetadata)
+ parser.close()
+ v
+ }
+
+ /**
+ * Parse an XML element from the XML event stream into a Variant.
+ * This method transforms the XML element along with its attributes and
child elements
+ * into a hierarchical Variant data structure that preserves the XML
structure.
+ *
+ * @param parser The XML event stream reader positioned after the start
element
+ * @param attributes The attributes of the current XML element to be
included in the Variant
+ * @param options Configuration options that control how XML is parsed into
Variants
+ * @return A Variant representing the XML element with its attributes and
child content
+ */
+ def convertVariant(
Review Comment:
pass `startElementName` and verify it in `EndElement`. This has helped catch
parser issues in `convertObject`.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/StaxXmlParser.scala:
##########
@@ -897,4 +914,219 @@ object StaxXmlParser {
curRecord
}
}
+
+ /**
+ * Parse the input XML string as a Variant value
+ */
+ def parseVariant(xml: String, options: XmlOptions): VariantVal = {
+ val parser = StaxXmlParserUtils.filteredReader(xml)
+ val rootEvent =
+ StaxXmlParserUtils.skipUntil(parser, XMLStreamConstants.START_ELEMENT)
+ val rootAttributes = rootEvent.asStartElement.getAttributes.asScala.toArray
+ val variant = convertVariant(parser, rootAttributes, options)
+ val v = new VariantVal(variant.getValue, variant.getMetadata)
+ parser.close()
+ v
+ }
+
+ /**
+ * Parse an XML element from the XML event stream into a Variant.
+ * This method transforms the XML element along with its attributes and
child elements
+ * into a hierarchical Variant data structure that preserves the XML
structure.
+ *
+ * @param parser The XML event stream reader positioned after the start
element
+ * @param attributes The attributes of the current XML element to be
included in the Variant
+ * @param options Configuration options that control how XML is parsed into
Variants
+ * @return A Variant representing the XML element with its attributes and
child content
+ */
+ def convertVariant(
+ parser: XMLEventReader,
+ attributes: Array[Attribute],
+ options: XmlOptions): Variant = {
+ // The variant builder for the root startElement
+ val rootBuilder = new VariantBuilder(false)
+ val start = rootBuilder.getWritePos
+
+ // Map to store the variant values of all child fields
+ // Each field could have multiple entries, which means it's an array
+ val fieldToVariants = collection.mutable.TreeMap.empty[String,
java.util.ArrayList[Variant]]
+
+ // Handle attributes first
+ StaxXmlParserUtils.convertAttributesToValuesMap(attributes,
options).foreach {
+ case (f, v) =>
+ val builder = new VariantBuilder(false)
+ appendXMLCharacterToVariant(builder, v, options)
+ val variants = fieldToVariants.getOrElseUpdate(f, new
java.util.ArrayList[Variant]())
+ variants.add(builder.result())
+ }
+
+ var shouldStop = false
+ while (!shouldStop) {
+ parser.nextEvent() match {
+ case s: StartElement =>
+ // For each child element, convert it to a variant and keep track of
it in
+ // fieldsToVariants
+ val attributes =
s.getAttributes.asScala.map(_.asInstanceOf[Attribute]).toArray
+ val field = StaxXmlParserUtils.getName(s.asStartElement.getName,
options)
+ val variants = fieldToVariants.getOrElseUpdate(field, new
java.util.ArrayList[Variant]())
+ variants.add(convertVariant(parser, attributes, options))
+
+ case c: Characters if !c.isWhiteSpace =>
+ // Treat the character as a value tag field, where we use the
[[XMLOptions.valueTag]] as
+ // the field key
+ val builder = new VariantBuilder(false)
+ appendXMLCharacterToVariant(builder, c.getData, options)
+ val variants = fieldToVariants.getOrElseUpdate(
+ options.valueTag,
+ new java.util.ArrayList[Variant]()
+ )
+ variants.add(builder.result())
+
+ case _: EndElement =>
+ if (fieldToVariants.nonEmpty) {
+ val onlyValueTagField = fieldToVariants.keySet.forall(_ ==
options.valueTag)
+ if (onlyValueTagField) {
+ // If the element only has value tag field, parse the element as
a variant primitive
+
rootBuilder.appendVariant(fieldToVariants(options.valueTag).get(0))
+ } else {
+ writeVariantObject(rootBuilder, start, fieldToVariants)
+ }
+ }
+ shouldStop = true
+
+ case _: EndDocument => shouldStop = true
+
+ case _ => // do nothing
+ }
+ }
+
+ // If the element is empty, we treat it as a Variant null
+ if (rootBuilder.getWritePos == start) {
+ rootBuilder.appendNull()
+ }
+
+ rootBuilder.result()
+ }
+
+ /**
+ * Write a variant object to the variant builder.
+ * This method handles the case sensitivity of field names when building the
variant object. When
+ * case sensitivity is disabled, a random field name will be used for all
fields with the same
+ * lowercase name.
+ *
+ * @param builder The variant builder to write to
+ * @param start The start position of the variant object in the builder
+ * @param fieldToVariants A map of field names to their corresponding
variant values of the object
+ */
+ private def writeVariantObject(
+ builder: VariantBuilder,
+ start: Int,
+ fieldToVariants: collection.mutable.TreeMap[String,
java.util.ArrayList[Variant]]): Unit = {
+ val rootFieldEntries = new java.util.ArrayList[FieldEntry]()
Review Comment:
```suggestion
fieldToVariants: collection.mutable.TreeMap[String,
java.util.ArrayList[Variant]]): Unit = {
val start = rootBuilder.getWritePos
val rootFieldEntries = new java.util.ArrayList[FieldEntry]()
```
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/xml/XmlVariantSuite.scala:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.xml
+
+import java.time.ZoneOffset
+
+import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row}
+import org.apache.spark.sql.catalyst.xml.{StaxXmlParser, XmlOptions}
+import org.apache.spark.sql.functions.{col, variant_get}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.SharedSparkSession
+
+class XmlVariantSuite extends QueryTest with SharedSparkSession with
TestXmlData {
+
+ private val baseOptions = Map("rowTag" -> "ROW", "valueTag" -> "_VALUE",
"attributePrefix" -> "_")
+
+ private val resDir = "test-data/xml-resources/"
+
+ // ==========================
+ // ====== Parser tests ======
+ // ==========================
+
+ private def testParser(
+ xml: String,
+ expectedJsonStr: String,
+ extraOptions: Map[String, String] = Map.empty): Unit = {
+ val parsed = StaxXmlParser.parseVariant(xml, XmlOptions(baseOptions ++
extraOptions))
+ assert(parsed.toJson(ZoneOffset.UTC) == expectedJsonStr)
+ }
+
+ test("Parser: parse primitive XML elements (long, decimal, double, etc.) as
variants") {
+ // Boolean -> Boolean
+ testParser("<ROW><isActive>false</isActive></ROW>",
"""{"isActive":false}""")
+ testParser("<ROW><isActive>true</isActive></ROW>", """{"isActive":true}""")
+
+ // Long -> Long
+ testParser("<ROW><id>2</id></ROW>", """{"id":2}""")
+ testParser("<ROW><id>+2</id></ROW>", """{"id":2}""")
+ testParser("<ROW><id>-2</id></ROW>", """{"id":-2}""")
+
+ // Decimal -> Decimal
+ testParser(
+ xml = "<ROW><price>158,058,049.001</price></ROW>",
+ expectedJsonStr = """{"price":158058049.001}"""
+ )
+ testParser(
+ xml = "<ROW><decimal>10.05</decimal></ROW>",
+ expectedJsonStr = """{"decimal":10.05}"""
+ )
+ testParser(
+ xml = "<ROW><amount>5.0</amount></ROW>",
+ expectedJsonStr = """{"amount":5}"""
+ )
+ // This is parsed as String, because it is too large for Decimal
+ testParser(
+ xml = "<ROW><amount>1e40</amount></ROW>",
+ expectedJsonStr = """{"amount":"1e40"}"""
+ )
+
+ // Date -> String
+ testParser(
+ xml = "<ROW><createdAt>2023-10-01</createdAt></ROW>",
+ expectedJsonStr = """{"createdAt":"2023-10-01"}"""
+ )
+
+ // Timestamp -> String
+ testParser(
+ xml = "<ROW><createdAt>2023-10-01T12:00:00Z</createdAt></ROW>",
+ expectedJsonStr = """{"createdAt":"2023-10-01T12:00:00Z"}"""
+ )
+
+ // String -> String
+ testParser("<ROW><name>Sam</name></ROW>", """{"name":"Sam"}""")
+ // Strings with spaces
+ testParser(
+ "<ROW><note> hello world </note></ROW>",
+ expectedJsonStr = """{"note":" hello world "}""",
+ extraOptions = Map("ignoreSurroundingSpaces" -> "false")
+ )
+ testParser(
+ xml = "<ROW><note> hello world </note></ROW>",
+ expectedJsonStr = """{"note":"hello world"}"""
+ )
+ }
+
+ test("Parser: parse XML attributes as variants") {
+ // XML elements with only attributes
+ testParser(
+ xml = "<ROW id=\"2\"></ROW>",
+ expectedJsonStr = """{"_id":2}"""
+ )
+ testParser(
+ xml = "<ROW><a><b attr=\"1\"></b></a></ROW>",
+ expectedJsonStr = """{"a":{"b":{"_attr":1}}}"""
+ )
+ testParser(
+ xml = "<ROW id=\"2\" name=\"Sam\" amount=\"93\"></ROW>",
+ expectedJsonStr = """{"_amount":93,"_id":2,"_name":"Sam"}"""
+ )
+
+ // XML elements with attributes and elements
+ testParser(
+ xml = "<ROW id=\"2\" name=\"Sam\"><amount>93</amount></ROW>",
+ expectedJsonStr = """{"_id":2,"_name":"Sam","amount":93}"""
+ )
+
+ // XML elements with attributes and nested elements
+ testParser(
+ xml = "<ROW id=\"2\"
name=\"Sam\"><info><amount>93</amount></info></ROW>",
+ expectedJsonStr = """{"_id":2,"_name":"Sam","info":{"amount":93}}"""
+ )
+
+ // XML elements with attributes and value tag
+ testParser(
+ xml = "<ROW id=\"2\" name=\"Sam\">93</ROW>",
+ expectedJsonStr = """{"_VALUE":93,"_id":2,"_name":"Sam"}"""
+ )
+ }
+
+ test("Parser: parse XML value tags as variants") {
+ // XML elements with value tags and attributes
+ testParser(
+ xml = "<ROW id=\"2\" name=\"Sam\">93</ROW>",
+ expectedJsonStr = """{"_VALUE":93,"_id":2,"_name":"Sam"}"""
+ )
+
+ // XML elements with value tags and nested elements
+ testParser(
+ xml = "<ROW><info>Sam<amount>93</amount></info></ROW>",
+ expectedJsonStr = """{"info":{"_VALUE":"Sam","amount":93}}"""
+ )
+ }
+
+ test("Parser: parse XML elements as variant object") {
+ testParser(
+ xml = "<ROW><info><name>Sam</name><amount>93</amount></info></ROW>",
+ expectedJsonStr = """{"info":{"amount":93,"name":"Sam"}}"""
+ )
+ }
+
+ test("Parser: parse XML elements as variant array") {
+ testParser(
+ xml = "<ROW><array>1</array><array>2</array></ROW>",
+ expectedJsonStr = """{"array":[1,2]}"""
+ )
+ }
+
+ test("Parser: null and empty XML elements are parsed as variant null") {
+ // XML elements with null and empty values
+ testParser(
+ xml = "<ROW><name></name><amount>93</amount></ROW>",
Review Comment:
Adding some more variations:
```suggestion
xml = """<ROW><name></name><amount>93</amount><space> </space><newline>
</newline></ROW>""",
```
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/StaxXmlParser.scala:
##########
@@ -897,4 +914,219 @@ object StaxXmlParser {
curRecord
}
}
+
+ /**
+ * Parse the input XML string as a Variant value
+ */
+ def parseVariant(xml: String, options: XmlOptions): VariantVal = {
+ val parser = StaxXmlParserUtils.filteredReader(xml)
+ val rootEvent =
+ StaxXmlParserUtils.skipUntil(parser, XMLStreamConstants.START_ELEMENT)
+ val rootAttributes = rootEvent.asStartElement.getAttributes.asScala.toArray
+ val variant = convertVariant(parser, rootAttributes, options)
+ val v = new VariantVal(variant.getValue, variant.getMetadata)
+ parser.close()
+ v
+ }
+
+ /**
+ * Parse an XML element from the XML event stream into a Variant.
+ * This method transforms the XML element along with its attributes and
child elements
+ * into a hierarchical Variant data structure that preserves the XML
structure.
+ *
+ * @param parser The XML event stream reader positioned after the start
element
+ * @param attributes The attributes of the current XML element to be
included in the Variant
+ * @param options Configuration options that control how XML is parsed into
Variants
+ * @return A Variant representing the XML element with its attributes and
child content
+ */
+ def convertVariant(
+ parser: XMLEventReader,
+ attributes: Array[Attribute],
+ options: XmlOptions): Variant = {
+ // The variant builder for the root startElement
+ val rootBuilder = new VariantBuilder(false)
+ val start = rootBuilder.getWritePos
+
+ // Map to store the variant values of all child fields
+ // Each field could have multiple entries, which means it's an array
+ val fieldToVariants = collection.mutable.TreeMap.empty[String,
java.util.ArrayList[Variant]]
Review Comment:
The following will avoid `groupBy` in `writeVariantObject`.
```suggestion
val caseSensitivityOrdering: Ordering[String] = if (caseSensitive) {
(x: String, y: String) => x.compareTo(y)
} else {
(x: String, y: String) => x.compareToIgnoreCase(y)
}
val fieldToVariants = collection.mutable.TreeMap.empty[String,
java.util.ArrayList[Variant]](caseSensitivityOrdering)
```
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/StaxXmlParser.scala:
##########
@@ -897,4 +914,219 @@ object StaxXmlParser {
curRecord
}
}
+
+ /**
+ * Parse the input XML string as a Variant value
+ */
+ def parseVariant(xml: String, options: XmlOptions): VariantVal = {
+ val parser = StaxXmlParserUtils.filteredReader(xml)
+ val rootEvent =
+ StaxXmlParserUtils.skipUntil(parser, XMLStreamConstants.START_ELEMENT)
+ val rootAttributes = rootEvent.asStartElement.getAttributes.asScala.toArray
+ val variant = convertVariant(parser, rootAttributes, options)
+ val v = new VariantVal(variant.getValue, variant.getMetadata)
+ parser.close()
+ v
+ }
+
+ /**
+ * Parse an XML element from the XML event stream into a Variant.
+ * This method transforms the XML element along with its attributes and
child elements
+ * into a hierarchical Variant data structure that preserves the XML
structure.
+ *
+ * @param parser The XML event stream reader positioned after the start
element
+ * @param attributes The attributes of the current XML element to be
included in the Variant
+ * @param options Configuration options that control how XML is parsed into
Variants
+ * @return A Variant representing the XML element with its attributes and
child content
+ */
+ def convertVariant(
+ parser: XMLEventReader,
+ attributes: Array[Attribute],
+ options: XmlOptions): Variant = {
+ // The variant builder for the root startElement
+ val rootBuilder = new VariantBuilder(false)
+ val start = rootBuilder.getWritePos
+
+ // Map to store the variant values of all child fields
+ // Each field could have multiple entries, which means it's an array
+ val fieldToVariants = collection.mutable.TreeMap.empty[String,
java.util.ArrayList[Variant]]
+
+ // Handle attributes first
+ StaxXmlParserUtils.convertAttributesToValuesMap(attributes,
options).foreach {
+ case (f, v) =>
+ val builder = new VariantBuilder(false)
+ appendXMLCharacterToVariant(builder, v, options)
+ val variants = fieldToVariants.getOrElseUpdate(f, new
java.util.ArrayList[Variant]())
+ variants.add(builder.result())
+ }
+
+ var shouldStop = false
+ while (!shouldStop) {
+ parser.nextEvent() match {
+ case s: StartElement =>
+ // For each child element, convert it to a variant and keep track of
it in
+ // fieldsToVariants
+ val attributes =
s.getAttributes.asScala.map(_.asInstanceOf[Attribute]).toArray
+ val field = StaxXmlParserUtils.getName(s.asStartElement.getName,
options)
+ val variants = fieldToVariants.getOrElseUpdate(field, new
java.util.ArrayList[Variant]())
+ variants.add(convertVariant(parser, attributes, options))
+
+ case c: Characters if !c.isWhiteSpace =>
+ // Treat the character as a value tag field, where we use the
[[XMLOptions.valueTag]] as
+ // the field key
+ val builder = new VariantBuilder(false)
+ appendXMLCharacterToVariant(builder, c.getData, options)
+ val variants = fieldToVariants.getOrElseUpdate(
+ options.valueTag,
+ new java.util.ArrayList[Variant]()
+ )
+ variants.add(builder.result())
+
+ case _: EndElement =>
+ if (fieldToVariants.nonEmpty) {
+ val onlyValueTagField = fieldToVariants.keySet.forall(_ ==
options.valueTag)
+ if (onlyValueTagField) {
+ // If the element only has value tag field, parse the element as
a variant primitive
+
rootBuilder.appendVariant(fieldToVariants(options.valueTag).get(0))
+ } else {
+ writeVariantObject(rootBuilder, start, fieldToVariants)
+ }
+ }
+ shouldStop = true
+
+ case _: EndDocument => shouldStop = true
+
+ case _ => // do nothing
+ }
+ }
+
+ // If the element is empty, we treat it as a Variant null
+ if (rootBuilder.getWritePos == start) {
+ rootBuilder.appendNull()
Review Comment:
Is this necessary?
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/StaxXmlParser.scala:
##########
@@ -45,7 +47,10 @@ import
org.apache.spark.sql.catalyst.xml.StaxXmlParser.convertStream
import org.apache.spark.sql.errors.QueryExecutionErrors
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
-import org.apache.spark.unsafe.types.UTF8String
+import org.apache.spark.types.variant.{Variant, VariantBuilder}
+import org.apache.spark.types.variant.VariantBuilder.FieldEntry
+import org.apache.spark.types.variant.VariantUtil.MAX_DECIMAL16_PRECISION
Review Comment:
nit: Easier to see where the field is coming from in the code
```suggestion
import org.apache.spark.types.variant.VariantUtil
```
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/StaxXmlParser.scala:
##########
@@ -897,4 +914,219 @@ object StaxXmlParser {
curRecord
}
}
+
+ /**
+ * Parse the input XML string as a Variant value
+ */
+ def parseVariant(xml: String, options: XmlOptions): VariantVal = {
+ val parser = StaxXmlParserUtils.filteredReader(xml)
+ val rootEvent =
+ StaxXmlParserUtils.skipUntil(parser, XMLStreamConstants.START_ELEMENT)
+ val rootAttributes = rootEvent.asStartElement.getAttributes.asScala.toArray
+ val variant = convertVariant(parser, rootAttributes, options)
+ val v = new VariantVal(variant.getValue, variant.getMetadata)
+ parser.close()
+ v
+ }
+
+ /**
+ * Parse an XML element from the XML event stream into a Variant.
+ * This method transforms the XML element along with its attributes and
child elements
+ * into a hierarchical Variant data structure that preserves the XML
structure.
+ *
+ * @param parser The XML event stream reader positioned after the start
element
+ * @param attributes The attributes of the current XML element to be
included in the Variant
+ * @param options Configuration options that control how XML is parsed into
Variants
+ * @return A Variant representing the XML element with its attributes and
child content
+ */
+ def convertVariant(
+ parser: XMLEventReader,
+ attributes: Array[Attribute],
+ options: XmlOptions): Variant = {
+ // The variant builder for the root startElement
+ val rootBuilder = new VariantBuilder(false)
+ val start = rootBuilder.getWritePos
+
+ // Map to store the variant values of all child fields
+ // Each field could have multiple entries, which means it's an array
+ val fieldToVariants = collection.mutable.TreeMap.empty[String,
java.util.ArrayList[Variant]]
+
+ // Handle attributes first
+ StaxXmlParserUtils.convertAttributesToValuesMap(attributes,
options).foreach {
+ case (f, v) =>
+ val builder = new VariantBuilder(false)
+ appendXMLCharacterToVariant(builder, v, options)
+ val variants = fieldToVariants.getOrElseUpdate(f, new
java.util.ArrayList[Variant]())
+ variants.add(builder.result())
+ }
+
+ var shouldStop = false
+ while (!shouldStop) {
+ parser.nextEvent() match {
+ case s: StartElement =>
+ // For each child element, convert it to a variant and keep track of
it in
+ // fieldsToVariants
+ val attributes =
s.getAttributes.asScala.map(_.asInstanceOf[Attribute]).toArray
+ val field = StaxXmlParserUtils.getName(s.asStartElement.getName,
options)
+ val variants = fieldToVariants.getOrElseUpdate(field, new
java.util.ArrayList[Variant]())
+ variants.add(convertVariant(parser, attributes, options))
+
+ case c: Characters if !c.isWhiteSpace =>
+ // Treat the character as a value tag field, where we use the
[[XMLOptions.valueTag]] as
+ // the field key
+ val builder = new VariantBuilder(false)
+ appendXMLCharacterToVariant(builder, c.getData, options)
+ val variants = fieldToVariants.getOrElseUpdate(
+ options.valueTag,
+ new java.util.ArrayList[Variant]()
+ )
+ variants.add(builder.result())
+
+ case _: EndElement =>
+ if (fieldToVariants.nonEmpty) {
+ val onlyValueTagField = fieldToVariants.keySet.forall(_ ==
options.valueTag)
+ if (onlyValueTagField) {
+ // If the element only has value tag field, parse the element as
a variant primitive
+
rootBuilder.appendVariant(fieldToVariants(options.valueTag).get(0))
+ } else {
+ writeVariantObject(rootBuilder, start, fieldToVariants)
+ }
+ }
+ shouldStop = true
+
+ case _: EndDocument => shouldStop = true
+
+ case _ => // do nothing
+ }
+ }
+
+ // If the element is empty, we treat it as a Variant null
+ if (rootBuilder.getWritePos == start) {
+ rootBuilder.appendNull()
+ }
+
+ rootBuilder.result()
+ }
+
+ /**
+ * Write a variant object to the variant builder.
+ * This method handles the case sensitivity of field names when building the
variant object. When
+ * case sensitivity is disabled, a random field name will be used for all
fields with the same
+ * lowercase name.
+ *
+ * @param builder The variant builder to write to
+ * @param start The start position of the variant object in the builder
+ * @param fieldToVariants A map of field names to their corresponding
variant values of the object
+ */
+ private def writeVariantObject(
+ builder: VariantBuilder,
+ start: Int,
+ fieldToVariants: collection.mutable.TreeMap[String,
java.util.ArrayList[Variant]]): Unit = {
+ val rootFieldEntries = new java.util.ArrayList[FieldEntry]()
+
+ fieldToVariants
+ // Group the fields by their keys and take case sensitivity into account
+ .groupBy {
Review Comment:
not required if the `TreeMap` in `convertVariant` is ordered with
`caseSensitivityOrdering`
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/StaxXmlParser.scala:
##########
@@ -897,4 +914,219 @@ object StaxXmlParser {
curRecord
}
}
+
+ /**
+ * Parse the input XML string as a Variant value
+ */
+ def parseVariant(xml: String, options: XmlOptions): VariantVal = {
+ val parser = StaxXmlParserUtils.filteredReader(xml)
+ val rootEvent =
+ StaxXmlParserUtils.skipUntil(parser, XMLStreamConstants.START_ELEMENT)
+ val rootAttributes = rootEvent.asStartElement.getAttributes.asScala.toArray
+ val variant = convertVariant(parser, rootAttributes, options)
+ val v = new VariantVal(variant.getValue, variant.getMetadata)
+ parser.close()
+ v
+ }
+
+ /**
+ * Parse an XML element from the XML event stream into a Variant.
+ * This method transforms the XML element along with its attributes and
child elements
+ * into a hierarchical Variant data structure that preserves the XML
structure.
+ *
+ * @param parser The XML event stream reader positioned after the start
element
+ * @param attributes The attributes of the current XML element to be
included in the Variant
+ * @param options Configuration options that control how XML is parsed into
Variants
+ * @return A Variant representing the XML element with its attributes and
child content
+ */
+ def convertVariant(
+ parser: XMLEventReader,
+ attributes: Array[Attribute],
+ options: XmlOptions): Variant = {
+ // The variant builder for the root startElement
+ val rootBuilder = new VariantBuilder(false)
+ val start = rootBuilder.getWritePos
+
+ // Map to store the variant values of all child fields
+ // Each field could have multiple entries, which means it's an array
+ val fieldToVariants = collection.mutable.TreeMap.empty[String,
java.util.ArrayList[Variant]]
+
+ // Handle attributes first
+ StaxXmlParserUtils.convertAttributesToValuesMap(attributes,
options).foreach {
+ case (f, v) =>
+ val builder = new VariantBuilder(false)
+ appendXMLCharacterToVariant(builder, v, options)
+ val variants = fieldToVariants.getOrElseUpdate(f, new
java.util.ArrayList[Variant]())
+ variants.add(builder.result())
+ }
+
+ var shouldStop = false
+ while (!shouldStop) {
+ parser.nextEvent() match {
+ case s: StartElement =>
+ // For each child element, convert it to a variant and keep track of
it in
+ // fieldsToVariants
+ val attributes =
s.getAttributes.asScala.map(_.asInstanceOf[Attribute]).toArray
+ val field = StaxXmlParserUtils.getName(s.asStartElement.getName,
options)
+ val variants = fieldToVariants.getOrElseUpdate(field, new
java.util.ArrayList[Variant]())
+ variants.add(convertVariant(parser, attributes, options))
+
+ case c: Characters if !c.isWhiteSpace =>
+ // Treat the character as a value tag field, where we use the
[[XMLOptions.valueTag]] as
+ // the field key
+ val builder = new VariantBuilder(false)
+ appendXMLCharacterToVariant(builder, c.getData, options)
+ val variants = fieldToVariants.getOrElseUpdate(
+ options.valueTag,
+ new java.util.ArrayList[Variant]()
+ )
+ variants.add(builder.result())
+
+ case _: EndElement =>
+ if (fieldToVariants.nonEmpty) {
+ val onlyValueTagField = fieldToVariants.keySet.forall(_ ==
options.valueTag)
+ if (onlyValueTagField) {
+ // If the element only has value tag field, parse the element as
a variant primitive
+
rootBuilder.appendVariant(fieldToVariants(options.valueTag).get(0))
+ } else {
+ writeVariantObject(rootBuilder, start, fieldToVariants)
+ }
+ }
+ shouldStop = true
+
+ case _: EndDocument => shouldStop = true
+
+ case _ => // do nothing
+ }
+ }
+
+ // If the element is empty, we treat it as a Variant null
+ if (rootBuilder.getWritePos == start) {
+ rootBuilder.appendNull()
+ }
+
+ rootBuilder.result()
+ }
+
+ /**
+ * Write a variant object to the variant builder.
+ * This method handles the case sensitivity of field names when building the
variant object. When
+ * case sensitivity is disabled, a random field name will be used for all
fields with the same
+ * lowercase name.
+ *
+ * @param builder The variant builder to write to
+ * @param start The start position of the variant object in the builder
+ * @param fieldToVariants A map of field names to their corresponding
variant values of the object
+ */
+ private def writeVariantObject(
+ builder: VariantBuilder,
+ start: Int,
+ fieldToVariants: collection.mutable.TreeMap[String,
java.util.ArrayList[Variant]]): Unit = {
+ val rootFieldEntries = new java.util.ArrayList[FieldEntry]()
+
+ fieldToVariants
+ // Group the fields by their keys and take case sensitivity into account
+ .groupBy {
+ case (k, _) if SQLConf.get.caseSensitiveAnalysis => k
+ case (k, _) => k.toLowerCase(Locale.ROOT)
+ }
+ .foreach {
+ case (_, fieldToVariantsGroup) =>
+ // Pick the first field key as the field name
+ val field = fieldToVariantsGroup.head._1
+
+ // Add the field key to the variant metadata
+ val fieldId = builder.addKey(field)
+ rootFieldEntries.add(
+ new FieldEntry(field, fieldId, builder.getWritePos - start)
+ )
+
+ // Aggregate all variant values for the field
+ val variants = fieldToVariantsGroup.values.flatMap(_.asScala)
+
+ val fieldValue = if (variants.size > 1) {
+ // If the field has more than one entry, it's an array field.
Build a Variant
+ // array as the field value
+ val arrayBuilder = new VariantBuilder(false)
+ val start = arrayBuilder.getWritePos
Review Comment:
To avoid confusion with the other `start` variable in this function.
```suggestion
val arrayStart = arrayBuilder.getWritePos
```
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/StaxXmlParser.scala:
##########
@@ -897,4 +914,219 @@ object StaxXmlParser {
curRecord
}
}
+
+ /**
+ * Parse the input XML string as a Variant value
+ */
+ def parseVariant(xml: String, options: XmlOptions): VariantVal = {
+ val parser = StaxXmlParserUtils.filteredReader(xml)
+ val rootEvent =
+ StaxXmlParserUtils.skipUntil(parser, XMLStreamConstants.START_ELEMENT)
+ val rootAttributes = rootEvent.asStartElement.getAttributes.asScala.toArray
+ val variant = convertVariant(parser, rootAttributes, options)
+ val v = new VariantVal(variant.getValue, variant.getMetadata)
+ parser.close()
+ v
+ }
+
+ /**
+ * Parse an XML element from the XML event stream into a Variant.
+ * This method transforms the XML element along with its attributes and
child elements
+ * into a hierarchical Variant data structure that preserves the XML
structure.
+ *
+ * @param parser The XML event stream reader positioned after the start
element
+ * @param attributes The attributes of the current XML element to be
included in the Variant
+ * @param options Configuration options that control how XML is parsed into
Variants
+ * @return A Variant representing the XML element with its attributes and
child content
+ */
+ def convertVariant(
+ parser: XMLEventReader,
+ attributes: Array[Attribute],
+ options: XmlOptions): Variant = {
+ // The variant builder for the root startElement
+ val rootBuilder = new VariantBuilder(false)
+ val start = rootBuilder.getWritePos
+
+ // Map to store the variant values of all child fields
+ // Each field could have multiple entries, which means it's an array
+ val fieldToVariants = collection.mutable.TreeMap.empty[String,
java.util.ArrayList[Variant]]
+
+ // Handle attributes first
+ StaxXmlParserUtils.convertAttributesToValuesMap(attributes,
options).foreach {
+ case (f, v) =>
+ val builder = new VariantBuilder(false)
+ appendXMLCharacterToVariant(builder, v, options)
+ val variants = fieldToVariants.getOrElseUpdate(f, new
java.util.ArrayList[Variant]())
+ variants.add(builder.result())
+ }
+
+ var shouldStop = false
+ while (!shouldStop) {
+ parser.nextEvent() match {
+ case s: StartElement =>
+ // For each child element, convert it to a variant and keep track of
it in
+ // fieldsToVariants
+ val attributes =
s.getAttributes.asScala.map(_.asInstanceOf[Attribute]).toArray
+ val field = StaxXmlParserUtils.getName(s.asStartElement.getName,
options)
+ val variants = fieldToVariants.getOrElseUpdate(field, new
java.util.ArrayList[Variant]())
+ variants.add(convertVariant(parser, attributes, options))
+
+ case c: Characters if !c.isWhiteSpace =>
+ // Treat the character as a value tag field, where we use the
[[XMLOptions.valueTag]] as
+ // the field key
+ val builder = new VariantBuilder(false)
+ appendXMLCharacterToVariant(builder, c.getData, options)
+ val variants = fieldToVariants.getOrElseUpdate(
+ options.valueTag,
+ new java.util.ArrayList[Variant]()
+ )
+ variants.add(builder.result())
+
+ case _: EndElement =>
+ if (fieldToVariants.nonEmpty) {
+ val onlyValueTagField = fieldToVariants.keySet.forall(_ ==
options.valueTag)
+ if (onlyValueTagField) {
+ // If the element only has value tag field, parse the element as
a variant primitive
+
rootBuilder.appendVariant(fieldToVariants(options.valueTag).get(0))
+ } else {
+ writeVariantObject(rootBuilder, start, fieldToVariants)
+ }
+ }
+ shouldStop = true
+
+ case _: EndDocument => shouldStop = true
+
+ case _ => // do nothing
+ }
+ }
+
+ // If the element is empty, we treat it as a Variant null
+ if (rootBuilder.getWritePos == start) {
+ rootBuilder.appendNull()
+ }
+
+ rootBuilder.result()
+ }
+
+ /**
+ * Write a variant object to the variant builder.
+ * This method handles the case sensitivity of field names when building the
variant object. When
+ * case sensitivity is disabled, a random field name will be used for all
fields with the same
+ * lowercase name.
+ *
+ * @param builder The variant builder to write to
+ * @param start The start position of the variant object in the builder
+ * @param fieldToVariants A map of field names to their corresponding
variant values of the object
+ */
+ private def writeVariantObject(
+ builder: VariantBuilder,
+ start: Int,
+ fieldToVariants: collection.mutable.TreeMap[String,
java.util.ArrayList[Variant]]): Unit = {
+ val rootFieldEntries = new java.util.ArrayList[FieldEntry]()
+
+ fieldToVariants
+ // Group the fields by their keys and take case sensitivity into account
+ .groupBy {
+ case (k, _) if SQLConf.get.caseSensitiveAnalysis => k
+ case (k, _) => k.toLowerCase(Locale.ROOT)
+ }
+ .foreach {
+ case (_, fieldToVariantsGroup) =>
+ // Pick the first field key as the field name
+ val field = fieldToVariantsGroup.head._1
+
+ // Add the field key to the variant metadata
+ val fieldId = builder.addKey(field)
+ rootFieldEntries.add(
+ new FieldEntry(field, fieldId, builder.getWritePos - start)
+ )
+
+ // Aggregate all variant values for the field
+ val variants = fieldToVariantsGroup.values.flatMap(_.asScala)
+
+ val fieldValue = if (variants.size > 1) {
+ // If the field has more than one entry, it's an array field.
Build a Variant
+ // array as the field value
+ val arrayBuilder = new VariantBuilder(false)
+ val start = arrayBuilder.getWritePos
+ val offsets = new util.ArrayList[Integer]()
+ variants.foreach { v =>
+ offsets.add(arrayBuilder.getWritePos - start)
+ arrayBuilder.appendVariant(v)
+ }
+ arrayBuilder.finishWritingArray(start, offsets)
+ arrayBuilder.result()
+ } else {
+ // Otherwise, just use the first variant as the field value
+ variants.head
+ }
+
+ // Append the field value to the variant builder
+ builder.appendVariant(fieldValue)
+ }
+
+ // Finish writing the variant object
+ builder.finishWritingObject(start, rootFieldEntries)
+ }
+
+ /**
+ * Convert an XML Character value `s` into a variant value and append the
result to `builder`.
+ * The result can only be one of a variant boolean/long/decimal/string.
Anything other than
+ * the supported types will be appended to the Variant builder as a string.
+ *
+ * Floating point types (double, float) are not considered to avoid
precision loss.
+ */
+ private def appendXMLCharacterToVariant(
+ builder: VariantBuilder,
+ s: String,
+ options: XmlOptions): Unit = {
+ if (s == null || s == options.nullValue) {
+ builder.appendNull()
+ return
+ }
+
+ val value = if (options.ignoreSurroundingSpaces) s.trim() else s
+
+ // Exit early for empty strings
+ if (value.isEmpty) {
+ builder.appendString(value)
+ return
+ }
+
+ // Try parsing the value as boolean first
+ if (value.toLowerCase(Locale.ROOT) == "true") {
+ builder.appendBoolean(true)
+ return
+ }
+ if (value.toLowerCase(Locale.ROOT) == "false") {
+ builder.appendBoolean(false)
+ return
+ }
+
+ // Try parsing the value as a long
+ allCatch opt value.toLong match {
+ case Some(l) =>
+ builder.appendLong(l)
+ return
+ case _ =>
+ }
+
+ // Try parsing the value as decimal
+ val decimalParser = ExprUtils.getDecimalParser(options.locale)
+ allCatch opt decimalParser(value) match {
+ case Some(decimalValue) =>
+ var d = decimalValue
+ if (d.scale() < 0) {
+ d = d.setScale(0)
+ }
+ if (d.scale <= MAX_DECIMAL16_PRECISION && d.precision <=
MAX_DECIMAL16_PRECISION) {
+ builder.appendDecimal(d)
+ return
+ }
+ case _ =>
+ }
+
+ // If the character is of other primitive types, parse it as a string
Review Comment:
Lets add other data types once we have incremental type casting.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/StaxXmlParser.scala:
##########
@@ -897,4 +914,219 @@ object StaxXmlParser {
curRecord
}
}
+
+ /**
+ * Parse the input XML string as a Variant value
+ */
+ def parseVariant(xml: String, options: XmlOptions): VariantVal = {
+ val parser = StaxXmlParserUtils.filteredReader(xml)
+ val rootEvent =
+ StaxXmlParserUtils.skipUntil(parser, XMLStreamConstants.START_ELEMENT)
+ val rootAttributes = rootEvent.asStartElement.getAttributes.asScala.toArray
+ val variant = convertVariant(parser, rootAttributes, options)
+ val v = new VariantVal(variant.getValue, variant.getMetadata)
+ parser.close()
+ v
+ }
+
+ /**
+ * Parse an XML element from the XML event stream into a Variant.
+ * This method transforms the XML element along with its attributes and
child elements
+ * into a hierarchical Variant data structure that preserves the XML
structure.
+ *
+ * @param parser The XML event stream reader positioned after the start
element
+ * @param attributes The attributes of the current XML element to be
included in the Variant
+ * @param options Configuration options that control how XML is parsed into
Variants
+ * @return A Variant representing the XML element with its attributes and
child content
+ */
+ def convertVariant(
+ parser: XMLEventReader,
+ attributes: Array[Attribute],
+ options: XmlOptions): Variant = {
+ // The variant builder for the root startElement
+ val rootBuilder = new VariantBuilder(false)
+ val start = rootBuilder.getWritePos
+
+ // Map to store the variant values of all child fields
+ // Each field could have multiple entries, which means it's an array
+ val fieldToVariants = collection.mutable.TreeMap.empty[String,
java.util.ArrayList[Variant]]
+
+ // Handle attributes first
+ StaxXmlParserUtils.convertAttributesToValuesMap(attributes,
options).foreach {
+ case (f, v) =>
+ val builder = new VariantBuilder(false)
+ appendXMLCharacterToVariant(builder, v, options)
+ val variants = fieldToVariants.getOrElseUpdate(f, new
java.util.ArrayList[Variant]())
+ variants.add(builder.result())
+ }
+
+ var shouldStop = false
+ while (!shouldStop) {
+ parser.nextEvent() match {
+ case s: StartElement =>
+ // For each child element, convert it to a variant and keep track of
it in
+ // fieldsToVariants
+ val attributes =
s.getAttributes.asScala.map(_.asInstanceOf[Attribute]).toArray
+ val field = StaxXmlParserUtils.getName(s.asStartElement.getName,
options)
+ val variants = fieldToVariants.getOrElseUpdate(field, new
java.util.ArrayList[Variant]())
+ variants.add(convertVariant(parser, attributes, options))
+
+ case c: Characters if !c.isWhiteSpace =>
+ // Treat the character as a value tag field, where we use the
[[XMLOptions.valueTag]] as
+ // the field key
+ val builder = new VariantBuilder(false)
+ appendXMLCharacterToVariant(builder, c.getData, options)
+ val variants = fieldToVariants.getOrElseUpdate(
+ options.valueTag,
+ new java.util.ArrayList[Variant]()
+ )
+ variants.add(builder.result())
+
+ case _: EndElement =>
+ if (fieldToVariants.nonEmpty) {
+ val onlyValueTagField = fieldToVariants.keySet.forall(_ ==
options.valueTag)
+ if (onlyValueTagField) {
+ // If the element only has value tag field, parse the element as
a variant primitive
+
rootBuilder.appendVariant(fieldToVariants(options.valueTag).get(0))
+ } else {
+ writeVariantObject(rootBuilder, start, fieldToVariants)
+ }
+ }
+ shouldStop = true
+
+ case _: EndDocument => shouldStop = true
+
+ case _ => // do nothing
+ }
+ }
+
+ // If the element is empty, we treat it as a Variant null
+ if (rootBuilder.getWritePos == start) {
+ rootBuilder.appendNull()
+ }
+
+ rootBuilder.result()
+ }
+
+ /**
+ * Write a variant object to the variant builder.
+ * This method handles the case sensitivity of field names when building the
variant object. When
+ * case sensitivity is disabled, a random field name will be used for all
fields with the same
+ * lowercase name.
+ *
+ * @param builder The variant builder to write to
+ * @param start The start position of the variant object in the builder
+ * @param fieldToVariants A map of field names to their corresponding
variant values of the object
+ */
+ private def writeVariantObject(
+ builder: VariantBuilder,
+ start: Int,
+ fieldToVariants: collection.mutable.TreeMap[String,
java.util.ArrayList[Variant]]): Unit = {
+ val rootFieldEntries = new java.util.ArrayList[FieldEntry]()
+
+ fieldToVariants
+ // Group the fields by their keys and take case sensitivity into account
+ .groupBy {
+ case (k, _) if SQLConf.get.caseSensitiveAnalysis => k
+ case (k, _) => k.toLowerCase(Locale.ROOT)
+ }
+ .foreach {
+ case (_, fieldToVariantsGroup) =>
+ // Pick the first field key as the field name
+ val field = fieldToVariantsGroup.head._1
+
+ // Add the field key to the variant metadata
+ val fieldId = builder.addKey(field)
+ rootFieldEntries.add(
+ new FieldEntry(field, fieldId, builder.getWritePos - start)
+ )
+
+ // Aggregate all variant values for the field
+ val variants = fieldToVariantsGroup.values.flatMap(_.asScala)
+
+ val fieldValue = if (variants.size > 1) {
+ // If the field has more than one entry, it's an array field.
Build a Variant
+ // array as the field value
+ val arrayBuilder = new VariantBuilder(false)
+ val start = arrayBuilder.getWritePos
+ val offsets = new util.ArrayList[Integer]()
+ variants.foreach { v =>
+ offsets.add(arrayBuilder.getWritePos - start)
+ arrayBuilder.appendVariant(v)
+ }
+ arrayBuilder.finishWritingArray(start, offsets)
+ arrayBuilder.result()
+ } else {
+ // Otherwise, just use the first variant as the field value
+ variants.head
+ }
+
+ // Append the field value to the variant builder
+ builder.appendVariant(fieldValue)
+ }
+
+ // Finish writing the variant object
+ builder.finishWritingObject(start, rootFieldEntries)
+ }
+
+ /**
+ * Convert an XML Character value `s` into a variant value and append the
result to `builder`.
+ * The result can only be one of a variant boolean/long/decimal/string.
Anything other than
+ * the supported types will be appended to the Variant builder as a string.
+ *
+ * Floating point types (double, float) are not considered to avoid
precision loss.
+ */
+ private def appendXMLCharacterToVariant(
+ builder: VariantBuilder,
+ s: String,
+ options: XmlOptions): Unit = {
+ if (s == null || s == options.nullValue) {
+ builder.appendNull()
+ return
+ }
+
+ val value = if (options.ignoreSurroundingSpaces) s.trim() else s
+
+ // Exit early for empty strings
+ if (value.isEmpty) {
+ builder.appendString(value)
+ return
+ }
+
+ // Try parsing the value as boolean first
+ if (value.toLowerCase(Locale.ROOT) == "true") {
+ builder.appendBoolean(true)
+ return
+ }
+ if (value.toLowerCase(Locale.ROOT) == "false") {
+ builder.appendBoolean(false)
+ return
+ }
+
+ // Try parsing the value as a long
+ allCatch opt value.toLong match {
+ case Some(l) =>
+ builder.appendLong(l)
+ return
+ case _ =>
+ }
+
+ // Try parsing the value as decimal
+ val decimalParser = ExprUtils.getDecimalParser(options.locale)
Review Comment:
Use the global `decimalParser`
```suggestion
```
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/xml/XmlVariantSuite.scala:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.xml
+
+import java.time.ZoneOffset
+
+import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row}
+import org.apache.spark.sql.catalyst.xml.{StaxXmlParser, XmlOptions}
+import org.apache.spark.sql.functions.{col, variant_get}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.SharedSparkSession
+
+class XmlVariantSuite extends QueryTest with SharedSparkSession with
TestXmlData {
+
+ private val baseOptions = Map("rowTag" -> "ROW", "valueTag" -> "_VALUE",
"attributePrefix" -> "_")
+
+ private val resDir = "test-data/xml-resources/"
+
+ // ==========================
+ // ====== Parser tests ======
+ // ==========================
+
+ private def testParser(
+ xml: String,
+ expectedJsonStr: String,
+ extraOptions: Map[String, String] = Map.empty): Unit = {
+ val parsed = StaxXmlParser.parseVariant(xml, XmlOptions(baseOptions ++
extraOptions))
+ assert(parsed.toJson(ZoneOffset.UTC) == expectedJsonStr)
+ }
+
+ test("Parser: parse primitive XML elements (long, decimal, double, etc.) as
variants") {
+ // Boolean -> Boolean
+ testParser("<ROW><isActive>false</isActive></ROW>",
"""{"isActive":false}""")
+ testParser("<ROW><isActive>true</isActive></ROW>", """{"isActive":true}""")
+
+ // Long -> Long
+ testParser("<ROW><id>2</id></ROW>", """{"id":2}""")
+ testParser("<ROW><id>+2</id></ROW>", """{"id":2}""")
+ testParser("<ROW><id>-2</id></ROW>", """{"id":-2}""")
+
+ // Decimal -> Decimal
+ testParser(
+ xml = "<ROW><price>158,058,049.001</price></ROW>",
+ expectedJsonStr = """{"price":158058049.001}"""
+ )
+ testParser(
+ xml = "<ROW><decimal>10.05</decimal></ROW>",
+ expectedJsonStr = """{"decimal":10.05}"""
+ )
+ testParser(
+ xml = "<ROW><amount>5.0</amount></ROW>",
+ expectedJsonStr = """{"amount":5}"""
+ )
+ // This is parsed as String, because it is too large for Decimal
+ testParser(
+ xml = "<ROW><amount>1e40</amount></ROW>",
+ expectedJsonStr = """{"amount":"1e40"}"""
+ )
+
+ // Date -> String
+ testParser(
+ xml = "<ROW><createdAt>2023-10-01</createdAt></ROW>",
+ expectedJsonStr = """{"createdAt":"2023-10-01"}"""
+ )
+
+ // Timestamp -> String
+ testParser(
+ xml = "<ROW><createdAt>2023-10-01T12:00:00Z</createdAt></ROW>",
+ expectedJsonStr = """{"createdAt":"2023-10-01T12:00:00Z"}"""
+ )
+
+ // String -> String
+ testParser("<ROW><name>Sam</name></ROW>", """{"name":"Sam"}""")
+ // Strings with spaces
+ testParser(
+ "<ROW><note> hello world </note></ROW>",
+ expectedJsonStr = """{"note":" hello world "}""",
+ extraOptions = Map("ignoreSurroundingSpaces" -> "false")
+ )
+ testParser(
+ xml = "<ROW><note> hello world </note></ROW>",
+ expectedJsonStr = """{"note":"hello world"}"""
+ )
+ }
+
+ test("Parser: parse XML attributes as variants") {
+ // XML elements with only attributes
+ testParser(
+ xml = "<ROW id=\"2\"></ROW>",
+ expectedJsonStr = """{"_id":2}"""
+ )
+ testParser(
+ xml = "<ROW><a><b attr=\"1\"></b></a></ROW>",
+ expectedJsonStr = """{"a":{"b":{"_attr":1}}}"""
+ )
+ testParser(
+ xml = "<ROW id=\"2\" name=\"Sam\" amount=\"93\"></ROW>",
+ expectedJsonStr = """{"_amount":93,"_id":2,"_name":"Sam"}"""
+ )
+
+ // XML elements with attributes and elements
+ testParser(
+ xml = "<ROW id=\"2\" name=\"Sam\"><amount>93</amount></ROW>",
+ expectedJsonStr = """{"_id":2,"_name":"Sam","amount":93}"""
+ )
+
+ // XML elements with attributes and nested elements
+ testParser(
+ xml = "<ROW id=\"2\"
name=\"Sam\"><info><amount>93</amount></info></ROW>",
+ expectedJsonStr = """{"_id":2,"_name":"Sam","info":{"amount":93}}"""
+ )
+
+ // XML elements with attributes and value tag
+ testParser(
+ xml = "<ROW id=\"2\" name=\"Sam\">93</ROW>",
+ expectedJsonStr = """{"_VALUE":93,"_id":2,"_name":"Sam"}"""
+ )
+ }
+
+ test("Parser: parse XML value tags as variants") {
+ // XML elements with value tags and attributes
+ testParser(
+ xml = "<ROW id=\"2\" name=\"Sam\">93</ROW>",
+ expectedJsonStr = """{"_VALUE":93,"_id":2,"_name":"Sam"}"""
+ )
+
+ // XML elements with value tags and nested elements
+ testParser(
+ xml = "<ROW><info>Sam<amount>93</amount></info></ROW>",
+ expectedJsonStr = """{"info":{"_VALUE":"Sam","amount":93}}"""
+ )
+ }
+
+ test("Parser: parse XML elements as variant object") {
+ testParser(
+ xml = "<ROW><info><name>Sam</name><amount>93</amount></info></ROW>",
+ expectedJsonStr = """{"info":{"amount":93,"name":"Sam"}}"""
+ )
+ }
+
+ test("Parser: parse XML elements as variant array") {
+ testParser(
+ xml = "<ROW><array>1</array><array>2</array></ROW>",
+ expectedJsonStr = """{"array":[1,2]}"""
+ )
+ }
+
+ test("Parser: null and empty XML elements are parsed as variant null") {
+ // XML elements with null and empty values
+ testParser(
+ xml = "<ROW><name></name><amount>93</amount></ROW>",
+ expectedJsonStr = """{"amount":93,"name":null}"""
+ )
+ testParser(
+ xml = "<ROW><name>Sam</name><amount>n/a</amount></ROW>",
+ expectedJsonStr = """{"amount":null,"name":"Sam"}""",
+ extraOptions = Map("nullValue" -> "n/a")
+ )
+ }
+
+ test("Parser: Parse whitespaces with quotes") {
+ // XML elements with whitespaces
+ testParser(
+ xml = s"""
+ |<ROW>
+ | <a>" "</a>
+ | <b>" "<c>1</c></b>
+ | <d><e attr=" "></e></d>
+ |</ROW>
+ |""".stripMargin,
+ expectedJsonStr = """{"a":"\" \"","b":{"_VALUE":"\"
\"","c":1},"d":{"e":{"_attr":" "}}}""",
+ extraOptions = Map("ignoreSurroundingSpaces" -> "false")
+ )
+ }
+
+ test("Parser: Comments are ignored") {
+ testParser(
+ xml = """
+ |<ROW>
+ | <!-- comment -->
+ | <name><!-- before value --> Sam <!-- after value --></name>
+ | <!-- comment -->
+ | <amount>93</amount>
+ |</ROW>
Review Comment:
Add element in a comment:
```suggestion
| <amount>93</amount>
| <!-- <a>1</a> -->
|</ROW>
```
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/StaxXmlParser.scala:
##########
@@ -897,4 +914,220 @@ object StaxXmlParser {
curRecord
}
}
+
+ /**
+ * Parse the input XML string as a Varaint value
+ */
+ def parseVariant(xml: String, options: XmlOptions): VariantVal = {
+ val parser = StaxXmlParserUtils.filteredReader(xml)
+ val rootEvent =
+ StaxXmlParserUtils.skipUntil(parser, XMLStreamConstants.START_ELEMENT)
+ val rootAttributes = rootEvent.asStartElement.getAttributes.asScala.toArray
+ val variant = convertVariant(parser, rootAttributes, options)
+ val v = new VariantVal(variant.getValue, variant.getMetadata)
+ parser.close()
+ v
+ }
+
+ /**
+ * Parse an XML element from the XML event stream into a Variant.
+ * This method transforms the XML element along with its attributes and
child elements
+ * into a hierarchical Variant data structure that preserves the XML
structure.
+ *
+ * @param parser The XML event stream reader positioned after the start
element
+ * @param attributes The attributes of the current XML element to be
included in the Variant
+ * @param options Configuration options that control how XML is parsed into
Variants
+ * @return A Variant representing the XML element with its attributes and
child content
+ */
+ def convertVariant(
+ parser: XMLEventReader,
+ attributes: Array[Attribute],
+ options: XmlOptions): Variant = {
+ // The variant builder for the root startElement
+ val rootBuilder = new VariantBuilder(false)
+ val start = rootBuilder.getWritePos
+
+ // Map to store the variant values of all child fields
+ // Each field could have multiple entries, which means it's an array
+ val fieldToVariants = collection.mutable.TreeMap.empty[String,
java.util.ArrayList[Variant]]
+
+ // Handle attributes first
+ StaxXmlParserUtils.convertAttributesToValuesMap(attributes,
options).foreach {
+ case (f, v) =>
+ val builder = new VariantBuilder(false)
+ appendXMLCharacterToVariant(builder, v, options)
+ addOrUpdateVariantFields(fieldToVariants, f, builder.result())
+ }
+
+ var shouldStop = false
+ while (!shouldStop) {
+ parser.nextEvent() match {
+ case s: StartElement =>
+ // For each child element, convert it to a variant and keep track of
it in
+ // fieldsToVariants
+ val attributes =
s.getAttributes.asScala.map(_.asInstanceOf[Attribute]).toArray
+ val field = StaxXmlParserUtils.getName(s.asStartElement.getName,
options)
+ addOrUpdateVariantFields(
+ fieldToVariants = fieldToVariants,
+ field = field,
+ variant = convertVariant(parser, attributes, options)
+ )
+
+ case c: Characters if !c.isWhiteSpace =>
+ // Treat the character as a value tag field, where we use the
[[XMLOptions.valueTag]] as
+ // the field key
+ val builder = new VariantBuilder(false)
+ appendXMLCharacterToVariant(builder, c.getData, options)
+ addOrUpdateVariantFields(
+ fieldToVariants = fieldToVariants,
+ field = options.valueTag,
+ variant = builder.result()
+ )
+
+ case _: EndElement =>
+ if (fieldToVariants.nonEmpty) {
+ val onlyValueTagField = fieldToVariants.keySet.forall(_ ==
options.valueTag)
+ if (onlyValueTagField) {
+ // If the element only has value tag field, parse the element as
a variant primitive
+
rootBuilder.appendVariant(fieldToVariants(options.valueTag).get(0))
+ } else {
+ // Otherwise, build the element as an object with all the fields
in fieldToVariants
+ val rootFieldEntries = new java.util.ArrayList[FieldEntry]()
+
+ fieldToVariants.foreach {
+ case (field, variants) =>
+ // Add the field key to the variant metadata
+ val fieldId = rootBuilder.addKey(field)
+ rootFieldEntries.add(
+ new FieldEntry(field, fieldId, rootBuilder.getWritePos -
start)
+ )
+
+ val fieldValue = if (variants.size() > 1) {
+ // If the field has more than one entry, it's an array
field. Build a Variant
+ // array as the field value
+ val arrayBuilder = new VariantBuilder(false)
+ val start = arrayBuilder.getWritePos
+ val offsets = new util.ArrayList[Integer]()
+ variants.asScala.foreach { v =>
+ offsets.add(arrayBuilder.getWritePos - start)
+ arrayBuilder.appendVariant(v)
+ }
+ arrayBuilder.finishWritingArray(start, offsets)
+ arrayBuilder.result()
+ } else {
+ // Otherwise, just use the first variant as the field value
+ variants.get(0)
+ }
+
+ // Append the field value to the variant builder
+ rootBuilder.appendVariant(fieldValue)
+ }
+
+ // Finish writing the root element as an object if it has more
than one child element
+ // or attribute
+ rootBuilder.finishWritingObject(start, rootFieldEntries)
+ }
+ }
+ shouldStop = true
+
+ case _: EndDocument => shouldStop = true
+
+ case _ => // do nothing
+ }
+ }
+
+ // If the element is empty, we treat it as a Variant null
+ if (rootBuilder.getWritePos == start) {
+ rootBuilder.appendNull()
+ }
+
+ rootBuilder.result()
+ }
+
+ /**
+ * Add or update the given field and its corresponding variant values in the
fieldToVariants map.
+ * If a field has multiple variant values, it will be parsed as a Variant
array.
+ *
+ * This method handles the case sensitivity of the field names based on the
SQLConf setting.
+ */
+ private def addOrUpdateVariantFields(
+ fieldToVariants: collection.mutable.TreeMap[String,
java.util.ArrayList[Variant]],
+ field: String,
+ variant: Variant): Unit = {
+ val variants = if (SQLConf.get.caseSensitiveAnalysis) {
+ // If case-sensitive analysis is enabled, we need to use the original
field name
+ // to avoid case-insensitive key collision
+ fieldToVariants.getOrElseUpdate(field, new
java.util.ArrayList[Variant]())
+ } else {
+ // Otherwise, we can use the lower-case field name for case-insensitive
key collision
+ fieldToVariants.get(field.toLowerCase(Locale.ROOT)) match {
+ case Some(variantList) => variantList
+ case _ =>
+ // If the field doesn't exist, create the entry with the original
field name
+ fieldToVariants.getOrElseUpdate(field, new
java.util.ArrayList[Variant]())
+ }
+ }
+ variants.add(variant)
+ }
+
+ /**
+ * Convert an XML Character value `s` into a variant value and append the
result to `builder`.
+ * The result can only be one of a variant boolean/long/decimal/string.
Anything other than
+ * the supported types will be appended to the Variant builder as a string.
+ *
+ * Floating point types (double, float) are not considered to avoid
precision loss.
+ */
+ private def appendXMLCharacterToVariant(
Review Comment:
Lets do CSV like incremental type casting in a follow-up PR.
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/xml/XmlVariantSuite.scala:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.xml
+
+import java.time.ZoneOffset
+
+import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row}
+import org.apache.spark.sql.catalyst.xml.{StaxXmlParser, XmlOptions}
+import org.apache.spark.sql.functions.{col, variant_get}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.SharedSparkSession
+
+class XmlVariantSuite extends QueryTest with SharedSparkSession with
TestXmlData {
+
+ private val baseOptions = Map("rowTag" -> "ROW", "valueTag" -> "_VALUE",
"attributePrefix" -> "_")
+
+ private val resDir = "test-data/xml-resources/"
+
+ // ==========================
+ // ====== Parser tests ======
+ // ==========================
+
+ private def testParser(
+ xml: String,
+ expectedJsonStr: String,
+ extraOptions: Map[String, String] = Map.empty): Unit = {
+ val parsed = StaxXmlParser.parseVariant(xml, XmlOptions(baseOptions ++
extraOptions))
+ assert(parsed.toJson(ZoneOffset.UTC) == expectedJsonStr)
+ }
+
+ test("Parser: parse primitive XML elements (long, decimal, double, etc.) as
variants") {
+ // Boolean -> Boolean
+ testParser("<ROW><isActive>false</isActive></ROW>",
"""{"isActive":false}""")
+ testParser("<ROW><isActive>true</isActive></ROW>", """{"isActive":true}""")
+
+ // Long -> Long
+ testParser("<ROW><id>2</id></ROW>", """{"id":2}""")
+ testParser("<ROW><id>+2</id></ROW>", """{"id":2}""")
+ testParser("<ROW><id>-2</id></ROW>", """{"id":-2}""")
+
+ // Decimal -> Decimal
+ testParser(
+ xml = "<ROW><price>158,058,049.001</price></ROW>",
+ expectedJsonStr = """{"price":158058049.001}"""
+ )
+ testParser(
+ xml = "<ROW><decimal>10.05</decimal></ROW>",
+ expectedJsonStr = """{"decimal":10.05}"""
+ )
+ testParser(
+ xml = "<ROW><amount>5.0</amount></ROW>",
+ expectedJsonStr = """{"amount":5}"""
+ )
+ // This is parsed as String, because it is too large for Decimal
+ testParser(
+ xml = "<ROW><amount>1e40</amount></ROW>",
+ expectedJsonStr = """{"amount":"1e40"}"""
+ )
+
+ // Date -> String
+ testParser(
+ xml = "<ROW><createdAt>2023-10-01</createdAt></ROW>",
+ expectedJsonStr = """{"createdAt":"2023-10-01"}"""
+ )
+
+ // Timestamp -> String
+ testParser(
+ xml = "<ROW><createdAt>2023-10-01T12:00:00Z</createdAt></ROW>",
+ expectedJsonStr = """{"createdAt":"2023-10-01T12:00:00Z"}"""
+ )
+
+ // String -> String
+ testParser("<ROW><name>Sam</name></ROW>", """{"name":"Sam"}""")
+ // Strings with spaces
+ testParser(
+ "<ROW><note> hello world </note></ROW>",
+ expectedJsonStr = """{"note":" hello world "}""",
+ extraOptions = Map("ignoreSurroundingSpaces" -> "false")
+ )
+ testParser(
+ xml = "<ROW><note> hello world </note></ROW>",
+ expectedJsonStr = """{"note":"hello world"}"""
+ )
+ }
+
+ test("Parser: parse XML attributes as variants") {
+ // XML elements with only attributes
+ testParser(
+ xml = "<ROW id=\"2\"></ROW>",
+ expectedJsonStr = """{"_id":2}"""
+ )
+ testParser(
+ xml = "<ROW><a><b attr=\"1\"></b></a></ROW>",
+ expectedJsonStr = """{"a":{"b":{"_attr":1}}}"""
+ )
+ testParser(
+ xml = "<ROW id=\"2\" name=\"Sam\" amount=\"93\"></ROW>",
+ expectedJsonStr = """{"_amount":93,"_id":2,"_name":"Sam"}"""
+ )
+
+ // XML elements with attributes and elements
+ testParser(
+ xml = "<ROW id=\"2\" name=\"Sam\"><amount>93</amount></ROW>",
+ expectedJsonStr = """{"_id":2,"_name":"Sam","amount":93}"""
+ )
+
+ // XML elements with attributes and nested elements
+ testParser(
+ xml = "<ROW id=\"2\"
name=\"Sam\"><info><amount>93</amount></info></ROW>",
+ expectedJsonStr = """{"_id":2,"_name":"Sam","info":{"amount":93}}"""
+ )
+
+ // XML elements with attributes and value tag
+ testParser(
+ xml = "<ROW id=\"2\" name=\"Sam\">93</ROW>",
+ expectedJsonStr = """{"_VALUE":93,"_id":2,"_name":"Sam"}"""
+ )
+ }
+
+ test("Parser: parse XML value tags as variants") {
+ // XML elements with value tags and attributes
+ testParser(
+ xml = "<ROW id=\"2\" name=\"Sam\">93</ROW>",
+ expectedJsonStr = """{"_VALUE":93,"_id":2,"_name":"Sam"}"""
+ )
+
+ // XML elements with value tags and nested elements
+ testParser(
+ xml = "<ROW><info>Sam<amount>93</amount></info></ROW>",
+ expectedJsonStr = """{"info":{"_VALUE":"Sam","amount":93}}"""
+ )
+ }
+
+ test("Parser: parse XML elements as variant object") {
+ testParser(
+ xml = "<ROW><info><name>Sam</name><amount>93</amount></info></ROW>",
+ expectedJsonStr = """{"info":{"amount":93,"name":"Sam"}}"""
+ )
+ }
+
+ test("Parser: parse XML elements as variant array") {
+ testParser(
+ xml = "<ROW><array>1</array><array>2</array></ROW>",
+ expectedJsonStr = """{"array":[1,2]}"""
+ )
+ }
+
+ test("Parser: null and empty XML elements are parsed as variant null") {
+ // XML elements with null and empty values
+ testParser(
+ xml = "<ROW><name></name><amount>93</amount></ROW>",
+ expectedJsonStr = """{"amount":93,"name":null}"""
+ )
+ testParser(
+ xml = "<ROW><name>Sam</name><amount>n/a</amount></ROW>",
+ expectedJsonStr = """{"amount":null,"name":"Sam"}""",
+ extraOptions = Map("nullValue" -> "n/a")
+ )
+ }
+
+ test("Parser: Parse whitespaces with quotes") {
+ // XML elements with whitespaces
+ testParser(
+ xml = s"""
+ |<ROW>
+ | <a>" "</a>
+ | <b>" "<c>1</c></b>
+ | <d><e attr=" "></e></d>
+ |</ROW>
+ |""".stripMargin,
+ expectedJsonStr = """{"a":"\" \"","b":{"_VALUE":"\"
\"","c":1},"d":{"e":{"_attr":" "}}}""",
+ extraOptions = Map("ignoreSurroundingSpaces" -> "false")
+ )
+ }
+
+ test("Parser: Comments are ignored") {
+ testParser(
+ xml = """
+ |<ROW>
+ | <!-- comment -->
+ | <name><!-- before value --> Sam <!-- after value --></name>
+ | <!-- comment -->
+ | <amount>93</amount>
+ |</ROW>
+ |""".stripMargin,
+ expectedJsonStr = """{"amount":93,"name":"Sam"}"""
+ )
+ }
+
+ test("Parser: CDATA should be handled properly") {
+ testParser(
+ xml = """
+ |<!-- CDATA outside row should be ignored -->
+ |<ROW>
+ | <name><![CDATA[Sam]]></name>
+ | <amount>93</amount>
+ |</ROW>
+ |""".stripMargin,
+ expectedJsonStr = """{"amount":93,"name":"Sam"}"""
+ )
+ }
+
+ test("Parser: parse mixed types as variants") {
+ val expectedJsonStr =
+ """
+ |{
+ | "arrayOfArray1":[
+ | {"item":[1,2,3]},
+ | {"item":["str1","str2"]}
+ | ],
+ | "arrayOfArray2":[
+ | {"item":[1,2,3]},
+ | {"item":[1.1,2.1,3.1]}
+ | ],
+ | "arrayOfBigInteger":[922337203685477580700,-922337203685477580800],
+ | "arrayOfBoolean":[true,false,true],
+ |
"arrayOfDouble":[1.2,1.7976931348623157,"4.9E-324","2.2250738585072014E-308"],
+ | "arrayOfInteger":[1,2147483647,-2147483648],
+ |
"arrayOfLong":[21474836470,9223372036854775807,-9223372036854775808],
+ | "arrayOfNull":[null,null],
+ | "arrayOfString":["str1","str2"],
+ | "arrayOfStruct":[
+ | {"field1":true,"field2":"str1"},
+ | {"field1":false},
+ | {"field3":null}
+ | ],
+ | "struct":{
+ | "field1":true,
+ | "field2":92233720368547758070
+ | },
+ | "structWithArrayFields":{
+ | "field1":[4,5,6],
+ | "field2":["str1","str2"]
+ | }
+ |}
+ |""".stripMargin.replaceAll("\\s+", "")
+ testParser(
+ xml = complexFieldAndType1.head,
+ expectedJsonStr = expectedJsonStr
+ )
+
+ val expectedJsonStr2 =
+ """
+ |{
+ | "arrayOfArray1":[
+ | {"array":{"item":5}},
+ | {
+ | "array":[
+ | {"item":[6,7]},
+ | {"item":8}
+ | ]
+ | }
+ | ],
+ | "arrayOfArray2":[
+ | {"array":{"item":{"inner1":"str1"}}},
+ | {
+ | "array":[
+ | null,
+ | {
+ | "item":[
+ | {"inner2":["str3","str33"]},
+ | {"inner1":"str11","inner2":"str4"}
+ | ]
+ | }
+ | ]
+ | },
+ | {
+ | "array":{
+ | "item":{
+ | "inner3":[
+ | {"inner4":[2,3]},
+ | null
+ | ]
+ | }
+ | }
+ | }
+ | ]
+ |}
+ """.stripMargin.replaceAll("\\s+", "")
+ testParser(
+ xml = complexFieldAndType2.head,
+ expectedJsonStr = expectedJsonStr2
+ )
+ }
+
+ test("Parser: Case sensitivity test") {
+ val xmlString =
+ """
+ |<ROW>
+ | <a>1<b>2</b></a>
+ | <A>3<b>4</b></A>
+ |</ROW>
+ |""".stripMargin
+ withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+ testParser(
+ xml = xmlString,
+ expectedJsonStr = """{"A":[{"_VALUE":3,"b":4},{"_VALUE":1,"b":2}]}"""
+ )
+ }
+ withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+ testParser(
+ xml = xmlString,
+ expectedJsonStr = """{"A":{"_VALUE":3,"b":4},"a":{"_VALUE":1,"b":2}}"""
+ )
+ }
+ }
+
+ test("Parser: XML array elements interspersed between other elements") {
+ testParser(
+ xml = """
+ |<ROW>
+ | <a>1</a>
+ | <b>2</b>
+ | <a>3</a>
+ |</ROW>
+ |""".stripMargin,
+ expectedJsonStr = """{"a":[1,3],"b":2}"""
+ )
+
+ testParser(
+ xml = """
+ |<ROW>
+ | value1
+ | <a>1</a>
+ | value2
+ | <a>2</a>
+ | value3
+ |</ROW>
+ |""".stripMargin,
+ expectedJsonStr = """{"_VALUE":["value1","value2","value3"],"a":[1,2]}"""
+ )
+
+ // long and double
+ testParser(
+ xml = """
+ |<ROW>
+ | <a>
+ | 1
+ | <b>2</b>
+ | 3
+ | <b>4</b>
+ | 5.0
+ | </a>
+ |</ROW>
+ |""".stripMargin,
+ expectedJsonStr = """{"a":{"_VALUE":[1,3,5],"b":[2,4]}}"""
+ )
+
+ // Comments
+ testParser(
+ xml = """
+ |<ROW>
+ | <!-- comment -->
+ | <a>1</a>
+ | <!-- comment -->
+ | <b>2</b>
+ | <!-- comment -->
+ | <a>3</a>
+ |</ROW>
+ |""".stripMargin,
+ expectedJsonStr = """{"a":[1,3],"b":2}"""
+ )
+ }
+
+ // =======================
+ // ====== DSL tests ======
+ // =======================
+
+ private def createDSLDataFrame(
+ fileName: String,
+ singleVariantColumn: Option[String] = None,
+ schemaDDL: Option[String] = None,
+ extraOptions: Map[String, String] = Map.empty): DataFrame = {
+ assert(
+ singleVariantColumn.isDefined || schemaDDL.isDefined,
+ "Either singleVariantColumn or schema must be defined to ingest XML
files as variants via DSL"
+ )
+ var reader = spark.read.format("xml").options(baseOptions ++ extraOptions)
+ singleVariantColumn.foreach(
+ singleVariantColumnName =>
+ reader = reader.option("singleVariantColumn", singleVariantColumnName)
+ )
+ schemaDDL.foreach(s => reader = reader.schema(s))
+
+ reader.load(getTestResourcePath(resDir + fileName))
+ }
+
+ test("DSL: read XML files using singleVariantColumn") {
+ val df = createDSLDataFrame(fileName = "cars.xml", singleVariantColumn =
Some("var"))
+ checkAnswer(
+ df.select(variant_get(col("var"), "$.year", "int")),
+ Seq(Row(2012), Row(1997), Row(2015))
+ )
+ }
+
+ test("DSL: read XML files with defined schema") {
+ val df = createDSLDataFrame(
+ fileName = "books-complicated.xml",
+ schemaDDL = Some(
+ "_id string, author string, title string, genre variant, price double,
" +
+ "publish_dates variant"
+ ),
+ extraOptions = Map("rowTag" -> "book")
+ )
+ checkAnswer(
+ df.select(variant_get(col("genre"), "$.name", "string")),
+ Seq(Row("Computer"), Row("Fantasy"), Row("Fantasy"))
+ )
+ }
+
+ // TODO: This should be allowed once we support variant ingestion with
malformed record handling
Review Comment:
Update this comment. The test has both schema and singleVariantColumn
specified. So it has an analysis error: `INVALID_SINGLE_VARIANT_COLUMN`.
For the same cars.xml, add another three tests scenarios:
- singleVariantColumn in PERMISSIVE, FAILFAST and DROPMALFORMED mode
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
