sandip-db commented on code in PR #50052:
URL: https://github.com/apache/spark/pull/50052#discussion_r1997472249
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala:
##########
@@ -187,6 +204,7 @@ object MultiLineCSVDataSource extends CSVDataSource with
Logging {
parser: UnivocityParser,
headerChecker: CSVHeaderChecker,
requiredSchema: StructType): Iterator[InternalRow] = {
+ CSVDataSource.readHeaderForSingleVariantColumn(conf, file, parser)
Review Comment:
ditto.. The extra read can be avoided if `file.start == 0`
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala:
##########
@@ -110,6 +126,7 @@ object TextInputCSVDataSource extends CSVDataSource {
}
}
+ CSVDataSource.readHeaderForSingleVariantColumn(conf, file, parser)
Review Comment:
The extra read can be skipped if `file.start == 0`
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala:
##########
@@ -473,4 +528,61 @@ private[sql] object UnivocityParser {
parser.options.columnNameOfCorruptRecord)
filteredLines.flatMap(safeParser.parse)
}
+
+ private def tryParseNumeric(builder: VariantBuilder, s: String): Boolean = {
+ // Use while-loop because this can be performance-critical.
+ var hasDot = false
+ val length = s.length
+ var i = 0
+ while (i < length) {
+ val ch = s.charAt(i)
+ if (ch != '-' && ch != '.' && !(ch >= '0' && ch <= '9')) return false
Review Comment:
What about scientific notation (like 3e2), '+' prefix (like +2) and comma in
numbers (like 2,000,000)?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
