andygrove commented on code in PR #1768:
URL: https://github.com/apache/datafusion-comet/pull/1768#discussion_r2114221607
##########
spark/src/test/scala/org/apache/comet/CometStringExpressionSuite.scala:
##########
@@ -114,92 +95,91 @@ class CometStringExpressionSuite extends CometTestBase {
}
test("InitCap") {
- Seq(false, true).foreach { dictionary =>
- withSQLConf("parquet.enable.dictionary" -> dictionary.toString) {
- val table = "names"
- withTable(table) {
- sql(s"create table $table(id int, name varchar(20)) using parquet")
- sql(
- s"insert into $table values(1, 'james smith'), (2, 'michael
rose'), " +
- "(3, 'robert williams'), (4, 'rames rose'), (5, 'james smith'),
" +
- "(6, 'robert rose-smith'), (7, 'james ähtäri')")
- if (CometConf.COMET_EXEC_INITCAP_ENABLED.get()) {
- // TODO: remove this if clause
https://github.com/apache/datafusion-comet/issues/1052
- checkSparkAnswerAndOperator(s"SELECT initcap(name) FROM $table")
- } else {
- checkSparkAnswer(s"SELECT initcap(name) FROM $table")
- }
- }
+ val table = "names"
+ withTable(table) {
+ sql(s"create table $table(id int, name varchar(20)) using parquet")
+ sql(
+ s"insert into $table values(1, 'james smith'), (2, 'michael rose'), " +
+ "(3, 'robert williams'), (4, 'rames rose'), (5, 'james smith'), " +
+ "(6, 'robert rose-smith'), (7, 'james ähtäri')")
+ if (CometConf.COMET_EXEC_INITCAP_ENABLED.get()) {
+ // TODO: remove this if clause
https://github.com/apache/datafusion-comet/issues/1052
+ checkSparkAnswerAndOperator(s"SELECT initcap(name) FROM $table")
+ } else {
+ checkSparkAnswer(s"SELECT initcap(name) FROM $table")
}
}
}
test("trim") {
- Seq(false, true).foreach { dictionary =>
- withSQLConf(
- "parquet.enable.dictionary" -> dictionary.toString,
- CometConf.COMET_CASE_CONVERSION_ENABLED.key -> "true") {
- val table = "test"
- withTable(table) {
- sql(s"create table $table(col varchar(20)) using parquet")
- sql(s"insert into $table values(' SparkSQL '), ('SSparkSQLS')")
+ withSQLConf(CometConf.COMET_CASE_CONVERSION_ENABLED.key -> "true") {
+ val table = "test"
+ withTable(table) {
+ sql(s"create table $table(col varchar(20)) using parquet")
+ sql(s"insert into $table values(' SparkSQL '), ('SSparkSQLS')")
- checkSparkAnswerAndOperator(s"SELECT upper(trim(col)) FROM $table")
- checkSparkAnswerAndOperator(s"SELECT trim('SL', col) FROM $table")
+ checkSparkAnswerAndOperator(s"SELECT upper(trim(col)) FROM $table")
+ checkSparkAnswerAndOperator(s"SELECT trim('SL', col) FROM $table")
- checkSparkAnswerAndOperator(s"SELECT upper(btrim(col)) FROM $table")
- checkSparkAnswerAndOperator(s"SELECT btrim('SL', col) FROM $table")
+ checkSparkAnswerAndOperator(s"SELECT upper(btrim(col)) FROM $table")
+ checkSparkAnswerAndOperator(s"SELECT btrim('SL', col) FROM $table")
- checkSparkAnswerAndOperator(s"SELECT upper(ltrim(col)) FROM $table")
- checkSparkAnswerAndOperator(s"SELECT ltrim('SL', col) FROM $table")
+ checkSparkAnswerAndOperator(s"SELECT upper(ltrim(col)) FROM $table")
+ checkSparkAnswerAndOperator(s"SELECT ltrim('SL', col) FROM $table")
- checkSparkAnswerAndOperator(s"SELECT upper(rtrim(col)) FROM $table")
- checkSparkAnswerAndOperator(s"SELECT rtrim('SL', col) FROM $table")
- }
+ checkSparkAnswerAndOperator(s"SELECT upper(rtrim(col)) FROM $table")
+ checkSparkAnswerAndOperator(s"SELECT rtrim('SL', col) FROM $table")
}
}
}
test("string concat_ws") {
- Seq(false, true).foreach { dictionary =>
- withSQLConf("parquet.enable.dictionary" -> dictionary.toString) {
- val table = "names"
- withTable(table) {
- sql(
- s"create table $table(id int, first_name varchar(20),
middle_initial char(1), last_name varchar(20)) using parquet")
- sql(
- s"insert into $table values(1, 'James', 'B', 'Taylor'), (2,
'Smith', 'C', 'Davis')," +
- " (3, NULL, NULL, NULL), (4, 'Smith', 'C', 'Davis')")
- checkSparkAnswerAndOperator(
- s"SELECT concat_ws(' ', first_name, middle_initial, last_name)
FROM $table")
- }
- }
+ val table = "names"
+ withTable(table) {
+ sql(
+ s"create table $table(id int, first_name varchar(20), middle_initial
char(1), last_name varchar(20)) using parquet")
+ sql(
+ s"insert into $table values(1, 'James', 'B', 'Taylor'), (2, 'Smith',
'C', 'Davis')," +
+ " (3, NULL, NULL, NULL), (4, 'Smith', 'C', 'Davis')")
+ checkSparkAnswerAndOperator(
+ s"SELECT concat_ws(' ', first_name, middle_initial, last_name) FROM
$table")
}
}
test("string repeat") {
- Seq(false, true).foreach { dictionary =>
- withSQLConf("parquet.enable.dictionary" -> dictionary.toString) {
- val table = "names"
- withTable(table) {
- sql(s"create table $table(id int, name varchar(20)) using parquet")
- sql(s"insert into $table values(1, 'James'), (2, 'Smith'), (3,
'Smith')")
- checkSparkAnswerAndOperator(s"SELECT repeat(name, 3) FROM $table")
- }
- }
+ val table = "names"
+ withTable(table) {
+ sql(s"create table $table(id int, name varchar(20)) using parquet")
+ sql(s"insert into $table values(1, 'James'), (2, 'Smith'), (3, 'Smith')")
+ checkSparkAnswerAndOperator(s"SELECT repeat(name, 3) FROM $table")
}
}
test("length, reverse, instr, replace, translate") {
- Seq(false, true).foreach { dictionary =>
- withSQLConf("parquet.enable.dictionary" -> dictionary.toString) {
- val table = "test"
+ val table = "test"
+ withTable(table) {
+ sql(s"create table $table(col string) using parquet")
+ sql(
+ s"insert into $table values('Spark SQL '), (NULL), (''), ('苹果手机'),
('Spark SQL '), (NULL), (''), ('苹果手机')")
+ checkSparkAnswerAndOperator("select length(col), reverse(col),
instr(col, 'SQL'), instr(col, '手机'), replace(col, 'SQL', '123')," +
+ s" replace(col, 'SQL'), replace(col, '手机', '平板'), translate(col,
'SL苹', '123') from $table")
+ }
+ }
+
+ // based on Spark SQL ParquetFilterSuite test "filter pushdown -
StringPredicate"
+ test("string predicate filter") {
Review Comment:
This is the new test that is added in this PR
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For additional commands, e-mail: github-h...@datafusion.apache.org
