ZhangchengHu0923 opened a new issue #1418:
URL: https://github.com/apache/incubator-seatunnel/issues/1418
I found we don't have a native spark-clickhouse-source, but I need read some
data from Clickhouse table,so I use spark-jdbc-source instead, and a problem
came across.
# problem desc
## SeaTunnel version: build by myself from dev branch
## everything is fine，when there is no DateTime64 field
table schema
```
CREATE TABLE pdi.st_source_pure
(
`cust_id` String,
`cust_name` String,
`cust_idcard` String,
`age` Int32,
`sex` String
)
ENGINE = StripeLog
```
SeaTunnel config
```
env {
spark.app.name = "SeaTunnel"
spark.executor.instances = 2
spark.executor.cores = 1
spark.executor.memory = "1g"
}
source {
jdbc {
driver = "ru.yandex.clickhouse.ClickHouseDriver"
url = "jdbc:clickhouse://192.168.10.204:8123/pdi"
table = "st_source_pure"
result_table_name = "SOURCE_VIEW"
user = "default"
password = "jRtXD8F8"
}
}
transform {
}
sink {
Console {}
}
```
## get an "Unsupported type OTHER" while there is a DateTime64 field
table schema
```
CREATE TABLE pdi.st_source_dates
(
`date04` DateTime64,
`cust_id` String,
`cust_name` String,
`cust_idcard` String,
`age` Int32,
`sex` String
)
ENGINE = StripeLog
```
SeaTunnel config
```
env {
# You can set spark configuration here
# see available properties defined by spark:
https://spark.apache.org/docs/latest/configuration.html#available-properties
spark.app.name = "SeaTunnel"
spark.executor.instances = 2
spark.executor.cores = 1
spark.executor.memory = "1g"
spark.master = local
}
source {
jdbc {
driver = "ru.yandex.clickhouse.ClickHouseDriver"
url = "jdbc:clickhouse://192.168.10.204:8123/pdi"
table = "st_source_dates"
result_table_name = "SOURCE_VIEW"
user = "default"
password = "jRtXD8F8"
}
}
transform {
}
sink {
Console {}
}
```
Error Log
22/03/07 17:53:57 INFO SharedState: Warehouse path is
'file:/Users/zhangchenghu/software/seatunnel/spark-warehouse/'.
22/03/07 17:53:57 INFO StateStoreCoordinatorRef: Registered
StateStoreCoordinator endpoint
22/03/07 17:53:58 INFO Version: Elasticsearch Hadoop v6.8.3 [8a5f44bf7d]
22/03/07 17:53:58 INFO ClickHouseDriver: Driver registered
22/03/07 17:53:59 ERROR Seatunnel:
===============================================================================
22/03/07 17:53:59 ERROR Seatunnel: Fatal Error,
22/03/07 17:53:59 ERROR Seatunnel: Please submit bug report in
https://github.com/apache/incubator-seatunnel/issues
22/03/07 17:53:59 ERROR Seatunnel: Reason:Unsupported type OTHER
22/03/07 17:53:59 ERROR Seatunnel: Exception
StackTrace:java.sql.SQLException: Unsupported type OTHER
at
org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.org$apache$spark$sql$execution$datasources$jdbc$JdbcUtils$$getCatalystType(JdbcUtils.scala:251)
at
org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$8.apply(JdbcUtils.scala:316)
at
org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$8.apply(JdbcUtils.scala:316)
at scala.Option.getOrElse(Option.scala:121)
at
org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.getSchema(JdbcUtils.scala:315)
at
org.apache.spark.sql.execution.datasources.jdbc.JDBCRDD$.resolveTable(JDBCRDD.scala:63)
at
org.apache.spark.sql.execution.datasources.jdbc.JDBCRelation$.getSchema(JDBCRelation.scala:210)
at
org.apache.spark.sql.execution.datasources.jdbc.JdbcRelationProvider.createRelation(JdbcRelationProvider.scala:35)
at
org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:332)
at
org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:242)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:230)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:186)
at org.apache.seatunnel.spark.source.Jdbc.getData(Jdbc.scala:33)
at org.apache.seatunnel.spark.source.Jdbc.getData(Jdbc.scala:28)
at
org.apache.seatunnel.spark.batch.SparkBatchExecution.registerInputTempView(SparkBatchExecution.java:54)
at
org.apache.seatunnel.spark.batch.SparkBatchExecution.lambda$start$0(SparkBatchExecution.java:95)
at java.util.ArrayList.forEach(ArrayList.java:1259)
at
org.apache.seatunnel.spark.batch.SparkBatchExecution.start(SparkBatchExecution.java:95)
at org.apache.seatunnel.Seatunnel.entryPoint(Seatunnel.java:103)
at org.apache.seatunnel.Seatunnel.run(Seatunnel.java:61)
at org.apache.seatunnel.SeatunnelSpark.main(SeatunnelSpark.java:29)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
at
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:855)
at
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:161)
at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:184)
at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
at
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:930)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:939)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
22/03/07 17:53:59 ERROR Seatunnel:
===============================================================================
Exception in thread "main" java.sql.SQLException: Unsupported type OTHER
at
org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.org$apache$spark$sql$execution$datasources$jdbc$JdbcUtils$$getCatalystType(JdbcUtils.scala:251)
at
org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$8.apply(JdbcUtils.scala:316)
at
org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$8.apply(JdbcUtils.scala:316)
at scala.Option.getOrElse(Option.scala:121)
at
org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.getSchema(JdbcUtils.scala:315)
at
org.apache.spark.sql.execution.datasources.jdbc.JDBCRDD$.resolveTable(JDBCRDD.scala:63)
at
org.apache.spark.sql.execution.datasources.jdbc.JDBCRelation$.getSchema(JDBCRelation.scala:210)
at
org.apache.spark.sql.execution.datasources.jdbc.JdbcRelationProvider.createRelation(JdbcRelationProvider.scala:35)
at
org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:332)
at
org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:242)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:230)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:186)
at org.apache.seatunnel.spark.source.Jdbc.getData(Jdbc.scala:33)
at org.apache.seatunnel.spark.source.Jdbc.getData(Jdbc.scala:28)
at
org.apache.seatunnel.spark.batch.SparkBatchExecution.registerInputTempView(SparkBatchExecution.java:54)
at
org.apache.seatunnel.spark.batch.SparkBatchExecution.lambda$start$0(SparkBatchExecution.java:95)
at java.util.ArrayList.forEach(ArrayList.java:1259)
at
org.apache.seatunnel.spark.batch.SparkBatchExecution.start(SparkBatchExecution.java:95)
at org.apache.seatunnel.Seatunnel.entryPoint(Seatunnel.java:103)
at org.apache.seatunnel.Seatunnel.run(Seatunnel.java:61)
at org.apache.seatunnel.SeatunnelSpark.main(SeatunnelSpark.java:29)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
at
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:855)
at
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:161)
at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:184)
at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
at
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:930)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:939)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
## other information: Date\DateTime field works well
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
