Hi I would like to store/save DataFrame in a database table which is created
already and want to insert into always without creating table every time.
Unfortunately Spark API forces me to create table every time I have seen
Spark source code the following calls uses same method beneath if you
carefully see code it always expects new table to be created any idea how do
I use existing table to save dataframe
dataframe.write().jdbc(); OR dataframe.insertInto()
def jdbc(url: String, table: String, connectionProperties: Properties): Unit
= {
val props = new Properties()
extraOptions.foreach { case (key, value) =>
props.put(key, value)
}
// connectionProperties should override settings in extraOptions
props.putAll(connectionProperties)
val conn = JdbcUtils.createConnection(url, props)
try {
var tableExists = JdbcUtils.tableExists(conn, url, table)
if (mode == SaveMode.Ignore && tableExists) {
return
}
if (mode == SaveMode.ErrorIfExists && tableExists) {
sys.error(s"Table $table already exists.")
}
if (mode == SaveMode.Overwrite && tableExists) {
JdbcUtils.dropTable(conn, table)
tableExists = false
}
// Create the table if the table didn't exist.
if (!tableExists) {
val schema = JdbcUtils.schemaString(df, url)
val sql = s"CREATE TABLE $table ($schema)"
conn.createStatement.executeUpdate(sql)
}
} finally {
conn.close()
}
JdbcUtils.saveTable(df, url, table, props)
}
--
View this message in context:
http://apache-spark-user-list.1001560.n3.nabble.com/No-support-to-save-DataFrame-in-existing-database-table-using-DataFrameWriter-jdbc-tp25603.html
Sent from the Apache Spark User List mailing list archive at Nabble.com.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
