Github user chiwanpark commented on a diff in the pull request:
https://github.com/apache/flink/pull/1079#discussion_r39033766
--- Diff:
flink-staging/flink-hcatalog/src/main/scala/org/apache/flink/hcatalog/scala/HCatOutputFormat.scala
---
@@ -0,0 +1,449 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.hcatalog.scala
+
+import java.io.IOException
+import java.util
+
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.typeutils.TupleTypeInfo
+import org.apache.flink.api.scala.createTypeInformation
+import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
+import org.apache.flink.hcatalog.HCatOutputFormatBase
+import org.apache.hadoop.conf.Configuration
+import org.apache.hive.hcatalog.data.{DefaultHCatRecord, HCatRecord}
+
+import scala.collection.JavaConversions.mapAsJavaMap
+import scala.collection.JavaConverters._
+
+/*
+ * A OutputFormat to write to HCatalog tables for scala api
+ * The outputFormat supports write to table and partitions through
partitionValue map.
+ *
+ * scala tuple types and {@link HCatRecord} are accepted
+ * as data source. If user want to write other type of data to HCatalog, a
preceding Map function
+ * is required to convert the user data type to {@link HCatRecord}
+ *
+ * The constructor provides type checking and requires import
org.apache.flink.api.scala._ in the
+ * import statement of the user's program when compiling to allow scala
macro to extract type info
+ */
+/**
+ *
+ * @param database hive database name, if null, default database is used
+ * @param table hive table name
+ * @param partitionValues Map of partition values, if null, whole table is
used
+ * @param config the configuration
+ * @tparam T
+ */
+class HCatOutputFormat[T: TypeInformation](
+ database: String,
+ table: String,
+ partitionValues: Map[String,
String],
+ config: Configuration
+ ) extends
+HCatOutputFormatBase[T](database, table, mapAsJavaMap(partitionValues),
config) {
+
+ //get the TypeInformation
+ val ti: TypeInformation[T] = createTypeInformation[T]
+
+
+ //check the types of columns if the input is tuple(case class) type
+ if (ti.isInstanceOf[CaseClassTypeInfo[_]]) {
+ //check the number of columns
+ if (ti.asInstanceOf[CaseClassTypeInfo[_]].getArity !=
this.reqType.getArity) {
+ throw new IOException("tuple has different arity from the table's
column numbers")
+ }
+
+ for (i <- 0 until this.reqType.getArity) {
+ val it = ti.asInstanceOf[CaseClassTypeInfo[_]].getTypeAt(i)
+ val ot = reqType.asInstanceOf[TupleTypeInfo[_]].getTypeAt(i)
+
+ if (!isCompatibleType(it, ot)) {
+ throw new IOException("field has different type from required")
+ }
+ }
+ }
+
+ /**
+ *
+ * @param database hive database name, if null, default database is used
+ * @param table hive table name
+ * @param partitionValues Map of partition values, if null, whole table
is used
+ */
+ def this(database: String, table: String, partitionValues: Map[String,
String]) {
+ this(database, table, partitionValues, new Configuration)
+ }
+
+ //convert the type T to HCatRecord, if T is already HCatRecord, do
nothing
+ override protected def TupleToHCatRecord(record: T): HCatRecord = {
+ if (ti.isInstanceOf[CaseClassTypeInfo[_]]) {
+
+ val fieldList: util.List[AnyRef] = new util.ArrayList[AnyRef]
+ //we've checked the type of the scala tuple matches the table, so
fill the list now
+ fillList(fieldList, record.asInstanceOf[AnyRef])
+ new DefaultHCatRecord(fieldList)
+ }
+ else if (record.isInstanceOf[HCatRecord]) {
--- End diff --
We can implement this with matching `ti` to `WritableTypeInfo[HCatRecord]`.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
