This is an automated email from the ASF dual-hosted git repository.
zirui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new f055e00c5a [INLONG-8994][Sort] Add hudi connector on flink 1.15 (#9168)
f055e00c5a is described below
commit f055e00c5a9d93345c3949f391888192e04e0a8a
Author: Sting <zpen...@connect.ust.hk>
AuthorDate: Tue Oct 31 16:32:38 2023 +0800
[INLONG-8994][Sort] Add hudi connector on flink 1.15 (#9168)
---
.../src/main/assemblies/sort-connectors-v1.15.xml | 8 +
inlong-sort/sort-core/pom.xml | 6 +
inlong-sort/sort-flink/sort-flink-v1.15/pom.xml | 2 +-
.../sort-flink-v1.15/sort-connectors/hudi/pom.xml | 106 ++++++++
.../hudi/table/sink/HudiTableInlongFactory.java | 55 ++++
.../org.apache.flink.table.factories.Factory | 16 ++
.../sort-flink-v1.15/sort-connectors/pom.xml | 1 +
.../notices/NOTICE-hudi-flink1.15-bundle.txt | 299 +++++++++++++++++++++
8 files changed, 492 insertions(+), 1 deletion(-)
diff --git a/inlong-distribution/src/main/assemblies/sort-connectors-v1.15.xml
b/inlong-distribution/src/main/assemblies/sort-connectors-v1.15.xml
index 3b8b56497d..c88fd0cd3d 100644
--- a/inlong-distribution/src/main/assemblies/sort-connectors-v1.15.xml
+++ b/inlong-distribution/src/main/assemblies/sort-connectors-v1.15.xml
@@ -99,5 +99,13 @@
</includes>
<fileMode>0644</fileMode>
</fileSet>
+ <fileSet>
+
<directory>../inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/hudi/target</directory>
+ <outputDirectory>inlong-sort/connectors</outputDirectory>
+ <includes>
+
<include>sort-connector-hudi-v1.15-${project.version}.jar</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
</fileSets>
</assembly>
diff --git a/inlong-sort/sort-core/pom.xml b/inlong-sort/sort-core/pom.xml
index e8303bc57d..7228bfb59c 100644
--- a/inlong-sort/sort-core/pom.xml
+++ b/inlong-sort/sort-core/pom.xml
@@ -287,6 +287,12 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-connector-hudi-v1.15</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/inlong-sort/sort-flink/sort-flink-v1.15/pom.xml
b/inlong-sort/sort-flink/sort-flink-v1.15/pom.xml
index d1e459e088..923bf01d4c 100644
--- a/inlong-sort/sort-flink/sort-flink-v1.15/pom.xml
+++ b/inlong-sort/sort-flink/sort-flink-v1.15/pom.xml
@@ -130,7 +130,7 @@
<!-- hudi -->
<dependency>
<groupId>org.apache.hudi</groupId>
- <artifactId>hudi-flink1.13-bundle</artifactId>
+ <artifactId>hudi-flink1.15-bundle</artifactId>
<version>${hudi.version}</version>
</dependency>
<dependency>
diff --git
a/inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/hudi/pom.xml
b/inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/hudi/pom.xml
new file mode 100644
index 0000000000..ee0118e330
--- /dev/null
+++ b/inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/hudi/pom.xml
@@ -0,0 +1,106 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-connectors-v1.15</artifactId>
+ <version>1.10.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>sort-connector-hudi-v1.15</artifactId>
+ <packaging>jar</packaging>
+ <name>Apache InLong - Sort-connector-hudi</name>
+
+ <properties>
+
<inlong.root.dir>${project.parent.parent.parent.parent.parent.basedir}</inlong.root.dir>
+ <stax2-api.version>4.2.1</stax2-api.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-connector-base</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hudi</groupId>
+ <artifactId>hudi-flink1.15-bundle</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libfb303</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.woodstox</groupId>
+ <artifactId>woodstox-core</artifactId>
+ <version>${woodstox-core.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.woodstox</groupId>
+ <artifactId>stax2-api</artifactId>
+ <version>${stax2-api.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>org.apache.hudi:*</include>
+
<include>org.apache.hive:hive-exec</include>
+ <include>org.apache.hadoop:*</include>
+ <include>com.fasterxml.woodstox:*</include>
+ <include>org.codehaus.woodstox:*</include>
+ <include>com.google.guava:*</include>
+ <include>com.google.protobuf:*</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+
<artifact>org.apache.inlong:sort-connector-*</artifact>
+ <includes>
+ <include>org/apache/inlong/**</include>
+
<include>META-INF/services/org.apache.flink.table.factories.Factory</include>
+ </includes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git
a/inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/hudi/src/main/java/org/apache/inlong/sort/hudi/table/sink/HudiTableInlongFactory.java
b/inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/hudi/src/main/java/org/apache/inlong/sort/hudi/table/sink/HudiTableInlongFactory.java
new file mode 100644
index 0000000000..d2985ac5d4
--- /dev/null
+++
b/inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/hudi/src/main/java/org/apache/inlong/sort/hudi/table/sink/HudiTableInlongFactory.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sort.hudi.table.sink;
+
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.hudi.table.HoodieTableFactory;
+
+import java.util.Set;
+
+import static org.apache.inlong.sort.base.Constants.INLONG_AUDIT;
+import static org.apache.inlong.sort.base.Constants.INLONG_METRIC;
+
+public class HudiTableInlongFactory extends HoodieTableFactory {
+
+ public static final String SORT_CONNECTOR_IDENTIFY_HUDI = "hudi-inlong";
+
+ public HudiTableInlongFactory() {
+ super();
+ }
+
+ @Override
+ public String factoryIdentifier() {
+ return SORT_CONNECTOR_IDENTIFY_HUDI;
+ }
+
+ @Override
+ public DynamicTableSink createDynamicTableSink(Context context) {
+ return super.createDynamicTableSink(context);
+ }
+
+ @Override
+ public Set<ConfigOption<?>> optionalOptions() {
+ Set<ConfigOption<?>> configOptions = super.optionalOptions();
+ configOptions.add(INLONG_METRIC);
+ configOptions.add(INLONG_AUDIT);
+ return configOptions;
+ }
+
+}
diff --git
a/inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/hudi/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory
b/inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/hudi/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory
new file mode 100644
index 0000000000..709a02e60c
--- /dev/null
+++
b/inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/hudi/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.inlong.sort.hudi.table.sink.HudiTableInlongFactory
diff --git a/inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/pom.xml
b/inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/pom.xml
index fda025a670..7aaa9444ea 100644
--- a/inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/pom.xml
+++ b/inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/pom.xml
@@ -41,6 +41,7 @@
<module>pulsar</module>
<module>tubemq</module>
<module>hbase</module>
+ <module>hudi</module>
</modules>
<properties>
diff --git
a/licenses/inlong-sort-connectors/notices/NOTICE-hudi-flink1.15-bundle.txt
b/licenses/inlong-sort-connectors/notices/NOTICE-hudi-flink1.15-bundle.txt
new file mode 100644
index 0000000000..5f5d11feaf
--- /dev/null
+++ b/licenses/inlong-sort-connectors/notices/NOTICE-hudi-flink1.15-bundle.txt
@@ -0,0 +1,299 @@
+// ------------------------------------------------------------------
+// NOTICE file corresponding to the section 4d of The Apache License,
+// Version 2.0, in this case for
+// ------------------------------------------------------------------
+
+The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd
+unless otherwise noted.
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+
+hudi-flink1.15-bundle
+Copyright 2023 The Apache Software Foundation
+
+hudi-common
+Copyright 2023 The Apache Software Foundation
+
+Apache HttpClient Fluent API
+Copyright 2011-2015 The Apache Software Foundation
+
+Apache HttpClient
+Copyright 1999-2015 The Apache Software Foundation
+
+Objenesis
+Copyright 2006-2017 Joe Walnes, Henri Tremblay, Leonardo Mesquita
+
+Apache HBase - Client
+Copyright 2007-2020 The Apache Software Foundation
+
+Apache HBase Patched and Relocated (Shaded) Protobuf
+Copyright 2017-2020 The Apache Software Foundation
+
+---
+
+Apache HBase - Common
+Copyright 2007-2020 The Apache Software Foundation
+
+Apache HBase - Hadoop Compatibility
+Copyright 2007-2020 The Apache Software Foundation
+
+Apache HBase - Hadoop Two Compatibility
+Copyright 2007-2020 The Apache Software Foundation
+
+Apache HBase - Shaded Protocol
+Copyright 2007-2020 The Apache Software Foundation
+
+Apache HBase - Protocol
+Copyright 2007-2020 The Apache Software Foundation
+
+Apache Commons Collections
+Copyright 2001-2019 The Apache Software Foundation
+
+Apache HBase Relocated (Shaded) Third-party Miscellaneous Libs
+Copyright 2017-2020 The Apache Software Foundation
+
+Apache Commons CLI
+Copyright 2001-2017 The Apache Software Foundation
+
+Apache HBase Relocated (Shaded) Netty Libs
+Copyright 2017-2020 The Apache Software Foundation
+
+htrace-core4
+Copyright 2016 The Apache Software Foundation
+
+Apache HBase - Server
+Copyright 2007-2020 The Apache Software Foundation
+
+Apache HBase - Metrics API
+Copyright 2007-2020 The Apache Software Foundation
+
+Apache HBase - Metrics Implementation
+Copyright 2007-2020 The Apache Software Foundation
+
+hudi-client-common
+Copyright 2023 The Apache Software Foundation
+
+hudi-aws
+Copyright 2023 The Apache Software Foundation
+
+hudi-flink-client
+Copyright 2023 The Apache Software Foundation
+
+hudi-flink1.15.x
+Copyright 2023 The Apache Software Foundation
+
+hudi-flink
+Copyright 2023 The Apache Software Foundation
+
+hudi-sync-common
+Copyright 2023 The Apache Software Foundation
+
+hudi-hadoop-mr
+Copyright 2023 The Apache Software Foundation
+
+hudi-hive-sync
+Copyright 2023 The Apache Software Foundation
+
+hudi-timeline-service
+Copyright 2023 The Apache Software Foundation
+
+==============================================================
+ Jetty Web Container
+ Copyright 1995-2018 Mort Bay Consulting Pty Ltd.
+==============================================================
+
+Jetty is dual licensed under both
+
+ * The Apache 2.0 License
+ http://www.apache.org/licenses/LICENSE-2.0.html
+
+ and
+
+ * The Eclipse Public 1.0 License
+ http://www.eclipse.org/legal/epl-v10.html
+
+Jetty may be distributed under either license.
+
+------
+Eclipse
+
+The following artifacts are EPL.
+ * org.eclipse.jetty.orbit:org.eclipse.jdt.core
+
+The following artifacts are EPL and ASL2.
+ * org.eclipse.jetty.orbit:javax.security.auth.message
+
+The following artifacts are EPL and CDDL 1.0.
+ * org.eclipse.jetty.orbit:javax.mail.glassfish
+
+------
+Oracle
+
+The following artifacts are CDDL + GPLv2 with classpath exception.
+https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html
+
+ * javax.servlet:javax.servlet-api
+ * javax.annotation:javax.annotation-api
+ * javax.transaction:javax.transaction-api
+ * javax.websocket:javax.websocket-api
+
+------
+Oracle OpenJDK
+
+If ALPN is used to negotiate HTTP/2 connections, then the following
+artifacts may be included in the distribution or downloaded when ALPN
+module is selected.
+
+ * java.sun.security.ssl
+
+These artifacts replace/modify OpenJDK classes. The modififications
+are hosted at github and both modified and original are under GPL v2 with
+classpath exceptions.
+http://openjdk.java.net/legal/gplv2+ce.html
+
+------
+OW2
+
+The following artifacts are licensed by the OW2 Foundation according to the
+terms of http://asm.ow2.org/license.html
+
+org.ow2.asm:asm-commons
+org.ow2.asm:asm
+
+------
+Apache
+
+The following artifacts are ASL2 licensed.
+
+org.apache.taglibs:taglibs-standard-spec
+org.apache.taglibs:taglibs-standard-impl
+
+------
+MortBay
+
+The following artifacts are ASL2 licensed. Based on selected classes from
+following Apache Tomcat jars, all ASL2 licensed.
+
+org.mortbay.jasper:apache-jsp
+ org.apache.tomcat:tomcat-jasper
+ org.apache.tomcat:tomcat-juli
+ org.apache.tomcat:tomcat-jsp-api
+ org.apache.tomcat:tomcat-el-api
+ org.apache.tomcat:tomcat-jasper-el
+ org.apache.tomcat:tomcat-api
+ org.apache.tomcat:tomcat-util-scan
+ org.apache.tomcat:tomcat-util
+
+org.mortbay.jasper:apache-el
+ org.apache.tomcat:tomcat-jasper-el
+ org.apache.tomcat:tomcat-el-api
+
+------
+Mortbay
+
+The following artifacts are CDDL + GPLv2 with classpath exception.
+
+https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html
+
+org.eclipse.jetty.toolchain:jetty-schemas
+
+------
+Assorted
+
+The UnixCrypt.java code implements the one way cryptography used by
+Unix systems for simple password protection. Copyright 1996 Aki Yoshida,
+modified April 2001 by Iris Van den Broeke, Daniel Deville.
+Permission to use, copy, modify and distribute UnixCrypt
+for non-commercial or commercial purposes and without fee is
+granted provided that the copyright notice appears in all copies.
+
+Apache Flink
+Copyright 2006-2022 The Apache Software Foundation
+
+Flink : Connectors : Hadoop compatibility
+Copyright 2014-2022 The Apache Software Foundation
+
+Flink : Formats : Parquet
+Copyright 2014-2022 The Apache Software Foundation
+
+Flink : Formats : Json
+Copyright 2014-2022 The Apache Software Foundation
+
+Flink : Format : Common
+Copyright 2014-2022 The Apache Software Foundation
+
+Apache Parquet MR (Incubating)
+Copyright 2014-2015 The Apache Software Foundation
+
+--------------------------------------------------------------------------------
+
+This product includes code from Apache Avro, which includes the following in
+its NOTICE file:
+
+ Apache Avro
+ Copyright 2010-2015 The Apache Software Foundation
+
+ This product includes software developed at
+ The Apache Software Foundation (http://www.apache.org/).
+
+# Jackson JSON processor
+
+Jackson is a high-performance, Free/Open Source JSON processing library.
+It was originally written by Tatu Saloranta (tatu.salora...@iki.fi), and has
+been in development since 2007.
+It is currently developed by a community of developers, as well as supported
+commercially by FasterXML.com.
+
+## Licensing
+
+Jackson core and extension components may licensed under different licenses.
+To find the details that apply to this artifact see the accompanying LICENSE
file.
+For more information, including possible other licensing options, contact
+FasterXML.com (http://fasterxml.com).
+
+## Credits
+
+A list of contributors may be found from CREDITS file, which is included
+in some artifacts (usually source distributions); but is always available
+from the source code management (SCM) system project uses.
+
+Apache Avro
+Copyright 2009-2020 The Apache Software Foundation
+
+Apache Commons Codec
+Copyright 2002-2009 The Apache Software Foundation
+
+This product includes software developed by
+The Apache Software Foundation (http://www.apache.org/).
+
+--------------------------------------------------------------------------------
+src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java contains
+test data from http://aspell.sourceforge.net/test/batch0.tab.
+
+Copyright (C) 2002 Kevin Atkinson (kev...@gnu.org). Verbatim copying
+and distribution of this entire article is permitted in any medium,
+provided this notice is preserved.
+--------------------------------------------------------------------------------
+
+Apache Commons IO
+Copyright 2002-2012 The Apache Software Foundation
+
+Apache HttpCore
+Copyright 2005-2015 The Apache Software Foundation
+
+This project contains annotations derived from JCIP-ANNOTATIONS
+Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net
+
+=============================================================================
+= NOTICE file corresponding to section 4d of the Apache License Version 2.0 =
+=============================================================================
+This product includes software developed by
+Joda.org (http://www.joda.org/).
+
+Jackson core and extension components may be licensed under different licenses.
+To find the details that apply to this artifact see the accompanying LICENSE
file.
+For more information, including possible other licensing options, contact
+FasterXML.com (http://fasterxml.com).
+
