b7wch opened a new issue, #12827:
URL: https://github.com/apache/hudi/issues/12827
**config-sink.json**
```
{
"name": "hudi-sink",
"config": {
"bootstrap.servers": "kafkabroker:9092",
"connector.class":
"org.apache.hudi.connect.HoodieSinkConnector",
"tasks.max": "4",
"key.converter":
"org.apache.kafka.connect.storage.StringConverter",
"value.converter":
"org.apache.kafka.connect.storage.StringConverter",
"value.converter.schemas.enable": "false",
"sasl.mechanism": "PLAIN",
"topics": "hudi-test-topic",
"hoodie.table.name": "hudi-test-topic",
"hoodie.table.type": "MERGE_ON_READ",
"hoodie.base.path": "s3a://test/hudi-test-topic",
"store.url": "http://xxx:9000";,
"com.amazonaws.services.s3.enableV4": "true",
"fs.s3a.impl": "org.apache.hadoop.fs.s3a.S3AFileSystem",
"fs.AbstractFileSystem.s3a.impl":
"org.apache.hadoop.fs.s3a.S3A",
"fs.defaultFS": "s3a://test/hudi-test-topic",
"fs.s3a.endpoint": "http://xxx:9000";,
"fs.s3a.access.key":"xxx",
"fs.s3a.secret.key":"xxx",
"fs.s3a.path.style.access": "true",
"fs.s3a.signing-algorithm": "S3SignerType",
"fs.s3a.connection.ssl.enabled": "false",
"fs.s3a.aws.credentials.provider":
"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider",
"fs.s3a.endpoint.region": "us-east-1",
"s3.bucket.name": "test",
"aws.access.key.id": "xxx",
"aws.secret.access.key": "xxx",
"hoodie.datasource.write.recordkey.field": "volume",
"hoodie.datasource.write.partitionpath.field": "date",
"hoodie.schemaprovider.class":
"org.apache.hudi.schema.SchemaRegistryProvider",
"hoodie.streamer.schemaprovider.registry.url":
"http://localhost:8082/subjects/hudi-test-topic/versions/latest";,
"hoodie.deltastreamer.schemaprovider.registry.url":
"http://localhost:8082/subjects/hudi-test-topic/versions/latest";,
"hoodie.kafka.commit.interval.secs": 60
}
}
```
**Error LOG**
```
[2025-02-11 09:16:25,814] ERROR [hudi-sink|task-1] Fatal error initializing
task null for partition 1 (org.apache.hudi.connect.HoodieSinkTask:202)
org.apache.hudi.exception.HoodieException: Fatal error instantiating Hudi
Write Provider
at
org.apache.hudi.connect.writers.KafkaConnectWriterProvider.<init>(KafkaConnectWriterProvider.java:105)
at
org.apache.hudi.connect.transaction.ConnectTransactionParticipant.<init>(ConnectTransactionParticipant.java:65)
at
org.apache.hudi.connect.HoodieSinkTask.bootstrap(HoodieSinkTask.java:198)
at org.apache.hudi.connect.HoodieSinkTask.open(HoodieSinkTask.java:151)
at
org.apache.kafka.connect.runtime.WorkerSinkTask.openPartitions(WorkerSinkTask.java:644)
at
org.apache.kafka.connect.runtime.WorkerSinkTask.access$1200(WorkerSinkTask.java:73)
at
org.apache.kafka.connect.runtime.WorkerSinkTask$HandleRebalance.onPartitionsAssigned(WorkerSinkTask.java:741)
at
org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.invokePartitionsAssigned(ConsumerCoordinator.java:322)
at
org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinComplete(ConsumerCoordinator.java:471)
at
org.apache.kafka.clients.consumer.internals.AbstractCoordinator.joinGroupIfNeeded(AbstractCoordinator.java:474)
at
org.apache.kafka.clients.consumer.internals.AbstractCoordinator.ensureActiveGroup(AbstractCoordinator.java:385)
at
org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.poll(ConsumerCoordinator.java:557)
at
org.apache.kafka.clients.consumer.KafkaConsumer.updateAssignmentMetadataIfNeeded(KafkaConsumer.java:1272)
at
org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1236)
at
org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1216)
at
org.apache.kafka.connect.runtime.WorkerSinkTask.pollConsumer(WorkerSinkTask.java:479)
at
org.apache.kafka.connect.runtime.WorkerSinkTask.poll(WorkerSinkTask.java:331)
at
org.apache.kafka.connect.runtime.WorkerSinkTask.iteration(WorkerSinkTask.java:237)
at
org.apache.kafka.connect.runtime.WorkerSinkTask.execute(WorkerSinkTask.java:206)
at
org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:202)
at org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:257)
at
org.apache.kafka.connect.runtime.isolation.Plugins.lambda$withClassLoader$1(Plugins.java:181)
at
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
Caused by: org.apache.hudi.exception.HoodieException: Unable to create
org.apache.hudi.storage.hadoop.HoodieHadoopStorage
at
org.apache.hudi.storage.HoodieStorageUtils.getStorage(HoodieStorageUtils.java:44)
at
org.apache.hudi.storage.HoodieStorageUtils.getStorage(HoodieStorageUtils.java:34)
at
org.apache.hudi.client.BaseHoodieClient.<init>(BaseHoodieClient.java:93)
at
org.apache.hudi.client.BaseHoodieWriteClient.<init>(BaseHoodieWriteClient.java:172)
at
org.apache.hudi.client.BaseHoodieWriteClient.<init>(BaseHoodieWriteClient.java:157)
at
org.apache.hudi.client.HoodieJavaWriteClient.<init>(HoodieJavaWriteClient.java:56)
at
org.apache.hudi.connect.writers.KafkaConnectWriterProvider.<init>(KafkaConnectWriterProvider.java:103)
... 26 more
Caused by: org.apache.hudi.exception.HoodieException: Unable to instantiate
class org.apache.hudi.storage.hadoop.HoodieHadoopStorage
at
org.apache.hudi.common.util.ReflectionUtils.loadClass(ReflectionUtils.java:75)
at
org.apache.hudi.storage.HoodieStorageUtils.getStorage(HoodieStorageUtils.java:41)
... 32 more
Caused by: java.lang.reflect.InvocationTargetException
at
java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native
Method)
at
java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at
java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:500)
at
java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:481)
at
org.apache.hudi.common.util.ReflectionUtils.loadClass(ReflectionUtils.java:73)
... 33 more
Caused by: org.apache.hudi.exception.HoodieIOException: Failed to get
instance of org.apache.hadoop.fs.FileSystem
at org.apache.hudi.hadoop.fs.HadoopFSUtils.getFs(HadoopFSUtils.java:128)
at org.apache.hudi.hadoop.fs.HadoopFSUtils.getFs(HadoopFSUtils.java:119)
at
org.apache.hudi.storage.hadoop.HoodieHadoopStorage.<init>(HoodieHadoopStorage.java:64)
... 39 more
Caused by: org.apache.hadoop.security.KerberosAuthException: failure to
login: javax.security.auth.login.LoginException:
java.lang.NullPointerException: invalid null input: name
at
jdk.security.auth/com.sun.security.auth.UnixPrincipal.<init>(UnixPrincipal.java:71)
at
jdk.security.auth/com.sun.security.auth.module.UnixLoginModule.login(UnixLoginModule.java:134)
at
java.base/javax.security.auth.login.LoginContext.invoke(LoginContext.java:755)
at
java.base/javax.security.auth.login.LoginContext$4.run(LoginContext.java:679)
at
java.base/javax.security.auth.login.LoginContext$4.run(LoginContext.java:677)
at
java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
at
java.base/javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:677)
at
java.base/javax.security.auth.login.LoginContext.login(LoginContext.java:587)
at
org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:871)
at
org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:833)
at
org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:702)
at
org.apache.hadoop.fs.FileSystem$Cache$Key.<init>(FileSystem.java:3445)
at
org.apache.hadoop.fs.FileSystem$Cache$Key.<init>(FileSystem.java:3435)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3277)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:475)
at org.apache.hadoop.fs.Path.getFileSystem(Path.java:356)
at org.apache.hudi.hadoop.fs.HadoopFSUtils.getFs(HadoopFSUtils.java:126)
at org.apache.hudi.hadoop.fs.HadoopFSUtils.getFs(HadoopFSUtils.java:119)
at
org.apache.hudi.storage.hadoop.HoodieHadoopStorage.<init>(HoodieHadoopStorage.java:64)
at
java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native
Method)
at
java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at
java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:500)
at
java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:481)
at
org.apache.hudi.common.util.ReflectionUtils.loadClass(ReflectionUtils.java:73)
at
org.apache.hudi.storage.HoodieStorageUtils.getStorage(HoodieStorageUtils.java:41)
at
org.apache.hudi.storage.HoodieStorageUtils.getStorage(HoodieStorageUtils.java:34)
at
org.apache.hudi.client.BaseHoodieClient.<init>(BaseHoodieClient.java:93)
at
org.apache.hudi.client.BaseHoodieWriteClient.<init>(BaseHoodieWriteClient.java:172)
at
org.apache.hudi.client.BaseHoodieWriteClient.<init>(BaseHoodieWriteClient.java:157)
at
org.apache.hudi.client.HoodieJavaWriteClient.<init>(HoodieJavaWriteClient.java:56)
at
org.apache.hudi.connect.writers.KafkaConnectWriterProvider.<init>(KafkaConnectWriterProvider.java:103)
at
org.apache.hudi.connect.transaction.ConnectTransactionParticipant.<init>(ConnectTransactionParticipant.java:65)
at
org.apache.hudi.connect.HoodieSinkTask.bootstrap(HoodieSinkTask.java:198)
at org.apache.hudi.connect.HoodieSinkTask.open(HoodieSinkTask.java:151)
at
org.apache.kafka.connect.runtime.WorkerSinkTask.openPartitions(WorkerSinkTask.java:644)
at
org.apache.kafka.connect.runtime.WorkerSinkTask.access$1200(WorkerSinkTask.java:73)
at
org.apache.kafka.connect.runtime.WorkerSinkTask$HandleRebalance.onPartitionsAssigned(WorkerSinkTask.java:741)
at
org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.invokePartitionsAssigned(ConsumerCoordinator.java:322)
at
org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinComplete(ConsumerCoordinator.java:471)
at
org.apache.kafka.clients.consumer.internals.AbstractCoordinator.joinGroupIfNeeded(AbstractCoordinator.java:474)
at
org.apache.kafka.clients.consumer.internals.AbstractCoordinator.ensureActiveGroup(AbstractCoordinator.java:385)
at
org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.poll(ConsumerCoordinator.java:557)
at
org.apache.kafka.clients.consumer.KafkaConsumer.updateAssignmentMetadataIfNeeded(KafkaConsumer.java:1272)
at
org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1236)
at
org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1216)
at
org.apache.kafka.connect.runtime.WorkerSinkTask.pollConsumer(WorkerSinkTask.java:479)
at
org.apache.kafka.connect.runtime.WorkerSinkTask.poll(WorkerSinkTask.java:331)
at
org.apache.kafka.connect.runtime.WorkerSinkTask.iteration(WorkerSinkTask.java:237)
at
org.apache.kafka.connect.runtime.WorkerSinkTask.execute(WorkerSinkTask.java:206)
at
org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:202)
at org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:257)
at
org.apache.kafka.connect.runtime.isolation.Plugins.lambda$withClassLoader$1(Plugins.java:181)
at
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
at
org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:928)
at
org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:833)
at
org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:702)
at
org.apache.hadoop.fs.FileSystem$Cache$Key.<init>(FileSystem.java:3445)
at
org.apache.hadoop.fs.FileSystem$Cache$Key.<init>(FileSystem.java:3435)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3277)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:475)
at org.apache.hadoop.fs.Path.getFileSystem(Path.java:356)
at org.apache.hudi.hadoop.fs.HadoopFSUtils.getFs(HadoopFSUtils.java:126)
... 41 more
Caused by: javax.security.auth.login.LoginException:
java.lang.NullPointerException: invalid null input: name
at
jdk.security.auth/com.sun.security.auth.UnixPrincipal.<init>(UnixPrincipal.java:71)
at
jdk.security.auth/com.sun.security.auth.module.UnixLoginModule.login(UnixLoginModule.java:134)
at
java.base/javax.security.auth.login.LoginContext.invoke(LoginContext.java:755)
at
java.base/javax.security.auth.login.LoginContext$4.run(LoginContext.java:679)
at
java.base/javax.security.auth.login.LoginContext$4.run(LoginContext.java:677)
at
java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
at
java.base/javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:677)
at
java.base/javax.security.auth.login.LoginContext.login(LoginContext.java:587)
at
org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:871)
at
org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:833)
at
org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:702)
at
org.apache.hadoop.fs.FileSystem$Cache$Key.<init>(FileSystem.java:3445)
at
org.apache.hadoop.fs.FileSystem$Cache$Key.<init>(FileSystem.java:3435)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3277)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:475)
at org.apache.hadoop.fs.Path.getFileSystem(Path.java:356)
at org.apache.hudi.hadoop.fs.HadoopFSUtils.getFs(HadoopFSUtils.java:126)
at org.apache.hudi.hadoop.fs.HadoopFSUtils.getFs(HadoopFSUtils.java:119)
at
org.apache.hudi.storage.hadoop.HoodieHadoopStorage.<init>(HoodieHadoopStorage.java:64)
at
java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native
Method)
at
java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at
java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:500)
at
java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:481)
at
org.apache.hudi.common.util.ReflectionUtils.loadClass(ReflectionUtils.java:73)
at
org.apache.hudi.storage.HoodieStorageUtils.getStorage(HoodieStorageUtils.java:41)
at
org.apache.hudi.storage.HoodieStorageUtils.getStorage(HoodieStorageUtils.java:34)
at
org.apache.hudi.client.BaseHoodieClient.<init>(BaseHoodieClient.java:93)
at
org.apache.hudi.client.BaseHoodieWriteClient.<init>(BaseHoodieWriteClient.java:172)
at
org.apache.hudi.client.BaseHoodieWriteClient.<init>(BaseHoodieWriteClient.java:157)
at
org.apache.hudi.client.HoodieJavaWriteClient.<init>(HoodieJavaWriteClient.java:56)
at
org.apache.hudi.connect.writers.KafkaConnectWriterProvider.<init>(KafkaConnectWriterProvider.java:103)
at
org.apache.hudi.connect.transaction.ConnectTransactionParticipant.<init>(ConnectTransactionParticipant.java:65)
at
org.apache.hudi.connect.HoodieSinkTask.bootstrap(HoodieSinkTask.java:198)
at org.apache.hudi.connect.HoodieSinkTask.open(HoodieSinkTask.java:151)
at
org.apache.kafka.connect.runtime.WorkerSinkTask.openPartitions(WorkerSinkTask.java:644)
at
org.apache.kafka.connect.runtime.WorkerSinkTask.access$1200(WorkerSinkTask.java:73)
at
org.apache.kafka.connect.runtime.WorkerSinkTask$HandleRebalance.onPartitionsAssigned(WorkerSinkTask.java:741)
at
org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.invokePartitionsAssigned(ConsumerCoordinator.java:322)
at
org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinComplete(ConsumerCoordinator.java:471)
at
org.apache.kafka.clients.consumer.internals.AbstractCoordinator.joinGroupIfNeeded(AbstractCoordinator.java:474)
at
org.apache.kafka.clients.consumer.internals.AbstractCoordinator.ensureActiveGroup(AbstractCoordinator.java:385)
at
org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.poll(ConsumerCoordinator.java:557)
at
org.apache.kafka.clients.consumer.KafkaConsumer.updateAssignmentMetadataIfNeeded(KafkaConsumer.java:1272)
at
org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1236)
at
org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1216)
at
org.apache.kafka.connect.runtime.WorkerSinkTask.pollConsumer(WorkerSinkTask.java:479)
at
org.apache.kafka.connect.runtime.WorkerSinkTask.poll(WorkerSinkTask.java:331)
at
org.apache.kafka.connect.runtime.WorkerSinkTask.iteration(WorkerSinkTask.java:237)
at
org.apache.kafka.connect.runtime.WorkerSinkTask.execute(WorkerSinkTask.java:206)
at
org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:202)
at org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:257)
at
org.apache.kafka.connect.runtime.isolation.Plugins.lambda$withClassLoader$1(Plugins.java:181)
at
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
at
java.base/javax.security.auth.login.LoginContext.invoke(LoginContext.java:850)
at
java.base/javax.security.auth.login.LoginContext$4.run(LoginContext.java:679)
at
java.base/javax.security.auth.login.LoginContext$4.run(LoginContext.java:677)
at
java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
at
java.base/javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:677)
at
java.base/javax.security.auth.login.LoginContext.login(LoginContext.java:587)
at
org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:871)
... 49 more
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
