[
https://issues.apache.org/jira/browse/HUDI-3790?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Alexey Kudinkin reassigned HUDI-3790:
-------------------------------------
Assignee: Danny Chen
> Flink MT writer does not properly partition records
> ---------------------------------------------------
>
> Key: HUDI-3790
> URL: https://issues.apache.org/jira/browse/HUDI-3790
> Project: Apache Hudi
> Issue Type: Bug
> Reporter: Alexey Kudinkin
> Assignee: Danny Chen
> Priority: Blocker
>
> Enabling Column Stats partition in MT leads to Flink MT Writer failing, since
> it doesn't do records partitioning properly.
>
> {code:java}
> 2022-04-02T22:00:25.3014528Z [ERROR] Tests run: 5, Failures: 3, Errors: 0,
> Skipped: 0, Time elapsed: 1.713 s <<< FAILURE! - in
> org.apache.hudi.source.TestFileIndex
> 2022-04-02T22:00:25.3020916Z [ERROR]
> org.apache.hudi.source.TestFileIndex.testFileListingUsingMetadata(boolean)[1]
> Time elapsed: 0.658 s <<< FAILURE!
> 2022-04-02T22:00:25.3021360Z java.lang.AssertionError:
> 2022-04-02T22:00:25.3021686Z should have 4 partitions
> 2022-04-02T22:00:25.3021966Z Expected: is "par1,par2,par3,par4"
> 2022-04-02T22:00:25.3022219Z but: was ""
> 2022-04-02T22:00:25.3022550Z at
> org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:20)
> 2022-04-02T22:00:25.3023042Z at
> org.apache.hudi.source.TestFileIndex.testFileListingUsingMetadata(TestFileIndex.java:67)
> 2022-04-02T22:00:25.3023504Z at
> sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 2022-04-02T22:00:25.3023930Z at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> 2022-04-02T22:00:25.3024667Z at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 2022-04-02T22:00:25.3025129Z at
> java.lang.reflect.Method.invoke(Method.java:498)
> 2022-04-02T22:00:25.3025579Z at
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> 2022-04-02T22:00:25.3026118Z at
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> 2022-04-02T22:00:25.3026728Z at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> 2022-04-02T22:00:25.3027317Z at
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> 2022-04-02T22:00:25.3027878Z at
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestableMethod(TimeoutExtension.java:140)
> 2022-04-02T22:00:25.3028467Z at
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestTemplateMethod(TimeoutExtension.java:92)
> 2022-04-02T22:00:25.3029111Z at
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> 2022-04-02T22:00:25.3029736Z at
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> 2022-04-02T22:00:25.3030368Z at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> 2022-04-02T22:00:25.3031013Z at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> 2022-04-02T22:00:25.3031606Z at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> 2022-04-02T22:00:25.3032231Z at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> 2022-04-02T22:00:25.3032789Z at
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> 2022-04-02T22:00:25.3033300Z at
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> 2022-04-02T22:00:25.3033903Z at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeTestMethod$6(TestMethodTestDescriptor.java:212)
> 2022-04-02T22:00:25.3034620Z at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> 2022-04-02T22:00:25.3035308Z at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeTestMethod(TestMethodTestDescriptor.java:208)
> 2022-04-02T22:00:25.3035918Z at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:137)
> 2022-04-02T22:00:25.3036503Z at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:71)
> 2022-04-02T22:00:25.3037089Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
> 2022-04-02T22:00:25.3037691Z at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> 2022-04-02T22:00:25.3038292Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> 2022-04-02T22:00:25.3038825Z at
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> 2022-04-02T22:00:25.3039378Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> 2022-04-02T22:00:25.3039978Z at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> 2022-04-02T22:00:25.3040559Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> 2022-04-02T22:00:25.3041106Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> 2022-04-02T22:00:25.3041750Z at
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
> 2022-04-02T22:00:25.3042501Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask$DefaultDynamicTestExecutor.execute(NodeTestTask.java:212)
> 2022-04-02T22:00:25.3043134Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask$DefaultDynamicTestExecutor.execute(NodeTestTask.java:192)
> 2022-04-02T22:00:25.3043770Z at
> org.junit.jupiter.engine.descriptor.TestTemplateTestDescriptor.execute(TestTemplateTestDescriptor.java:139)
> 2022-04-02T22:00:25.3044390Z at
> org.junit.jupiter.engine.descriptor.TestTemplateTestDescriptor.lambda$execute$2(TestTemplateTestDescriptor.java:107)
> 2022-04-02T22:00:25.3044942Z at
> java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
> 2022-04-02T22:00:25.3045416Z at
> java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> 2022-04-02T22:00:25.3045891Z at
> java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:175)
> 2022-04-02T22:00:25.3046361Z at
> java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> 2022-04-02T22:00:25.3046837Z at
> java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
> 2022-04-02T22:00:25.3047317Z at
> java.util.stream.ReferencePipeline$11$1.accept(ReferencePipeline.java:440)
> 2022-04-02T22:00:25.3047785Z at
> java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> 2022-04-02T22:00:25.3048255Z at
> java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> 2022-04-02T22:00:25.3048727Z at
> java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> 2022-04-02T22:00:25.3049182Z at
> java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
> 2022-04-02T22:00:25.3049652Z at
> java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> 2022-04-02T22:00:25.3050131Z at
> java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948)
> 2022-04-02T22:00:25.3050598Z at
> java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
> 2022-04-02T22:00:25.3051066Z at
> java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
> 2022-04-02T22:00:25.3051557Z at
> java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
> 2022-04-02T22:00:25.3052056Z at
> java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
> 2022-04-02T22:00:25.3052597Z at
> java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
> 2022-04-02T22:00:25.3053056Z at
> java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:485)
> 2022-04-02T22:00:25.3053516Z at
> java.util.stream.ReferencePipeline$7$1.accept(ReferencePipeline.java:272)
> 2022-04-02T22:00:25.3053988Z at
> java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> 2022-04-02T22:00:25.3054463Z at
> java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> 2022-04-02T22:00:25.3054919Z at
> java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> 2022-04-02T22:00:25.3055405Z at
> java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1384)
> 2022-04-02T22:00:25.3055877Z at
> java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
> 2022-04-02T22:00:25.3056337Z at
> java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
> 2022-04-02T22:00:25.3056831Z at
> java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
> 2022-04-02T22:00:25.3057337Z at
> java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
> 2022-04-02T22:00:25.3057812Z at
> java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
> 2022-04-02T22:00:25.3058272Z at
> java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:485)
> 2022-04-02T22:00:25.3058833Z at
> java.util.stream.ReferencePipeline$7$1.accept(ReferencePipeline.java:272)
> 2022-04-02T22:00:25.3059313Z at
> java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1384)
> 2022-04-02T22:00:25.3059786Z at
> java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
> 2022-04-02T22:00:25.3060259Z at
> java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
> 2022-04-02T22:00:25.3060801Z at
> java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
> 2022-04-02T22:00:25.3061308Z at
> java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
> 2022-04-02T22:00:25.3061796Z at
> java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
> 2022-04-02T22:00:25.3062253Z at
> java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:485)
> 2022-04-02T22:00:25.3062779Z at
> org.junit.jupiter.engine.descriptor.TestTemplateTestDescriptor.execute(TestTemplateTestDescriptor.java:107)
> 2022-04-02T22:00:25.3063374Z at
> org.junit.jupiter.engine.descriptor.TestTemplateTestDescriptor.execute(TestTemplateTestDescriptor.java:42)
> 2022-04-02T22:00:25.3063979Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
> 2022-04-02T22:00:25.3064565Z at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> 2022-04-02T22:00:25.3065165Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> 2022-04-02T22:00:25.3065702Z at
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> 2022-04-02T22:00:25.3066238Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> 2022-04-02T22:00:25.3066830Z at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> 2022-04-02T22:00:25.3067407Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> 2022-04-02T22:00:25.3069894Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> 2022-04-02T22:00:25.3070360Z at
> java.util.ArrayList.forEach(ArrayList.java:1259)
> 2022-04-02T22:00:25.3073116Z at
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> 2022-04-02T22:00:25.3073903Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> 2022-04-02T22:00:25.3074514Z at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> 2022-04-02T22:00:25.3075247Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> 2022-04-02T22:00:25.3075787Z at
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> 2022-04-02T22:00:25.3076330Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> 2022-04-02T22:00:25.3076911Z at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> 2022-04-02T22:00:25.3077492Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> 2022-04-02T22:00:25.3078045Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> 2022-04-02T22:00:25.3078494Z at
> java.util.ArrayList.forEach(ArrayList.java:1259)
> 2022-04-02T22:00:25.3079071Z at
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> 2022-04-02T22:00:25.3079768Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> 2022-04-02T22:00:25.3080358Z at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> 2022-04-02T22:00:25.3080952Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> 2022-04-02T22:00:25.3081495Z at
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> 2022-04-02T22:00:25.3082033Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> 2022-04-02T22:00:25.3082683Z at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> 2022-04-02T22:00:25.3083266Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> 2022-04-02T22:00:25.3083821Z at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> 2022-04-02T22:00:25.3084448Z at
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
> 2022-04-02T22:00:25.3085124Z at
> org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
> 2022-04-02T22:00:25.3085728Z at
> org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
> 2022-04-02T22:00:25.3086319Z at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:107)
> 2022-04-02T22:00:25.3086907Z at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:87)
> 2022-04-02T22:00:25.3087541Z at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:53)
> 2022-04-02T22:00:25.3088171Z at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:66)
> 2022-04-02T22:00:25.3088788Z at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:51)
> 2022-04-02T22:00:25.3089344Z at
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:87)
> 2022-04-02T22:00:25.3089848Z at
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:66)
> 2022-04-02T22:00:25.3090389Z at
> org.apache.maven.surefire.junitplatform.JUnitPlatformProvider.invokeAllTests(JUnitPlatformProvider.java:154)
> 2022-04-02T22:00:25.3090977Z at
> org.apache.maven.surefire.junitplatform.JUnitPlatformProvider.invoke(JUnitPlatformProvider.java:127)
> 2022-04-02T22:00:25.3091533Z at
> org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:377)
> 2022-04-02T22:00:25.3092029Z at
> org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:138)
> 2022-04-02T22:00:25.3092568Z at
> org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:465)
> 2022-04-02T22:00:25.3093033Z at
> org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:451)
> {code}
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
