[
https://issues.apache.org/jira/browse/FLINK-19447?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17203995#comment-17203995
]
Dian Fu edited comment on FLINK-19447 at 9/29/20, 2:48 PM:
-----------------------------------------------------------
[~mgergely] I found the following exception in the log file. Could you take a
look at if it would help?
{code}
21:49:04,691 [RS_CLOSE_META-regionserver/71af2d647bb3:0-0] WARN
org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler [] - Fatal
error occurred while opening region hbase:meta,,1.1588230740
, aborting...
java.lang.NoClassDefFoundError: Could not initialize class
org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper
at
org.apache.hadoop.hbase.io.asyncfs.AsyncFSOutputHelper.createOutput(AsyncFSOutputHelper.java:51)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.initOutput(AsyncProtobufLogWriter.java:169)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:166)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.wal.AsyncFSWALProvider.createAsyncWriter(AsyncFSWALProvider.java:113)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.createWriterInstance(AsyncFSWAL.java:643)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.createWriterInstance(AsyncFSWAL.java:126)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:767)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:501)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.init(AbstractFSWAL.java:442)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWAL(AbstractFSWALProvider.java:156)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWAL(AbstractFSWALProvider.java:61)
~[hbase-server-2.2.3.jar:2.2.3]
at org.apache.hadoop.hbase.wal.WALFactory.getWAL(WALFactory.java:284)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.regionserver.HRegionServer.getWAL(HRegionServer.java:2181)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:133)
[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104)
[hbase-server-2.2.3.jar:2.2.3]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[?:1.8.0_242]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[?:1.8.0_242]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_242]
21:49:04,692 [RS_CLOSE_META-regionserver/71af2d647bb3:0-0] ERROR
org.apache.hadoop.hbase.regionserver.HRegionServer [] - *****
ABORTING region server 71af2d647bb3,35715,1601329739543: Failed to open region
hbase:meta,,1.1588230740 and can not recover *****
java.lang.NoClassDefFoundError: Could not initialize class
org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper
at
org.apache.hadoop.hbase.io.asyncfs.AsyncFSOutputHelper.createOutput(AsyncFSOutputHelper.java:51)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.initOutput(AsyncProtobufLogWriter.java:169)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:166)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.wal.AsyncFSWALProvider.createAsyncWriter(AsyncFSWALProvider.java:113)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.createWriterInstance(AsyncFSWAL.java:643)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.createWriterInstance(AsyncFSWAL.java:126)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:767)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:501)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.init(AbstractFSWAL.java:442)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWAL(AbstractFSWALProvider.java:156)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWAL(AbstractFSWALProvider.java:61)
~[hbase-server-2.2.3.jar:2.2.3]
at org.apache.hadoop.hbase.wal.WALFactory.getWAL(WALFactory.java:284)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.regionserver.HRegionServer.getWAL(HRegionServer.java:2181)
~[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:133)
[hbase-server-2.2.3.jar:2.2.3]
at
org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104)
[hbase-server-2.2.3.jar:2.2.3]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[?:1.8.0_242]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[?:1.8.0_242]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_242]
{code}
was (Author: dian.fu):
[~mgergely] I found the following exception in the log file. Could you take a
look at if it would help?
{code}
Cause:
java.lang.NoClassDefFoundError: Could not initialize class
org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper
at
org.apache.hadoop.hbase.io.asyncfs.AsyncFSOutputHelper.createOutput(AsyncFSOutputHelper.java:51)
at
org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.initOutput(AsyncProtobufLogWriter.java:169)
at
org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:166)
at
org.apache.hadoop.hbase.wal.AsyncFSWALProvider.createAsyncWriter(AsyncFSWALProvider.java:113)
at
org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.createWriterInstance(AsyncFSWAL.java:643)
at
org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.createWriterInstance(AsyncFSWAL.java:126)
at
org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:767)
at
org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:501)
at
org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.init(AbstractFSWAL.java:442)
at
org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWAL(AbstractFSWALProvider.java:156)
at
org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWAL(AbstractFSWALProvider.java:61)
at org.apache.hadoop.hbase.wal.WALFactory.getWAL(WALFactory.java:284)
at
org.apache.hadoop.hbase.regionserver.HRegionServer.getWAL(HRegionServer.java:2181)
at
org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:133)
at
org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
{code}
> HBaseConnectorITCase.HBaseTestingClusterAutoStarter failed with "Master not
> initialized after 200000ms"
> -------------------------------------------------------------------------------------------------------
>
> Key: FLINK-19447
> URL: https://issues.apache.org/jira/browse/FLINK-19447
> Project: Flink
> Issue Type: Bug
> Components: Connectors / HBase, Tests
> Affects Versions: 1.12.0
> Reporter: Dian Fu
> Priority: Major
> Labels: test-stability
>
> https://dev.azure.com/apache-flink/apache-flink/_build/results?buildId=7042&view=logs&j=961f8f81-6b52-53df-09f6-7291a2e4af6a&t=60581941-0138-53c0-39fe-86d62be5f407
> {code}
> 2020-09-28T21:52:21.2146147Z
> org.apache.flink.connector.hbase2.HBaseConnectorITCase Time elapsed: 208.382
> sec <<< ERROR!
> 2020-09-28T21:52:21.2146638Z java.io.IOException: Shutting down
> 2020-09-28T21:52:21.2147004Z at
> org.apache.hadoop.hbase.MiniHBaseCluster.init(MiniHBaseCluster.java:266)
> 2020-09-28T21:52:21.2147637Z at
> org.apache.hadoop.hbase.MiniHBaseCluster.<init>(MiniHBaseCluster.java:116)
> 2020-09-28T21:52:21.2148120Z at
> org.apache.hadoop.hbase.HBaseTestingUtility.startMiniHBaseCluster(HBaseTestingUtility.java:1142)
> 2020-09-28T21:52:21.2148831Z at
> org.apache.hadoop.hbase.HBaseTestingUtility.startMiniCluster(HBaseTestingUtility.java:1107)
> 2020-09-28T21:52:21.2149347Z at
> org.apache.hadoop.hbase.HBaseTestingUtility.startMiniCluster(HBaseTestingUtility.java:1061)
> 2020-09-28T21:52:21.2149896Z at
> org.apache.flink.connector.hbase2.util.HBaseTestingClusterAutoStarter.setUp(HBaseTestingClusterAutoStarter.java:122)
> 2020-09-28T21:52:21.2150721Z at
> sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 2020-09-28T21:52:21.2151136Z at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> 2020-09-28T21:52:21.2151609Z at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 2020-09-28T21:52:21.2152039Z at
> java.lang.reflect.Method.invoke(Method.java:498)
> 2020-09-28T21:52:21.2152462Z at
> org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
> 2020-09-28T21:52:21.2152941Z at
> org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
> 2020-09-28T21:52:21.2153489Z at
> org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
> 2020-09-28T21:52:21.2153962Z at
> org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24)
> 2020-09-28T21:52:21.2154406Z at
> org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
> 2020-09-28T21:52:21.2154828Z at
> org.junit.runners.ParentRunner.run(ParentRunner.java:363)
> 2020-09-28T21:52:21.2155381Z at
> org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:367)
> 2020-09-28T21:52:21.2155864Z at
> org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:274)
> 2020-09-28T21:52:21.2156378Z at
> org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238)
> 2020-09-28T21:52:21.2156865Z at
> org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:161)
> 2020-09-28T21:52:21.2157458Z at
> org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:290)
> 2020-09-28T21:52:21.2157993Z at
> org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:242)
> 2020-09-28T21:52:21.2158470Z at
> org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:121)
> 2020-09-28T21:52:21.2158890Z Caused by: java.lang.RuntimeException: Master
> not initialized after 200000ms
> 2020-09-28T21:52:21.2159350Z at
> org.apache.hadoop.hbase.util.JVMClusterUtil.waitForEvent(JVMClusterUtil.java:229)
> 2020-09-28T21:52:21.2159823Z at
> org.apache.hadoop.hbase.util.JVMClusterUtil.startup(JVMClusterUtil.java:197)
> 2020-09-28T21:52:21.2160270Z at
> org.apache.hadoop.hbase.LocalHBaseCluster.startup(LocalHBaseCluster.java:413)
> 2020-09-28T21:52:21.2160800Z at
> org.apache.hadoop.hbase.MiniHBaseCluster.init(MiniHBaseCluster.java:259)
> 2020-09-28T21:52:21.2161096Z ... 22 more
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
