kgeisz opened a new pull request, #8503: URL: https://github.com/apache/hbase/pull/8503
https://issues.apache.org/jira/browse/HBASE-30301 ## AI Usage The TestReadOnlyControllerFlush.java test class was generated using Claude Opus 4.6 ## Summary This pull request fixes an issue where running `flush <table>` on a read-only cluster in a Read-Replica setup results in the `flush` command hanging indefinitely and the master log getting spammed with `WriteAttemptedOnReadOnlyClusterException`. This fix is done by overriding `preTableFlush` in `MasterReadOnlyController` and adding the `internalReadOnlyGuard()` (similar to other overrides in this class). Since the read-only guard is being used, no error is written to the master's log when the user tries to flush a table on a replica cluster. They will only see the following error in the HBase shell: ``` hbase:010:0> flush 't1' 2026-07-24 17:28:05,673 INFO client.RawAsyncHBaseAdmin: Unrecoverable error in master side. Fallback to FlushTableProcedure V1 org.apache.hadoop.hbase.WriteAttemptedOnReadOnlyClusterException: org.apache.hadoop.hbase.WriteAttemptedOnReadOnlyClusterException: Operation not allowed in Read-Only Mode at org.apache.hadoop.hbase.security.access.AbstractReadOnlyController.internalReadOnlyGuard(AbstractReadOnlyController.java:60) at org.apache.hadoop.hbase.security.access.MasterReadOnlyController.preTableFlush(MasterReadOnlyController.java:268) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$98.call(MasterCoprocessorHost.java:1271) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$98.call(MasterCoprocessorHost.java:1268) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:592) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:676) at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preTableFlush(MasterCoprocessorHost.java:1268) at org.apache.hadoop.hbase.master.HMaster$23.run(HMaster.java:4687) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.flushTable(HMaster.java:4684) at org.apache.hadoop.hbase.master.MasterRpcServices.flushTable(MasterRpcServices.java:3710) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:461) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:500) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:481) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:164) at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:87) at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:475) at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:402) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:435) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:482) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.WriteAttemptedOnReadOnlyClusterException): org.apache.hadoop.hbase.WriteAttemptedOnReadOnlyClusterException: Operation not allowed in Read-Only Mode at org.apache.hadoop.hbase.security.access.AbstractReadOnlyController.internalReadOnlyGuard(AbstractReadOnlyController.java:60) at org.apache.hadoop.hbase.security.access.MasterReadOnlyController.preTableFlush(MasterReadOnlyController.java:268) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$98.call(MasterCoprocessorHost.java:1271) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$98.call(MasterCoprocessorHost.java:1268) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:592) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:676) at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preTableFlush(MasterCoprocessorHost.java:1268) at org.apache.hadoop.hbase.master.HMaster$23.run(HMaster.java:4687) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.flushTable(HMaster.java:4684) at org.apache.hadoop.hbase.master.MasterRpcServices.flushTable(MasterRpcServices.java:3710) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:461) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ... 32 more 2026-07-24 17:28:05,679 INFO client.RawAsyncHBaseAdmin: Operation: FLUSH, Table Name: default:t1 failed with org.apache.hadoop.hbase.WriteAttemptedOnReadOnlyClusterException: Operation not allowed in Read-Only Mode at org.apache.hadoop.hbase.security.access.AbstractReadOnlyController.internalReadOnlyGuard(AbstractReadOnlyController.java:60) at org.apache.hadoop.hbase.security.access.MasterReadOnlyController.preTableFlush(MasterReadOnlyController.java:268) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$98.call(MasterCoprocessorHost.java:1271) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$98.call(MasterCoprocessorHost.java:1268) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:592) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:676) at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preTableFlush(MasterCoprocessorHost.java:1268) at org.apache.hadoop.hbase.master.HMaster$23.run(HMaster.java:4687) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.flushTable(HMaster.java:4684) at org.apache.hadoop.hbase.master.MasterRpcServices.flushTable(MasterRpcServices.java:3710) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:461) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ERROR: org.apache.hadoop.hbase.WriteAttemptedOnReadOnlyClusterException: Operation not allowed in Read-Only Mode at org.apache.hadoop.hbase.security.access.AbstractReadOnlyController.internalReadOnlyGuard(AbstractReadOnlyController.java:60) at org.apache.hadoop.hbase.security.access.MasterReadOnlyController.preTableFlush(MasterReadOnlyController.java:268) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$98.call(MasterCoprocessorHost.java:1271) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$98.call(MasterCoprocessorHost.java:1268) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:592) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:676) at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preTableFlush(MasterCoprocessorHost.java:1268) at org.apache.hadoop.hbase.procedure.flush.MasterFlushTableProcedureManager.execProcedure(MasterFlushTableProcedureManager.java:132) at org.apache.hadoop.hbase.master.MasterRpcServices.execProcedure(MasterRpcServices.java:1043) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:461) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) For usage try 'help "flush"' Took 0.0622 seconds ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
