[
https://issues.apache.org/jira/browse/HIVE-29436?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
ASF GitHub Bot updated HIVE-29436:
----------------------------------
Labels: pull-request-available (was: )
> Iceberg materialized view rebuild locking issue
> ------------------------------------------------
>
> Key: HIVE-29436
> URL: https://issues.apache.org/jira/browse/HIVE-29436
> Project: Hive
> Issue Type: Bug
> Components: HiveServer2
> Reporter: Zsolt Miskolczi
> Priority: Major
> Labels: pull-request-available
>
> AlterMaterializedViewRebuildAnalyzer fails acquiring
> MATERIALIZATION_REBUILD_LOCKS entry.
> TransactionId (MRL_TXN_ID) is the primary key in this table.
> Unfortunately, for Iceberg materialized views, Hive currently does not open a
> transaction, so when AlterMaterializedViewRebuildAnalyzer reads the current
> transaction id, it receives 0 (this is the default value).
>
> Repro steps:
>
> {noformat}
> set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
> set hive.support.concurrency=true;
> drop materialized view if exists mat1;
> drop table if exists tbl_ice;
> create table tbl_ice(a int, b string, c int) stored by iceberg stored as orc
> tblproperties ('format-version'='1');
> insert into tbl_ice values (1, 'one', 50), (2, 'two', 51), (3, 'three', 52),
> (4, 'four', 53), (5, 'five', 54);
> create materialized view mat1 stored by iceberg stored as orc tblproperties
> ('format-version'='1') as
> select tbl_ice.b, tbl_ice.c from tbl_ice where tbl_ice.c > 52;
> create materialized view mat2 stored by iceberg stored as orc tblproperties
> ('format-version'='1') as
> select tbl_ice.b, tbl_ice.c from tbl_ice where tbl_ice.c > 53;
> insert into tbl_ice (a, b, c)
> values (6, 'six', 55);
> alter materialized view mat1 rebuild;
> alter materialized view mat2 rebuild;
> {noformat}
> Exception:
> {noformat}
> [ERROR] Failures:[ERROR] TestIcebergLlapLocalCliDriver.testCliDriver:60
> Client execution failed with error code = 40000runningalter materialized view
> mat2 rebuildfname=mv_iceberg_bug.q
> See ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log,
> or check ./ql/target/surefire-reports or
> ./itests/qtest/target/surefire-reports/ for specific test cases logs.
> org.apache.hadoop.hive.ql.parse.SemanticException:
> java.lang.RuntimeException: org.springframework.dao.DuplicateKeyException:
> PreparedStatementCallback; SQL [INSERT INTO "MATERIALIZATION_REBUILD_LOCKS"
> ("MRL_TXN_ID", "MRL_DB_NAME", "MRL_TBL_NAME", "MRL_LAST_HEARTBEAT") VALUES
> (?, ?, ?, 1770142215446)]; The statement was aborted because it would have
> caused a duplicate key value in a unique or primary key constraint or unique
> index identified by 'SQL0000000189-1d5e44af-019c-24b2-799b-00002931ada8'
> defined on 'MATERIALIZATION_REBUILD_LOCKS'.; nested exception is
> org.apache.derby.shared.common.error.DerbySQLIntegrityConstraintViolationException:
> The statement was aborted because it would have caused a duplicate key value
> in a unique or primary key constraint or unique index identified by
> 'SQL0000000189-1d5e44af-019c-24b2-799b-00002931ada8' defined on
> 'MATERIALIZATION_REBUILD_LOCKS'. at
> org.apache.hadoop.hive.ql.ddl.view.materialized.alter.rebuild.AlterMaterializedViewRebuildAnalyzer.getRewrittenAST(AlterMaterializedViewRebuildAnalyzer.java:227)
> at
> org.apache.hadoop.hive.ql.ddl.view.materialized.alter.rebuild.AlterMaterializedViewRebuildAnalyzer.analyzeInternal(AlterMaterializedViewRebuildAnalyzer.java:175)
> at
> org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:358)
> at org.apache.hadoop.hive.ql.Compiler.analyze(Compiler.java:224) at
> org.apache.hadoop.hive.ql.Compiler.compile(Compiler.java:109) at
> org.apache.hadoop.hive.ql.Driver.compile(Driver.java:498) at
> org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:450) at
> org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:414) at
> org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:408) at
> org.apache.hadoop.hive.ql.reexec.ReExecDriver.compileAndRespond(ReExecDriver.java:126)
> at
> org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:234)
> at org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:259)
> at org.apache.hadoop.hive.cli.CliDriver.processCmd1(CliDriver.java:203) at
> org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:129) at
> org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:430) at
> org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:358) at
> org.apache.hadoop.hive.ql.QTestUtil.executeClientInternal(QTestUtil.java:760)
> at
> org.apache.hadoop.hive.ql.QTestUtil.executeClient(QTestUtil.java:730)
> at
> org.apache.hadoop.hive.cli.control.CoreCliDriver.runTest(CoreCliDriver.java:115)
> at
> org.apache.hadoop.hive.cli.control.CliAdapter.runTest(CliAdapter.java:139)
> at
> org.apache.hadoop.hive.cli.TestIcebergLlapLocalCliDriver.testCliDriver(TestIcebergLlapLocalCliDriver.java:60)
> at
> java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
> at java.base/java.lang.reflect.Method.invoke(Method.java:580) at
> org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
> at
> org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
> at
> org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
> at
> org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
> at
> org.apache.hadoop.hive.cli.control.CliAdapter$2$1.evaluate(CliAdapter.java:118)
> at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
> at
> org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
> at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
> at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
> at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at
> org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at
> org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at
> org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at
> org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at
> org.junit.runners.ParentRunner.run(ParentRunner.java:413) at
> org.junit.runners.Suite.runChild(Suite.java:128) at
> org.junit.runners.Suite.runChild(Suite.java:27) at
> org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at
> org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at
> org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at
> org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at
> org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at
> org.apache.hadoop.hive.cli.control.CliAdapter$1$1.evaluate(CliAdapter.java:89)
> at org.junit.rules.RunRules.evaluate(RunRules.java:20) at
> org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at
> org.junit.runners.ParentRunner.run(ParentRunner.java:413) at
> org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:316)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:240)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:214)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:155)
> at
> org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:385)
> at
> org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:162)
> at org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:507)
> at
> org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:495)Caused
> by: java.lang.RuntimeException:
> org.springframework.dao.DuplicateKeyException: PreparedStatementCallback; SQL
> [INSERT INTO "MATERIALIZATION_REBUILD_LOCKS" ("MRL_TXN_ID", "MRL_DB_NAME",
> "MRL_TBL_NAME", "MRL_LAST_HEARTBEAT") VALUES (?, ?, ?, 1770142215446)]; The
> statement was aborted because it would have caused a duplicate key value in a
> unique or primary key constraint or unique index identified by
> 'SQL0000000189-1d5e44af-019c-24b2-799b-00002931ada8' defined on
> 'MATERIALIZATION_REBUILD_LOCKS'.; nested exception is
> org.apache.derby.shared.common.error.DerbySQLIntegrityConstraintViolationException:
> The statement was aborted because it would have caused a duplicate key value
> in a unique or primary key constraint or unique index identified by
> 'SQL0000000189-1d5e44af-019c-24b2-799b-00002931ada8' defined on
> 'MATERIALIZATION_REBUILD_LOCKS'. at
> org.apache.hadoop.hive.metastore.txn.jdbc.functions.LockMaterializationRebuildFunction.execute(LockMaterializationRebuildFunction.java:89)
> at
> org.apache.hadoop.hive.metastore.txn.TxnHandler.lockMaterializationRebuild(TxnHandler.java:774)
> at
> java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
> at java.base/java.lang.reflect.Method.invoke(Method.java:580) at
> org.apache.hadoop.hive.metastore.txn.TransactionalRetryProxy.lambda$invoke$0(TransactionalRetryProxy.java:100)
> at
> org.apache.hadoop.hive.metastore.txn.TransactionalRetryProxy.lambda$invoke$5(TransactionalRetryProxy.java:114)
> at
> org.apache.hadoop.hive.metastore.txn.TransactionalRetryProxy.lambda$invoke$6(TransactionalRetryProxy.java:163)
> at
> org.apache.hadoop.hive.metastore.txn.retry.SqlRetryHandler.executeWithRetryInternal(SqlRetryHandler.java:136)
> at
> org.apache.hadoop.hive.metastore.txn.retry.SqlRetryHandler.executeWithRetry(SqlRetryHandler.java:122)
> at
> org.apache.hadoop.hive.metastore.txn.TransactionalRetryProxy.invoke(TransactionalRetryProxy.java:186)
> at jdk.proxy2/jdk.proxy2.$Proxy74.lockMaterializationRebuild(Unknown
> Source) at
> org.apache.hadoop.hive.metastore.HMSHandler.get_lock_materialization_rebuild(HMSHandler.java:8846)
> at
> java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
> at java.base/java.lang.reflect.Method.invoke(Method.java:580) at
> org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:91)
> at
> org.apache.hadoop.hive.metastore.AbstractHMSHandlerProxy.invoke(AbstractHMSHandlerProxy.java:82)
> at
> jdk.proxy2/jdk.proxy2.$Proxy57.get_lock_materialization_rebuild(Unknown
> Source) at
> org.apache.hadoop.hive.metastore.client.ThriftHiveMetaStoreClient.lockMaterializationRebuild(ThriftHiveMetaStoreClient.java:3795)
> at
> org.apache.hadoop.hive.metastore.client.MetaStoreClientWrapper.lockMaterializationRebuild(MetaStoreClientWrapper.java:1357)
> at
> org.apache.hadoop.hive.metastore.client.MetaStoreClientWrapper.lockMaterializationRebuild(MetaStoreClientWrapper.java:1357)
> at
> org.apache.hadoop.hive.metastore.client.MetaStoreClientWrapper.lockMaterializationRebuild(MetaStoreClientWrapper.java:1357)
> at
> java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
> at java.base/java.lang.reflect.Method.invoke(Method.java:580) at
> org.apache.hadoop.hive.metastore.client.SynchronizedMetaStoreClient$SynchronizedHandler.invoke(SynchronizedMetaStoreClient.java:69)
> at jdk.proxy2/jdk.proxy2.$Proxy58.lockMaterializationRebuild(Unknown
> Source) at
> org.apache.hadoop.hive.metastore.client.MetaStoreClientWrapper.lockMaterializationRebuild(MetaStoreClientWrapper.java:1357)
> at
> java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
> at java.base/java.lang.reflect.Method.invoke(Method.java:580) at
> org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:232)
> at jdk.proxy2/jdk.proxy2.$Proxy58.lockMaterializationRebuild(Unknown
> Source) at
> org.apache.hadoop.hive.ql.lockmgr.DbTxnManager.acquireMaterializationRebuildLock(DbTxnManager.java:1031)
> at
> org.apache.hadoop.hive.ql.ddl.view.materialized.alter.rebuild.AlterMaterializedViewRebuildAnalyzer.getRewrittenAST(AlterMaterializedViewRebuildAnalyzer.java:216)
> ... 57 moreCaused by: org.springframework.dao.DuplicateKeyException:
> PreparedStatementCallback; SQL [INSERT INTO "MATERIALIZATION_REBUILD_LOCKS"
> ("MRL_TXN_ID", "MRL_DB_NAME", "MRL_TBL_NAME", "MRL_LAST_HEARTBEAT") VALUES
> (?, ?, ?, 1770142215446)]; The statement was aborted because it would have
> caused a duplicate key value in a unique or primary key constraint or unique
> index identified by 'SQL0000000189-1d5e44af-019c-24b2-799b-00002931ada8'
> defined on 'MATERIALIZATION_REBUILD_LOCKS'.; nested exception is
> org.apache.derby.shared.common.error.DerbySQLIntegrityConstraintViolationException:
> The statement was aborted because it would have caused a duplicate key value
> in a unique or primary key constraint or unique index identified by
> 'SQL0000000189-1d5e44af-019c-24b2-799b-00002931ada8' defined on
> 'MATERIALIZATION_REBUILD_LOCKS'. at
> org.springframework.jdbc.support.SQLErrorCodeSQLExceptionTranslator.doTranslate(SQLErrorCodeSQLExceptionTranslator.java:244)
> at
> org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:73)
> at
> org.springframework.jdbc.core.JdbcTemplate.translateException(JdbcTemplate.java:1575)
> at
> org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:667)
> at org.springframework.jdbc.core.JdbcTemplate.update(JdbcTemplate.java:960)
> at org.springframework.jdbc.core.JdbcTemplate.update(JdbcTemplate.java:981)
> at
> org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate.update(NamedParameterJdbcTemplate.java:331)
> at
> org.apache.hadoop.hive.metastore.txn.jdbc.functions.LockMaterializationRebuildFunction.execute(LockMaterializationRebuildFunction.java:86)
> ... 88 moreCaused by:
> org.apache.derby.shared.common.error.DerbySQLIntegrityConstraintViolationException:
> The statement was aborted because it would have caused a duplicate key value
> in a unique or primary key constraint or unique index identified by
> 'SQL0000000189-1d5e44af-019c-24b2-799b-00002931ada8' defined on
> 'MATERIALIZATION_REBUILD_LOCKS'. at
> org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(SQLExceptionFactory.java:93)
> at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Util.java:230)
> at
> org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(TransactionResourceImpl.java:431)
> at
> org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(TransactionResourceImpl.java:360)
> at
> org.apache.derby.impl.jdbc.EmbedConnection.handleException(EmbedConnection.java:2400)
> at
> org.apache.derby.impl.jdbc.ConnectionChild.handleException(ConnectionChild.java:86)
> at
> org.apache.derby.impl.jdbc.EmbedStatement.executeStatement(EmbedStatement.java:1436)
> at
> org.apache.derby.impl.jdbc.EmbedPreparedStatement.executeStatement(EmbedPreparedStatement.java:1709)
> at
> org.apache.derby.impl.jdbc.EmbedPreparedStatement.executeLargeUpdate(EmbedPreparedStatement.java:320)
> at
> org.apache.derby.impl.jdbc.EmbedPreparedStatement.executeUpdate(EmbedPreparedStatement.java:309)
> at
> com.zaxxer.hikari.pool.ProxyPreparedStatement.executeUpdate(ProxyPreparedStatement.java:61)
> at
> com.zaxxer.hikari.pool.HikariProxyPreparedStatement.executeUpdate(HikariProxyPreparedStatement.java)
> at
> org.springframework.jdbc.core.JdbcTemplate.lambda$update$2(JdbcTemplate.java:965)
> at
> org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:648)
> ... 92 moreCaused by: ERROR 23505: The statement was aborted because it would
> have caused a duplicate key value in a unique or primary key constraint or
> unique index identified by
> 'SQL0000000189-1d5e44af-019c-24b2-799b-00002931ada8' defined on
> 'MATERIALIZATION_REBUILD_LOCKS'. at
> org.apache.derby.shared.common.error.StandardException.newException(StandardException.java:299)
> at
> org.apache.derby.shared.common.error.StandardException.newException(StandardException.java:294)
> at
> org.apache.derby.impl.sql.execute.IndexChanger.insertAndCheckDups(IndexChanger.java:593)
> at
> org.apache.derby.impl.sql.execute.IndexChanger.doInsert(IndexChanger.java:393)
> at
> org.apache.derby.impl.sql.execute.IndexChanger.insert(IndexChanger.java:713)
> at
> org.apache.derby.impl.sql.execute.IndexSetChanger.insert(IndexSetChanger.java:268)
> at
> org.apache.derby.impl.sql.execute.RowChangerImpl.insertRow(RowChangerImpl.java:458)
> at
> org.apache.derby.impl.sql.execute.InsertResultSet.normalInsertCore(InsertResultSet.java:871)
> at
> org.apache.derby.impl.sql.execute.InsertResultSet.open(InsertResultSet.java:448)
> at
> org.apache.derby.impl.sql.GenericPreparedStatement.executeStmt(GenericPreparedStatement.java:470)
> at
> org.apache.derby.impl.sql.GenericPreparedStatement.execute(GenericPreparedStatement.java:349)
> at
> org.apache.derby.impl.jdbc.EmbedStatement.executeStatement(EmbedStatement.java:1344)
> ... 99 more{noformat}
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
