yujun777 commented on code in PR #68180:
URL: https://github.com/apache/doris/pull/68180#discussion_r4058982584
##########
fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmBaselineRebuildTest.java:
##########
@@ -122,35 +126,172 @@ public void
testDropColumnMarksBaselineRebuildOnlyWhenReferenced() throws Except
Assertions.assertTrue(mtmv.getIvmInfo().isBaselineRebuildRequired());
}
+ /**
+ * Which MV partitions must be rebuilt is decided by the MV's partition
mapping, not by what the
+ * refresh snapshot happens to record. This test publishes no snapshot at
all: an MV whose partitions
+ * follow the base table's still narrows the rebuild down to the
partitions that read the dropped one.
+ */
@Test
- public void testPublishedPctPartitionUsesPartitionsBaselineRebuild()
throws Exception {
+ public void testDropPartitionMarksOnlyMvPartitionsThatReadIt() throws
Exception {
String db = "ivm_partitions_baseline_rebuild";
- createPartitionedIvmTableAndMv(db);
+ createPartitionedIvmTableAndPartitionedMv(db);
MTMV mtmv = getMtmv(db);
- OlapTable baseTable = getBaseTable(db);
- publishPctPartitionSnapshot(mtmv, baseTable, "p202001");
+ Assertions.assertEquals(2, mtmv.getPartitionNames().size());
+ Set<String> expected = mvPartitionsWithSameRange(mtmv,
getBaseTable(db), "p202001");
+ Assertions.assertEquals(1, expected.size());
executeSql("ALTER TABLE ivm_base DROP PARTITION p202001");
Assertions.assertFalse(mtmv.getIvmInfo().requiresCompleteBaselineRebuild());
- Assertions.assertEquals(Collections.singleton("mv_partition"),
- mtmv.getIvmInfo().getPendingBaselineRebuildPartitions());
+ Assertions.assertEquals(expected,
mtmv.getIvmInfo().getPendingBaselineRebuildPartitions());
}
+ /**
+ * A base partition that no MV partition reads: dropping it cannot leave
any of its rows in the MV, so
+ * there is nothing to rebuild. The previous selection could not tell this
apart from "the snapshot does
+ * not know this partition" and rebuilt the whole MV instead.
+ */
@Test
- public void testMissingPctSnapshotRequiresCompleteBaselineRebuild() throws
Exception {
- String db = "ivm_complete_baseline_rebuild";
- createPartitionedIvmTableAndMv(db);
+ public void testDropPartitionOutsideMvPartitionsMarksNothing() throws
Exception {
+ String db = "ivm_partition_outside_mv";
+ createPartitionedIvmTableAndPartitionedMv(db);
+ MTMV mtmv = getMtmv(db);
+ // Added after the MV was created and never synced into it, so no MV
partition reads it.
+ executeSql("ALTER TABLE ivm_base ADD PARTITION p202003 "
+ + "VALUES [('2020-03-01'), ('2020-04-01'))");
+ Assertions.assertTrue(mvPartitionsWithSameRange(mtmv,
getBaseTable(db), "p202003").isEmpty());
+
+ executeSql("ALTER TABLE ivm_base DROP PARTITION p202003");
+
+ Assertions.assertFalse(mtmv.getIvmInfo().isBaselineRebuildRequired());
+ }
+
+ /**
+ * The partition mapping is built from the MV's PCT tables only. A changed
partition of a joined table
+ * the MV's partition column does not reach is invisible to it, and
missing such a change leaves rows
+ * of the dropped partition in the MV forever, so the whole MV has to be
rebuilt.
+ */
+ @Test
+ public void
testNonPctBaseTablePartitionChangeRequiresCompleteBaselineRebuild() throws
Exception {
+ String db = "ivm_non_pct_partition_change";
+ createPartitionedIvmTable(db);
+ createTable("CREATE TABLE " + db + ".ivm_dim (\n"
+ + " dt date NOT NULL,\n"
+ + " id int NOT NULL,\n"
+ + " v int\n"
+ + ")\n"
+ + "DUPLICATE KEY(dt, id)\n"
+ + "PARTITION BY RANGE(dt) (\n"
+ + " PARTITION d202001 VALUES [('2020-01-01'),
('2020-02-01')),\n"
+ + " PARTITION d202002 VALUES [('2020-02-01'),
('2020-03-01'))\n"
+ + ")\n"
+ + "DISTRIBUTED BY HASH(id) BUCKETS 1\n"
+ + "PROPERTIES ('replication_num' = '1', 'binlog.enable' =
'true', "
+ + "'binlog.format' = 'ROW')");
+ // The join is on a non-partition column, so ivm_dim is a base table
of the MV but not a PCT table.
+ createMvByNereids("CREATE MATERIALIZED VIEW ivm_mv\n"
+ + "BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n"
+ + "PARTITION BY(dt)\n"
+ + "DISTRIBUTED BY RANDOM BUCKETS 1\n"
+ + "PROPERTIES ('replication_num' = '1')\n"
+ + "AS SELECT b.dt, b.k1, b.v1 FROM ivm_base b JOIN ivm_dim d
ON b.k1 = d.id");
+ MTMV mtmv = getMtmv(db);
+ Assertions.assertTrue(mtmv.isIvm());
+ Assertions.assertEquals(Sets.newHashSet("ivm_base"),
+ mtmv.getMvPartitionInfo().getPctInfos().stream()
+ .map(pctInfo -> pctInfo.getTableInfo().getTableName())
+ .collect(Collectors.toSet()));
+
+ executeSql("ALTER TABLE ivm_dim DROP PARTITION d202001");
+
+
Assertions.assertTrue(mtmv.getIvmInfo().requiresCompleteBaselineRebuild());
+ }
+
+ /**
+ * A join whose condition carries the MV's partition column makes both
tables PCT tables, so the mapping
+ * reads both of them. The marker already holds this table's write lock,
so it takes the other one with a
+ * bounded tryLock: while that table is free, the rebuild is still
narrowed to the partitions that read
+ * the dropped one.
+ */
+ @Test
+ public void testMultiPctTablePartitionChangeStillNarrows() throws
Exception {
+ String db = "ivm_multi_pct_partition_change";
+ createTwoPctTableIvm(db);
MTMV mtmv = getMtmv(db);
-
mtmv.getMvPartitionInfo().setPartitionType(MTMVPartitionType.FOLLOW_BASE_TABLE);
- mtmv.getMvPartitionInfo().setPctInfos(Collections.singletonList(
- new BaseColInfo("dt", new BaseTableInfo(getBaseTable(db)))));
+ Set<String> expected = mvPartitionsWithSameRange(mtmv,
getBaseTable(db), "p202001");
+ Assertions.assertEquals(1, expected.size());
executeSql("ALTER TABLE ivm_base DROP PARTITION p202001");
+
Assertions.assertFalse(mtmv.getIvmInfo().requiresCompleteBaselineRebuild());
+ Assertions.assertEquals(expected,
mtmv.getIvmInfo().getPendingBaselineRebuildPartitions());
+ }
+
+ /**
+ * The same MV, but the other PCT table is being written while the
partition DDL marks. Waiting for it
+ * would close a cycle with the DDL that holds it -- each would hold the
write lock the other one needs --
+ * so the marker gives up on the mapping and the whole MV is rebuilt.
+ */
+ @Test
+ public void testMultiPctTableBusyOtherTableRebuildsWholeMv() throws
Exception {
+ String db = "ivm_multi_pct_busy";
+ createTwoPctTableIvm(db);
+ MTMV mtmv = getMtmv(db);
+ OlapTable otherPctTable = (OlapTable)
getDb(db).getTableOrMetaException("ivm_dim");
+ CountDownLatch locked = new CountDownLatch(1);
+ CountDownLatch released = new CountDownLatch(1);
Review Comment:
Addressed in 7ad093ee957, at the helper level as you suggested.
`MetaLockUtilsTest.testTryReadLockTablesReleasesWhatItTookWhenALaterTableIsBusy`
takes a readable table with the lower id and a busy one with the higher id, so
the batch acquires the first read lock and only then meets the busy table; it
returns false, and the release is proved by taking the first table's write lock
afterwards, which a held read lock would block.
Writing it turned up a trap worth recording: the busy lock has to be held by
*another* thread, because a thread that holds the write lock may take the read
lock underneath it -- my first version held it on the test thread and the batch
returned true, so nothing was busy at all. Both this test and
`IvmBaselineRebuildTest.testMultiPctTableBusyOtherTableRebuildsWholeMv` now
acquire with a bounded tryLock and join the worker with a termination
assertion, so a failure cannot strand a non-daemon thread. The integration test
still fails the batch on its first table, which is what its comment now says:
it covers the whole-MV fallback, and the rollback is covered by the helper test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
