This is an automated email from the ASF dual-hosted git repository.
yuqi1129 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 93edbe5abe [#14282] fix(core): Clean securable objects, tag relations
and policies when dropping a metalake (#14285)
93edbe5abe is described below
commit 93edbe5abe1fbef1f4853f7a676d82df4ed49cf9
Author: Qi Yu <[email protected]>
AuthorDate: Wed Oct 28 08:36:17 2026 +0800
[#14282] fix(core): Clean securable objects, tag relations and policies
when dropping a metalake (#14285)
### What changes were proposed in this pull request?
`MetalakeMetaService.deleteMetalake` now cleans securable objects and
tag relations before the roles and tags they hang off, on both the
cascade and the non-cascade path. The non-cascade path also soft-deletes
policies and policy versions.
### Why are the changes needed?
Both cleanup statements find their rows through a live role or tag, so
running them after the roles and tags were soft-deleted updated nothing.
The rows of a dropped metalake stayed live and were never collected, and
so did its policies after a non-cascade drop.
Fixes #14282
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
- New `TestMetalakeMetaService.testDeleteMetalakeRemovesGovernanceRows`
(cascade and non-cascade), which fails without the fix.
- `TestRoleMetaService.testDeleteRoleMetasByLegacyTimeline` now also
counts the securable object of each role among the rows hard-deleted
after the metalake drop. Before the fix the cascade drop left these rows
live, so the test didn't count them.
- `TestMetalakeMetaService`, `TestMetalakeManager`,
`TestRoleMetaService`, `TestUserMetaService`, `TestGroupMetaService`,
`TestTagMetaService` and `TestPolicyMetaService` pass on H2; MySQL and
PostgreSQL run in CI.
---
.../relational/service/MetalakeMetaService.java | 34 ++++++---
.../service/TestMetalakeMetaService.java | 82 ++++++++++++++++++++++
.../relational/service/TestRoleMetaService.java | 5 +-
3 files changed, 108 insertions(+), 13 deletions(-)
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetalakeMetaService.java
b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetalakeMetaService.java
index 02f94121ae..1cdbc5b854 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetalakeMetaService.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetalakeMetaService.java
@@ -277,22 +277,24 @@ public class MetalakeMetaService {
SessionUtils.doWithoutCommit(
GroupMetaMapper.class,
mapper ->
mapper.softDeleteGroupMetasByMetalakeId(metalakeId)),
- () ->
- SessionUtils.doWithoutCommit(
- RoleMetaMapper.class,
- mapper ->
mapper.softDeleteRoleMetasByMetalakeId(metalakeId)),
+ // Securable objects and tag relations find their rows through a
live role or tag, so
+ // they are removed before the roles and tags themselves.
() ->
SessionUtils.doWithoutCommit(
SecurableObjectMapper.class,
mapper ->
mapper.softDeleteSecurableObjectsByMetalakeId(metalakeId)),
() ->
SessionUtils.doWithoutCommit(
- TagMetaMapper.class,
- mapper ->
mapper.softDeleteTagMetasByMetalakeId(metalakeId)),
+ RoleMetaMapper.class,
+ mapper ->
mapper.softDeleteRoleMetasByMetalakeId(metalakeId)),
() ->
SessionUtils.doWithoutCommit(
TagMetadataObjectRelMapper.class,
mapper ->
mapper.softDeleteTagMetadataObjectRelsByMetalakeId(metalakeId)),
+ () ->
+ SessionUtils.doWithoutCommit(
+ TagMetaMapper.class,
+ mapper ->
mapper.softDeleteTagMetasByMetalakeId(metalakeId)),
() ->
SessionUtils.doWithoutCommit(
PolicyTagRelMapper.class, mapper ->
mapper.softDeleteByMetalakeId(metalakeId)),
@@ -387,25 +389,35 @@ public class MetalakeMetaService {
SessionUtils.doWithoutCommit(
GroupMetaMapper.class,
mapper ->
mapper.softDeleteGroupMetasByMetalakeId(metalakeId)),
+ // Securable objects and tag relations find their rows through a
live role or tag, so
+ // they are removed before the roles and tags themselves.
+ () ->
+ SessionUtils.doWithoutCommit(
+ SecurableObjectMapper.class,
+ mapper ->
mapper.softDeleteSecurableObjectsByMetalakeId(metalakeId)),
() ->
SessionUtils.doWithoutCommit(
RoleMetaMapper.class,
mapper ->
mapper.softDeleteRoleMetasByMetalakeId(metalakeId)),
() ->
SessionUtils.doWithoutCommit(
- SecurableObjectMapper.class,
- mapper ->
mapper.softDeleteSecurableObjectsByMetalakeId(metalakeId)),
+ TagMetadataObjectRelMapper.class,
+ mapper ->
mapper.softDeleteTagMetadataObjectRelsByMetalakeId(metalakeId)),
() ->
SessionUtils.doWithoutCommit(
TagMetaMapper.class,
mapper ->
mapper.softDeleteTagMetasByMetalakeId(metalakeId)),
() ->
SessionUtils.doWithoutCommit(
- TagMetadataObjectRelMapper.class,
- mapper ->
mapper.softDeleteTagMetadataObjectRelsByMetalakeId(metalakeId)),
+ PolicyTagRelMapper.class, mapper ->
mapper.softDeleteByMetalakeId(metalakeId)),
() ->
SessionUtils.doWithoutCommit(
- PolicyTagRelMapper.class, mapper ->
mapper.softDeleteByMetalakeId(metalakeId)),
+ PolicyMetaMapper.class,
+ mapper ->
mapper.softDeletePolicyMetasByMetalakeId(metalakeId)),
+ () ->
+ SessionUtils.doWithoutCommit(
+ PolicyVersionMapper.class,
+ mapper ->
mapper.softDeletePolicyVersionsByMetalakeId(metalakeId)),
() ->
SessionUtils.doWithoutCommit(
OwnerMetaMapper.class,
diff --git
a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestMetalakeMetaService.java
b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestMetalakeMetaService.java
index ed560ee704..28d1693b1a 100644
---
a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestMetalakeMetaService.java
+++
b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestMetalakeMetaService.java
@@ -23,6 +23,8 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
import java.io.IOException;
import java.sql.Connection;
import java.sql.ResultSet;
@@ -38,16 +40,24 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.apache.gravitino.Entity;
import org.apache.gravitino.EntityAlreadyExistsException;
+import org.apache.gravitino.MetadataObject;
+import org.apache.gravitino.NameIdentifier;
import org.apache.gravitino.Namespace;
+import org.apache.gravitino.authorization.Privileges;
+import org.apache.gravitino.authorization.SecurableObjects;
import org.apache.gravitino.exceptions.NoSuchEntityException;
import org.apache.gravitino.exceptions.NonEmptyEntityException;
import org.apache.gravitino.exceptions.OptimisticLockException;
import org.apache.gravitino.meta.BaseMetalake;
import org.apache.gravitino.meta.CatalogEntity;
import org.apache.gravitino.meta.ColumnEntity;
+import org.apache.gravitino.meta.PolicyEntity;
+import org.apache.gravitino.meta.RoleEntity;
import org.apache.gravitino.meta.SchemaEntity;
import org.apache.gravitino.meta.SchemaVersion;
import org.apache.gravitino.meta.TableEntity;
+import org.apache.gravitino.meta.TagEntity;
+import org.apache.gravitino.policy.PolicyContents;
import org.apache.gravitino.rel.types.Types;
import org.apache.gravitino.storage.RandomIdGenerator;
import org.apache.gravitino.storage.relational.TestJDBCBackend;
@@ -58,6 +68,7 @@ import org.apache.gravitino.storage.relational.po.SchemaPO;
import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper;
import org.apache.gravitino.storage.relational.utils.POConverters;
import org.apache.gravitino.storage.relational.utils.SessionUtils;
+import org.apache.gravitino.utils.NameIdentifierUtil;
import org.apache.gravitino.utils.NamespaceUtil;
import org.apache.ibatis.session.SqlSession;
import org.junit.jupiter.api.Assertions;
@@ -469,6 +480,61 @@ public class TestMetalakeMetaService extends
TestJDBCBackend {
assertEquals(0, countActiveTableVersionRows(table.id()));
}
+ @TestTemplate
+ public void testDeleteMetalakeRemovesGovernanceRows() throws IOException {
+ for (boolean cascade : new boolean[] {true, false}) {
+ String metalakeName = METALAKE_NAME + (cascade ? "_cascade" :
"_non_cascade");
+ BaseMetalake metalake = createAndInsertMakeLake(metalakeName);
+
+ RoleEntity role =
+ createRoleEntity(
+ RandomIdGenerator.INSTANCE.nextId(),
+ NamespaceUtil.ofRole(metalakeName),
+ "role",
+ AUDIT_INFO,
+ List.of(
+ SecurableObjects.ofMetalake(
+ metalakeName,
List.of(Privileges.CreateCatalog.allow()))),
+ null);
+ backend.insert(role, false);
+
+ // Tags attach to catalogs and below, and a non-cascade drop needs a
metalake without
+ // catalogs, so only the cascade drop can still have live tag relations.
+ TagEntity tag = createAndInsertTagEntity("tag", "tag", metalakeName);
+ if (cascade) {
+ CatalogEntity catalog = createAndInsertCatalog(metalakeName,
"catalog");
+ TagMetaService.getInstance()
+ .associateTagsWithMetadataObject(
+ catalog.nameIdentifier(),
+ Entity.EntityType.CATALOG,
+ new NameIdentifier[] {NameIdentifierUtil.ofTag(metalakeName,
tag.name())},
+ new NameIdentifier[0]);
+ }
+
+ PolicyEntity policy =
+ createAndInsertPolicyEntity(
+ "policy",
+ "policy",
+ PolicyContents.custom(
+ ImmutableMap.of("field", 1),
ImmutableSet.of(MetadataObject.Type.TABLE), null),
+ metalakeName);
+
+ assertEquals(1, countActiveRows("role_meta_securable_object", "role_id",
role.id()));
+ assertEquals(cascade ? 1 : 0, countActiveRows("tag_relation_meta",
"tag_id", tag.id()));
+ assertEquals(1, countActiveRows("policy_meta", "policy_id",
policy.id()));
+ assertEquals(1, countActiveRows("policy_version_info", "policy_id",
policy.id()));
+
+ assertTrue(
+
MetalakeMetaService.getInstance().deleteMetalake(metalake.nameIdentifier(),
cascade));
+
+ // Rows left with deleted_at = 0 are never collected by the legacy-data
cleanup.
+ assertEquals(0, countActiveRows("role_meta_securable_object", "role_id",
role.id()));
+ assertEquals(0, countActiveRows("tag_relation_meta", "tag_id",
tag.id()));
+ assertEquals(0, countActiveRows("policy_meta", "policy_id",
policy.id()));
+ assertEquals(0, countActiveRows("policy_version_info", "policy_id",
policy.id()));
+ }
+ }
+
@TestTemplate
public void testMetaLifeCycleFromCreationToDeletion() throws IOException {
// meta data creation
@@ -497,6 +563,22 @@ public class TestMetalakeMetaService extends
TestJDBCBackend {
assertFalse(legacyRecordExistsInDB(metalake.id(),
Entity.EntityType.METALAKE));
}
+ private int countActiveRows(String table, String idColumn, Long id) {
+ try (SqlSession sqlSession =
+
SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true);
+ Connection connection = sqlSession.getConnection();
+ Statement statement = connection.createStatement();
+ ResultSet rs =
+ statement.executeQuery(
+ String.format(
+ "SELECT count(*) FROM %s WHERE %s = %d AND deleted_at = 0",
+ table, idColumn, id))) {
+ return rs.next() ? rs.getInt(1) : 0;
+ } catch (SQLException e) {
+ throw new RuntimeException("SQL execution failed", e);
+ }
+ }
+
private int countActiveTableVersionRows(Long tableId) {
try (SqlSession sqlSession =
SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true);
diff --git
a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestRoleMetaService.java
b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestRoleMetaService.java
index a7fbd048bd..a203c371e4 100644
---
a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestRoleMetaService.java
+++
b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestRoleMetaService.java
@@ -1629,14 +1629,15 @@ class TestRoleMetaService extends TestJDBCBackend {
// hard delete after soft delete
deletedCount =
roleMetaService.deleteRoleMetasByLegacyTimeline(Instant.now().toEpochMilli() +
1000, 1);
- assertEquals(3, deletedCount); // delete 1 role + 1 userRoleRel + 1
groupRoleRel
+ // delete 1 role + 1 userRoleRel + 1 groupRoleRel + 1 securableObject
+ assertEquals(4, deletedCount);
assertEquals(1, countRoles(metalake.id())); // 2 - 1
assertEquals(1, countUserRoleRels()); // 2 - 1
assertEquals(1, countGroupRoleRels()); // 2 - 1
deletedCount =
roleMetaService.deleteRoleMetasByLegacyTimeline(Instant.now().toEpochMilli() +
1000, 1);
- assertEquals(3, deletedCount);
+ assertEquals(4, deletedCount);
assertEquals(0, countRoles(metalake.id()));
assertEquals(0, countUserRoleRels());
assertEquals(0, countGroupRoleRels());
