This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new d74ce36970 [#6380] fix(postgres-sql): Fix errors for PG backend about
`delete...limit..` clause. (#6393)
d74ce36970 is described below
commit d74ce369701ce6a950ef4be485395c814ae34baf
Author: Qi Yu <y...@datastrato.com>
AuthorDate: Thu Feb 6 10:53:31 2025 +0800
[#6380] fix(postgres-sql): Fix errors for PG backend about
`delete...limit..` clause. (#6393)
### What changes were proposed in this pull request?
PostgreSQL does not support SQL sentences like `DELETE FROM xxxx_table
where xxxx limit 10` , Cluase `limit xxx` is not allowed in the `Delete
syntax`
### Why are the changes needed?
it's a bug.
Fix: #6380
### Does this PR introduce _any_ user-facing change?
N/A
### How was this patch tested?
UT
---
.../storage/relational/RelationalGarbageCollector.java | 3 ++-
.../postgresql/CatalogMetaPostgreSQLProvider.java | 10 ++++++++++
.../postgresql/FilesetMetaPostgreSQLProvider.java | 11 +++++++++++
.../postgresql/FilesetVersionPostgreSQLProvider.java | 11 +++++++++++
.../provider/postgresql/GroupMetaPostgreSQLProvider.java | 11 +++++++++++
.../postgresql/GroupRoleRelPostgreSQLProvider.java | 11 +++++++++++
.../postgresql/MetalakeMetaPostgreSQLProvider.java | 10 ++++++++++
.../provider/postgresql/ModelMetaPostgreSQLProvider.java | 10 ++++++++++
.../ModelVersionAliasRelPostgreSQLProvider.java | 10 ++++++++++
.../postgresql/ModelVersionMetaPostgreSQLProvider.java | 10 ++++++++++
.../provider/postgresql/OwnerMetaPostgreSQLProvider.java | 11 +++++++++++
.../provider/postgresql/RoleMetaPostgreSQLProvider.java | 11 +++++++++++
.../provider/postgresql/SchemaMetaPostgreSQLProvider.java | 10 ++++++++++
.../postgresql/SecurableObjectPostgreSQLProvider.java | 10 ++++++++++
.../postgresql/TableColumnPostgreSQLProvider.java | 10 ++++++++++
.../provider/postgresql/TableMetaPostgreSQLProvider.java | 11 +++++++++++
.../provider/postgresql/TagMetaPostgreSQLProvider.java | 10 ++++++++++
.../TagMetadataObjectRelPostgreSQLProvider.java | 10 ++++++++++
.../provider/postgresql/TopicMetaPostgreSQLProvider.java | 11 +++++++++++
.../provider/postgresql/UserMetaPostgreSQLProvider.java | 11 +++++++++++
.../postgresql/UserRoleRelPostgreSQLProvider.java | 11 +++++++++++
.../org/apache/gravitino/storage/TestEntityStorage.java | 15 ++++++++++++++-
22 files changed, 226 insertions(+), 2 deletions(-)
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalGarbageCollector.java
b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalGarbageCollector.java
index 3cf72aeae5..135f253629 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalGarbageCollector.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalGarbageCollector.java
@@ -68,7 +68,8 @@ public final class RelationalGarbageCollector implements
Closeable {
garbageCollectorPool.scheduleAtFixedRate(this::collectAndClean, 5,
frequency, TimeUnit.MINUTES);
}
- private void collectAndClean() {
+ @VisibleForTesting
+ public void collectAndClean() {
long threadId = Thread.currentThread().getId();
LOG.info("Thread {} start to collect garbage...", threadId);
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/CatalogMetaPostgreSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/CatalogMetaPostgreSQLProvider.java
index 77bf3c4e28..a945a709c7 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/CatalogMetaPostgreSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/CatalogMetaPostgreSQLProvider.java
@@ -43,6 +43,16 @@ public class CatalogMetaPostgreSQLProvider extends
CatalogMetaBaseSQLProvider {
+ " WHERE metalake_id = #{metalakeId} AND deleted_at = 0";
}
+ @Override
+ public String deleteCatalogMetasByLegacyTimeline(
+ @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit)
{
+ return "DELETE FROM "
+ + TABLE_NAME
+ + " WHERE catalog_id IN (SELECT catalog_id FROM "
+ + TABLE_NAME
+ + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT
#{limit})";
+ }
+
@Override
public String insertCatalogMetaOnDuplicateKeyUpdate(CatalogPO catalogPO) {
return "INSERT INTO "
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/FilesetMetaPostgreSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/FilesetMetaPostgreSQLProvider.java
index c3200d6351..7b55549c1c 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/FilesetMetaPostgreSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/FilesetMetaPostgreSQLProvider.java
@@ -22,6 +22,7 @@ import static
org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper.M
import
org.apache.gravitino.storage.relational.mapper.provider.base.FilesetMetaBaseSQLProvider;
import org.apache.gravitino.storage.relational.po.FilesetPO;
+import org.apache.ibatis.annotations.Param;
public class FilesetMetaPostgreSQLProvider extends FilesetMetaBaseSQLProvider {
@Override
@@ -60,6 +61,16 @@ public class FilesetMetaPostgreSQLProvider extends
FilesetMetaBaseSQLProvider {
+ " WHERE fileset_id = #{filesetId} AND deleted_at = 0";
}
+ @Override
+ public String deleteFilesetMetasByLegacyTimeline(
+ @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit)
{
+ return "DELETE FROM "
+ + META_TABLE_NAME
+ + " WHERE fileset_id IN (SELECT fileset_id FROM "
+ + META_TABLE_NAME
+ + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT
#{limit})";
+ }
+
@Override
public String insertFilesetMetaOnDuplicateKeyUpdate(FilesetPO filesetPO) {
return "INSERT INTO "
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/FilesetVersionPostgreSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/FilesetVersionPostgreSQLProvider.java
index b9e6d4c5b1..b7c165cc04 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/FilesetVersionPostgreSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/FilesetVersionPostgreSQLProvider.java
@@ -22,6 +22,7 @@ import static
org.apache.gravitino.storage.relational.mapper.FilesetVersionMappe
import
org.apache.gravitino.storage.relational.mapper.provider.base.FilesetVersionBaseSQLProvider;
import org.apache.gravitino.storage.relational.po.FilesetVersionPO;
+import org.apache.ibatis.annotations.Param;
public class FilesetVersionPostgreSQLProvider extends
FilesetVersionBaseSQLProvider {
@Override
@@ -60,6 +61,16 @@ public class FilesetVersionPostgreSQLProvider extends
FilesetVersionBaseSQLProvi
+ " WHERE fileset_id = #{filesetId} AND deleted_at = 0";
}
+ @Override
+ public String deleteFilesetVersionsByLegacyTimeline(
+ @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit)
{
+ return "DELETE FROM "
+ + VERSION_TABLE_NAME
+ + " WHERE id IN (SELECT id FROM "
+ + VERSION_TABLE_NAME
+ + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT
#{limit})";
+ }
+
@Override
public String softDeleteFilesetVersionsByRetentionLine(
Long filesetId, long versionRetentionLine, int limit) {
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/GroupMetaPostgreSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/GroupMetaPostgreSQLProvider.java
index 3ace33f6f8..063136c0b8 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/GroupMetaPostgreSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/GroupMetaPostgreSQLProvider.java
@@ -24,6 +24,7 @@ import static
org.apache.gravitino.storage.relational.mapper.RoleMetaMapper.ROLE
import
org.apache.gravitino.storage.relational.mapper.provider.base.GroupMetaBaseSQLProvider;
import org.apache.gravitino.storage.relational.po.GroupPO;
+import org.apache.ibatis.annotations.Param;
public class GroupMetaPostgreSQLProvider extends GroupMetaBaseSQLProvider {
@Override
@@ -95,4 +96,14 @@ public class GroupMetaPostgreSQLProvider extends
GroupMetaBaseSQLProvider {
+ " gt.metalake_id = #{metalakeId}"
+ " GROUP BY gt.group_id";
}
+
+ @Override
+ public String deleteGroupMetasByLegacyTimeline(
+ @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit)
{
+ return "DELETE FROM "
+ + GROUP_TABLE_NAME
+ + " WHERE group_id IN (SELECT group_id FROM "
+ + GROUP_TABLE_NAME
+ + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT
#{limit})";
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/GroupRoleRelPostgreSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/GroupRoleRelPostgreSQLProvider.java
index 4051412393..8148c95728 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/GroupRoleRelPostgreSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/GroupRoleRelPostgreSQLProvider.java
@@ -23,6 +23,7 @@ import static
org.apache.gravitino.storage.relational.mapper.GroupRoleRelMapper.
import java.util.List;
import
org.apache.gravitino.storage.relational.mapper.provider.base.GroupRoleRelBaseSQLProvider;
+import org.apache.ibatis.annotations.Param;
public class GroupRoleRelPostgreSQLProvider extends
GroupRoleRelBaseSQLProvider {
@Override
@@ -70,4 +71,14 @@ public class GroupRoleRelPostgreSQLProvider extends
GroupRoleRelBaseSQLProvider
+ " timestamp '1970-01-01 00:00:00')*1000)))"
+ " WHERE role_id = #{roleId} AND deleted_at = 0";
}
+
+ @Override
+ public String deleteGroupRoleRelMetasByLegacyTimeline(
+ @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit)
{
+ return "DELETE FROM "
+ + GROUP_ROLE_RELATION_TABLE_NAME
+ + " WHERE id IN (SELECT id FROM "
+ + GROUP_ROLE_RELATION_TABLE_NAME
+ + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT
#{limit})";
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/MetalakeMetaPostgreSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/MetalakeMetaPostgreSQLProvider.java
index 06dde29751..f52eafb8a2 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/MetalakeMetaPostgreSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/MetalakeMetaPostgreSQLProvider.java
@@ -87,4 +87,14 @@ public class MetalakeMetaPostgreSQLProvider extends
MetalakeMetaBaseSQLProvider
+ " AND last_version = #{oldMetalakeMeta.lastVersion}"
+ " AND deleted_at = 0";
}
+
+ @Override
+ public String deleteMetalakeMetasByLegacyTimeline(
+ @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit)
{
+ return "DELETE FROM "
+ + TABLE_NAME
+ + " WHERE metalake_id IN (SELECT metalake_id FROM "
+ + TABLE_NAME
+ + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT
#{limit})";
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/ModelMetaPostgreSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/ModelMetaPostgreSQLProvider.java
index 8f62252aa8..b7db5a4d70 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/ModelMetaPostgreSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/ModelMetaPostgreSQLProvider.java
@@ -83,4 +83,14 @@ public class ModelMetaPostgreSQLProvider extends
ModelMetaBaseSQLProvider {
+ " timestamp '1970-01-01 00:00:00')*1000)))"
+ " WHERE schema_id = #{schemaId} AND deleted_at = 0";
}
+
+ @Override
+ public String deleteModelMetasByLegacyTimeline(
+ @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit)
{
+ return "DELETE FROM "
+ + ModelMetaMapper.TABLE_NAME
+ + " WHERE model_id IN (SELECT model_id FROM "
+ + ModelMetaMapper.TABLE_NAME
+ + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT
#{limit})";
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/ModelVersionAliasRelPostgreSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/ModelVersionAliasRelPostgreSQLProvider.java
index da23bdca2d..7bcac308af 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/ModelVersionAliasRelPostgreSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/ModelVersionAliasRelPostgreSQLProvider.java
@@ -98,4 +98,14 @@ public class ModelVersionAliasRelPostgreSQLProvider extends
ModelVersionAliasRel
+ ModelMetaMapper.TABLE_NAME
+ " WHERE metalake_id = #{metalakeId} AND deleted_at = 0) AND
deleted_at = 0";
}
+
+ @Override
+ public String deleteModelVersionAliasRelsByLegacyTimeline(
+ @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit)
{
+ return "DELETE FROM "
+ + ModelVersionAliasRelMapper.TABLE_NAME
+ + " WHERE id IN (SELECT id FROM "
+ + ModelVersionAliasRelMapper.TABLE_NAME
+ + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT
#{limit})";
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/ModelVersionMetaPostgreSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/ModelVersionMetaPostgreSQLProvider.java
index 4183a53617..28fa2d5001 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/ModelVersionMetaPostgreSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/ModelVersionMetaPostgreSQLProvider.java
@@ -90,4 +90,14 @@ public class ModelVersionMetaPostgreSQLProvider extends
ModelVersionMetaBaseSQLP
+ " timestamp '1970-01-01 00:00:00')*1000)))"
+ " WHERE metalake_id = #{metalakeId} AND deleted_at = 0";
}
+
+ @Override
+ public String deleteModelVersionMetasByLegacyTimeline(
+ @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit)
{
+ return "DELETE FROM "
+ + ModelVersionMetaMapper.TABLE_NAME
+ + " WHERE id IN (SELECT id FROM "
+ + ModelMetaMapper.TABLE_NAME
+ + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT
#{limit})";
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/OwnerMetaPostgreSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/OwnerMetaPostgreSQLProvider.java
index ba594f7732..c0f5d73d8e 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/OwnerMetaPostgreSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/OwnerMetaPostgreSQLProvider.java
@@ -26,6 +26,7 @@ import
org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper;
import org.apache.gravitino.storage.relational.mapper.TableMetaMapper;
import org.apache.gravitino.storage.relational.mapper.TopicMetaMapper;
import
org.apache.gravitino.storage.relational.mapper.provider.base.OwnerMetaBaseSQLProvider;
+import org.apache.ibatis.annotations.Param;
public class OwnerMetaPostgreSQLProvider extends OwnerMetaBaseSQLProvider {
@Override
@@ -117,4 +118,14 @@ public class OwnerMetaPostgreSQLProvider extends
OwnerMetaBaseSQLProvider {
+ "ft.fileset_id = ot.metadata_object_id AND ot.metadata_object_type =
'FILESET'"
+ ")";
}
+
+ @Override
+ public String deleteOwnerMetasByLegacyTimeline(
+ @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit)
{
+ return "DELETE FROM "
+ + OWNER_TABLE_NAME
+ + " WHERE id IN (SELECT id FROM "
+ + OWNER_TABLE_NAME
+ + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT
#{limit})";
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/RoleMetaPostgreSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/RoleMetaPostgreSQLProvider.java
index 0558c0b3c6..b4b1054be7 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/RoleMetaPostgreSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/RoleMetaPostgreSQLProvider.java
@@ -22,6 +22,7 @@ import static
org.apache.gravitino.storage.relational.mapper.RoleMetaMapper.ROLE
import
org.apache.gravitino.storage.relational.mapper.provider.base.RoleMetaBaseSQLProvider;
import org.apache.gravitino.storage.relational.po.RolePO;
+import org.apache.ibatis.annotations.Param;
public class RoleMetaPostgreSQLProvider extends RoleMetaBaseSQLProvider {
@Override
@@ -67,4 +68,14 @@ public class RoleMetaPostgreSQLProvider extends
RoleMetaBaseSQLProvider {
+ " last_version = #{roleMeta.lastVersion},"
+ " deleted_at = #{roleMeta.deletedAt}";
}
+
+ @Override
+ public String deleteRoleMetasByLegacyTimeline(
+ @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit)
{
+ return "DELETE FROM "
+ + ROLE_TABLE_NAME
+ + " WHERE role_id IN (SELECT role_id FROM "
+ + ROLE_TABLE_NAME
+ + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT
#{limit})";
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/SchemaMetaPostgreSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/SchemaMetaPostgreSQLProvider.java
index c39b244581..8e96a7024d 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/SchemaMetaPostgreSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/SchemaMetaPostgreSQLProvider.java
@@ -22,6 +22,7 @@ import static
org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper.TA
import
org.apache.gravitino.storage.relational.mapper.provider.base.SchemaMetaBaseSQLProvider;
import org.apache.gravitino.storage.relational.po.SchemaPO;
+import org.apache.ibatis.annotations.Param;
public class SchemaMetaPostgreSQLProvider extends SchemaMetaBaseSQLProvider {
@Override
@@ -81,4 +82,13 @@ public class SchemaMetaPostgreSQLProvider extends
SchemaMetaBaseSQLProvider {
+ " timestamp '1970-01-01 00:00:00')*1000)))"
+ " WHERE catalog_id = #{catalogId} AND deleted_at = 0";
}
+
+ public String deleteSchemaMetasByLegacyTimeline(
+ @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit)
{
+ return "DELETE FROM "
+ + TABLE_NAME
+ + " WHERE schema_id IN (SELECT schema_id FROM "
+ + TABLE_NAME
+ + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT
#{limit})";
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/SecurableObjectPostgreSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/SecurableObjectPostgreSQLProvider.java
index 6de57dbdc4..c67324f9a7 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/SecurableObjectPostgreSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/SecurableObjectPostgreSQLProvider.java
@@ -144,4 +144,14 @@ public class SecurableObjectPostgreSQLProvider extends
SecurableObjectBaseSQLPro
+ "ft.fileset_id = sect.metadata_object_id AND sect.type = 'FILESET'"
+ ")";
}
+
+ @Override
+ public String deleteSecurableObjectsByLegacyTimeline(
+ @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit)
{
+ return "DELETE FROM "
+ + SECURABLE_OBJECT_TABLE_NAME
+ + " WHERE id IN (SELECT id FROM "
+ + ROLE_TABLE_NAME
+ + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT
#{limit})";
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TableColumnPostgreSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TableColumnPostgreSQLProvider.java
index bf99438c8b..e8ae522ac7 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TableColumnPostgreSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TableColumnPostgreSQLProvider.java
@@ -59,4 +59,14 @@ public class TableColumnPostgreSQLProvider extends
TableColumnBaseSQLProvider {
+ " timestamp '1970-01-01 00:00:00')*1000)))"
+ " WHERE schema_id = #{schemaId} AND deleted_at = 0";
}
+
+ @Override
+ public String deleteColumnPOsByLegacyTimeline(
+ @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit)
{
+ return "DELETE FROM "
+ + TableColumnMapper.COLUMN_TABLE_NAME
+ + " WHERE id IN (SELECT id FROM "
+ + TableColumnMapper.COLUMN_TABLE_NAME
+ + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT
#{limit})";
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TableMetaPostgreSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TableMetaPostgreSQLProvider.java
index 94cb571c3f..227f73f585 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TableMetaPostgreSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TableMetaPostgreSQLProvider.java
@@ -22,6 +22,7 @@ import static
org.apache.gravitino.storage.relational.mapper.TableMetaMapper.TAB
import
org.apache.gravitino.storage.relational.mapper.provider.base.TableMetaBaseSQLProvider;
import org.apache.gravitino.storage.relational.po.TablePO;
+import org.apache.ibatis.annotations.Param;
public class TableMetaPostgreSQLProvider extends TableMetaBaseSQLProvider {
@Override
@@ -88,4 +89,14 @@ public class TableMetaPostgreSQLProvider extends
TableMetaBaseSQLProvider {
+ " timestamp '1970-01-01 00:00:00')*1000)))"
+ " WHERE schema_id = #{schemaId} AND deleted_at = 0";
}
+
+ @Override
+ public String deleteTableMetasByLegacyTimeline(
+ @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit)
{
+ return "DELETE FROM "
+ + TABLE_NAME
+ + " WHERE table_id IN (SELECT table_id FROM "
+ + TABLE_NAME
+ + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT
#{limit})";
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TagMetaPostgreSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TagMetaPostgreSQLProvider.java
index 582307b6b1..9f30b3f3f6 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TagMetaPostgreSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TagMetaPostgreSQLProvider.java
@@ -100,4 +100,14 @@ public class TagMetaPostgreSQLProvider extends
TagMetaBaseSQLProvider {
+ " AND last_version = #{oldTagMeta.lastVersion}"
+ " AND deleted_at = 0";
}
+
+ @Override
+ public String deleteTagMetasByLegacyTimeline(
+ @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit)
{
+ return "DELETE FROM "
+ + TAG_TABLE_NAME
+ + " WHERE tag_id IN (SELECT tag_id FROM "
+ + TAG_TABLE_NAME
+ + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT
#{limit})";
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TagMetadataObjectRelPostgreSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TagMetadataObjectRelPostgreSQLProvider.java
index 827098b6e2..9045ead8ea 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TagMetadataObjectRelPostgreSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TagMetadataObjectRelPostgreSQLProvider.java
@@ -198,4 +198,14 @@ public class TagMetadataObjectRelPostgreSQLProvider
extends TagMetadataObjectRel
+ " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name =
#{tagName}"
+ " AND te.deleted_at = 0 AND tm.deleted_at = 0 AND mm.deleted_at = 0";
}
+
+ @Override
+ public String deleteTagEntityRelsByLegacyTimeline(
+ @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit)
{
+ return "DELETE FROM "
+ + TagMetadataObjectRelMapper.TAG_METADATA_OBJECT_RELATION_TABLE_NAME
+ + " WHERE id IN (SELECT id FROM "
+ + TagMetadataObjectRelMapper.TAG_METADATA_OBJECT_RELATION_TABLE_NAME
+ + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT
#{limit})";
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TopicMetaPostgreSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TopicMetaPostgreSQLProvider.java
index 411662ef65..5ddcb4eb41 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TopicMetaPostgreSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/TopicMetaPostgreSQLProvider.java
@@ -22,6 +22,7 @@ import static
org.apache.gravitino.storage.relational.mapper.TopicMetaMapper.TAB
import
org.apache.gravitino.storage.relational.mapper.provider.base.TopicMetaBaseSQLProvider;
import org.apache.gravitino.storage.relational.po.TopicPO;
+import org.apache.ibatis.annotations.Param;
public class TopicMetaPostgreSQLProvider extends TopicMetaBaseSQLProvider {
@@ -93,4 +94,14 @@ public class TopicMetaPostgreSQLProvider extends
TopicMetaBaseSQLProvider {
+ " last_version = #{topicMeta.lastVersion},"
+ " deleted_at = #{topicMeta.deletedAt}";
}
+
+ @Override
+ public String deleteTopicMetasByLegacyTimeline(
+ @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit)
{
+ return "DELETE FROM "
+ + TABLE_NAME
+ + " WHERE topic_id IN (SELECT topic_id FROM "
+ + TABLE_NAME
+ + " WHERE deleted_at != 0 AND deleted_at < #{legacyTimeline} LIMIT
#{limit})";
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/UserMetaPostgreSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/UserMetaPostgreSQLProvider.java
index 84ab965582..e5c2427791 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/UserMetaPostgreSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/UserMetaPostgreSQLProvider.java
@@ -24,6 +24,7 @@ import static
org.apache.gravitino.storage.relational.mapper.UserRoleRelMapper.U
import
org.apache.gravitino.storage.relational.mapper.provider.base.UserMetaBaseSQLProvider;
import org.apache.gravitino.storage.relational.po.UserPO;
+import org.apache.ibatis.annotations.Param;
public class UserMetaPostgreSQLProvider extends UserMetaBaseSQLProvider {
@Override
@@ -95,4 +96,14 @@ public class UserMetaPostgreSQLProvider extends
UserMetaBaseSQLProvider {
+ " ut.metalake_id = #{metalakeId}"
+ " GROUP BY ut.user_id";
}
+
+ @Override
+ public String deleteUserMetasByLegacyTimeline(
+ @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit)
{
+ return "DELETE FROM "
+ + USER_TABLE_NAME
+ + " WHERE user_id IN (SELECT user_id FROM "
+ + USER_TABLE_NAME
+ + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT
#{limit})";
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/UserRoleRelPostgreSQLProvider.java
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/UserRoleRelPostgreSQLProvider.java
index 48a1be2233..c5f1d7124a 100644
---
a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/UserRoleRelPostgreSQLProvider.java
+++
b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/UserRoleRelPostgreSQLProvider.java
@@ -25,6 +25,7 @@ import static
org.apache.gravitino.storage.relational.mapper.UserRoleRelMapper.U
import java.util.List;
import
org.apache.gravitino.storage.relational.mapper.provider.base.UserRoleRelBaseSQLProvider;
import org.apache.gravitino.storage.relational.po.UserRoleRelPO;
+import org.apache.ibatis.annotations.Param;
public class UserRoleRelPostgreSQLProvider extends UserRoleRelBaseSQLProvider {
@Override
@@ -99,4 +100,14 @@ public class UserRoleRelPostgreSQLProvider extends
UserRoleRelBaseSQLProvider {
+ " deleted_at = VALUES(deleted_at)"
+ "</script>";
}
+
+ @Override
+ public String deleteUserRoleRelMetasByLegacyTimeline(
+ @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit)
{
+ return "DELETE FROM "
+ + USER_ROLE_RELATION_TABLE_NAME
+ + " WHERE id IN (SELECT id FROM "
+ + USER_ROLE_RELATION_TABLE_NAME
+ + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT
#{limit})";
+ }
}
diff --git
a/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java
b/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java
index a85f896281..c1a52c5154 100644
--- a/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java
+++ b/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java
@@ -37,6 +37,7 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import java.io.File;
import java.io.IOException;
+import java.lang.reflect.Field;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -83,6 +84,9 @@ import org.apache.gravitino.meta.TopicEntity;
import org.apache.gravitino.meta.UserEntity;
import org.apache.gravitino.rel.types.Type;
import org.apache.gravitino.rel.types.Types;
+import org.apache.gravitino.storage.relational.RelationalBackend;
+import org.apache.gravitino.storage.relational.RelationalEntityStore;
+import org.apache.gravitino.storage.relational.RelationalGarbageCollector;
import org.apache.gravitino.storage.relational.TestJDBCBackend;
import org.apache.gravitino.storage.relational.converters.H2ExceptionConverter;
import
org.apache.gravitino.storage.relational.converters.MySQLExceptionConverter;
@@ -136,7 +140,7 @@ public class TestEntityStorage {
if (type.equalsIgnoreCase("h2")) {
// The following properties are used to create the JDBC connection;
they are just for test,
// in the real world, they will be set automatically by the
configuration file if you set
- // ENTITY_RELATIONAL_STOR as EMBEDDED_ENTITY_RELATIONAL_STORE.
+ // ENTITY_RELATIONAL_STORE as EMBEDDED_ENTITY_RELATIONAL_STORE.
Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_URL))
.thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", DB_DIR));
Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_USER)).thenReturn("gravitino");
@@ -171,6 +175,15 @@ public class TestEntityStorage {
new PostgreSQLExceptionConverter(),
true);
+ RelationalEntityStore store =
+ (RelationalEntityStore)
EntityStoreFactory.createEntityStore(config);
+ store.initialize(config);
+ Field f = FieldUtils.getField(RelationalEntityStore.class, "backend",
true);
+ RelationalBackend backend = (RelationalBackend) f.get(store);
+ RelationalGarbageCollector garbageCollector =
+ new RelationalGarbageCollector(backend, config);
+ garbageCollector.collectAndClean();
+
} else {
throw new UnsupportedOperationException("Unsupported entity store
type: " + type);
}
