This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 40aab9d43a [#13765] feat(policy): Add TMS policy content fields and
REST round-trip (#13766)
40aab9d43a is described below
commit 40aab9d43ac2cd3389e1f28265f3fc4cc61290de
Author: MaSai <[email protected]>
AuthorDate: Tue Oct 13 09:56:21 2026 +0800
[#13765] feat(policy): Add TMS policy content fields and REST round-trip
(#13766)
### What changes were proposed in this pull request?
- Register built-in policy types `system_iceberg_rewrite_manifests` and
`system_iceberg_snapshot_expiration` with placeholder content
(`trigger-expr=false`)
- Wire REST/DTO `@JsonSubTypes` and OpenAPI schemas for those types
- `PolicyContents` factories and DTO converters for the placeholders
**Out of scope (deferred):** attaching `schedule` / `minIntervalMs` /
`jobOptions` onto built-in policy content, TMS shared field models,
nearest-policy resolution, and credential-shaped option validation.
Those land in follow-up PRs once content wiring is ready.
### Why are the changes needed?
First subtask of Epic #13764, narrowed to BuiltInType registration so
TMS can map task kinds without shipping unattached public field models.
Fix: #13765
### Does this PR introduce _any_ user-facing change?
Yes. Two new built-in policy types can be created via the policies API
with empty placeholder content. They do not auto-trigger until real
thresholds land.
### How was this patch tested?
```bash
./gradlew :api:test --tests
'org.apache.gravitino.policy.TestPolicyBuiltInType' \
:common:test --tests 'org.apache.gravitino.dto.policy.TestPolicyDTO' \
--tests
'org.apache.gravitino.dto.policy.TestRewriteManifestsAndSnapshotExpirationPolicyDTO'
-PskipITs
```
---------
Co-authored-by: Cursor <[email protected]>
---
.../policy/IcebergRewriteManifestsContent.java | 90 +++++++++++++++++
.../policy/IcebergSnapshotExpirationContent.java | 90 +++++++++++++++++
.../java/org/apache/gravitino/policy/Policy.java | 9 ++
.../apache/gravitino/policy/PolicyContents.java | 18 ++++
.../gravitino/policy/TestPolicyBuiltInType.java | 18 ++++
.../gravitino/dto/policy/PolicyContentDTO.java | 78 +++++++++++++++
.../org/apache/gravitino/dto/policy/PolicyDTO.java | 6 ++
.../dto/requests/PolicyCreateRequest.java | 6 ++
.../dto/requests/PolicyUpdateRequest.java | 6 ++
.../apache/gravitino/dto/util/DTOConverters.java | 14 +++
...iteManifestsAndSnapshotExpirationPolicyDTO.java | 68 +++++++++++++
.../org/apache/gravitino/policy/PolicyManager.java | 4 +
docs/open-api/policies.yaml | 109 ++++++++++++++++++++-
13 files changed, 512 insertions(+), 4 deletions(-)
diff --git
a/api/src/main/java/org/apache/gravitino/policy/IcebergRewriteManifestsContent.java
b/api/src/main/java/org/apache/gravitino/policy/IcebergRewriteManifestsContent.java
new file mode 100644
index 0000000000..53d59202cb
--- /dev/null
+++
b/api/src/main/java/org/apache/gravitino/policy/IcebergRewriteManifestsContent.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.policy;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+import org.apache.gravitino.MetadataObject;
+
+/**
+ * Placeholder built-in policy content for Iceberg manifest rewrite.
+ *
+ * <p>Configurable thresholds will be added when the rewrite-manifests policy
lands; this class
+ * registers the built-in type for TMS task mapping.
+ */
+public class IcebergRewriteManifestsContent implements PolicyContent {
+ /** Property key for strategy type. */
+ public static final String STRATEGY_TYPE_KEY = "strategy.type";
+ /** Property key for the job template. */
+ public static final String JOB_TEMPLATE_NAME_KEY = "job.template-name";
+ /** Rule key for the trigger expression. */
+ public static final String TRIGGER_EXPR_KEY = "trigger-expr";
+ /** Rule key for the score expression. */
+ public static final String SCORE_EXPR_KEY = "score-expr";
+ /** Strategy type for manifest rewrite. */
+ public static final String STRATEGY_TYPE_VALUE = "iceberg-rewrite-manifests";
+ /** Built-in job template name for rewrite manifests. */
+ public static final String JOB_TEMPLATE_NAME_VALUE =
"builtin-iceberg-rewrite-manifests";
+
+ private static final Set<MetadataObject.Type> SUPPORTED_OBJECT_TYPES =
+ ImmutableSet.of(
+ MetadataObject.Type.CATALOG, MetadataObject.Type.SCHEMA,
MetadataObject.Type.TABLE);
+
+ /** Default constructor for Jackson deserialization only. */
+ public IcebergRewriteManifestsContent() {}
+
+ @Override
+ public Set<MetadataObject.Type> supportedObjectTypes() {
+ return SUPPORTED_OBJECT_TYPES;
+ }
+
+ @Override
+ public Map<String, String> properties() {
+ return ImmutableMap.of(
+ STRATEGY_TYPE_KEY, STRATEGY_TYPE_VALUE, JOB_TEMPLATE_NAME_KEY,
JOB_TEMPLATE_NAME_VALUE);
+ }
+
+ @Override
+ public Map<String, Object> rules() {
+ Map<String, Object> rules = new LinkedHashMap<>();
+ // Placeholders must not auto-trigger until real thresholds land in a
follow-up.
+ rules.put(TRIGGER_EXPR_KEY, "false");
+ rules.put(SCORE_EXPR_KEY, "1");
+ return Collections.unmodifiableMap(rules);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return other instanceof IcebergRewriteManifestsContent;
+ }
+
+ @Override
+ public int hashCode() {
+ return IcebergRewriteManifestsContent.class.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "IcebergRewriteManifestsContent{}";
+ }
+}
diff --git
a/api/src/main/java/org/apache/gravitino/policy/IcebergSnapshotExpirationContent.java
b/api/src/main/java/org/apache/gravitino/policy/IcebergSnapshotExpirationContent.java
new file mode 100644
index 0000000000..d752d8f905
--- /dev/null
+++
b/api/src/main/java/org/apache/gravitino/policy/IcebergSnapshotExpirationContent.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.policy;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+import org.apache.gravitino.MetadataObject;
+
+/**
+ * Placeholder built-in policy content for Iceberg snapshot expiration.
+ *
+ * <p>Configurable thresholds will be added when the expire-snapshots policy
lands; this class
+ * registers the built-in type for TMS task mapping.
+ */
+public class IcebergSnapshotExpirationContent implements PolicyContent {
+ /** Property key for strategy type. */
+ public static final String STRATEGY_TYPE_KEY = "strategy.type";
+ /** Property key for the job template. */
+ public static final String JOB_TEMPLATE_NAME_KEY = "job.template-name";
+ /** Rule key for the trigger expression. */
+ public static final String TRIGGER_EXPR_KEY = "trigger-expr";
+ /** Rule key for the score expression. */
+ public static final String SCORE_EXPR_KEY = "score-expr";
+ /** Strategy type for snapshot expiration. */
+ public static final String STRATEGY_TYPE_VALUE =
"iceberg-snapshot-expiration";
+ /** Built-in job template name for expire snapshots. */
+ public static final String JOB_TEMPLATE_NAME_VALUE =
"builtin-iceberg-expire-snapshots";
+
+ private static final Set<MetadataObject.Type> SUPPORTED_OBJECT_TYPES =
+ ImmutableSet.of(
+ MetadataObject.Type.CATALOG, MetadataObject.Type.SCHEMA,
MetadataObject.Type.TABLE);
+
+ /** Default constructor for Jackson deserialization only. */
+ public IcebergSnapshotExpirationContent() {}
+
+ @Override
+ public Set<MetadataObject.Type> supportedObjectTypes() {
+ return SUPPORTED_OBJECT_TYPES;
+ }
+
+ @Override
+ public Map<String, String> properties() {
+ return ImmutableMap.of(
+ STRATEGY_TYPE_KEY, STRATEGY_TYPE_VALUE, JOB_TEMPLATE_NAME_KEY,
JOB_TEMPLATE_NAME_VALUE);
+ }
+
+ @Override
+ public Map<String, Object> rules() {
+ Map<String, Object> rules = new LinkedHashMap<>();
+ // Placeholders must not auto-trigger until real thresholds land in a
follow-up.
+ rules.put(TRIGGER_EXPR_KEY, "false");
+ rules.put(SCORE_EXPR_KEY, "1");
+ return Collections.unmodifiableMap(rules);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return other instanceof IcebergSnapshotExpirationContent;
+ }
+
+ @Override
+ public int hashCode() {
+ return IcebergSnapshotExpirationContent.class.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "IcebergSnapshotExpirationContent{}";
+ }
+}
diff --git a/api/src/main/java/org/apache/gravitino/policy/Policy.java
b/api/src/main/java/org/apache/gravitino/policy/Policy.java
index 80519d1157..c71a9c213b 100644
--- a/api/src/main/java/org/apache/gravitino/policy/Policy.java
+++ b/api/src/main/java/org/apache/gravitino/policy/Policy.java
@@ -47,6 +47,15 @@ public interface Policy extends Auditable {
BUILT_IN_TYPE_PREFIX + "iceberg_orphan_file_removal",
IcebergOrphanFileRemovalContent.class),
+ /** Iceberg manifest rewrite policy. */
+ ICEBERG_REWRITE_MANIFESTS(
+ BUILT_IN_TYPE_PREFIX + "iceberg_rewrite_manifests",
IcebergRewriteManifestsContent.class),
+
+ /** Iceberg snapshot expiration policy. */
+ ICEBERG_SNAPSHOT_EXPIRATION(
+ BUILT_IN_TYPE_PREFIX + "iceberg_snapshot_expiration",
+ IcebergSnapshotExpirationContent.class),
+
/** Built-in policy type for filtering rows of tagged tables. */
ROW_FILTER(BUILT_IN_TYPE_PREFIX + "row_filter", RowFilterContent.class),
diff --git a/api/src/main/java/org/apache/gravitino/policy/PolicyContents.java
b/api/src/main/java/org/apache/gravitino/policy/PolicyContents.java
index 501475c51f..8ca63b1f85 100644
--- a/api/src/main/java/org/apache/gravitino/policy/PolicyContents.java
+++ b/api/src/main/java/org/apache/gravitino/policy/PolicyContents.java
@@ -168,6 +168,24 @@ public class PolicyContents {
return new IcebergOrphanFileRemovalContent(olderThanDays, location,
dryRun);
}
+ /**
+ * Creates placeholder Iceberg rewrite-manifests policy content.
+ *
+ * @return rewrite-manifests policy content
+ */
+ public static PolicyContent icebergRewriteManifests() {
+ return new IcebergRewriteManifestsContent();
+ }
+
+ /**
+ * Creates placeholder Iceberg snapshot-expiration policy content.
+ *
+ * @return snapshot-expiration policy content
+ */
+ public static PolicyContent icebergSnapshotExpiration() {
+ return new IcebergSnapshotExpirationContent();
+ }
+
/**
* Creates a row-filter policy for tagged tables.
*
diff --git
a/api/src/test/java/org/apache/gravitino/policy/TestPolicyBuiltInType.java
b/api/src/test/java/org/apache/gravitino/policy/TestPolicyBuiltInType.java
index 9e49f61326..93df4a1d6a 100644
--- a/api/src/test/java/org/apache/gravitino/policy/TestPolicyBuiltInType.java
+++ b/api/src/test/java/org/apache/gravitino/policy/TestPolicyBuiltInType.java
@@ -29,6 +29,12 @@ public class TestPolicyBuiltInType {
Assertions.assertEquals(
Policy.BuiltInType.ICEBERG_COMPACTION,
Policy.BuiltInType.fromPolicyType("system_iceberg_compaction"));
+ Assertions.assertEquals(
+ Policy.BuiltInType.ICEBERG_REWRITE_MANIFESTS,
+ Policy.BuiltInType.fromPolicyType("system_iceberg_rewrite_manifests"));
+ Assertions.assertEquals(
+ Policy.BuiltInType.ICEBERG_SNAPSHOT_EXPIRATION,
+
Policy.BuiltInType.fromPolicyType("system_iceberg_snapshot_expiration"));
Assertions.assertEquals(
Policy.BuiltInType.ROW_FILTER,
Policy.BuiltInType.fromPolicyType("system_row_filter"));
Assertions.assertEquals(
@@ -55,6 +61,12 @@ public class TestPolicyBuiltInType {
void testBuiltInTypePolicyTypeValue() {
Assertions.assertEquals(
"system_iceberg_compaction",
Policy.BuiltInType.ICEBERG_COMPACTION.policyType());
+ Assertions.assertEquals(
+ "system_iceberg_rewrite_manifests",
+ Policy.BuiltInType.ICEBERG_REWRITE_MANIFESTS.policyType());
+ Assertions.assertEquals(
+ "system_iceberg_snapshot_expiration",
+ Policy.BuiltInType.ICEBERG_SNAPSHOT_EXPIRATION.policyType());
Assertions.assertEquals("system_row_filter",
Policy.BuiltInType.ROW_FILTER.policyType());
Assertions.assertEquals("system_column_mask",
Policy.BuiltInType.COLUMN_MASK.policyType());
}
@@ -63,6 +75,12 @@ public class TestPolicyBuiltInType {
void testBuiltInTypeContentClass() {
Assertions.assertEquals(
IcebergDataCompactionContent.class,
Policy.BuiltInType.ICEBERG_COMPACTION.contentClass());
+ Assertions.assertEquals(
+ IcebergRewriteManifestsContent.class,
+ Policy.BuiltInType.ICEBERG_REWRITE_MANIFESTS.contentClass());
+ Assertions.assertEquals(
+ IcebergSnapshotExpirationContent.class,
+ Policy.BuiltInType.ICEBERG_SNAPSHOT_EXPIRATION.contentClass());
Assertions.assertEquals(RowFilterContent.class,
Policy.BuiltInType.ROW_FILTER.contentClass());
Assertions.assertEquals(ColumnMaskContent.class,
Policy.BuiltInType.COLUMN_MASK.contentClass());
}
diff --git
a/common/src/main/java/org/apache/gravitino/dto/policy/PolicyContentDTO.java
b/common/src/main/java/org/apache/gravitino/dto/policy/PolicyContentDTO.java
index 65fae911ad..4ac19d7be4 100644
--- a/common/src/main/java/org/apache/gravitino/dto/policy/PolicyContentDTO.java
+++ b/common/src/main/java/org/apache/gravitino/dto/policy/PolicyContentDTO.java
@@ -304,6 +304,84 @@ public interface PolicyContentDTO extends PolicyContent {
}
}
+ /** Placeholder DTO for Iceberg rewrite-manifests policy content. */
+ @EqualsAndHashCode
+ @ToString
+ class IcebergRewriteManifestsContentDTO implements PolicyContentDTO {
+ private IcebergRewriteManifestsContentDTO() {}
+
+ /**
+ * @return empty rewrite-manifests content DTO
+ */
+ public static IcebergRewriteManifestsContentDTO instance() {
+ return new IcebergRewriteManifestsContentDTO();
+ }
+
+ @Override
+ public Set<MetadataObject.Type> supportedObjectTypes() {
+ return toDomainContent().supportedObjectTypes();
+ }
+
+ @Override
+ public Map<String, String> properties() {
+ return toDomainContent().properties();
+ }
+
+ @Override
+ public Map<String, Object> rules() {
+ return toDomainContent().rules();
+ }
+
+ @Override
+ public void validate() {
+ PolicyContentDTO.super.validate();
+ toDomainContent().validate();
+ }
+
+ private PolicyContent toDomainContent() {
+ return PolicyContents.icebergRewriteManifests();
+ }
+ }
+
+ /** Placeholder DTO for Iceberg snapshot-expiration policy content. */
+ @EqualsAndHashCode
+ @ToString
+ class IcebergSnapshotExpirationContentDTO implements PolicyContentDTO {
+ private IcebergSnapshotExpirationContentDTO() {}
+
+ /**
+ * @return empty snapshot-expiration content DTO
+ */
+ public static IcebergSnapshotExpirationContentDTO instance() {
+ return new IcebergSnapshotExpirationContentDTO();
+ }
+
+ @Override
+ public Set<MetadataObject.Type> supportedObjectTypes() {
+ return toDomainContent().supportedObjectTypes();
+ }
+
+ @Override
+ public Map<String, String> properties() {
+ return toDomainContent().properties();
+ }
+
+ @Override
+ public Map<String, Object> rules() {
+ return toDomainContent().rules();
+ }
+
+ @Override
+ public void validate() {
+ PolicyContentDTO.super.validate();
+ toDomainContent().validate();
+ }
+
+ private PolicyContent toDomainContent() {
+ return PolicyContents.icebergSnapshotExpiration();
+ }
+ }
+
/** Represents typed row-filter policy content. */
@EqualsAndHashCode
@ToString
diff --git
a/common/src/main/java/org/apache/gravitino/dto/policy/PolicyDTO.java
b/common/src/main/java/org/apache/gravitino/dto/policy/PolicyDTO.java
index 2f75784868..8ceff3a327 100644
--- a/common/src/main/java/org/apache/gravitino/dto/policy/PolicyDTO.java
+++ b/common/src/main/java/org/apache/gravitino/dto/policy/PolicyDTO.java
@@ -61,6 +61,12 @@ public class PolicyDTO implements Policy {
@JsonSubTypes.Type(
value = PolicyContentDTO.IcebergOrphanFileRemovalContentDTO.class,
name = "system_iceberg_orphan_file_removal"),
+ @JsonSubTypes.Type(
+ value = PolicyContentDTO.IcebergRewriteManifestsContentDTO.class,
+ name = "system_iceberg_rewrite_manifests"),
+ @JsonSubTypes.Type(
+ value = PolicyContentDTO.IcebergSnapshotExpirationContentDTO.class,
+ name = "system_iceberg_snapshot_expiration"),
@JsonSubTypes.Type(
value = PolicyContentDTO.RowFilterContentDTO.class,
name = "system_row_filter"),
diff --git
a/common/src/main/java/org/apache/gravitino/dto/requests/PolicyCreateRequest.java
b/common/src/main/java/org/apache/gravitino/dto/requests/PolicyCreateRequest.java
index d5ef0430a7..96a2357289 100644
---
a/common/src/main/java/org/apache/gravitino/dto/requests/PolicyCreateRequest.java
+++
b/common/src/main/java/org/apache/gravitino/dto/requests/PolicyCreateRequest.java
@@ -65,6 +65,12 @@ public class PolicyCreateRequest implements RESTRequest {
@JsonSubTypes.Type(
value = PolicyContentDTO.IcebergOrphanFileRemovalContentDTO.class,
name = "system_iceberg_orphan_file_removal"),
+ @JsonSubTypes.Type(
+ value = PolicyContentDTO.IcebergRewriteManifestsContentDTO.class,
+ name = "system_iceberg_rewrite_manifests"),
+ @JsonSubTypes.Type(
+ value = PolicyContentDTO.IcebergSnapshotExpirationContentDTO.class,
+ name = "system_iceberg_snapshot_expiration"),
@JsonSubTypes.Type(
value = PolicyContentDTO.RowFilterContentDTO.class,
name = "system_row_filter"),
diff --git
a/common/src/main/java/org/apache/gravitino/dto/requests/PolicyUpdateRequest.java
b/common/src/main/java/org/apache/gravitino/dto/requests/PolicyUpdateRequest.java
index 28a37aa25f..265296110b 100644
---
a/common/src/main/java/org/apache/gravitino/dto/requests/PolicyUpdateRequest.java
+++
b/common/src/main/java/org/apache/gravitino/dto/requests/PolicyUpdateRequest.java
@@ -146,6 +146,12 @@ public interface PolicyUpdateRequest extends RESTRequest {
@JsonSubTypes.Type(
value = PolicyContentDTO.IcebergOrphanFileRemovalContentDTO.class,
name = "system_iceberg_orphan_file_removal"),
+ @JsonSubTypes.Type(
+ value = PolicyContentDTO.IcebergRewriteManifestsContentDTO.class,
+ name = "system_iceberg_rewrite_manifests"),
+ @JsonSubTypes.Type(
+ value = PolicyContentDTO.IcebergSnapshotExpirationContentDTO.class,
+ name = "system_iceberg_snapshot_expiration"),
@JsonSubTypes.Type(
value = PolicyContentDTO.RowFilterContentDTO.class,
name = "system_row_filter"),
diff --git
a/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java
b/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java
index fd45252a6a..ebf54be030 100644
--- a/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java
+++ b/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java
@@ -107,6 +107,8 @@ import org.apache.gravitino.model.ModelVersion;
import org.apache.gravitino.policy.ColumnMaskContent;
import org.apache.gravitino.policy.IcebergDataCompactionContent;
import org.apache.gravitino.policy.IcebergOrphanFileRemovalContent;
+import org.apache.gravitino.policy.IcebergRewriteManifestsContent;
+import org.apache.gravitino.policy.IcebergSnapshotExpirationContent;
import org.apache.gravitino.policy.PolicyContent;
import org.apache.gravitino.policy.PolicyContents;
import org.apache.gravitino.policy.RowFilterContent;
@@ -686,6 +688,12 @@ public class DTOConverters {
.withDryRun(content.dryRun())
.build();
}
+ if (policyContent instanceof IcebergRewriteManifestsContent) {
+ return PolicyContentDTO.IcebergRewriteManifestsContentDTO.instance();
+ }
+ if (policyContent instanceof IcebergSnapshotExpirationContent) {
+ return PolicyContentDTO.IcebergSnapshotExpirationContentDTO.instance();
+ }
if (policyContent instanceof IcebergDataCompactionContent) {
IcebergDataCompactionContent icebergCompactionContent =
(IcebergDataCompactionContent) policyContent;
@@ -1565,6 +1573,12 @@ public class DTOConverters {
return PolicyContents.icebergOrphanFileRemoval(
content.olderThanDays(), content.location(), content.dryRun());
}
+ if (policyContentDTO instanceof
PolicyContentDTO.IcebergRewriteManifestsContentDTO) {
+ return PolicyContents.icebergRewriteManifests();
+ }
+ if (policyContentDTO instanceof
PolicyContentDTO.IcebergSnapshotExpirationContentDTO) {
+ return PolicyContents.icebergSnapshotExpiration();
+ }
if (policyContentDTO instanceof
PolicyContentDTO.IcebergCompactionContentDTO) {
PolicyContentDTO.IcebergCompactionContentDTO icebergCompactionContentDTO
=
(PolicyContentDTO.IcebergCompactionContentDTO) policyContentDTO;
diff --git
a/common/src/test/java/org/apache/gravitino/dto/policy/TestRewriteManifestsAndSnapshotExpirationPolicyDTO.java
b/common/src/test/java/org/apache/gravitino/dto/policy/TestRewriteManifestsAndSnapshotExpirationPolicyDTO.java
new file mode 100644
index 0000000000..2087eef48a
--- /dev/null
+++
b/common/src/test/java/org/apache/gravitino/dto/policy/TestRewriteManifestsAndSnapshotExpirationPolicyDTO.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.dto.policy;
+
+import org.apache.gravitino.dto.requests.PolicyCreateRequest;
+import org.apache.gravitino.dto.requests.PolicyUpdateRequest;
+import org.apache.gravitino.dto.util.DTOConverters;
+import org.apache.gravitino.json.JsonUtils;
+import org.apache.gravitino.policy.PolicyContent;
+import org.apache.gravitino.policy.PolicyContents;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class TestRewriteManifestsAndSnapshotExpirationPolicyDTO {
+
+ @Test
+ void rewriteManifestsCreateAndUpdateRequestTypedContent() throws Exception {
+ assertCreateAndUpdateRoundTrip(
+ "system_iceberg_rewrite_manifests",
PolicyContents.icebergRewriteManifests());
+ }
+
+ @Test
+ void snapshotExpirationCreateAndUpdateRequestTypedContent() throws Exception
{
+ assertCreateAndUpdateRoundTrip(
+ "system_iceberg_snapshot_expiration",
PolicyContents.icebergSnapshotExpiration());
+ }
+
+ private static void assertCreateAndUpdateRoundTrip(String policyType,
PolicyContent expected)
+ throws Exception {
+ PolicyCreateRequest create =
+ JsonUtils.objectMapper()
+ .readValue(
+ "{\"name\":\"p\",\"policyType\":\""
+ + policyType
+ + "\",\"enabled\":true,\"content\":{}}",
+ PolicyCreateRequest.class);
+ create.validate();
+ Assertions.assertEquals(expected,
DTOConverters.fromDTO(create.getPolicyContent()));
+
+ PolicyUpdateRequest update =
+ JsonUtils.objectMapper()
+ .readValue(
+ "{\"@type\":\"updateContent\",\"policyType\":\""
+ + policyType
+ + "\",\"newContent\":{}}",
+ PolicyUpdateRequest.class);
+ update.validate();
+ PolicyUpdateRequest.UpdatePolicyContentRequest contentUpdate =
+ (PolicyUpdateRequest.UpdatePolicyContentRequest) update;
+ Assertions.assertEquals(expected,
DTOConverters.fromDTO(contentUpdate.getNewContent()));
+ }
+}
diff --git a/core/src/main/java/org/apache/gravitino/policy/PolicyManager.java
b/core/src/main/java/org/apache/gravitino/policy/PolicyManager.java
index b742ef98e7..5c10ac000e 100644
--- a/core/src/main/java/org/apache/gravitino/policy/PolicyManager.java
+++ b/core/src/main/java/org/apache/gravitino/policy/PolicyManager.java
@@ -115,6 +115,8 @@ public class PolicyManager implements PolicyDispatcher {
throws PolicyAlreadyExistsException {
NameIdentifier metalakeIdent = NameIdentifierUtil.ofMetalake(metalake);
checkMetalake(metalakeIdent, entityStore);
+ content.validate();
+
PolicyEntity policyEntity =
PolicyEntity.builder()
.withId(idGenerator.nextId())
@@ -338,6 +340,8 @@ public class PolicyManager implements PolicyDispatcher {
}
}
+ newContent.validate();
+
PolicyEntity.Builder builder = newPolicyBuilder(policyEntity);
builder.withName(newName);
builder.withComment(newComment);
diff --git a/docs/open-api/policies.yaml b/docs/open-api/policies.yaml
index 3a1a943fb3..c8089274f0 100644
--- a/docs/open-api/policies.yaml
+++ b/docs/open-api/policies.yaml
@@ -336,7 +336,7 @@ components:
IcebergCompactionContent:
type: object
- description: Built-in Iceberg compaction thresholds, scoring weights,
and job options.
+ description: Built-in Iceberg compaction thresholds, scoring weights,
and rewrite options.
properties:
minDataFileMse:
type: integer
@@ -363,6 +363,15 @@ components:
format: int64
minimum: 1
default: 50
+ rewriteStrategy:
+ type: string
+ enum: [binpack, sort]
+ default: binpack
+ description: Iceberg rewrite_data_files top-level strategy.
+ sortOrder:
+ type: string
+ default: ""
+ description: Iceberg rewrite_data_files top-level sort_order.
Required when rewriteStrategy is sort; must be empty for binpack.
rewriteOptions:
type: object
default: {}
@@ -439,6 +448,64 @@ components:
content:
$ref: '#/components/schemas/IcebergOrphanFileRemovalContent'
+ IcebergRewriteManifestsContent:
+ type: object
+ description: Placeholder content for system_iceberg_rewrite_manifests.
Configurable thresholds land in a follow-up.
+ additionalProperties: false
+
+ IcebergRewriteManifestsPolicy:
+ allOf:
+ - $ref: '#/components/schemas/PolicyBase'
+ - type: object
+ required: [content]
+ properties:
+ policyType:
+ type: string
+ enum: [system_iceberg_rewrite_manifests]
+ content:
+ $ref: '#/components/schemas/IcebergRewriteManifestsContent'
+
+ IcebergRewriteManifestsPolicyCreateRequest:
+ allOf:
+ - $ref: '#/components/schemas/PolicyCreateRequestBase'
+ - type: object
+ required: [content]
+ properties:
+ policyType:
+ type: string
+ enum: [system_iceberg_rewrite_manifests]
+ content:
+ $ref: '#/components/schemas/IcebergRewriteManifestsContent'
+
+ IcebergSnapshotExpirationContent:
+ type: object
+ description: Placeholder content for system_iceberg_snapshot_expiration.
Configurable thresholds land in a follow-up.
+ additionalProperties: false
+
+ IcebergSnapshotExpirationPolicy:
+ allOf:
+ - $ref: '#/components/schemas/PolicyBase'
+ - type: object
+ required: [content]
+ properties:
+ policyType:
+ type: string
+ enum: [system_iceberg_snapshot_expiration]
+ content:
+ $ref: '#/components/schemas/IcebergSnapshotExpirationContent'
+
+ IcebergSnapshotExpirationPolicyCreateRequest:
+ allOf:
+ - $ref: '#/components/schemas/PolicyCreateRequestBase'
+ - type: object
+ required: [content]
+ properties:
+ policyType:
+ type: string
+ enum: [system_iceberg_snapshot_expiration]
+ content:
+ $ref: '#/components/schemas/IcebergSnapshotExpirationContent'
+
RowFilterContent:
type: object
description: Expression content for a built-in row-filter policy.
@@ -593,7 +660,7 @@ components:
policyType:
type: string
not:
- enum: [system_iceberg_compaction,
system_iceberg_orphan_file_removal, system_row_filter, system_column_mask]
+ enum: [system_iceberg_compaction,
system_iceberg_orphan_file_removal, system_iceberg_rewrite_manifests,
system_iceberg_snapshot_expiration, system_row_filter, system_column_mask]
content:
$ref: '#/components/schemas/CustomPolicyContent'
@@ -604,6 +671,8 @@ components:
- $ref: '#/components/schemas/CustomPolicy'
- $ref: '#/components/schemas/IcebergOrphanFileRemovalPolicy'
- $ref: '#/components/schemas/IcebergCompactionPolicy'
+ - $ref: '#/components/schemas/IcebergRewriteManifestsPolicy'
+ - $ref: '#/components/schemas/IcebergSnapshotExpirationPolicy'
- $ref: '#/components/schemas/RowFilterPolicy'
- $ref: '#/components/schemas/ColumnMaskPolicy'
discriminator:
@@ -612,6 +681,8 @@ components:
custom: '#/components/schemas/CustomPolicy'
system_iceberg_orphan_file_removal:
'#/components/schemas/IcebergOrphanFileRemovalPolicy'
system_iceberg_compaction:
'#/components/schemas/IcebergCompactionPolicy'
+ system_iceberg_rewrite_manifests:
'#/components/schemas/IcebergRewriteManifestsPolicy'
+ system_iceberg_snapshot_expiration:
'#/components/schemas/IcebergSnapshotExpirationPolicy'
system_row_filter: '#/components/schemas/RowFilterPolicy'
system_column_mask: '#/components/schemas/ColumnMaskPolicy'
@@ -684,7 +755,7 @@ components:
policyType:
type: string
not:
- enum: [system_iceberg_compaction,
system_iceberg_orphan_file_removal, system_row_filter, system_column_mask]
+ enum: [system_iceberg_compaction,
system_iceberg_orphan_file_removal, system_iceberg_rewrite_manifests,
system_iceberg_snapshot_expiration, system_row_filter, system_column_mask]
content:
$ref: '#/components/schemas/CustomPolicyContent'
@@ -694,6 +765,8 @@ components:
oneOf:
- $ref: '#/components/schemas/CustomPolicyCreateRequest'
- $ref:
'#/components/schemas/IcebergOrphanFileRemovalPolicyCreateRequest'
+ - $ref:
'#/components/schemas/IcebergRewriteManifestsPolicyCreateRequest'
+ - $ref:
'#/components/schemas/IcebergSnapshotExpirationPolicyCreateRequest'
- $ref: '#/components/schemas/IcebergCompactionPolicyCreateRequest'
- $ref: '#/components/schemas/RowFilterPolicyCreateRequest'
- $ref: '#/components/schemas/ColumnMaskPolicyCreateRequest'
@@ -702,6 +775,8 @@ components:
mapping:
custom: '#/components/schemas/CustomPolicyCreateRequest'
system_iceberg_orphan_file_removal:
'#/components/schemas/IcebergOrphanFileRemovalPolicyCreateRequest'
+ system_iceberg_rewrite_manifests:
'#/components/schemas/IcebergRewriteManifestsPolicyCreateRequest'
+ system_iceberg_snapshot_expiration:
'#/components/schemas/IcebergSnapshotExpirationPolicyCreateRequest'
system_iceberg_compaction:
'#/components/schemas/IcebergCompactionPolicyCreateRequest'
system_row_filter:
'#/components/schemas/RowFilterPolicyCreateRequest'
system_column_mask:
'#/components/schemas/ColumnMaskPolicyCreateRequest'
@@ -794,7 +869,7 @@ components:
policyType:
type: string
not:
- enum: [system_iceberg_compaction,
system_iceberg_orphan_file_removal, system_row_filter, system_column_mask]
+ enum: [system_iceberg_compaction,
system_iceberg_orphan_file_removal, system_iceberg_rewrite_manifests,
system_iceberg_snapshot_expiration, system_row_filter, system_column_mask]
newContent:
$ref: '#/components/schemas/CustomPolicyContent'
@@ -820,6 +895,28 @@ components:
newContent:
$ref: '#/components/schemas/IcebergOrphanFileRemovalContent'
+ UpdateIcebergRewriteManifestsPolicyContentRequest:
+ allOf:
+ - $ref: '#/components/schemas/UpdatePolicyContentRequestBase'
+ - type: object
+ properties:
+ policyType:
+ type: string
+ enum: [system_iceberg_rewrite_manifests]
+ newContent:
+ $ref: '#/components/schemas/IcebergRewriteManifestsContent'
+
+ UpdateIcebergSnapshotExpirationPolicyContentRequest:
+ allOf:
+ - $ref: '#/components/schemas/UpdatePolicyContentRequestBase'
+ - type: object
+ properties:
+ policyType:
+ type: string
+ enum: [system_iceberg_snapshot_expiration]
+ newContent:
+ $ref: '#/components/schemas/IcebergSnapshotExpirationContent'
+
UpdateRowFilterPolicyContentRequest:
allOf:
- $ref: '#/components/schemas/UpdatePolicyContentRequestBase'
@@ -848,6 +945,8 @@ components:
- $ref: '#/components/schemas/UpdateCustomPolicyContentRequest'
- $ref:
'#/components/schemas/UpdateIcebergCompactionPolicyContentRequest'
- $ref:
'#/components/schemas/UpdateIcebergOrphanFileRemovalPolicyContentRequest'
+ - $ref:
'#/components/schemas/UpdateIcebergRewriteManifestsPolicyContentRequest'
+ - $ref:
'#/components/schemas/UpdateIcebergSnapshotExpirationPolicyContentRequest'
- $ref: '#/components/schemas/UpdateRowFilterPolicyContentRequest'
- $ref: '#/components/schemas/UpdateColumnMaskPolicyContentRequest'
discriminator:
@@ -856,6 +955,8 @@ components:
custom: '#/components/schemas/UpdateCustomPolicyContentRequest'
system_iceberg_compaction:
'#/components/schemas/UpdateIcebergCompactionPolicyContentRequest'
system_iceberg_orphan_file_removal:
'#/components/schemas/UpdateIcebergOrphanFileRemovalPolicyContentRequest'
+ system_iceberg_rewrite_manifests:
'#/components/schemas/UpdateIcebergRewriteManifestsPolicyContentRequest'
+ system_iceberg_snapshot_expiration:
'#/components/schemas/UpdateIcebergSnapshotExpirationPolicyContentRequest'
system_row_filter:
'#/components/schemas/UpdateRowFilterPolicyContentRequest'
system_column_mask:
'#/components/schemas/UpdateColumnMaskPolicyContentRequest'
example: {
