This is an automated email from the ASF dual-hosted git repository. mchades pushed a commit to branch codex/issue-13540-ossie-import-export in repository https://gitbox.apache.org/repos/asf/gravitino.git
commit da4c2503cd5fc5cf4aedd2d521a987f6f45a52f6 Author: mchades <[email protected]> AuthorDate: Mon Sep 28 21:58:28 2026 +0800 [#13540] feat(semantic-model): Add Ossie import and export --- .../apache/gravitino/semantic/SemanticModel.java | 6 + core/build.gradle.kts | 2 + .../catalog/SemanticModelOperationDispatcher.java | 34 +- .../OssieSemanticModelDocumentConverter.java | 715 +++++++++++++++++++++ .../semantic/SemanticModelPropertiesMetadata.java | 64 ++ .../TestSemanticModelOperationDispatcher.java | 51 +- .../TestOssieSemanticModelDocumentConverter.java | 405 ++++++++++++ .../TestSemanticModelPropertiesMetadata.java | 46 ++ .../server/web/rest/SemanticModelOperations.java | 155 ++++- .../web/rest/TestSemanticModelOperations.java | 171 +++++ 10 files changed, 1634 insertions(+), 15 deletions(-) diff --git a/api/src/main/java/org/apache/gravitino/semantic/SemanticModel.java b/api/src/main/java/org/apache/gravitino/semantic/SemanticModel.java index f0650155be..1fac8ce883 100644 --- a/api/src/main/java/org/apache/gravitino/semantic/SemanticModel.java +++ b/api/src/main/java/org/apache/gravitino/semantic/SemanticModel.java @@ -32,6 +32,12 @@ import org.apache.gravitino.annotation.Evolving; @Evolving public interface SemanticModel extends Auditable { + /** The property key for the Apache Ossie document version. */ + String PROPERTY_OSSIE_VERSION = "ossie-version"; + + /** The default Apache Ossie document version. */ + String DEFAULT_OSSIE_VERSION = "0.2.0.dev0"; + /** * Returns the Semantic Model name. * diff --git a/core/build.gradle.kts b/core/build.gradle.kts index a8511341f0..7d88be97a0 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -43,6 +43,8 @@ dependencies { implementation(libs.concurrent.trees) implementation(libs.guava) implementation(libs.h2db) + implementation(libs.jackson.databind) + implementation(libs.jackson.dataformat.yaml) implementation(libs.jackson.jaxrs.json.provider) // This is required by lance implementation(libs.lance) { exclude(group = "com.fasterxml.jackson.core", module = "*") // provided by gravitino diff --git a/core/src/main/java/org/apache/gravitino/catalog/SemanticModelOperationDispatcher.java b/core/src/main/java/org/apache/gravitino/catalog/SemanticModelOperationDispatcher.java index 4608645a18..132a195830 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/SemanticModelOperationDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/catalog/SemanticModelOperationDispatcher.java @@ -18,7 +18,12 @@ */ package org.apache.gravitino.catalog; +import static org.apache.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForAlter; +import static org.apache.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForCreate; +import static org.apache.gravitino.semantic.SemanticModel.PROPERTY_OSSIE_VERSION; + import com.google.common.base.Preconditions; +import java.util.HashMap; import java.util.Map; import javax.annotation.Nullable; import org.apache.gravitino.Catalog; @@ -33,12 +38,16 @@ import org.apache.gravitino.secret.SecretManager; import org.apache.gravitino.semantic.SemanticModel; import org.apache.gravitino.semantic.SemanticModelChange; import org.apache.gravitino.semantic.SemanticModelDefinition; +import org.apache.gravitino.semantic.SemanticModelPropertiesMetadata; import org.apache.gravitino.storage.IdGenerator; /** Dispatches always-managed Semantic Model operations to Gravitino's EntityStore. */ public class SemanticModelOperationDispatcher extends OperationDispatcher implements SemanticModelDispatcher { + private static final SemanticModelPropertiesMetadata PROPERTIES_METADATA = + new SemanticModelPropertiesMetadata(); + private final CatalogManager catalogManager; private final SchemaDispatcher schemaDispatcher; private final ManagedSemanticModelOperations managedOperations; @@ -94,10 +103,16 @@ public class SemanticModelOperationDispatcher extends OperationDispatcher throws NoSuchSchemaException, SemanticModelAlreadyExistsException, IllegalSemanticModelException { Preconditions.checkArgument(properties != null, "Properties must not be null"); + validatePropertyForCreate(PROPERTIES_METADATA, properties); + Map<String, String> effectiveProperties = new HashMap<>(properties); + effectiveProperties.putIfAbsent( + PROPERTY_OSSIE_VERSION, + (String) PROPERTIES_METADATA.getDefaultValue(PROPERTY_OSSIE_VERSION)); + checkRelationalCatalog(ident.namespace()); NameIdentifier schemaIdent = schemaIdentifier(ident); schemaDispatcher.loadSchema(schemaIdent); - return managedOperations.createSemanticModel(ident, comment, definition, properties); + return managedOperations.createSemanticModel(ident, comment, definition, effectiveProperties); } @Override @@ -107,6 +122,7 @@ public class SemanticModelOperationDispatcher extends OperationDispatcher if (changes == null || changes.length == 0) { throw new IllegalSemanticModelException("At least one Semantic Model change is required"); } + validatePropertyChanges(changes); checkRelationalCatalog(ident.namespace()); NameIdentifier schemaIdent = schemaIdentifier(ident); if (!schemaDispatcher.schemaExists(schemaIdent)) { @@ -138,4 +154,20 @@ public class SemanticModelOperationDispatcher extends OperationDispatcher private static NameIdentifier schemaIdentifier(NameIdentifier ident) { return NameIdentifier.of(ident.namespace().levels()); } + + private static void validatePropertyChanges(SemanticModelChange[] changes) { + Map<String, String> upserts = new HashMap<>(); + Map<String, String> deletes = new HashMap<>(); + for (SemanticModelChange change : changes) { + if (change instanceof SemanticModelChange.SetProperty) { + SemanticModelChange.SetProperty setProperty = (SemanticModelChange.SetProperty) change; + upserts.put(setProperty.getProperty(), setProperty.getValue()); + } else if (change instanceof SemanticModelChange.RemoveProperty) { + SemanticModelChange.RemoveProperty removeProperty = + (SemanticModelChange.RemoveProperty) change; + deletes.put(removeProperty.getProperty(), removeProperty.getProperty()); + } + } + validatePropertyForAlter(PROPERTIES_METADATA, upserts, deletes); + } } diff --git a/core/src/main/java/org/apache/gravitino/semantic/OssieSemanticModelDocumentConverter.java b/core/src/main/java/org/apache/gravitino/semantic/OssieSemanticModelDocumentConverter.java new file mode 100644 index 0000000000..98343ab19f --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/semantic/OssieSemanticModelDocumentConverter.java @@ -0,0 +1,715 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.semantic; + +import static org.apache.gravitino.semantic.SemanticModel.DEFAULT_OSSIE_VERSION; +import static org.apache.gravitino.semantic.SemanticModel.PROPERTY_OSSIE_VERSION; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.core.StreamReadFeature; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import javax.annotation.Nullable; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.dto.requests.SemanticModelCreateRequest; +import org.apache.gravitino.dto.semantic.SemanticModelDefinitionDTO; +import org.apache.gravitino.exceptions.IllegalSemanticModelException; + +/** Converts between standalone Apache Ossie documents and Gravitino Semantic Models. */ +public final class OssieSemanticModelDocumentConverter { + + /** Maximum accepted document length in characters. */ + public static final int MAX_DOCUMENT_LENGTH = 4 * 1024 * 1024; + + private static final String GRAVITINO_VENDOR = "GRAVITINO"; + private static final String INTERCHANGE_MARKER = "_apache_gravitino_interchange"; + private static final int INTERCHANGE_MARKER_VERSION = 1; + private static final int MAX_NESTING_DEPTH = 100; + + private static final Set<String> ROOT_PROPERTIES = + Set.of( + "version", + "name", + "description", + "ai_context", + "datasets", + "relationships", + "metrics", + "custom_extensions"); + private static final Set<String> DATASET_PROPERTIES = + Set.of( + "name", + "source", + "primary_key", + "unique_keys", + "description", + "ai_context", + "fields", + "custom_extensions"); + private static final Set<String> FIELD_PROPERTIES = + Set.of( + "name", + "expression", + "dimension", + "label", + "description", + "datatype", + "ai_context", + "custom_extensions"); + private static final Set<String> RELATIONSHIP_PROPERTIES = + Set.of("name", "from", "to", "from_columns", "to_columns", "ai_context", "custom_extensions"); + private static final Set<String> METRIC_PROPERTIES = + Set.of("name", "expression", "description", "datatype", "ai_context", "custom_extensions"); + private static final Set<String> EXPRESSION_PROPERTIES = Set.of("dialects"); + private static final Set<String> DIALECT_EXPRESSION_PROPERTIES = Set.of("dialect", "expression"); + private static final Set<String> DIMENSION_PROPERTIES = Set.of("is_time"); + private static final Set<String> CUSTOM_EXTENSION_PROPERTIES = Set.of("vendor_name", "data"); + private static final Set<String> OSSIE_DIALECTS = + Set.of( + "ANSI_SQL", + "SNOWFLAKE", + "MDX", + "TABLEAU", + "DATABRICKS", + "MAQL", + "BIGQUERY", + "SIGMA", + "THOUGHTSPOT", + "DAX", + "OSSIE_SQL_2026"); + + private static final ObjectMapper JSON_MAPPER = createJsonMapper(); + private static final ObjectMapper YAML_MAPPER = createYamlMapper(); + + /** Supported Ossie document serialization formats. */ + public enum Format { + /** YAML serialization. */ + YAML, + /** JSON serialization. */ + JSON + } + + private OssieSemanticModelDocumentConverter() {} + + /** + * Converts one standalone Apache Ossie YAML or JSON document into a create request. + * + * @param document The standalone Ossie document. + * @return The converted Semantic Model create request. + * @throws IllegalSemanticModelException If the document cannot be parsed or represented by + * Gravitino. + */ + public static SemanticModelCreateRequest importDocument(String document) { + ObjectNode root = parseDocument(document); + return toCreateRequest(root); + } + + /** + * Exports one Gravitino Semantic Model as a standalone Apache Ossie document. + * + * @param semanticModel The Semantic Model to export. + * @param format The requested serialization format. + * @return The serialized Ossie document. + * @throws IllegalSemanticModelException If the model cannot be represented as a valid Ossie + * document. + */ + public static String exportDocument(SemanticModel semanticModel, Format format) { + Objects.requireNonNull(semanticModel, "semanticModel must not be null"); + Objects.requireNonNull(format, "format must not be null"); + + ObjectNode definition = + JSON_MAPPER.valueToTree( + SemanticModelDefinitionDTO.fromDefinition(semanticModel.definition())); + transformNativeDefinition(definition); + + ObjectNode root = JSON_MAPPER.createObjectNode(); + root.put("version", ossieVersion(semanticModel.properties())); + root.put("name", semanticModel.name()); + if (semanticModel.comment() != null) { + root.put("description", semanticModel.comment()); + } + definition.fields().forEachRemaining(entry -> root.set(entry.getKey(), entry.getValue())); + stashProperties(root, semanticModel.properties()); + + // Validate the generated document through the same conversion path used for imports. + toCreateRequest(root.deepCopy()); + + try { + if (format == Format.JSON) { + return JSON_MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(root) + "\n"; + } + return YAML_MAPPER.writeValueAsString(root); + } catch (JsonProcessingException e) { + throw new IllegalSemanticModelException( + e, "Cannot serialize Apache Ossie document: %s", e.getOriginalMessage()); + } + } + + private static ObjectMapper createJsonMapper() { + return JsonMapper.builder() + .enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS) + .enable(DeserializationFeature.USE_BIG_INTEGER_FOR_INTS) + .build() + .setSerializationInclusion(JsonInclude.Include.NON_NULL); + } + + private static ObjectMapper createYamlMapper() { + YAMLFactory factory = + YAMLFactory.builder() + .enable(StreamReadFeature.STRICT_DUPLICATE_DETECTION) + .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER) + .build(); + factory.setStreamReadConstraints( + StreamReadConstraints.builder() + .maxNestingDepth(MAX_NESTING_DEPTH) + .maxStringLength(MAX_DOCUMENT_LENGTH) + .build()); + return new ObjectMapper(factory) + .enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS) + .enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS) + .enable(DeserializationFeature.USE_BIG_INTEGER_FOR_INTS) + .setSerializationInclusion(JsonInclude.Include.NON_NULL); + } + + private static ObjectNode parseDocument(String document) { + if (StringUtils.isBlank(document)) { + throw new IllegalSemanticModelException("Apache Ossie document must not be empty"); + } + if (document.length() > MAX_DOCUMENT_LENGTH) { + throw new IllegalSemanticModelException( + "Apache Ossie document exceeds the maximum length of %s characters", MAX_DOCUMENT_LENGTH); + } + + try { + JsonNode parsed = YAML_MAPPER.readTree(document); + if (!(parsed instanceof ObjectNode)) { + throw invalid("$", "document root must be an object"); + } + return (ObjectNode) parsed; + } catch (JsonProcessingException e) { + throw new IllegalSemanticModelException( + e, "Cannot parse Apache Ossie YAML or JSON: %s", e.getOriginalMessage()); + } + } + + private static SemanticModelCreateRequest toCreateRequest(ObjectNode root) { + validateObject(root, "$", ROOT_PROPERTIES); + String ossieVersion = validateVersion(root.get("version")); + + Map<String, String> properties = new LinkedHashMap<>(extractProperties(root)); + String extensionVersion = properties.get(PROPERTY_OSSIE_VERSION); + if (extensionVersion != null && !extensionVersion.equals(ossieVersion)) { + throw invalid( + "$.custom_extensions", + "Gravitino property '" + PROPERTY_OSSIE_VERSION + "' conflicts with $.version"); + } + properties.put(PROPERTY_OSSIE_VERSION, ossieVersion); + ObjectNode definition = JSON_MAPPER.createObjectNode(); + copy(root, definition, "ai_context"); + copy(root, definition, "datasets"); + copy(root, definition, "relationships"); + copy(root, definition, "metrics"); + copy(root, definition, "custom_extensions"); + transformOssieDefinition(definition, "$"); + + ObjectNode requestNode = JSON_MAPPER.createObjectNode(); + copy(root, requestNode, "name"); + if (root.has("description")) { + requestNode.set("comment", root.get("description")); + } + requestNode.set("definition", definition); + requestNode.set("properties", JSON_MAPPER.valueToTree(properties)); + + try { + SemanticModelCreateRequest request = + JSON_MAPPER.treeToValue(requestNode, SemanticModelCreateRequest.class); + request.validate(); + request.toDefinition(); + return request; + } catch (JsonProcessingException | IllegalArgumentException e) { + throw new IllegalSemanticModelException( + e, + "Cannot convert Apache Ossie document to a Gravitino Semantic Model: %s", + originalMessage(e)); + } + } + + private static String validateVersion(@Nullable JsonNode version) { + if (version == null || !version.isTextual() || StringUtils.isBlank(version.textValue())) { + throw invalid("$.version", "must be a non-empty string"); + } + return version.textValue(); + } + + private static String ossieVersion(Map<String, String> properties) { + String version = properties.getOrDefault(PROPERTY_OSSIE_VERSION, DEFAULT_OSSIE_VERSION); + if (StringUtils.isBlank(version)) { + throw invalid( + "$.version", + "Semantic Model property '" + PROPERTY_OSSIE_VERSION + "' must not be blank"); + } + return version; + } + + private static void transformOssieDefinition(ObjectNode definition, String path) { + validateAIContext(definition.get("ai_context"), path + ".ai_context"); + transformObjectArray( + definition.get("datasets"), + path + ".datasets", + OssieSemanticModelDocumentConverter::transformOssieDataset); + transformObjectArray( + definition.get("relationships"), + path + ".relationships", + OssieSemanticModelDocumentConverter::transformOssieRelationship); + transformObjectArray( + definition.get("metrics"), + path + ".metrics", + OssieSemanticModelDocumentConverter::transformOssieMetric); + transformOssieCustomExtensions( + definition.get("custom_extensions"), path + ".custom_extensions"); + rename(definition, "ai_context", "aiContext"); + rename(definition, "custom_extensions", "customExtensions"); + } + + private static void transformOssieDataset(ObjectNode dataset, String path) { + validateObject(dataset, path, DATASET_PROPERTIES); + JsonNode source = dataset.get("source"); + if (source != null) { + if (!source.isTextual()) { + throw invalid(path + ".source", "must be a string"); + } + String sourceValue = source.textValue(); + String[] parts = sourceValue.split("\\.", -1); + if (parts.length != 3 || StringUtils.isAnyBlank(parts)) { + throw invalid( + path + ".source", + "must be a three-part catalog.schema.entity identifier; query sources are not supported"); + } + ObjectNode identifier = JSON_MAPPER.createObjectNode(); + identifier.putArray("namespace").add(parts[0]).add(parts[1]); + identifier.put("name", parts[2]); + dataset.set("source", identifier); + } + + validateAIContext(dataset.get("ai_context"), path + ".ai_context"); + transformObjectArray( + dataset.get("fields"), + path + ".fields", + OssieSemanticModelDocumentConverter::transformOssieField); + transformOssieCustomExtensions(dataset.get("custom_extensions"), path + ".custom_extensions"); + rename(dataset, "primary_key", "primaryKey"); + rename(dataset, "unique_keys", "uniqueKeys"); + rename(dataset, "ai_context", "aiContext"); + rename(dataset, "custom_extensions", "customExtensions"); + } + + private static void transformOssieField(ObjectNode field, String path) { + validateObject(field, path, FIELD_PROPERTIES); + transformOssieExpression(field.get("expression"), path + ".expression"); + JsonNode dimension = field.get("dimension"); + if (dimension instanceof ObjectNode) { + validateObject((ObjectNode) dimension, path + ".dimension", DIMENSION_PROPERTIES); + rename((ObjectNode) dimension, "is_time", "isTime"); + } + validateAIContext(field.get("ai_context"), path + ".ai_context"); + transformOssieCustomExtensions(field.get("custom_extensions"), path + ".custom_extensions"); + rename(field, "ai_context", "aiContext"); + rename(field, "custom_extensions", "customExtensions"); + } + + private static void transformOssieRelationship(ObjectNode relationship, String path) { + validateObject(relationship, path, RELATIONSHIP_PROPERTIES); + validateAIContext(relationship.get("ai_context"), path + ".ai_context"); + transformOssieCustomExtensions( + relationship.get("custom_extensions"), path + ".custom_extensions"); + rename(relationship, "from_columns", "fromColumns"); + rename(relationship, "to_columns", "toColumns"); + rename(relationship, "ai_context", "aiContext"); + rename(relationship, "custom_extensions", "customExtensions"); + } + + private static void transformOssieMetric(ObjectNode metric, String path) { + validateObject(metric, path, METRIC_PROPERTIES); + transformOssieExpression(metric.get("expression"), path + ".expression"); + validateAIContext(metric.get("ai_context"), path + ".ai_context"); + transformOssieCustomExtensions(metric.get("custom_extensions"), path + ".custom_extensions"); + rename(metric, "ai_context", "aiContext"); + rename(metric, "custom_extensions", "customExtensions"); + } + + private static void transformOssieExpression(@Nullable JsonNode expression, String path) { + if (!(expression instanceof ObjectNode)) { + return; + } + ObjectNode expressionObject = (ObjectNode) expression; + validateObject(expressionObject, path, EXPRESSION_PROPERTIES); + transformObjectArray( + expressionObject.get("dialects"), + path + ".dialects", + (dialectExpression, dialectPath) -> { + validateObject(dialectExpression, dialectPath, DIALECT_EXPRESSION_PROPERTIES); + JsonNode dialect = dialectExpression.get("dialect"); + if (dialect != null + && dialect.isTextual() + && !OSSIE_DIALECTS.contains(dialect.textValue())) { + throw invalid( + dialectPath + ".dialect", + "unsupported Apache Ossie dialect '" + dialect.textValue() + "'"); + } + }); + } + + private static void transformOssieCustomExtensions(@Nullable JsonNode extensions, String path) { + transformObjectArray( + extensions, + path, + (extension, extensionPath) -> { + validateObject(extension, extensionPath, CUSTOM_EXTENSION_PROPERTIES); + rename(extension, "vendor_name", "vendorName"); + }); + } + + private static void validateAIContext(@Nullable JsonNode context, String path) { + if (context == null) { + return; + } + if (context.isTextual()) { + return; + } + if (!(context instanceof ObjectNode)) { + throw invalid(path, "must be a string or object"); + } + + ObjectNode object = (ObjectNode) context; + validateOptionalText(object, "instructions", path); + validateOptionalStringArray(object, "synonyms", path); + validateOptionalStringArray(object, "examples", path); + } + + private static void transformNativeDefinition(ObjectNode definition) { + transformObjectArray( + definition.get("datasets"), + "$.datasets", + OssieSemanticModelDocumentConverter::transformNativeDataset); + transformObjectArray( + definition.get("relationships"), + "$.relationships", + OssieSemanticModelDocumentConverter::transformNativeRelationship); + transformObjectArray( + definition.get("metrics"), + "$.metrics", + OssieSemanticModelDocumentConverter::transformNativeMetric); + transformNativeCustomExtensions(definition.get("customExtensions"), "$.custom_extensions"); + rename(definition, "aiContext", "ai_context"); + rename(definition, "customExtensions", "custom_extensions"); + } + + private static void transformNativeDataset(ObjectNode dataset, String path) { + JsonNode source = dataset.get("source"); + if (!(source instanceof ObjectNode)) { + throw invalid(path + ".source", "must be a Gravitino source identifier"); + } + JsonNode namespace = source.get("namespace"); + JsonNode name = source.get("name"); + if (!(namespace instanceof ArrayNode) + || namespace.size() != 2 + || !namespace.get(0).isTextual() + || !namespace.get(1).isTextual() + || name == null + || !name.isTextual()) { + throw invalid(path + ".source", "must contain exactly catalog.schema.name"); + } + dataset.put( + "source", + namespace.get(0).textValue() + "." + namespace.get(1).textValue() + "." + name.textValue()); + + transformObjectArray( + dataset.get("fields"), + path + ".fields", + OssieSemanticModelDocumentConverter::transformNativeField); + transformNativeCustomExtensions(dataset.get("customExtensions"), path + ".custom_extensions"); + rename(dataset, "primaryKey", "primary_key"); + rename(dataset, "uniqueKeys", "unique_keys"); + rename(dataset, "aiContext", "ai_context"); + rename(dataset, "customExtensions", "custom_extensions"); + } + + private static void transformNativeField(ObjectNode field, String path) { + JsonNode dimension = field.get("dimension"); + if (dimension instanceof ObjectNode) { + rename((ObjectNode) dimension, "isTime", "is_time"); + } + transformNativeCustomExtensions(field.get("customExtensions"), path + ".custom_extensions"); + rename(field, "aiContext", "ai_context"); + rename(field, "customExtensions", "custom_extensions"); + } + + private static void transformNativeRelationship(ObjectNode relationship, String path) { + transformNativeCustomExtensions( + relationship.get("customExtensions"), path + ".custom_extensions"); + rename(relationship, "fromColumns", "from_columns"); + rename(relationship, "toColumns", "to_columns"); + rename(relationship, "aiContext", "ai_context"); + rename(relationship, "customExtensions", "custom_extensions"); + } + + private static void transformNativeMetric(ObjectNode metric, String path) { + transformNativeCustomExtensions(metric.get("customExtensions"), path + ".custom_extensions"); + rename(metric, "aiContext", "ai_context"); + rename(metric, "customExtensions", "custom_extensions"); + } + + private static void transformNativeCustomExtensions(@Nullable JsonNode extensions, String path) { + transformObjectArray( + extensions, + path, + (extension, extensionPath) -> rename(extension, "vendorName", "vendor_name")); + } + + private static void stashProperties(ObjectNode root, Map<String, String> properties) { + Map<String, String> interchangeProperties = new LinkedHashMap<>(); + if (properties != null) { + properties.forEach( + (key, value) -> { + if (!PROPERTY_OSSIE_VERSION.equals(key)) { + interchangeProperties.put(key, value); + } + }); + } + + JsonNode existingExtensions = root.get("custom_extensions"); + ArrayNode extensions; + if (existingExtensions == null) { + if (interchangeProperties.isEmpty()) { + return; + } + extensions = root.putArray("custom_extensions"); + } else if (existingExtensions instanceof ArrayNode) { + extensions = (ArrayNode) existingExtensions; + } else { + throw invalid("$.custom_extensions", "must be an array"); + } + ensureNoReservedMarker(extensions); + + if (interchangeProperties.isEmpty()) { + return; + } + + ObjectNode payload = JSON_MAPPER.createObjectNode(); + ObjectNode marker = payload.putObject(INTERCHANGE_MARKER); + marker.put("version", INTERCHANGE_MARKER_VERSION); + marker.set("properties", JSON_MAPPER.valueToTree(interchangeProperties)); + + ObjectNode extension = extensions.addObject(); + extension.put("vendor_name", GRAVITINO_VENDOR); + extension.put("data", writeExtensionData(payload)); + } + + private static Map<String, String> extractProperties(ObjectNode root) { + JsonNode extensionsNode = root.get("custom_extensions"); + if (!(extensionsNode instanceof ArrayNode)) { + return Map.of(); + } + + ArrayNode extensions = (ArrayNode) extensionsNode; + Map<String, String> properties = new LinkedHashMap<>(); + boolean found = false; + Iterator<JsonNode> iterator = extensions.iterator(); + while (iterator.hasNext()) { + JsonNode candidate = iterator.next(); + if (!(candidate instanceof ObjectNode)) { + continue; + } + ObjectNode extension = (ObjectNode) candidate; + if (!GRAVITINO_VENDOR.equals(extension.path("vendor_name").asText(null))) { + continue; + } + + ObjectNode payload = parseExtensionData(extension.path("data").asText(null)); + JsonNode markerNode = payload == null ? null : payload.get(INTERCHANGE_MARKER); + if (markerNode == null) { + continue; + } + if (found) { + throw invalid("$.custom_extensions", "contains multiple Gravitino interchange markers"); + } + if (!(markerNode instanceof ObjectNode) + || markerNode.path("version").asInt(-1) != INTERCHANGE_MARKER_VERSION + || !(markerNode.get("properties") instanceof ObjectNode)) { + throw invalid( + "$.custom_extensions", "contains an unsupported Gravitino interchange marker"); + } + + ObjectNode propertyNode = (ObjectNode) markerNode.get("properties"); + propertyNode + .fields() + .forEachRemaining( + entry -> { + if (!entry.getValue().isTextual()) { + throw invalid( + "$.custom_extensions", + "Gravitino property '" + entry.getKey() + "' must be a string"); + } + properties.put(entry.getKey(), entry.getValue().textValue()); + }); + found = true; + + payload.remove(INTERCHANGE_MARKER); + if (payload.isEmpty()) { + iterator.remove(); + } else { + extension.put("data", writeExtensionData(payload)); + } + } + if (extensions.isEmpty()) { + root.remove("custom_extensions"); + } + return properties; + } + + private static void ensureNoReservedMarker(ArrayNode extensions) { + for (JsonNode candidate : extensions) { + if (!(candidate instanceof ObjectNode) + || !GRAVITINO_VENDOR.equals(candidate.path("vendor_name").asText(null))) { + continue; + } + ObjectNode payload = parseExtensionData(candidate.path("data").asText(null)); + if (payload != null && payload.has(INTERCHANGE_MARKER)) { + throw invalid( + "$.custom_extensions", "already contains a reserved Gravitino interchange marker"); + } + } + } + + @Nullable + private static ObjectNode parseExtensionData(@Nullable String data) { + if (data == null) { + return null; + } + try { + JsonNode parsed = JSON_MAPPER.readTree(data); + return parsed instanceof ObjectNode ? (ObjectNode) parsed : null; + } catch (JsonProcessingException e) { + return null; + } + } + + private static String writeExtensionData(ObjectNode payload) { + try { + return JSON_MAPPER.writeValueAsString(payload); + } catch (JsonProcessingException e) { + throw new IllegalSemanticModelException( + e, "Cannot serialize Gravitino interchange extension: %s", e.getOriginalMessage()); + } + } + + private static void validateObject(ObjectNode object, String path, Set<String> allowed) { + object + .fields() + .forEachRemaining( + entry -> { + if (!allowed.contains(entry.getKey())) { + throw invalid( + path + "." + entry.getKey(), "property is not defined by Apache Ossie"); + } + if (entry.getValue().isNull()) { + throw invalid(path + "." + entry.getKey(), "must not be null"); + } + }); + } + + private static void validateOptionalText(ObjectNode object, String name, String path) { + JsonNode value = object.get(name); + if (value != null && !value.isTextual()) { + throw invalid(path + "." + name, "must be a string"); + } + } + + private static void validateOptionalStringArray(ObjectNode object, String name, String path) { + JsonNode value = object.get(name); + if (value == null) { + return; + } + if (!(value instanceof ArrayNode)) { + throw invalid(path + "." + name, "must be an array of strings"); + } + for (int index = 0; index < value.size(); index++) { + if (!value.get(index).isTextual()) { + throw invalid(path + "." + name + "[" + index + "]", "must be a string"); + } + } + } + + private static void transformObjectArray( + @Nullable JsonNode node, String path, ObjectTransformer transformer) { + if (!(node instanceof ArrayNode)) { + return; + } + ArrayNode array = (ArrayNode) node; + for (int index = 0; index < array.size(); index++) { + JsonNode item = array.get(index); + if (item instanceof ObjectNode) { + transformer.transform((ObjectNode) item, path + "[" + index + "]"); + } + } + } + + private static void copy(ObjectNode source, ObjectNode target, String name) { + JsonNode value = source.get(name); + if (value != null) { + target.set(name, value.deepCopy()); + } + } + + private static void rename(ObjectNode object, String source, String target) { + JsonNode value = object.remove(source); + if (value != null) { + object.set(target, value); + } + } + + private static String originalMessage(Exception exception) { + if (exception instanceof JsonProcessingException) { + return ((JsonProcessingException) exception).getOriginalMessage(); + } + return exception.getMessage(); + } + + private static IllegalSemanticModelException invalid(String path, String detail) { + return new IllegalSemanticModelException("%s: %s", path, detail); + } + + @FunctionalInterface + private interface ObjectTransformer { + void transform(ObjectNode object, String path); + } +} diff --git a/core/src/main/java/org/apache/gravitino/semantic/SemanticModelPropertiesMetadata.java b/core/src/main/java/org/apache/gravitino/semantic/SemanticModelPropertiesMetadata.java new file mode 100644 index 0000000000..42c3123e2f --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/semantic/SemanticModelPropertiesMetadata.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.semantic; + +import static org.apache.gravitino.semantic.SemanticModel.DEFAULT_OSSIE_VERSION; +import static org.apache.gravitino.semantic.SemanticModel.PROPERTY_OSSIE_VERSION; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import java.util.function.Function; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.connector.PropertiesMetadata; +import org.apache.gravitino.connector.PropertyEntry; + +/** Property metadata shared by all Gravitino-managed Semantic Models. */ +public final class SemanticModelPropertiesMetadata implements PropertiesMetadata { + + private static final Map<String, PropertyEntry<?>> PROPERTY_ENTRIES = + ImmutableMap.of( + PROPERTY_OSSIE_VERSION, + new PropertyEntry.Builder<String>() + .withName(PROPERTY_OSSIE_VERSION) + .withDescription("The Apache Ossie document version used for import and export") + .withRequired(false) + .withImmutable(false) + .withJavaType(String.class) + .withDefaultValue(DEFAULT_OSSIE_VERSION) + .withDecoder(SemanticModelPropertiesMetadata::decodeOssieVersion) + .withEncoder(Function.identity()) + .withHidden(false) + .withReserved(false) + .build()); + + /** Creates Semantic Model property metadata. */ + public SemanticModelPropertiesMetadata() {} + + @Override + public Map<String, PropertyEntry<?>> propertyEntries() { + return PROPERTY_ENTRIES; + } + + private static String decodeOssieVersion(String value) { + Preconditions.checkArgument( + StringUtils.isNotBlank(value), "Apache Ossie version must not be blank"); + return value; + } +} diff --git a/core/src/test/java/org/apache/gravitino/catalog/TestSemanticModelOperationDispatcher.java b/core/src/test/java/org/apache/gravitino/catalog/TestSemanticModelOperationDispatcher.java index f4be170d51..dd757dfceb 100644 --- a/core/src/test/java/org/apache/gravitino/catalog/TestSemanticModelOperationDispatcher.java +++ b/core/src/test/java/org/apache/gravitino/catalog/TestSemanticModelOperationDispatcher.java @@ -21,6 +21,8 @@ package org.apache.gravitino.catalog; import static org.apache.gravitino.Configs.TREE_LOCK_CLEAN_INTERVAL; import static org.apache.gravitino.Configs.TREE_LOCK_MAX_NODE_IN_MEMORY; import static org.apache.gravitino.Configs.TREE_LOCK_MIN_NODE_IN_MEMORY; +import static org.apache.gravitino.semantic.SemanticModel.DEFAULT_OSSIE_VERSION; +import static org.apache.gravitino.semantic.SemanticModel.PROPERTY_OSSIE_VERSION; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -109,9 +111,54 @@ public class TestSemanticModelOperationDispatcher { assertEquals("sales_model", created.name()); assertEquals(2, created.definition().datasets().length); + assertEquals(DEFAULT_OSSIE_VERSION, created.properties().get(PROPERTY_OSSIE_VERSION)); assertSame(created, dispatcher.loadSemanticModel(MODEL_IDENT)); } + @Test + public void testCreatePreservesExplicitOssieVersion() { + SemanticModel created = + dispatcher.createSemanticModel( + MODEL_IDENT, + "Sales", + validDefinition(), + Map.of(PROPERTY_OSSIE_VERSION, "future-version")); + + assertEquals("future-version", created.properties().get(PROPERTY_OSSIE_VERSION)); + } + + @Test + public void testCreateRejectsBlankOssieVersionBeforeCatalogLookup() { + assertThrows( + IllegalArgumentException.class, + () -> + dispatcher.createSemanticModel( + MODEL_IDENT, null, validDefinition(), Map.of(PROPERTY_OSSIE_VERSION, " "))); + verify(catalogManager, never()).loadCatalog(METADATA_CATALOG_IDENT); + } + + @Test + public void testAlterRejectsBlankOssieVersionBeforeCatalogLookup() { + assertThrows( + IllegalArgumentException.class, + () -> + dispatcher.alterSemanticModel( + MODEL_IDENT, SemanticModelChange.setProperty(PROPERTY_OSSIE_VERSION, " "))); + verify(catalogManager, never()).loadCatalog(METADATA_CATALOG_IDENT); + } + + @Test + public void testAlterCanRemoveOssieVersion() { + dispatcher.createSemanticModel( + MODEL_IDENT, "Sales", validDefinition(), Map.of(PROPERTY_OSSIE_VERSION, "future-version")); + + SemanticModel altered = + dispatcher.alterSemanticModel( + MODEL_IDENT, SemanticModelChange.removeProperty(PROPERTY_OSSIE_VERSION)); + + assertFalse(altered.properties().containsKey(PROPERTY_OSSIE_VERSION)); + } + @Test public void testSchemaFailureRemainsTyped() { when(schemaDispatcher.loadSchema(SCHEMA_IDENT)) @@ -146,7 +193,9 @@ public class TestSemanticModelOperationDispatcher { SemanticModelChange.rename("renamed_sales_model"), SemanticModelChange.updateComment("Updated")); NameIdentifier renamedIdent = NameIdentifier.of(NAMESPACE, renamed.name()); - assertEquals(Map.of("owner", "analytics"), propertyUpdated.properties()); + assertEquals( + Map.of("owner", "analytics", PROPERTY_OSSIE_VERSION, DEFAULT_OSSIE_VERSION), + propertyUpdated.properties()); assertEquals("Updated", renamed.comment()); SemanticModel replaced = diff --git a/core/src/test/java/org/apache/gravitino/semantic/TestOssieSemanticModelDocumentConverter.java b/core/src/test/java/org/apache/gravitino/semantic/TestOssieSemanticModelDocumentConverter.java new file mode 100644 index 0000000000..f570430e0f --- /dev/null +++ b/core/src/test/java/org/apache/gravitino/semantic/TestOssieSemanticModelDocumentConverter.java @@ -0,0 +1,405 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.semantic; + +import static org.apache.gravitino.semantic.SemanticModel.DEFAULT_OSSIE_VERSION; +import static org.apache.gravitino.semantic.SemanticModel.PROPERTY_OSSIE_VERSION; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.json.JsonMapper; +import java.time.Instant; +import java.util.Map; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.dto.requests.SemanticModelCreateRequest; +import org.apache.gravitino.exceptions.IllegalSemanticModelException; +import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.meta.SemanticModelEntity; +import org.junit.jupiter.api.Test; + +/** Tests standalone Apache Ossie document import, export, and conversion diagnostics. */ +public class TestOssieSemanticModelDocumentConverter { + + private static final JsonMapper JSON_MAPPER = JsonMapper.builder().build(); + + @Test + public void testImportYamlAndJsonDocuments() { + String yaml = + """ + version: 0.2.0.dev0 + name: sales + description: Governed sales definitions + ai_context: + instructions: Use certified definitions + confidence: 0.95 + datasets: + - name: orders + source: sales.mart.orders + primary_key: [order_id] + fields: + - name: order_id + expression: + dialects: + - dialect: ANSI_SQL + expression: order_id + datatype: String + dimension: {} + relationships: [] + metrics: + - name: revenue + expression: + dialects: + - dialect: DAX + expression: SUM(orders.amount) + datatype: Decimal + custom_extensions: + - vendor_name: EXAMPLE + data: '{"certified":true}' + - vendor_name: GRAVITINO + data: '{"_apache_gravitino_interchange":{"version":1,"properties":{"domain":"sales"}}}' + """; + + SemanticModelCreateRequest yamlRequest = + OssieSemanticModelDocumentConverter.importDocument(yaml); + assertEquals("sales", yamlRequest.getName()); + assertEquals("Governed sales definitions", yamlRequest.getComment()); + assertEquals( + Map.of("domain", "sales", PROPERTY_OSSIE_VERSION, DEFAULT_OSSIE_VERSION), + yamlRequest.getProperties()); + SemanticModelDefinition yamlDefinition = yamlRequest.toDefinition(); + assertEquals( + NameIdentifier.of("sales", "mart", "orders"), yamlDefinition.datasets()[0].source()); + assertEquals("DAX", yamlDefinition.metrics()[0].expression().dialects()[0].dialect()); + assertEquals(1, yamlDefinition.customExtensions().length); + assertEquals("EXAMPLE", yamlDefinition.customExtensions()[0].vendorName()); + assertEquals("Use certified definitions", yamlDefinition.aiContext().object().instructions()); + assertEquals( + "0.95", + yamlDefinition.aiContext().object().additionalProperties().get("confidence").toString()); + + String json = + """ + { + "version": "future-version", + "name": "inventory", + "datasets": [ + {"name": "items", "source": "sales.mart.items", "fields": []} + ] + } + """; + SemanticModelCreateRequest jsonRequest = + OssieSemanticModelDocumentConverter.importDocument(json); + assertEquals("inventory", jsonRequest.getName()); + assertEquals("future-version", jsonRequest.getProperties().get(PROPERTY_OSSIE_VERSION)); + assertEquals( + NameIdentifier.of("sales", "mart", "items"), + jsonRequest.toDefinition().datasets()[0].source()); + } + + @Test + public void testExportJsonAndYamlRoundTrip() throws Exception { + SemanticModel semanticModel = + semanticModel( + definition(), Map.of("domain", "sales", PROPERTY_OSSIE_VERSION, "future-version")); + + String jsonDocument = + OssieSemanticModelDocumentConverter.exportDocument( + semanticModel, OssieSemanticModelDocumentConverter.Format.JSON); + JsonNode json = JSON_MAPPER.readTree(jsonDocument); + assertEquals("future-version", json.path("version").textValue()); + assertEquals("sales", json.path("name").textValue()); + assertEquals("Governed sales definitions", json.path("description").textValue()); + assertEquals("sales.mart.orders", json.at("/datasets/0/source").textValue()); + assertEquals("order_id", json.at("/datasets/0/unique_keys/0/0").textValue()); + assertTrue(json.at("/datasets/0/fields/0/dimension").isObject()); + assertTrue(json.at("/datasets/0/fields/0/dimension").has("is_time")); + assertFalse(json.at("/datasets/0/fields/0/dimension/is_time").booleanValue()); + assertEquals("Use order identifiers", json.at("/datasets/0/fields/0/ai_context").textValue()); + assertEquals( + "FIELD_VENDOR", + json.at("/datasets/0/fields/0/custom_extensions/0/vendor_name").textValue()); + assertFalse(json.has("semantic_model")); + assertFalse(json.has("definition")); + assertFalse(json.has("properties")); + assertFalse(json.has("audit")); + assertEquals("GRAVITINO", json.at("/custom_extensions/1/vendor_name").textValue()); + JsonNode marker = JSON_MAPPER.readTree(json.at("/custom_extensions/1/data").textValue()); + assertEquals( + "sales", marker.at("/_apache_gravitino_interchange/properties/domain").textValue()); + assertFalse(marker.at("/_apache_gravitino_interchange/properties").has(PROPERTY_OSSIE_VERSION)); + + SemanticModelCreateRequest jsonRoundTrip = + OssieSemanticModelDocumentConverter.importDocument(jsonDocument); + assertEquals(semanticModel.name(), jsonRoundTrip.getName()); + assertEquals(semanticModel.comment(), jsonRoundTrip.getComment()); + assertEquals(semanticModel.definition(), jsonRoundTrip.toDefinition()); + assertEquals(semanticModel.properties(), jsonRoundTrip.getProperties()); + + String yamlDocument = + OssieSemanticModelDocumentConverter.exportDocument( + semanticModel, OssieSemanticModelDocumentConverter.Format.YAML); + assertTrue(yamlDocument.contains("version:")); + assertFalse(yamlDocument.contains("semantic_model:")); + SemanticModelCreateRequest yamlRoundTrip = + OssieSemanticModelDocumentConverter.importDocument(yamlDocument); + assertEquals(semanticModel.definition(), yamlRoundTrip.toDefinition()); + assertEquals(semanticModel.properties(), yamlRoundTrip.getProperties()); + } + + @Test + public void testExportDefaultsMissingOssieVersion() throws Exception { + SemanticModel semanticModel = semanticModel(definition(), Map.of()); + + String document = + OssieSemanticModelDocumentConverter.exportDocument( + semanticModel, OssieSemanticModelDocumentConverter.Format.JSON); + JsonNode json = JSON_MAPPER.readTree(document); + + assertEquals(DEFAULT_OSSIE_VERSION, json.path("version").textValue()); + SemanticModelCreateRequest roundTrip = + OssieSemanticModelDocumentConverter.importDocument(document); + assertEquals(DEFAULT_OSSIE_VERSION, roundTrip.getProperties().get(PROPERTY_OSSIE_VERSION)); + } + + @Test + public void testRejectsMalformedAndUnsupportedDocuments() { + assertInvalid("", "must not be empty"); + assertInvalid("[]", "root must be an object"); + assertInvalid("name: sales\ndatasets: []\n", "$.version: must be a non-empty string"); + assertInvalid("version: ' '\nname: sales\ndatasets: []\n", "must be a non-empty string"); + assertInvalid("version: 0.2.0.dev0\nsemantic_model: []\n", "semantic_model"); + assertInvalid( + "version: 0.2.0.dev0\nname: sales\ndescription: null\ndatasets: []\n", + "description: must not be null"); + assertInvalid("version: 0.2.0.dev0\nname: sales\nunknown: true\ndatasets: []\n", "unknown"); + assertInvalid( + "version: 0.2.0.dev0\n" + + "name: sales\n" + + "datasets:\n" + + " - name: orders\n" + + " source: SELECT * FROM orders\n", + "query sources are not supported"); + assertInvalid( + "version: 0.2.0.dev0\n" + + "name: sales\n" + + "datasets:\n" + + " - name: orders\n" + + " source: sales.mart.orders\n" + + " fields:\n" + + " - name: id\n" + + " expression:\n" + + " dialects:\n" + + " - dialect: TRINO\n" + + " expression: id\n", + "unsupported Apache Ossie dialect 'TRINO'"); + assertInvalid( + "version: 0.2.0.dev0\n" + "name: first\n" + "name: second\n" + "datasets: []\n", + "Duplicate field 'name'"); + assertInvalid( + "version: 0.2.0.dev0\nname: first\ndatasets: []\n" + + "---\nversion: 0.2.0.dev0\nname: second\ndatasets: []\n", + "Trailing token"); + assertInvalid( + "x".repeat(OssieSemanticModelDocumentConverter.MAX_DOCUMENT_LENGTH + 1), + "exceeds the maximum length"); + } + + @Test + public void testRejectsConflictingOssieVersionPropertyOnImport() { + String document = + """ + version: root-version + name: sales + datasets: + - name: orders + source: sales.mart.orders + custom_extensions: + - vendor_name: GRAVITINO + data: '{"_apache_gravitino_interchange":{"version":1,"properties":{"ossie-version":"extension-version"}}}' + """; + + assertInvalid(document, "property 'ossie-version' conflicts with $.version"); + } + + @Test + public void testRejectsBlankOssieVersionPropertyOnExport() { + IllegalSemanticModelException exception = + assertThrows( + IllegalSemanticModelException.class, + () -> + OssieSemanticModelDocumentConverter.exportDocument( + semanticModel(definition(), Map.of(PROPERTY_OSSIE_VERSION, " ")), + OssieSemanticModelDocumentConverter.Format.JSON)); + + assertTrue(exception.getMessage().contains("property 'ossie-version' must not be blank")); + } + + @Test + public void testRejectsReservedExtensionCollisionOnExport() { + CustomExtension reserved = + CustomExtension.builder() + .withVendorName("GRAVITINO") + .withData("{\"_apache_gravitino_interchange\":{\"version\":1,\"properties\":{}}}") + .build(); + SemanticModelDefinition definition = + SemanticModelDefinition.builder() + .withDatasets(new Dataset[] {dataset()}) + .withCustomExtensions(new CustomExtension[] {reserved}) + .build(); + + IllegalSemanticModelException exception = + assertThrows( + IllegalSemanticModelException.class, + () -> + OssieSemanticModelDocumentConverter.exportDocument( + semanticModel(definition, Map.of()), + OssieSemanticModelDocumentConverter.Format.JSON)); + assertTrue(exception.getMessage().contains("reserved Gravitino interchange marker")); + } + + @Test + public void testRejectsUnsupportedNativeDialectOnExport() { + Metric metric = + Metric.builder() + .withName("revenue") + .withExpression(expression("TRINO", "SUM(orders.amount)")) + .build(); + SemanticModelDefinition definition = + SemanticModelDefinition.builder() + .withDatasets(new Dataset[] {dataset()}) + .withMetrics(new Metric[] {metric}) + .build(); + + IllegalSemanticModelException exception = + assertThrows( + IllegalSemanticModelException.class, + () -> + OssieSemanticModelDocumentConverter.exportDocument( + semanticModel(definition, Map.of()), + OssieSemanticModelDocumentConverter.Format.JSON)); + assertTrue(exception.getMessage().contains("unsupported Apache Ossie dialect 'TRINO'")); + } + + private static SemanticModel semanticModel( + SemanticModelDefinition definition, Map<String, String> properties) { + return SemanticModelEntity.builder() + .withId(1L) + .withName("sales") + .withNamespace(Namespace.of("lake", "sales", "mart")) + .withComment("Governed sales definitions") + .withDefinition(definition) + .withProperties(properties) + .withAuditInfo( + AuditInfo.builder() + .withCreator("tester") + .withCreateTime(Instant.parse("2026-09-28T00:00:00Z")) + .build()) + .build(); + } + + private static SemanticModelDefinition definition() { + Dataset orders = dataset(); + Dataset customers = + Dataset.builder() + .withName("customers") + .withSource(NameIdentifier.of("sales", "mart", "customers")) + .build(); + Relationship relationship = + Relationship.builder() + .withName("orders_to_customers") + .withFrom("orders") + .withTo("customers") + .withFromColumns(new String[] {"customer_id"}) + .withToColumns(new String[] {"customer_id"}) + .withAIContext(AIContext.of("Join orders to customers")) + .withCustomExtensions(new CustomExtension[] {extension("RELATIONSHIP_VENDOR")}) + .build(); + Metric metric = + Metric.builder() + .withName("revenue") + .withExpression(expression("SUM(orders.amount)")) + .withDescription("Recognized revenue") + .withDatatype(DataType.DECIMAL) + .withAIContext(AIContext.of("Use the certified revenue metric")) + .withCustomExtensions(new CustomExtension[] {extension("METRIC_VENDOR")}) + .build(); + return SemanticModelDefinition.builder() + .withAIContext(AIContext.of("Use certified definitions")) + .withDatasets(new Dataset[] {orders, customers}) + .withRelationships(new Relationship[] {relationship}) + .withMetrics(new Metric[] {metric}) + .withCustomExtensions(new CustomExtension[] {extension("EXAMPLE")}) + .build(); + } + + private static Dataset dataset() { + Field field = + Field.builder() + .withName("order_id") + .withExpression(expression("order_id")) + .withDimension(Dimension.builder().withIsTime(false).build()) + .withLabel("Order ID") + .withDescription("Unique order identifier") + .withDatatype(DataType.STRING) + .withAIContext(AIContext.of("Use order identifiers")) + .withCustomExtensions(new CustomExtension[] {extension("FIELD_VENDOR")}) + .build(); + return Dataset.builder() + .withName("orders") + .withSource(NameIdentifier.of("sales", "mart", "orders")) + .withPrimaryKey(new String[] {"order_id"}) + .withUniqueKeys(new String[][] {{"order_id"}}) + .withDescription("Orders dataset") + .withAIContext(AIContext.of("Use certified orders")) + .withFields(new Field[] {field}) + .withCustomExtensions(new CustomExtension[] {extension("DATASET_VENDOR")}) + .build(); + } + + private static CustomExtension extension(String vendorName) { + return CustomExtension.builder() + .withVendorName(vendorName) + .withData("{\"certified\":true}") + .build(); + } + + private static Expression expression(String value) { + return expression(Dialects.ANSI_SQL, value); + } + + private static Expression expression(String dialectName, String value) { + DialectExpression dialect = + DialectExpression.builder().withDialect(dialectName).withExpression(value).build(); + return Expression.builder().withDialects(new DialectExpression[] {dialect}).build(); + } + + private static void assertInvalid(String document, String expectedMessage) { + IllegalSemanticModelException exception = + assertThrows( + IllegalSemanticModelException.class, + () -> OssieSemanticModelDocumentConverter.importDocument(document)); + assertTrue( + exception.getMessage().contains(expectedMessage), + () -> "Expected '" + expectedMessage + "' in: " + exception.getMessage()); + } +} diff --git a/core/src/test/java/org/apache/gravitino/semantic/TestSemanticModelPropertiesMetadata.java b/core/src/test/java/org/apache/gravitino/semantic/TestSemanticModelPropertiesMetadata.java new file mode 100644 index 0000000000..22e0f1830e --- /dev/null +++ b/core/src/test/java/org/apache/gravitino/semantic/TestSemanticModelPropertiesMetadata.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.semantic; + +import static org.apache.gravitino.semantic.SemanticModel.DEFAULT_OSSIE_VERSION; +import static org.apache.gravitino.semantic.SemanticModel.PROPERTY_OSSIE_VERSION; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.apache.gravitino.connector.PropertyEntry; +import org.junit.jupiter.api.Test; + +/** Tests the property metadata shared by Semantic Models. */ +public class TestSemanticModelPropertiesMetadata { + + private static final SemanticModelPropertiesMetadata METADATA = + new SemanticModelPropertiesMetadata(); + + @Test + public void testOssieVersionProperty() { + PropertyEntry<?> entry = METADATA.getPropertyEntry(PROPERTY_OSSIE_VERSION); + + assertFalse(entry.isRequired()); + assertFalse(entry.isImmutable()); + assertEquals(DEFAULT_OSSIE_VERSION, entry.getDefaultValue()); + assertEquals("future-version", entry.decode("future-version")); + assertThrows(IllegalArgumentException.class, () -> entry.decode(" ")); + } +} diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/SemanticModelOperations.java b/server/src/main/java/org/apache/gravitino/server/web/rest/SemanticModelOperations.java index 1c011be250..831801c141 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/SemanticModelOperations.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/SemanticModelOperations.java @@ -21,14 +21,19 @@ package org.apache.gravitino.server.web.rest; import com.codahale.metrics.annotation.ResponseMetered; import com.codahale.metrics.annotation.Timed; import java.util.Collections; +import java.util.Locale; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.Consumes; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.catalog.SemanticModelDispatcher; @@ -36,6 +41,7 @@ import org.apache.gravitino.dto.requests.SemanticModelCreateRequest; import org.apache.gravitino.dto.responses.SemanticModelResponse; import org.apache.gravitino.dto.util.DTOConverters; import org.apache.gravitino.metrics.MetricNames; +import org.apache.gravitino.semantic.OssieSemanticModelDocumentConverter; import org.apache.gravitino.semantic.SemanticModel; import org.apache.gravitino.semantic.SemanticModelDefinition; import org.apache.gravitino.server.web.Utils; @@ -48,6 +54,9 @@ import org.slf4j.LoggerFactory; public class SemanticModelOperations { private static final Logger LOG = LoggerFactory.getLogger(SemanticModelOperations.class); + private static final String VND_GRAVITINO_V1_JSON = "application/vnd.gravitino.v1+json"; + private static final String OSSIE_YAML_MEDIA_TYPE = "application/yaml"; + private static final String OSSIE_X_YAML_MEDIA_TYPE = "application/x-yaml"; private final SemanticModelDispatcher dispatcher; @@ -73,7 +82,7 @@ public class SemanticModelOperations { * @return A response containing the created Semantic Model. */ @POST - @Produces("application/vnd.gravitino.v1+json") + @Produces(VND_GRAVITINO_V1_JSON) @Timed(name = "create-semantic-model." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "create-semantic-model", absolute = true) public Response createSemanticModel( @@ -91,18 +100,8 @@ public class SemanticModelOperations { if (request == null) { throw new IllegalArgumentException("Request body must not be null"); } - request.validate(); - SemanticModelDefinition definition = request.toDefinition(); - NameIdentifier ident = - NameIdentifierUtil.ofSemanticModel(metalake, catalog, schema, request.getName()); SemanticModel semanticModel = - dispatcher.createSemanticModel( - ident, - request.getComment(), - definition, - request.getProperties() == null - ? Collections.emptyMap() - : request.getProperties()); + createSemanticModelEntity(metalake, catalog, schema, request); LOG.info( "Semantic Model created: {}.{}.{}.{}", metalake, @@ -116,6 +115,53 @@ public class SemanticModelOperations { } } + /** + * Imports a standalone Apache Ossie YAML or JSON document as a Semantic Model. + * + * @param metalake The metalake name. + * @param catalog The catalog name. + * @param schema The schema name. + * @param document The standalone Ossie document. + * @return A native response containing the created Semantic Model. + */ + @POST + @Path("ossie") + @Consumes({MediaType.APPLICATION_JSON, OSSIE_YAML_MEDIA_TYPE, OSSIE_X_YAML_MEDIA_TYPE}) + @Produces(VND_GRAVITINO_V1_JSON) + @Timed(name = "import-ossie-semantic-model." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "import-ossie-semantic-model", absolute = true) + public Response importOssieSemanticModel( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + String document) { + LOG.info( + "Received import Apache Ossie Semantic Model request for schema: {}.{}.{}", + metalake, + catalog, + schema); + try { + return Utils.doAs( + httpRequest, + () -> { + SemanticModelCreateRequest request = + OssieSemanticModelDocumentConverter.importDocument(document); + SemanticModel semanticModel = + createSemanticModelEntity(metalake, catalog, schema, request); + LOG.info( + "Apache Ossie Semantic Model imported: {}.{}.{}.{}", + metalake, + catalog, + schema, + semanticModel.name()); + return Utils.ok(new SemanticModelResponse(DTOConverters.toDTO(semanticModel))); + }); + } catch (Exception e) { + return ExceptionHandlers.handleSemanticModelException( + OperationType.CREATE, "Apache Ossie document", schema, e); + } + } + /** * Loads a Semantic Model. * @@ -127,7 +173,7 @@ public class SemanticModelOperations { */ @GET @Path("{semanticModel}") - @Produces("application/vnd.gravitino.v1+json") + @Produces(VND_GRAVITINO_V1_JSON) @Timed(name = "load-semantic-model." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "load-semantic-model", absolute = true) public Response loadSemanticModel( @@ -157,4 +203,87 @@ public class SemanticModelOperations { OperationType.LOAD, semanticModel, schema, e); } } + + /** + * Exports a Semantic Model as a standalone Apache Ossie YAML or JSON document. + * + * @param metalake The metalake name. + * @param catalog The catalog name. + * @param schema The schema name. + * @param semanticModel The Semantic Model name. + * @param format The output format, either {@code yaml} or {@code json}. + * @return A response containing the serialized Ossie document. + */ + @GET + @Path("{semanticModel}/ossie") + @Timed(name = "export-ossie-semantic-model." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "export-ossie-semantic-model", absolute = true) + public Response exportOssieSemanticModel( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("semanticModel") String semanticModel, + @DefaultValue("yaml") @QueryParam("format") String format) { + LOG.info( + "Received export Semantic Model as Apache Ossie request: {}.{}.{}.{}, format: {}", + metalake, + catalog, + schema, + semanticModel, + format); + try { + return Utils.doAs( + httpRequest, + () -> { + OssieSemanticModelDocumentConverter.Format outputFormat = parseOssieFormat(format); + NameIdentifier ident = + NameIdentifierUtil.ofSemanticModel(metalake, catalog, schema, semanticModel); + SemanticModel loaded = dispatcher.loadSemanticModel(ident); + String document = + OssieSemanticModelDocumentConverter.exportDocument(loaded, outputFormat); + String mediaType = + outputFormat == OssieSemanticModelDocumentConverter.Format.JSON + ? MediaType.APPLICATION_JSON + : OSSIE_YAML_MEDIA_TYPE; + return Response.ok(document, mediaType) + .header("Content-Disposition", ossieContentDisposition(semanticModel, outputFormat)) + .build(); + }); + } catch (Exception e) { + return ExceptionHandlers.handleSemanticModelException( + OperationType.LOAD, semanticModel, schema, e); + } + } + + private SemanticModel createSemanticModelEntity( + String metalake, String catalog, String schema, SemanticModelCreateRequest request) { + request.validate(); + SemanticModelDefinition definition = request.toDefinition(); + NameIdentifier ident = + NameIdentifierUtil.ofSemanticModel(metalake, catalog, schema, request.getName()); + return dispatcher.createSemanticModel( + ident, + request.getComment(), + definition, + request.getProperties() == null ? Collections.emptyMap() : request.getProperties()); + } + + private static OssieSemanticModelDocumentConverter.Format parseOssieFormat(String format) { + if (format == null) { + throw new IllegalArgumentException("Ossie format must be yaml or json"); + } + try { + return OssieSemanticModelDocumentConverter.Format.valueOf(format.toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + String.format("Unsupported Ossie format '%s'; expected yaml or json", format), e); + } + } + + private static String ossieContentDisposition( + String semanticModel, OssieSemanticModelDocumentConverter.Format format) { + String safeName = semanticModel.replaceAll("[^A-Za-z0-9._-]", "_"); + String extension = format.name().toLowerCase(Locale.ROOT); + return String.format("attachment; filename=\"%s.ossie.%s\"", safeName, extension); + } } diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestSemanticModelOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestSemanticModelOperations.java index a463c33775..7c550c5c09 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestSemanticModelOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestSemanticModelOperations.java @@ -18,15 +18,20 @@ */ package org.apache.gravitino.server.web.rest; +import static org.apache.gravitino.semantic.SemanticModel.DEFAULT_OSSIE_VERSION; +import static org.apache.gravitino.semantic.SemanticModel.PROPERTY_OSSIE_VERSION; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.json.JsonMapper; import java.io.IOException; import java.time.Instant; import java.util.Map; @@ -79,6 +84,7 @@ import org.mockito.ArgumentCaptor; public class TestSemanticModelOperations extends BaseOperationsTest { private static final String VND_V1_JSON = "application/vnd.gravitino.v1+json"; + private static final JsonMapper JSON_MAPPER = JsonMapper.builder().build(); private static class MockServletRequestFactory extends ServletRequestFactoryBase { @Override @@ -382,6 +388,160 @@ public class TestSemanticModelOperations extends BaseOperationsTest { "catalog is not relational"); } + @Test + void testImportOssieYamlAndJsonDocuments() { + NameIdentifier salesIdent = semanticModelIdentifier("sales"); + NameIdentifier inventoryIdent = semanticModelIdentifier("inventory"); + when(dispatcher.createSemanticModel( + eq(salesIdent), + eq("Sales definitions"), + any(SemanticModelDefinition.class), + eq(Map.of("domain", "sales", PROPERTY_OSSIE_VERSION, DEFAULT_OSSIE_VERSION)))) + .thenReturn(semanticModel("sales", "Sales definitions")); + when(dispatcher.createSemanticModel( + eq(inventoryIdent), + eq(null), + any(SemanticModelDefinition.class), + eq(Map.of(PROPERTY_OSSIE_VERSION, "future-version")))) + .thenReturn(semanticModel("inventory", null)); + + String yaml = + """ + version: 0.2.0.dev0 + name: sales + description: Sales definitions + datasets: + - name: orders + source: semantic_model_catalog.semantic_model_schema.orders + primary_key: [order_id] + fields: + - name: order_id + expression: + dialects: + - dialect: ANSI_SQL + expression: orders.order_id + datatype: String + custom_extensions: + - vendor_name: GRAVITINO + data: '{"_apache_gravitino_interchange":{"version":1,"properties":{"domain":"sales"}}}' + """; + Response yamlResponse = postDocument(semanticModelPath() + "/ossie", yaml, "application/yaml"); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), yamlResponse.getStatus()); + SemanticModelResponse yamlBody = yamlResponse.readEntity(SemanticModelResponse.class); + yamlBody.validate(); + Assertions.assertEquals("sales", yamlBody.getSemanticModel().name()); + + String json = + """ + { + "version": "future-version", + "name": "inventory", + "datasets": [ + { + "name": "orders", + "source": "semantic_model_catalog.semantic_model_schema.orders", + "fields": [] + } + ] + } + """; + Response jsonResponse = + postDocument(semanticModelPath() + "/ossie", json, MediaType.APPLICATION_JSON); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), jsonResponse.getStatus()); + SemanticModelResponse jsonBody = jsonResponse.readEntity(SemanticModelResponse.class); + jsonBody.validate(); + Assertions.assertEquals("inventory", jsonBody.getSemanticModel().name()); + + ArgumentCaptor<SemanticModelDefinition> definitionCaptor = + ArgumentCaptor.forClass(SemanticModelDefinition.class); + verify(dispatcher) + .createSemanticModel( + eq(salesIdent), + eq("Sales definitions"), + definitionCaptor.capture(), + eq(Map.of("domain", "sales", PROPERTY_OSSIE_VERSION, DEFAULT_OSSIE_VERSION))); + Assertions.assertEquals( + NameIdentifier.of(catalog, schema, "orders"), + definitionCaptor.getValue().datasets()[0].source()); + Assertions.assertEquals( + Dialects.ANSI_SQL, + definitionCaptor.getValue().datasets()[0].fields()[0].expression().dialects()[0].dialect()); + verify(dispatcher) + .createSemanticModel( + eq(inventoryIdent), + eq(null), + any(SemanticModelDefinition.class), + eq(Map.of(PROPERTY_OSSIE_VERSION, "future-version"))); + verifyNoMoreInteractions(dispatcher); + } + + @Test + void testExportOssieYamlAndJsonDocuments() throws Exception { + NameIdentifier ident = semanticModelIdentifier("sales"); + when(dispatcher.loadSemanticModel(ident)) + .thenReturn(semanticModel("sales", "Sales definitions")); + + Response yamlResponse = getOssie(semanticModelPath() + "/sales/ossie"); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), yamlResponse.getStatus()); + Assertions.assertEquals("application/yaml", yamlResponse.getMediaType().toString()); + Assertions.assertEquals( + "attachment; filename=\"sales.ossie.yaml\"", + yamlResponse.getHeaderString("Content-Disposition")); + String yaml = yamlResponse.readEntity(String.class); + Assertions.assertTrue(yaml.contains("version:")); + Assertions.assertTrue(yaml.contains("name:")); + Assertions.assertFalse(yaml.contains("semantic_model:")); + Assertions.assertFalse(yaml.contains("definition:")); + + Response jsonResponse = + target(semanticModelPath() + "/sales/ossie").queryParam("format", "json").request().get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), jsonResponse.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON, jsonResponse.getMediaType().toString()); + Assertions.assertEquals( + "attachment; filename=\"sales.ossie.json\"", + jsonResponse.getHeaderString("Content-Disposition")); + JsonNode json = JSON_MAPPER.readTree(jsonResponse.readEntity(String.class)); + Assertions.assertEquals("0.2.0.dev0", json.path("version").textValue()); + Assertions.assertEquals("sales", json.path("name").textValue()); + Assertions.assertEquals( + "semantic_model_catalog.semantic_model_schema.orders", + json.at("/datasets/0/source").textValue()); + Assertions.assertFalse(json.has("semantic_model")); + Assertions.assertFalse(json.has("definition")); + verify(dispatcher, times(2)).loadSemanticModel(ident); + verifyNoMoreInteractions(dispatcher); + } + + @Test + void testOssieConversionErrorsDoNotCallDispatcher() { + String querySource = + """ + version: 0.2.0.dev0 + name: sales + datasets: + - name: orders + source: SELECT * FROM orders + """; + assertError( + postDocument(semanticModelPath() + "/ossie", querySource, "application/x-yaml"), + Response.Status.BAD_REQUEST, + ErrorConstants.ILLEGAL_ARGUMENTS_CODE, + IllegalSemanticModelException.class.getSimpleName(), + "query sources are not supported"); + + assertError( + target(semanticModelPath() + "/sales/ossie").queryParam("format", "csv").request().get(), + Response.Status.BAD_REQUEST, + ErrorConstants.ILLEGAL_ARGUMENTS_CODE, + IllegalArgumentException.class.getSimpleName(), + "expected yaml or json"); + verifyNoMoreInteractions(dispatcher); + } + private SemanticModelCreateRequest createRequest( String name, String comment, SemanticModelDefinition definition) { return new SemanticModelCreateRequest( @@ -479,6 +639,17 @@ public class TestSemanticModelOperations extends BaseOperationsTest { .post(Entity.entity(json, MediaType.APPLICATION_JSON_TYPE)); } + private Response postDocument(String path, String document, String mediaType) { + return target(path) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept(VND_V1_JSON) + .post(Entity.entity(document, mediaType)); + } + + private Response getOssie(String path) { + return target(path).request().get(); + } + private static ErrorResponse assertError( Response response, Response.Status expectedStatus,
