This is an automated email from the ASF dual-hosted git repository. xxyu pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 3aff4aa461340b709694a8ac610c24188b954ac9 Author: ChenLiang.Lu <[email protected]> AuthorDate: Fri Dec 30 16:58:26 2022 +0800 KYLIN-5447 wrap logical view response KYLIN-5447 Wrap Logical View list --- .../kylin/rest/controller/NAdminController.java | 2 +- .../kylin/rest/response/LogicalViewResponse.java | 54 ++++++++++++++++++++++ .../apache/kylin/rest/service/SparkDDLService.java | 9 ++-- .../apache/kylin/rest/service/TableExtService.java | 4 +- .../org/apache/kylin/rest/ddl/ViewCheck.scala | 2 +- .../apache/kylin/rest/service/SparkDDLTest.java | 21 +++++---- .../org/apache/kylin/newten/LogicalViewTest.java | 4 ++ .../kylin/rest/controller/SparkDDLController.java | 6 +-- .../kylin/rest/service/MetaStoreService.java | 13 +++++- 9 files changed, 95 insertions(+), 20 deletions(-) diff --git a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NAdminController.java b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NAdminController.java index 2d0440f52f..c2f26bd8aa 100644 --- a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NAdminController.java +++ b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NAdminController.java @@ -76,7 +76,7 @@ public class NAdminController extends NBasicController { propertyKeys.add("kylin.security.remove-ldap-custom-security-limit-enabled"); propertyKeys.add("kylin.source.ddl.logical-view.enabled"); propertyKeys.add("kylin.source.ddl.hive.enabled"); - propertyKeys.add("kylin.source.ddl.logical-view-database"); + propertyKeys.add("kylin.source.ddl.logical-view.database"); propertyKeys.add("kylin.storage.check-quota-enabled"); // add second storage diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/response/LogicalViewResponse.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/response/LogicalViewResponse.java new file mode 100644 index 0000000000..2343207aa3 --- /dev/null +++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/response/LogicalViewResponse.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.rest.response; + +import org.apache.kylin.metadata.view.LogicalView; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, + isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) +public class LogicalViewResponse { + @JsonProperty("table_name") + private String tableName; + + @JsonProperty("created_sql") + private String createdSql; + + @JsonProperty("modified_user") + private String modifiedUser; + + @JsonProperty("created_project") + private String createdProject; + + public LogicalViewResponse(LogicalView view) { + this.tableName = view.getTableName(); + this.createdSql = view.getCreatedSql(); + this.modifiedUser = view.getModifiedUser(); + this.createdProject = view.getCreatedProject(); + } +} diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/SparkDDLService.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/SparkDDLService.java index a1c00fdcc9..ef76e21e55 100644 --- a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/SparkDDLService.java +++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/SparkDDLService.java @@ -41,6 +41,7 @@ import org.apache.kylin.metadata.view.LogicalViewManager; import org.apache.kylin.rest.ddl.SourceTableCheck; import org.apache.kylin.rest.ddl.ViewCheck; import org.apache.kylin.rest.request.ViewRequest; +import org.apache.kylin.rest.response.LogicalViewResponse; import org.apache.kylin.rest.util.AclPermissionUtil; import org.apache.spark.ddl.DDLCheck; @@ -135,18 +136,20 @@ public class SparkDDLService extends BasicService { LogicalViewLoader.unloadView(context.getLogicalViewName(), SparderEnv.getSparkSession()); } - public List<LogicalView> listAll(String project, String tableName) { + public List<LogicalViewResponse> listAll(String project, String tableName) { List<LogicalView> logicalViews = LogicalViewManager.getInstance(KylinConfig.getInstanceFromEnv()).list(); if (StringUtils.isNotBlank(tableName)) { logicalViews = logicalViews.stream() .filter(table -> table.getTableName().toLowerCase().contains(tableName.toLowerCase())) .collect(Collectors.toList()); } - logicalViews.forEach(table -> { + List<LogicalViewResponse> viewResponses = + logicalViews.stream().map(LogicalViewResponse::new).collect(Collectors.toList()); + viewResponses.forEach(table -> { if (!table.getCreatedProject().equalsIgnoreCase(project)) { table.setCreatedSql("***"); } }); - return logicalViews; + return viewResponses; } } \ No newline at end of file diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java index 595f5c0b6f..aabb60b613 100644 --- a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java +++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java @@ -18,7 +18,6 @@ package org.apache.kylin.rest.service; -import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_LOGICAL_VIEW; import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_TABLE_NAME; import static org.apache.kylin.common.exception.code.ErrorCodeServer.EXCLUDED_TABLE_REQUEST_NOT_ALLOWED; @@ -152,8 +151,7 @@ public class TableExtService extends BasicService { if (logicalTable != null && viewProject.equalsIgnoreCase(project)) { canLoadTables.add(table); } else { - throw new KylinException(INVALID_LOGICAL_VIEW, MsgPicker.getMsg() - .getLoadLogicalViewError(tableName, viewProject)); + tableResponse.getFailed().add(tableName); } }); } diff --git a/src/datasource-service/src/main/scala/org/apache/kylin/rest/ddl/ViewCheck.scala b/src/datasource-service/src/main/scala/org/apache/kylin/rest/ddl/ViewCheck.scala index 82935a66e5..879283b427 100644 --- a/src/datasource-service/src/main/scala/org/apache/kylin/rest/ddl/ViewCheck.scala +++ b/src/datasource-service/src/main/scala/org/apache/kylin/rest/ddl/ViewCheck.scala @@ -65,7 +65,7 @@ class ViewCheck extends DDLCheck { cnDescription.append(s"创建不要加 database 名称，系统自动创建到 ${config.getDDLLogicalViewDB} 库中，" + s"删除要加 ${config.getDDLLogicalViewDB} 库名称 \n") enDescription.append(s"Creating does not require adding database, it is automatically created in" - + s" ${config.getDDLLogicalViewDB} ,\n deleting should add ${config.getDDLLogicalViewDB} database") + + s" ${config.getDDLLogicalViewDB} , deleting should add ${config.getDDLLogicalViewDB} database\n") syntaxSupport.append(" `create logical view`, `drop logical view` ") cnDescription .append(s"仅支持 ${syntaxSupport} 语法\n") diff --git a/src/datasource-service/src/test/java/org/apache/kylin/rest/service/SparkDDLTest.java b/src/datasource-service/src/test/java/org/apache/kylin/rest/service/SparkDDLTest.java index 8e00df161f..80e9a5f114 100644 --- a/src/datasource-service/src/test/java/org/apache/kylin/rest/service/SparkDDLTest.java +++ b/src/datasource-service/src/test/java/org/apache/kylin/rest/service/SparkDDLTest.java @@ -40,6 +40,7 @@ import org.apache.kylin.metadata.view.LogicalViewManager; import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.rest.request.ViewRequest; import org.apache.kylin.rest.response.LoadTableResponse; +import org.apache.kylin.rest.response.LogicalViewResponse; import org.apache.spark.sql.LogicalViewLoader; import org.apache.spark.sql.SparderEnv; @@ -283,18 +284,23 @@ public class SparkDDLTest extends NLocalFileMetadataTestCase { Assert.assertEquals(3, description.get(0).size()); // view list in project - List<LogicalView> logicalViewsInProject = ddlService.listAll("ssb", ""); - List<LogicalView> logicalViewsInProject2 = ddlService.listAll("ssb", "table2"); + List<LogicalViewResponse> logicalViewsInProject = ddlService.listAll("ssb", ""); + List<LogicalViewResponse> logicalViewsInProject2 = ddlService.listAll("ssb", "table2"); + List<LogicalViewResponse> logicalViewsInProject3 = ddlService.listAll("demo", ""); Assert.assertEquals(3, logicalViewsInProject.size()); Assert.assertEquals(1, logicalViewsInProject2.size()); - LogicalView confidentialTable = + LogicalViewResponse confidentialTable = logicalViewsInProject.stream().filter(table -> table.getCreatedProject().equals("demo")).collect( Collectors.toList()).get(0); - LogicalView noConfidentialTable = + LogicalViewResponse noConfidentialTable = logicalViewsInProject.stream().filter(table -> table.getCreatedProject().equals("ssb")).collect( Collectors.toList()).get(0); + LogicalViewResponse noConfidentialTable2 = + logicalViewsInProject3.stream().filter(table -> table.getCreatedProject().equals("demo")).collect( + Collectors.toList()).get(0); Assert.assertEquals("***", confidentialTable.getCreatedSql()); Assert.assertNotEquals("***", noConfidentialTable.getCreatedSql()); + Assert.assertNotEquals("***", noConfidentialTable2.getCreatedSql()); // load table list String[] failedLoadTables = {"KYLIN_LOGICAL_VIEW.logical_view_table2", @@ -306,9 +312,8 @@ public class SparkDDLTest extends NLocalFileMetadataTestCase { LoadTableResponse tableResponse = new LoadTableResponse(); tableExtService.filterAccessTables(successLoadTables, canLoadTables, tableResponse, "ssb"); Assert.assertEquals(2, canLoadTables.size()); - assertKylinExeption( - () -> - tableExtService.filterAccessTables(failedLoadTables, canLoadTables, tableResponse, "ssb"), - "Can't load table"); + canLoadTables.clear(); + tableExtService.filterAccessTables(failedLoadTables, canLoadTables, tableResponse, "ssb"); + Assert.assertEquals(2, canLoadTables.size()); } } diff --git a/src/kylin-it/src/test/java/org/apache/kylin/newten/LogicalViewTest.java b/src/kylin-it/src/test/java/org/apache/kylin/newten/LogicalViewTest.java index df98c689ab..58d1109a55 100644 --- a/src/kylin-it/src/test/java/org/apache/kylin/newten/LogicalViewTest.java +++ b/src/kylin-it/src/test/java/org/apache/kylin/newten/LogicalViewTest.java @@ -26,6 +26,7 @@ import org.apache.kylin.common.util.Pair; import org.apache.kylin.engine.spark.NLocalWithSparkSessionTest; import org.apache.kylin.job.engine.JobEngineConfig; import org.apache.kylin.job.impl.threadpool.NDefaultScheduler; +import org.apache.kylin.metadata.cube.model.LayoutEntity; import org.apache.kylin.metadata.cube.model.NDataflow; import org.apache.kylin.metadata.cube.model.NDataflowManager; import org.apache.kylin.metadata.model.SegmentRange; @@ -34,6 +35,7 @@ import org.apache.kylin.util.ExecAndComp; import org.apache.spark.sql.SparderEnv; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -71,6 +73,8 @@ public class LogicalViewTest extends NLocalWithSparkSessionTest { public void testLogicalView() throws Exception { String dfID = "451e127a-b684-1474-744b-c9afc14378af"; NDataflow dataflow = dfMgr.getDataflow(dfID); + LayoutEntity layout = dataflow.getIndexPlan().getLayoutEntity(20000000001L); + Assert.assertNotNull(layout); populateSSWithCSVData(getTestConfig(), getProject(), SparderEnv.getSparkSession()); indexDataConstructor.buildIndex(dfID, SegmentRange.TimePartitionedSegmentRange.createInfinite(), Sets.newHashSet( diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/SparkDDLController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/SparkDDLController.java index aa4ebbd9a6..4395f8c4c2 100644 --- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/SparkDDLController.java +++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/SparkDDLController.java @@ -23,9 +23,9 @@ import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLI import java.util.List; import org.apache.kylin.common.exception.KylinException; -import org.apache.kylin.metadata.view.LogicalView; import org.apache.kylin.rest.request.ViewRequest; import org.apache.kylin.rest.response.EnvelopeResponse; +import org.apache.kylin.rest.response.LogicalViewResponse; import org.apache.kylin.rest.service.SparkDDLService; import org.apache.spark.sql.LogicalViewLoader; @@ -83,11 +83,11 @@ public class SparkDDLController extends NBasicController { @ApiOperation(value = "ddl_desc") @GetMapping(value = "/ddl/view_list") @ResponseBody - public EnvelopeResponse<List<LogicalView>> list( + public EnvelopeResponse<List<LogicalViewResponse>> list( @RequestParam("project") String project, @RequestParam(value = "table", required = false, defaultValue = "") String tableName) { project = checkProjectName(project); - List<LogicalView> logicalViews = sparkDDLService.listAll(project, tableName); + List<LogicalViewResponse> logicalViews = sparkDDLService.listAll(project, tableName); return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, logicalViews, ""); } } diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/MetaStoreService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/MetaStoreService.java index 3680b15cc4..7d6599b389 100644 --- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/MetaStoreService.java +++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/MetaStoreService.java @@ -19,6 +19,7 @@ package org.apache.kylin.rest.service; import static org.apache.kylin.common.constant.Constants.KE_VERSION; +import static org.apache.kylin.common.exception.ServerErrorCode.FAILED_CREATE_MODEL; import static org.apache.kylin.common.exception.ServerErrorCode.MODEL_EXPORT_ERROR; import static org.apache.kylin.common.exception.ServerErrorCode.MODEL_IMPORT_ERROR; import static org.apache.kylin.common.exception.ServerErrorCode.MODEL_METADATA_FILE_ERROR; @@ -97,6 +98,8 @@ import org.apache.kylin.metadata.project.NProjectManager; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.query.util.QueryHisStoreUtil; import org.apache.kylin.metadata.realization.RealizationStatusEnum; +import org.apache.kylin.metadata.view.LogicalView; +import org.apache.kylin.metadata.view.LogicalViewManager; import org.apache.kylin.rest.aspect.Transaction; import org.apache.kylin.rest.constant.ModelStatusToDisplayEnum; import org.apache.kylin.rest.request.ModelImportRequest; @@ -440,7 +443,7 @@ public class MetaStoreService extends BasicService { ProjectInstance projectInstance = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv()) .getProject(targetProject); ISourceMetadataExplorer explorer = SourceFactory.getSource(projectInstance).getSourceMetadataExplorer(); - + KylinConfig config = KylinConfig.getInstanceFromEnv(); List<TableDesc> existTableSet = Lists.newArrayList(); for (TableDesc missTableDesc : missTableList) { try { @@ -454,6 +457,14 @@ public class MetaStoreService extends BasicService { } catch (Exception e) { logger.warn("try load table: {} failed.", missTableDesc.getIdentity(), e); } + if (config.isDDLLogicalViewEnabled() && missTableDesc.isLogicalView()) { + LogicalView logicalView = LogicalViewManager.getInstance(config).get(missTableDesc.getName()); + if (logicalView != null && !targetProject.equalsIgnoreCase(logicalView.getCreatedProject())) { + throw new KylinException(FAILED_CREATE_MODEL, String.format(Locale.ROOT, + " Logical View %s can only add in project %s", + missTableDesc.getName(), logicalView.getCreatedProject())); + } + } } return existTableSet; }
