This is an automated email from the ASF dual-hosted git repository.
pinal pushed a commit to branch ATLAS-5021_u
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/ATLAS-5021_u by this push:
new 70f3d6ac1 ATLAS-5021 : review comments addressed
70f3d6ac1 is described below
commit 70f3d6ac1a299328dbb4c1fcd6c16144cccd5ac9
Author: Pinal Shah <[email protected]>
AuthorDate: Thu Sep 4 12:52:35 2025 +0530
ATLAS-5021 : review comments addressed
---
.../src/main/bin/run-trino-extractor.sh | 3 +-
.../src/main/conf/atlas-application.properties | 18 +++++++---
.../trino-extractor/src/main/conf/atlas-log4j.xml | 42 ----------------------
.../atlas-trino-extractor-logback.xml} | 7 +++-
.../apache/atlas/trino/cli/ExtractorContext.java | 25 ++++++++++---
.../src/main/assemblies/atlas-trino-extractor.xml | 2 +-
.../graphdb/janus/AtlasJanusGraphManagement.java | 2 +-
.../repository/graph/GraphBackedSearchIndexer.java | 1 -
.../repository/patches/UniqueAttributePatch.java | 1 -
9 files changed, 43 insertions(+), 58 deletions(-)
diff --git a/addons/trino-extractor/src/main/bin/run-trino-extractor.sh
b/addons/trino-extractor/src/main/bin/run-trino-extractor.sh
index a7d4c8d71..7c4589a47 100755
--- a/addons/trino-extractor/src/main/bin/run-trino-extractor.sh
+++ b/addons/trino-extractor/src/main/bin/run-trino-extractor.sh
@@ -45,7 +45,6 @@ if [ ! -e "${JAVA_BIN}" ] || [ ! -e "${JAR_BIN}" ]; then
exit 1
fi
-# Construct Atlas classpath using jars from hook/hive/atlas-hive-plugin-impl/
directory.
for i in "${BASEDIR}/lib/"*.jar; do
ATLASCPPATH="${ATLASCPPATH}:$i"
done
@@ -75,7 +74,7 @@ then
fi
JAVA_PROPERTIES="$ATLAS_OPTS -Datlas.log.dir=$ATLAS_LOG_DIR
-Datlas.log.file=atlas-trino-extractor.log
--Dlog4j.configuration=atlas-log4j.xml -Djdk.httpclient.HttpClient.log=requests
-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006"
+-Dlogback.configurationFile=atlas-trino-extractor-logback.xml
-Djdk.httpclient.HttpClient.log=requests"
IMPORT_ARGS=()
JVM_ARGS=
diff --git a/addons/trino-extractor/src/main/conf/atlas-application.properties
b/addons/trino-extractor/src/main/conf/atlas-application.properties
index d709ad016..7a09a7e36 100755
--- a/addons/trino-extractor/src/main/conf/atlas-application.properties
+++ b/addons/trino-extractor/src/main/conf/atlas-application.properties
@@ -19,14 +19,22 @@
atlas.rest.address=http://localhost:21000/
######## Trino connection ############
+######## If SSL is enabled, append below value with ";SSL=true" ##########
atlas.trino.jdbc.address=jdbc:trino://<host>:<port>/
+######## Trino Server Authentication ############
atlas.trino.jdbc.user=<username>
-######## Trino environment name ######
-atlas.trino.namespace=cm
+######## Trino environment name - default value "cm" ######
+#atlas.trino.namespace=cm
+
+######## Catalogs for which extractor is to run ######
#atlas.trino.catalogs.registered=
-######## Datasource for which ########
-######## Atlas hook is enabled #######
+######## Datasource for which Atlas hook is enabled########
#atlas.trino.catalog.hook.enabled.hive_catalog=true
-#atlas.trino.catalog.hook.enabled.hive_catalog.namespace=cm
\ No newline at end of file
+#atlas.trino.catalog.hook.enabled.hive_catalog.namespace=cm
+
+atlas.trino.extractor.catalog=
+atlas.trino.extractor.schema=
+atlas.trino.extractor.table=
+atlas.trino.extractor.schedule=
\ No newline at end of file
diff --git a/addons/trino-extractor/src/main/conf/atlas-log4j.xml
b/addons/trino-extractor/src/main/conf/atlas-log4j.xml
deleted file mode 100644
index 371c3b3e3..000000000
--- a/addons/trino-extractor/src/main/conf/atlas-log4j.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/";>
- <appender name="FILE" class="org.apache.log4j.RollingFileAppender">
- <param name="File" value="${atlas.log.dir}/${atlas.log.file}"/>
- <param name="Append" value="true"/>
- <param name="maxFileSize" value="100MB" />
- <param name="maxBackupIndex" value="20" />
- <layout class="org.apache.log4j.PatternLayout">
- <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m
(%C{1}:%L)%n"/>
- </layout>
- </appender>
-
- <logger name="org.apache.atlas.trino" additivity="false">
- <level value="info"/>
- <appender-ref ref="FILE"/>
- </logger>
-
- <root>
- <priority value="warn"/>
- <appender-ref ref="FILE"/>
- </root>
-</log4j:configuration>
diff --git a/addons/trino-extractor/src/main/resources/atlas-logback.xml
b/addons/trino-extractor/src/main/conf/atlas-trino-extractor-logback.xml
similarity index 85%
rename from addons/trino-extractor/src/main/resources/atlas-logback.xml
rename to addons/trino-extractor/src/main/conf/atlas-trino-extractor-logback.xml
index 2a616c524..4cb6e4259 100644
--- a/addons/trino-extractor/src/main/resources/atlas-logback.xml
+++ b/addons/trino-extractor/src/main/conf/atlas-trino-extractor-logback.xml
@@ -41,7 +41,12 @@
</rollingPolicy>
</appender>
- <logger name="org.apache.atlas.trino" additivity="false" level="info">
+ <logger name="org.apache.atlas" additivity="false" level="info">
+ <appender-ref ref="FILE"/>
+ </logger>
+
+ <!-- to avoid logs - The configuration log.flush.interval.messages = 1 was
supplied but isn't a known config -->
+ <logger name="org.apache.kafka.common.config.AbstractConfig"
additivity="false" level="error">
<appender-ref ref="FILE"/>
</logger>
diff --git
a/addons/trino-extractor/src/main/java/org/apache/atlas/trino/cli/ExtractorContext.java
b/addons/trino-extractor/src/main/java/org/apache/atlas/trino/cli/ExtractorContext.java
index 23d483ae6..dee04300c 100644
---
a/addons/trino-extractor/src/main/java/org/apache/atlas/trino/cli/ExtractorContext.java
+++
b/addons/trino-extractor/src/main/java/org/apache/atlas/trino/cli/ExtractorContext.java
@@ -23,12 +23,17 @@ import org.apache.atlas.trino.client.AtlasClientHelper;
import org.apache.atlas.trino.client.TrinoClientHelper;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.configuration.Configuration;
+import org.apache.commons.lang.StringUtils;
import java.io.IOException;
public class ExtractorContext {
static final String TRINO_NAMESPACE_CONF = "atlas.trino.namespace";
static final String DEFAULT_TRINO_NAMESPACE = "cm";
+ static final String TRINO_CATALOG_CONF =
"atlas.trino.extractor.catalog";
+ static final String TRINO_SCHEMA_CONF =
"atlas.trino.extractor.schema";
+ static final String TRINO_TABLE_CONF =
"atlas.trino.extractor.table";
+ static final String TRINO_SCHEDULE_CONF =
"atlas.trino.extractor.schedule";
static final String OPTION_CATALOG_SHORT = "c";
static final String OPTION_CATALOG_LONG = "catalog";
static final String OPTION_SCHEMA_SHORT = "s";
@@ -54,10 +59,22 @@ public class ExtractorContext {
this.atlasClientHelper = createAtlasClientHelper();
this.trinoClientHelper = createTrinoClientHelper();
this.namespace = atlasConf.getString(TRINO_NAMESPACE_CONF,
DEFAULT_TRINO_NAMESPACE);
- this.catalog = cmd.getOptionValue(OPTION_CATALOG_SHORT);
- this.schema = cmd.getOptionValue(OPTION_SCHEMA_SHORT);
- this.table = cmd.getOptionValue(OPTION_TABLE_SHORT);
- this.cronExpression =
cmd.getOptionValue(OPTION_CRON_EXPRESSION_SHORT);
+
+ String cmdCatalog = cmd.getOptionValue(OPTION_CATALOG_SHORT);
+ String cmdSchema = cmd.getOptionValue(OPTION_SCHEMA_SHORT);
+ String cmdTable = cmd.getOptionValue(OPTION_TABLE_SHORT);
+ String cmdSchedule =
cmd.getOptionValue(OPTION_CRON_EXPRESSION_SHORT);
+ this.cronExpression = StringUtils.isNotEmpty(cmdSchedule) ?
cmdSchedule : atlasConf.getString(TRINO_SCHEDULE_CONF);
+
+ if (StringUtils.isEmpty(cmdCatalog)) {
+ this.catalog = atlasConf.getString(TRINO_CATALOG_CONF);
+ this.schema = atlasConf.getString(TRINO_SCHEMA_CONF);
+ this.table = atlasConf.getString(TRINO_TABLE_CONF);
+ } else {
+ this.catalog = cmdCatalog;
+ this.schema = cmdSchema;
+ this.table = cmdTable;
+ }
}
public Configuration getAtlasConf() {
diff --git a/distro/src/main/assemblies/atlas-trino-extractor.xml
b/distro/src/main/assemblies/atlas-trino-extractor.xml
index 923c8dc95..205d43c9c 100644
--- a/distro/src/main/assemblies/atlas-trino-extractor.xml
+++ b/distro/src/main/assemblies/atlas-trino-extractor.xml
@@ -34,7 +34,7 @@
<directory>../addons/trino-extractor/src/main/conf</directory>
<outputDirectory>/conf</outputDirectory>
<includes>
- <include>atlas-log4j.xml</include>
+ <include>atlas-trino-extractor-logback.xml</include>
<include>atlas-application.properties</include>
</includes>
<fileMode>0755</fileMode>
diff --git
a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
index c9fa2e4cd..a1c6cf87e 100644
---
a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
+++
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
@@ -87,7 +87,7 @@ public class AtlasJanusGraphManagement implements
AtlasGraphManagement {
private final AtlasJanusGraph graph;
private final JanusGraphManagement management;
private final Set<String> newMultProperties = new HashSet<>();
- private boolean isSuccess = false;
+ private boolean isSuccess;
public AtlasJanusGraphManagement(AtlasJanusGraph graph,
JanusGraphManagement managementSystem) {
this.management = managementSystem;
diff --git
a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index 191dea810..d6aaab025 100755
---
a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++
b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -20,7 +20,6 @@ package org.apache.atlas.repository.graph;
import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.ApplicationProperties;
-import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.discovery.SearchIndexer;
import org.apache.atlas.exception.AtlasBaseException;
diff --git
a/repository/src/main/java/org/apache/atlas/repository/patches/UniqueAttributePatch.java
b/repository/src/main/java/org/apache/atlas/repository/patches/UniqueAttributePatch.java
index cf364a01d..0f9a4831e 100644
---
a/repository/src/main/java/org/apache/atlas/repository/patches/UniqueAttributePatch.java
+++
b/repository/src/main/java/org/apache/atlas/repository/patches/UniqueAttributePatch.java
@@ -22,7 +22,6 @@ import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.pc.WorkItemManager;
import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.IndexException;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer.UniqueKind;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasGraph;
