This is an automated email from the ASF dual-hosted git repository.
dishatalreja pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 2eafacd49 ATLAS-5084: provide a way to transform hdfs paths during
atlas import (#416)
2eafacd49 is described below
commit 2eafacd49ebc1acb8aebe7d33cccc50ebde3536b
Author: Disha Talreja <[email protected]>
AuthorDate: Wed Aug 20 10:35:45 2025 -0700
ATLAS-5084: provide a way to transform hdfs paths during atlas import (#416)
---
.../atlas/repository/impexp/ImportTransformer.java | 13 +++++++++----
.../repository/impexp/AtlasImportRequestTest.java | 4 ++--
.../atlas/repository/impexp/ImportServiceTest.java | 4 ++--
.../impexp/ImportTransformerJSONTest.java | 2 +-
.../repository/impexp/ImportTransformerTest.java | 12 ++++++------
.../impexp/ImportTransformsShaperTest.java | 2 +-
.../repository/impexp/ImportTransformsTest.java | 22 +++++++++++-----------
.../atlas/repository/impexp/ZipSourceTest.java | 2 +-
8 files changed, 33 insertions(+), 28 deletions(-)
diff --git
a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java
index c472b9444..7cb5948bc 100644
---
a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java
+++
b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java
@@ -31,7 +31,7 @@ import java.util.Map;
import java.util.Objects;
public abstract class ImportTransformer {
- private static final String TRANSFORMER_PARAMETER_SEPARATOR = "\\:";
+ private static final String TRANSFORMER_PARAMETER_SEPARATOR = "~";
private static final String TRANSFORMER_NAME_ADD = "add";
private static final String TRANSFORMER_NAME_CLEAR_ATTR =
"clearAttrValue";
@@ -49,6 +49,11 @@ public abstract class ImportTransformer {
}
public static ImportTransformer getTransformer(String transformerSpec)
throws AtlasBaseException {
+ if (transformerSpec.contains(":") &&
!transformerSpec.contains(TRANSFORMER_PARAMETER_SEPARATOR)) {
+ throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE,
+ "Invalid transformer specification. The parameter
separator ':' is no longer supported. Use '~' instead. Transformer
specification: " + transformerSpec);
+ }
+
String[] params = StringUtils.split(transformerSpec,
TRANSFORMER_PARAMETER_SEPARATOR);
String key = (params == null || params.length < 1) ?
transformerSpec : params[0];
@@ -66,13 +71,13 @@ public abstract class ImportTransformer {
} else if (key.equals(TRANSFORMER_NAME_UPPERCASE)) {
ret = new Uppercase();
} else if (key.equals(TRANSFORMER_NAME_REMOVE_CLASSIFICATION)) {
- String name = (params == null || params.length < 1) ? "" :
StringUtils.join(params, ":", 1, params.length);
+ String name = (params == null || params.length < 1) ? "" :
StringUtils.join(params, "~", 1, params.length);
ret = new RemoveClassification(name);
} else if (key.equals(TRANSFORMER_NAME_ADD)) {
- String name = (params == null || params.length < 1) ? "" :
StringUtils.join(params, ":", 1, params.length);
+ String name = (params == null || params.length < 1) ? "" :
StringUtils.join(params, "~", 1, params.length);
ret = new AddValueToAttribute(name);
} else if (key.equals(TRANSFORMER_NAME_CLEAR_ATTR)) {
- String name = (params == null || params.length < 1) ? "" :
StringUtils.join(params, ":", 1, params.length);
+ String name = (params == null || params.length < 1) ? "" :
StringUtils.join(params, "~", 1, params.length);
ret = new ClearAttributes(name);
} else if (key.equals(TRANSFORMER_SET_DELETED)) {
ret = new SetDeleted();
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java
index 0d04d6724..7c91ef441 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java
@@ -102,7 +102,7 @@ public class AtlasImportRequestTest {
@Test
public void serializeAtlasImportRequstFromJsonWith1Transform() {
- String jsonData = "{ \"options\": { \"transforms\": \"{
\\\"hive_db\\\": { \\\"qualifiedName\\\": [ \\\"replace:@cl1:@cl2\\\" ] } }\" }
}";
+ String jsonData = "{ \"options\": { \"transforms\": \"{
\\\"hive_db\\\": { \\\"qualifiedName\\\": [ \\\"replace~@cl1~@cl2\\\" ] } }\" }
}";
AtlasImportRequest request = AtlasType.fromJson(jsonData,
AtlasImportRequest.class);
@@ -123,7 +123,7 @@ public class AtlasImportRequestTest {
@Test
public void serializeAtlasImportRequstFromJson() {
- String jsonData = "{ \"options\": { \"transforms\": \"{
\\\"hive_db\\\": { \\\"qualifiedName\\\": [ \\\"replace:@cl1:@cl2\\\" ] },
\\\"hive_table\\\": { \\\"qualifiedName\\\": [ \\\"lowercase\\\",
\\\"replace:@cl1:@cl2\\\" ] } }\" } } }";
+ String jsonData = "{ \"options\": { \"transforms\": \"{
\\\"hive_db\\\": { \\\"qualifiedName\\\": [ \\\"replace~@cl1~@cl2\\\" ] },
\\\"hive_table\\\": { \\\"qualifiedName\\\": [ \\\"lowercase\\\",
\\\"replace~@cl1~@cl2\\\" ] } }\" } } }";
AtlasImportRequest request = AtlasType.fromJson(jsonData,
AtlasImportRequest.class);
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
index dbf84dbde..66cada7de 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
@@ -844,7 +844,7 @@ public class ImportServiceTest extends AtlasTestBase {
loadBaseModel();
loadHiveModel();
- String transformJSON = "{ \"Asset\": { \"qualifiedName\":[
\"lowercase\", \"replace:@cl1:@cl2\" ] } }";
+ String transformJSON = "{ \"Asset\": { \"qualifiedName\":[
\"lowercase\", \"replace~@cl1~@cl2\" ] } }";
ZipSource zipSource = new ZipSource(inputStream);
importService.setImportTransform(zipSource, transformJSON);
ImportTransforms importTransforms = zipSource.getImportTransform();
@@ -859,7 +859,7 @@ public class ImportServiceTest extends AtlasTestBase {
loadBaseModel();
loadHiveModel();
- String transformJSON = "{ \"Asset\": { \"qualifiedName\":[
\"replace:@cl1:@cl2\" ] }, \"hive_table\": { \"qualifiedName\":[ \"lowercase\"
] } }";
+ String transformJSON = "{ \"Asset\": { \"qualifiedName\":[
\"replace~@cl1~@cl2\" ] }, \"hive_table\": { \"qualifiedName\":[ \"lowercase\"
] } }";
ZipSource zipSource = new ZipSource(inputStream);
importService.setImportTransform(zipSource, transformJSON);
ImportTransforms importTransforms = zipSource.getImportTransform();
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerJSONTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerJSONTest.java
index 4a99f440c..5d34208f8 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerJSONTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerJSONTest.java
@@ -28,7 +28,7 @@ public class ImportTransformerJSONTest {
public void createAtlasImportTransformFromJson() throws Exception {
String hiveTableType = "hive_table";
String qualifiedName = "qualifiedName";
- String jsonTransforms = "{ \"hive_table\": { \"qualifiedName\":[
\"lowercase\", \"replace:@cl1:@cl2\" ] } }";
+ String jsonTransforms = "{ \"hive_table\": { \"qualifiedName\":[
\"lowercase\", \"replace~@cl1~@cl2\" ] } }";
ImportTransforms transforms =
ImportTransforms.fromJson(jsonTransforms);
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerTest.java
index 5a2e5c761..819cf2f9c 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerTest.java
@@ -30,7 +30,7 @@ public class ImportTransformerTest {
String param1 = "@cl1";
String param2 = "@cl2";
- ImportTransformer e =
ImportTransformer.getTransformer(String.format("%s:%s:%s", "replace", param1,
param2));
+ ImportTransformer e =
ImportTransformer.getTransformer(String.format("%s~%s~%s", "replace", param1,
param2));
assertTrue(e instanceof ImportTransformer.Replace);
assertEquals(((ImportTransformer.Replace) e).getToFindStr(), param1);
@@ -42,8 +42,8 @@ public class ImportTransformerTest {
String param1 = "@cl1";
String param2 = "@cl2";
- ImportTransformer e1 =
ImportTransformer.getTransformer(String.format("%s:%s:%s", "replace", param1,
param2));
- ImportTransformer e2 =
ImportTransformer.getTransformer("replace:tt1:tt2");
+ ImportTransformer e1 =
ImportTransformer.getTransformer(String.format("%s~%s~%s", "replace", param1,
param2));
+ ImportTransformer e2 =
ImportTransformer.getTransformer(String.format("replace~tt1~tt2"));
assertTrue(e1 instanceof ImportTransformer.Replace);
assertEquals(((ImportTransformer.Replace) e1).getToFindStr(), param1);
@@ -56,7 +56,7 @@ public class ImportTransformerTest {
@Test
public void createWithDefaultParameters() throws AtlasBaseException {
- ImportTransformer e1 =
ImportTransformer.getTransformer("replace:@cl1");
+ ImportTransformer e1 =
ImportTransformer.getTransformer("replace~@cl1");
ImportTransformer e2 = ImportTransformer.getTransformer("replace");
assertTrue(e1 instanceof ImportTransformer.Replace);
@@ -92,7 +92,7 @@ public class ImportTransformerTest {
@Test
public void applyReplaceTransformer1() throws AtlasBaseException {
- ImportTransformer e =
ImportTransformer.getTransformer("replace:@cl1:@cl2");
+ ImportTransformer e =
ImportTransformer.getTransformer("replace~@cl1~@cl2");
assertEquals(e.apply("@cl1"), "@cl2");
assertEquals(e.apply("default@cl1"), "default@cl2");
@@ -105,7 +105,7 @@ public class ImportTransformerTest {
@Test
public void applyReplaceTransformer2() throws AtlasBaseException {
- ImportTransformer e = ImportTransformer.getTransformer("replace:@cl1");
+ ImportTransformer e = ImportTransformer.getTransformer("replace~@cl1");
assertEquals(e.apply("@cl1"), "");
assertEquals(e.apply("default@cl1"), "default");
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsShaperTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsShaperTest.java
index 54fc4e002..f9f1dcc05 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsShaperTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsShaperTest.java
@@ -100,7 +100,7 @@ public class ImportTransformsShaperTest extends
AtlasTestBase {
private AtlasImportRequest getImporRequest() {
AtlasImportRequest request = new AtlasImportRequest();
- request.getOptions().put("transforms", "{ \"Referenceable\": { \"*\":[
\"addClassification:REPLICATED\" ] } }");
+ request.getOptions().put("transforms", "{ \"Referenceable\": { \"*\":[
\"addClassification~REPLICATED\" ] } }");
return request;
}
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java
index 200bcad86..48ca56bc6 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java
@@ -47,16 +47,16 @@ public class ImportTransformsTest {
private final String lowerCaseCL1 = "@cl1";
private final String lowerCaseCL2 = "@cl2";
- private final String jsonLowerCaseReplace = "{ \"hive_table\":
{ \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }";
- private final String jsonReplaceLowerCase = "{ \"Asset\": {
\"qualifiedName\":[ \"replace:@cl1:@cl2\" ] }, \"hive_table\": {
\"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }";
- private final String jsonReplaceRemoveClassification = "{ \"hive_table\":
{ \"qualifiedName\":[ \"replace:@%s:@%s\"], \"*\":[
\"removeClassification:%s_to_%s\" ] } }";
- private final String jsonReplaceAndAddAttrValue = "{ \"hive_table\":
{ \"qualifiedName\":[ \"replace:@%s:@%s\"], \"*\":[ \"add:%s=list:%s\" ] } }";
- private final String jsonSingleClearAttrValue = "{ \"hive_table\":
{ \"*\":[ \"clearAttrValue:replicatedToCluster\",
\"clearAttrValue:replicatedFromCluster\" ] } }";
- private final String jsonMultipleClearAttrValue = "{ \"hive_table\":
{ \"*\":[ \"clearAttrValue:replicatedToCluster,replicatedFromCluster\" ] } }";
- private final String jsonSetDeleted = "{ \"hive_table\":
{ \"*\":[ \"setDeleted\" ] } }";
- private final String jsonAddClasification = "{ \"hive_table\":
{ \"*\":[ \"addClassification:REPLICATED\" ] } }";
- private final String jsonAddClasification2 = "{ \"hive_table\":
{ \"*\":[ \"addClassification:REPLICATED_2\" ] } }";
- private final String jsonAddClasificationScoped = "{ \"hive_column\":
{ \"*\":[ \"addClassification:REPLICATED_2:topLevel\" ] } }";
+ private final String jsonLowerCaseReplace = "{ \"hive_table\": {
\"qualifiedName\":[ \"lowercase\", \"replace~@cl1~@cl2\" ] } }";
+ private final String jsonReplaceLowerCase = "{ \"Asset\": {
\"qualifiedName\":[ \"replace~@cl1~@cl2\" ] }, \"hive_table\": {
\"qualifiedName\":[ \"lowercase\", \"replace~@cl1~@cl2\" ] } }";
+ private final String jsonReplaceRemoveClassification = "{ \"hive_table\":
{ \"qualifiedName\":[ \"replace~@%s~@%s\"], \"*\":[
\"removeClassification~%s_to_%s\" ] } }";
+ private final String jsonReplaceAndAddAttrValue = "{ \"hive_table\": {
\"qualifiedName\":[ \"replace~@%s~@%s\"], \"*\":[ \"add~%s=list:%s\" ] } }";
+ private final String jsonSingleClearAttrValue = "{ \"hive_table\": {
\"*\":[ \"clearAttrValue~replicatedToCluster\",
\"clearAttrValue~replicatedFromCluster\" ] } }";
+ private final String jsonMultipleClearAttrValue = "{ \"hive_table\": {
\"*\":[ \"clearAttrValue~replicatedToCluster,replicatedFromCluster\" ] } }";
+ private final String jsonSetDeleted = "{ \"hive_table\": { \"*\":[
\"setDeleted\" ] } }";
+ private final String jsonAddClasification = "{ \"hive_table\": { \"*\":[
\"addClassification~REPLICATED\" ] } }";
+ private final String jsonAddClasification2 = "{ \"hive_table\": { \"*\":[
\"addClassification~REPLICATED_2\" ] } }";
+ private final String jsonAddClasificationScoped = "{ \"hive_column\": {
\"*\":[ \"addClassification~REPLICATED_2~topLevel\" ] } }";
private ImportTransforms transform;
@@ -278,7 +278,7 @@ public class ImportTransformsTest {
Map<String, List<ImportTransformer>> tr = new HashMap<>();
List<ImportTransformer> trList = new ArrayList<>();
-
trList.add(ImportTransformer.getTransformer(String.format("replace:%s:%s",
lowerCaseCL1, lowerCaseCL2)));
+
trList.add(ImportTransformer.getTransformer(String.format("replace~%s~%s",
lowerCaseCL1, lowerCaseCL2)));
tr.put(ATTR_NAME_QUALIFIED_NAME, trList);
transform.getTransforms().put("hive_column", tr);
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
index 2a9e4f6bf..b948b45c4 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
@@ -202,6 +202,6 @@ public class ZipSourceTest {
}
private ImportTransforms getTransformForHiveDB() {
- return ImportTransforms.fromJson("{ \"hive_db\": { \"qualifiedName\":
[ \"replace:@cl1:@cl2\" ] } }");
+ return ImportTransforms.fromJson("{ \"hive_db\": { \"qualifiedName\":
[ \"replace~@cl1~@cl2\" ] } }");
}
}
