This is an automated email from the ASF dual-hosted git repository. madhan pushed a commit to branch atlas-2.5 in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 29f24f108d6301c0e40206a6b41526830d3212f7 Author: Disha Talreja <[email protected]> AuthorDate: Wed Aug 20 10:35:45 2025 -0700 ATLAS-5084: provide a way to transform hdfs paths during atlas import (#416) (cherry picked from commit 2eafacd49ebc1acb8aebe7d33cccc50ebde3536b) --- .../atlas/repository/impexp/ImportTransformer.java | 13 +++++++++---- .../repository/impexp/AtlasImportRequestTest.java | 4 ++-- .../atlas/repository/impexp/ImportServiceTest.java | 4 ++-- .../impexp/ImportTransformerJSONTest.java | 2 +- .../repository/impexp/ImportTransformerTest.java | 12 ++++++------ .../impexp/ImportTransformsShaperTest.java | 2 +- .../repository/impexp/ImportTransformsTest.java | 22 +++++++++++----------- .../atlas/repository/impexp/ZipSourceTest.java | 2 +- 8 files changed, 33 insertions(+), 28 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java index c472b9444..7cb5948bc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java @@ -31,7 +31,7 @@ import java.util.Map; import java.util.Objects; public abstract class ImportTransformer { - private static final String TRANSFORMER_PARAMETER_SEPARATOR = "\\:"; + private static final String TRANSFORMER_PARAMETER_SEPARATOR = "~"; private static final String TRANSFORMER_NAME_ADD = "add"; private static final String TRANSFORMER_NAME_CLEAR_ATTR = "clearAttrValue"; @@ -49,6 +49,11 @@ public abstract class ImportTransformer { } public static ImportTransformer getTransformer(String transformerSpec) throws AtlasBaseException { + if (transformerSpec.contains(":") && !transformerSpec.contains(TRANSFORMER_PARAMETER_SEPARATOR)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, + "Invalid transformer specification. The parameter separator ':' is no longer supported. Use '~' instead. Transformer specification: " + transformerSpec); + } + String[] params = StringUtils.split(transformerSpec, TRANSFORMER_PARAMETER_SEPARATOR); String key = (params == null || params.length < 1) ? transformerSpec : params[0]; @@ -66,13 +71,13 @@ public abstract class ImportTransformer { } else if (key.equals(TRANSFORMER_NAME_UPPERCASE)) { ret = new Uppercase(); } else if (key.equals(TRANSFORMER_NAME_REMOVE_CLASSIFICATION)) { - String name = (params == null || params.length < 1) ? "" : StringUtils.join(params, ":", 1, params.length); + String name = (params == null || params.length < 1) ? "" : StringUtils.join(params, "~", 1, params.length); ret = new RemoveClassification(name); } else if (key.equals(TRANSFORMER_NAME_ADD)) { - String name = (params == null || params.length < 1) ? "" : StringUtils.join(params, ":", 1, params.length); + String name = (params == null || params.length < 1) ? "" : StringUtils.join(params, "~", 1, params.length); ret = new AddValueToAttribute(name); } else if (key.equals(TRANSFORMER_NAME_CLEAR_ATTR)) { - String name = (params == null || params.length < 1) ? "" : StringUtils.join(params, ":", 1, params.length); + String name = (params == null || params.length < 1) ? "" : StringUtils.join(params, "~", 1, params.length); ret = new ClearAttributes(name); } else if (key.equals(TRANSFORMER_SET_DELETED)) { ret = new SetDeleted(); diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java index 0d04d6724..7c91ef441 100644 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java @@ -102,7 +102,7 @@ public class AtlasImportRequestTest { @Test public void serializeAtlasImportRequstFromJsonWith1Transform() { - String jsonData = "{ \"options\": { \"transforms\": \"{ \\\"hive_db\\\": { \\\"qualifiedName\\\": [ \\\"replace:@cl1:@cl2\\\" ] } }\" } }"; + String jsonData = "{ \"options\": { \"transforms\": \"{ \\\"hive_db\\\": { \\\"qualifiedName\\\": [ \\\"replace~@cl1~@cl2\\\" ] } }\" } }"; AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class); @@ -123,7 +123,7 @@ public class AtlasImportRequestTest { @Test public void serializeAtlasImportRequstFromJson() { - String jsonData = "{ \"options\": { \"transforms\": \"{ \\\"hive_db\\\": { \\\"qualifiedName\\\": [ \\\"replace:@cl1:@cl2\\\" ] }, \\\"hive_table\\\": { \\\"qualifiedName\\\": [ \\\"lowercase\\\", \\\"replace:@cl1:@cl2\\\" ] } }\" } } }"; + String jsonData = "{ \"options\": { \"transforms\": \"{ \\\"hive_db\\\": { \\\"qualifiedName\\\": [ \\\"replace~@cl1~@cl2\\\" ] }, \\\"hive_table\\\": { \\\"qualifiedName\\\": [ \\\"lowercase\\\", \\\"replace~@cl1~@cl2\\\" ] } }\" } } }"; AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class); diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java index dbf84dbde..66cada7de 100644 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java @@ -844,7 +844,7 @@ public class ImportServiceTest extends AtlasTestBase { loadBaseModel(); loadHiveModel(); - String transformJSON = "{ \"Asset\": { \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }"; + String transformJSON = "{ \"Asset\": { \"qualifiedName\":[ \"lowercase\", \"replace~@cl1~@cl2\" ] } }"; ZipSource zipSource = new ZipSource(inputStream); importService.setImportTransform(zipSource, transformJSON); ImportTransforms importTransforms = zipSource.getImportTransform(); @@ -859,7 +859,7 @@ public class ImportServiceTest extends AtlasTestBase { loadBaseModel(); loadHiveModel(); - String transformJSON = "{ \"Asset\": { \"qualifiedName\":[ \"replace:@cl1:@cl2\" ] }, \"hive_table\": { \"qualifiedName\":[ \"lowercase\" ] } }"; + String transformJSON = "{ \"Asset\": { \"qualifiedName\":[ \"replace~@cl1~@cl2\" ] }, \"hive_table\": { \"qualifiedName\":[ \"lowercase\" ] } }"; ZipSource zipSource = new ZipSource(inputStream); importService.setImportTransform(zipSource, transformJSON); ImportTransforms importTransforms = zipSource.getImportTransform(); diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerJSONTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerJSONTest.java index 4a99f440c..5d34208f8 100644 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerJSONTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerJSONTest.java @@ -28,7 +28,7 @@ public class ImportTransformerJSONTest { public void createAtlasImportTransformFromJson() throws Exception { String hiveTableType = "hive_table"; String qualifiedName = "qualifiedName"; - String jsonTransforms = "{ \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }"; + String jsonTransforms = "{ \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace~@cl1~@cl2\" ] } }"; ImportTransforms transforms = ImportTransforms.fromJson(jsonTransforms); diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerTest.java index 5a2e5c761..819cf2f9c 100644 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerTest.java @@ -30,7 +30,7 @@ public class ImportTransformerTest { String param1 = "@cl1"; String param2 = "@cl2"; - ImportTransformer e = ImportTransformer.getTransformer(String.format("%s:%s:%s", "replace", param1, param2)); + ImportTransformer e = ImportTransformer.getTransformer(String.format("%s~%s~%s", "replace", param1, param2)); assertTrue(e instanceof ImportTransformer.Replace); assertEquals(((ImportTransformer.Replace) e).getToFindStr(), param1); @@ -42,8 +42,8 @@ public class ImportTransformerTest { String param1 = "@cl1"; String param2 = "@cl2"; - ImportTransformer e1 = ImportTransformer.getTransformer(String.format("%s:%s:%s", "replace", param1, param2)); - ImportTransformer e2 = ImportTransformer.getTransformer("replace:tt1:tt2"); + ImportTransformer e1 = ImportTransformer.getTransformer(String.format("%s~%s~%s", "replace", param1, param2)); + ImportTransformer e2 = ImportTransformer.getTransformer(String.format("replace~tt1~tt2")); assertTrue(e1 instanceof ImportTransformer.Replace); assertEquals(((ImportTransformer.Replace) e1).getToFindStr(), param1); @@ -56,7 +56,7 @@ public class ImportTransformerTest { @Test public void createWithDefaultParameters() throws AtlasBaseException { - ImportTransformer e1 = ImportTransformer.getTransformer("replace:@cl1"); + ImportTransformer e1 = ImportTransformer.getTransformer("replace~@cl1"); ImportTransformer e2 = ImportTransformer.getTransformer("replace"); assertTrue(e1 instanceof ImportTransformer.Replace); @@ -92,7 +92,7 @@ public class ImportTransformerTest { @Test public void applyReplaceTransformer1() throws AtlasBaseException { - ImportTransformer e = ImportTransformer.getTransformer("replace:@cl1:@cl2"); + ImportTransformer e = ImportTransformer.getTransformer("replace~@cl1~@cl2"); assertEquals(e.apply("@cl1"), "@cl2"); assertEquals(e.apply("default@cl1"), "default@cl2"); @@ -105,7 +105,7 @@ public class ImportTransformerTest { @Test public void applyReplaceTransformer2() throws AtlasBaseException { - ImportTransformer e = ImportTransformer.getTransformer("replace:@cl1"); + ImportTransformer e = ImportTransformer.getTransformer("replace~@cl1"); assertEquals(e.apply("@cl1"), ""); assertEquals(e.apply("default@cl1"), "default"); diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsShaperTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsShaperTest.java index 54fc4e002..f9f1dcc05 100644 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsShaperTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsShaperTest.java @@ -100,7 +100,7 @@ public class ImportTransformsShaperTest extends AtlasTestBase { private AtlasImportRequest getImporRequest() { AtlasImportRequest request = new AtlasImportRequest(); - request.getOptions().put("transforms", "{ \"Referenceable\": { \"*\":[ \"addClassification:REPLICATED\" ] } }"); + request.getOptions().put("transforms", "{ \"Referenceable\": { \"*\":[ \"addClassification~REPLICATED\" ] } }"); return request; } diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java index 200bcad86..48ca56bc6 100644 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java @@ -47,16 +47,16 @@ public class ImportTransformsTest { private final String lowerCaseCL1 = "@cl1"; private final String lowerCaseCL2 = "@cl2"; - private final String jsonLowerCaseReplace = "{ \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }"; - private final String jsonReplaceLowerCase = "{ \"Asset\": { \"qualifiedName\":[ \"replace:@cl1:@cl2\" ] }, \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }"; - private final String jsonReplaceRemoveClassification = "{ \"hive_table\": { \"qualifiedName\":[ \"replace:@%s:@%s\"], \"*\":[ \"removeClassification:%s_to_%s\" ] } }"; - private final String jsonReplaceAndAddAttrValue = "{ \"hive_table\": { \"qualifiedName\":[ \"replace:@%s:@%s\"], \"*\":[ \"add:%s=list:%s\" ] } }"; - private final String jsonSingleClearAttrValue = "{ \"hive_table\": { \"*\":[ \"clearAttrValue:replicatedToCluster\", \"clearAttrValue:replicatedFromCluster\" ] } }"; - private final String jsonMultipleClearAttrValue = "{ \"hive_table\": { \"*\":[ \"clearAttrValue:replicatedToCluster,replicatedFromCluster\" ] } }"; - private final String jsonSetDeleted = "{ \"hive_table\": { \"*\":[ \"setDeleted\" ] } }"; - private final String jsonAddClasification = "{ \"hive_table\": { \"*\":[ \"addClassification:REPLICATED\" ] } }"; - private final String jsonAddClasification2 = "{ \"hive_table\": { \"*\":[ \"addClassification:REPLICATED_2\" ] } }"; - private final String jsonAddClasificationScoped = "{ \"hive_column\": { \"*\":[ \"addClassification:REPLICATED_2:topLevel\" ] } }"; + private final String jsonLowerCaseReplace = "{ \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace~@cl1~@cl2\" ] } }"; + private final String jsonReplaceLowerCase = "{ \"Asset\": { \"qualifiedName\":[ \"replace~@cl1~@cl2\" ] }, \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace~@cl1~@cl2\" ] } }"; + private final String jsonReplaceRemoveClassification = "{ \"hive_table\": { \"qualifiedName\":[ \"replace~@%s~@%s\"], \"*\":[ \"removeClassification~%s_to_%s\" ] } }"; + private final String jsonReplaceAndAddAttrValue = "{ \"hive_table\": { \"qualifiedName\":[ \"replace~@%s~@%s\"], \"*\":[ \"add~%s=list:%s\" ] } }"; + private final String jsonSingleClearAttrValue = "{ \"hive_table\": { \"*\":[ \"clearAttrValue~replicatedToCluster\", \"clearAttrValue~replicatedFromCluster\" ] } }"; + private final String jsonMultipleClearAttrValue = "{ \"hive_table\": { \"*\":[ \"clearAttrValue~replicatedToCluster,replicatedFromCluster\" ] } }"; + private final String jsonSetDeleted = "{ \"hive_table\": { \"*\":[ \"setDeleted\" ] } }"; + private final String jsonAddClasification = "{ \"hive_table\": { \"*\":[ \"addClassification~REPLICATED\" ] } }"; + private final String jsonAddClasification2 = "{ \"hive_table\": { \"*\":[ \"addClassification~REPLICATED_2\" ] } }"; + private final String jsonAddClasificationScoped = "{ \"hive_column\": { \"*\":[ \"addClassification~REPLICATED_2~topLevel\" ] } }"; private ImportTransforms transform; @@ -278,7 +278,7 @@ public class ImportTransformsTest { Map<String, List<ImportTransformer>> tr = new HashMap<>(); List<ImportTransformer> trList = new ArrayList<>(); - trList.add(ImportTransformer.getTransformer(String.format("replace:%s:%s", lowerCaseCL1, lowerCaseCL2))); + trList.add(ImportTransformer.getTransformer(String.format("replace~%s~%s", lowerCaseCL1, lowerCaseCL2))); tr.put(ATTR_NAME_QUALIFIED_NAME, trList); transform.getTransforms().put("hive_column", tr); diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java index 2a9e4f6bf..b948b45c4 100644 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java @@ -202,6 +202,6 @@ public class ZipSourceTest { } private ImportTransforms getTransformForHiveDB() { - return ImportTransforms.fromJson("{ \"hive_db\": { \"qualifiedName\": [ \"replace:@cl1:@cl2\" ] } }"); + return ImportTransforms.fromJson("{ \"hive_db\": { \"qualifiedName\": [ \"replace~@cl1~@cl2\" ] } }"); } }
