This is an automated email from the ASF dual-hosted git repository.
pinal pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 02617b271 ATLAS-4892: Export/Import: Unable to import shell entities
02617b271 is described below
commit 02617b27190357500acfb508c11cd47b0e8b3630
Author: priyanshi-shah26 <[email protected]>
AuthorDate: Fri Aug 23 19:14:16 2024 +0530
ATLAS-4892: Export/Import: Unable to import shell entities
Signed-off-by: Pinal Shah <[email protected]>
---
.../org/apache/atlas/type/AtlasEntityType.java | 3 +++
.../org/apache/atlas/type/AtlasStructType.java | 3 +++
.../graph/v2/AtlasEntityGraphDiscoveryV2.java | 2 +-
.../store/graph/v2/AtlasEntityStoreV2.java | 13 +++++++---
.../store/graph/v2/EntityGraphMapper.java | 10 +++++++-
.../store/graph/v2/IDBasedEntityResolver.java | 30 +++++++++++++++-------
6 files changed, 47 insertions(+), 14 deletions(-)
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
index 60e57a379..2f1959ef8 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
@@ -1190,6 +1190,9 @@ public class AtlasEntityType extends AtlasStructType {
}
if (value == null) {
+ if (entityObj.getIsIncomplete() != null &&
entityObj.getIsIncomplete()) {// In case of import shell entities, avoid
checking of mandatory attributes
+ continue;
+ }
ret = false;
messages.add(objName + "." + attributeName +
": mandatory attribute value missing in type " + getTypeName());
}
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
index ffbe9e7d8..82e15fdd7 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
@@ -421,6 +421,9 @@ public class AtlasStructType extends AtlasType {
AtlasEntity entityObj = (AtlasEntity)
structObj;
if
(entityObj.getRelationshipAttribute(attrName) == null) {
+ if (entityObj.getIsIncomplete() != null &&
entityObj.getIsIncomplete()) { // In case of import shell entities, avoid
checking of mandatory attributes
+ continue;
+ }
ret = false;
messages.add(fieldName + ": mandatory
attribute value missing in type " + getTypeName());
}
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java
index b51951fe2..6ae3374ae 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java
@@ -192,7 +192,7 @@ public class AtlasEntityGraphDiscoveryV2 implements
EntityGraphDiscovery {
protected void resolveReferences() throws AtlasBaseException {
MetricRecorder metric =
RequestContext.get().startMetricRecord("resolveReferences");
- EntityResolver[] entityResolvers = new EntityResolver[] { new
IDBasedEntityResolver(this.graph, typeRegistry),
+ EntityResolver[] entityResolvers = new EntityResolver[] { new
IDBasedEntityResolver(this.graph, typeRegistry, entityGraphMapper),
new
UniqAttrBasedEntityResolver(this.graph, typeRegistry, entityGraphMapper)
};
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
index 656c9d14b..d113cf768 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
@@ -1254,6 +1254,10 @@ public class AtlasEntityStoreV2 implements
AtlasEntityStore {
// change entity 'isInComplete' to 'false' during full
update
if (isEntityIncomplete(vertex)) {
+ if (RequestContext.get().isImportInProgress() &&
(entity.getIsIncomplete() != null && entity.getIsIncomplete())) {
+ continue;
+ }
+
vertex.removeProperty(IS_INCOMPLETE_PROPERTY_KEY);
entity.setIsIncomplete(FALSE);
@@ -1269,8 +1273,9 @@ public class AtlasEntityStoreV2 implements
AtlasEntityStore {
requestContext.recordEntityGuidUpdate(entity, guid);
}
-
- context.addUpdated(guid, entity, entityType, vertex);
+ if (!isEntityIncomplete(vertex)) { // In case of an import
shell entities, skip updating to entitiesCreated, to avoid
mapAttributesAndClassification // In case of hook shell entities, it will not
reach to this case
+ context.addUpdated(guid, entity, entityType, vertex);
+ }
} else {
graphDiscoverer.validateAndNormalize(entity);
@@ -1291,7 +1296,9 @@ public class AtlasEntityStoreV2 implements
AtlasEntityStore {
requestContext.recordEntityGuidUpdate(entity, guid);
- context.addCreated(guid, entity, entityType, vertex);
+ if (!isEntityIncomplete(vertex)) { // In case of an import
shell entities, skip adding to entitiesUpdated, to avoid
mapAttributesAndClassification // In case of hook shell entities, it will not
reach to this case
+ context.addCreated(guid, entity, entityType, vertex);
+ }
}
// during import, update the system attributes
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
index 77ab99a96..e58f96663 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
@@ -194,7 +194,10 @@ public class EntityGraphMapper {
LOG.debug("==> createShellEntityVertex({})",
objectId.getTypeName());
}
- final String guid = UUID.randomUUID().toString();
+ String guid = objectId.getGuid();
+ if (!AtlasTypeUtil.isAssignedGuid(guid)) {
+ guid = UUID.randomUUID().toString();
+ }
AtlasEntityType entityType =
typeRegistry.getEntityTypeByName(objectId.getTypeName());
AtlasVertex ret = createStructVertex(objectId);
@@ -225,6 +228,11 @@ public class EntityGraphMapper {
return ret;
}
+ public AtlasVertex createShellEntityVertex(AtlasEntity entity,
EntityGraphDiscoveryContext context) throws AtlasBaseException {
+ AtlasObjectId objectId = new AtlasObjectId(entity.getGuid(),
entity.getTypeName(), entity.getAttributes());
+ return createShellEntityVertex(objectId, context);
+ }
+
public AtlasVertex createVertexWithGuid(AtlasEntity entity, String guid)
throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> createVertexWithGuid({})", entity.getTypeName());
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IDBasedEntityResolver.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IDBasedEntityResolver.java
index 2f37eac6b..ae0d2ecdd 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IDBasedEntityResolver.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IDBasedEntityResolver.java
@@ -38,10 +38,16 @@ public class IDBasedEntityResolver implements
EntityResolver {
private final AtlasGraph graph;
private final AtlasTypeRegistry typeRegistry;
+ private final EntityGraphMapper entityGraphMapper;
public IDBasedEntityResolver(AtlasGraph graph, AtlasTypeRegistry
typeRegistry) {
+ this(graph, typeRegistry, null);
+ }
+
+ public IDBasedEntityResolver(AtlasGraph graph, AtlasTypeRegistry
typeRegistry, EntityGraphMapper entityGraphMapper) {
this.graph = graph;
this.typeRegistry = typeRegistry;
+ this.entityGraphMapper = entityGraphMapper;
}
public EntityGraphDiscoveryContext
resolveEntityReferences(EntityGraphDiscoveryContext context) throws
AtlasBaseException {
@@ -55,19 +61,25 @@ public class IDBasedEntityResolver implements
EntityResolver {
boolean isAssignedGuid = AtlasTypeUtil.isAssignedGuid(guid);
AtlasVertex vertex = isAssignedGuid ?
AtlasGraphUtilsV2.findByGuid(this.graph, guid) : null;
- if (vertex == null && !RequestContext.get().isImportInProgress())
{ // if not found in the store, look if the entity is present in the stream
+ if (vertex == null) { // if not found in the store, look if the
entity is present in the stream
AtlasEntity entity = entityStream.getByGuid(guid);
+ if (!RequestContext.get().isImportInProgress()) {
- if (entity != null) { // look for the entity in the store
using unique-attributes
- AtlasEntityType entityType =
typeRegistry.getEntityTypeByName(entity.getTypeName());
+ if (entity != null) { // look for the entity in the store
using unique-attributes
+ AtlasEntityType entityType =
typeRegistry.getEntityTypeByName(entity.getTypeName());
- if (entityType == null) {
- throw new
AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID,
TypeCategory.ENTITY.name(), entity.getTypeName());
- }
+ if (entityType == null) {
+ throw new
AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID,
TypeCategory.ENTITY.name(), entity.getTypeName());
+ }
- vertex =
AtlasGraphUtilsV2.findByUniqueAttributes(this.graph, entityType,
entity.getAttributes());
- } else if (!isAssignedGuid) { // for local-guids, entity must
be in the stream
- throw new
AtlasBaseException(AtlasErrorCode.REFERENCED_ENTITY_NOT_FOUND, guid);
+ vertex =
AtlasGraphUtilsV2.findByUniqueAttributes(this.graph, entityType,
entity.getAttributes());
+ } else if (!isAssignedGuid) { // for local-guids, entity
must be in the stream
+ throw new
AtlasBaseException(AtlasErrorCode.REFERENCED_ENTITY_NOT_FOUND, guid);
+ }
+ } else if (entity != null && entity.getIsIncomplete() != null
&& entity.getIsIncomplete()) {
+ if (entityGraphMapper != null) {
+ vertex =
entityGraphMapper.createShellEntityVertex(entity, context);
+ }
}
}
