Skip to content
This repository was archived by the owner on Jun 10, 2024. It is now read-only.

Commit 9a41928

Browse files
cmgrotecmgrote
authored
Merge pull request #136 from cmgrote/main
Bug fix and unit tests
2 parents ffa9f8f + 63c8178 commit 9a41928

File tree

10 files changed

+273
-97
lines changed

10 files changed

+273
-97
lines changed

‎connector/pom.xml

+3-3
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,8 @@
7575
<artifactId>slf4j-api</artifactId>
7676
</dependency>
7777
<dependency>
78-
<groupId>org.slf4j</groupId>
79-
<artifactId>slf4j-nop</artifactId>
78+
<groupId>ch.qos.logback</groupId>
79+
<artifactId>logback-classic</artifactId>
8080
<scope>test</scope>
8181
</dependency>
8282
<dependency>
@@ -121,7 +121,7 @@
121121
<ignoredUnusedDeclaredDependency>junit:*</ignoredUnusedDeclaredDependency>
122122
<ignoredUnusedDeclaredDependency>org.junit.platform:*</ignoredUnusedDeclaredDependency>
123123
<!-- Used with slf4j as default implementation for testing -->
124-
<ignoredUnusedDeclaredDependency>org.slf4j:slf4j-nop:*</ignoredUnusedDeclaredDependency>
124+
<ignoredUnusedDeclaredDependency>ch.qos.logback:logback-classic:*</ignoredUnusedDeclaredDependency>
125125
<!-- Ignore these as usage depends on runtime configuration -->
126126
<ignoredUnusedDeclaredDependency>juxt:crux-metrics:*</ignoredUnusedDeclaredDependency>
127127
<ignoredUnusedDeclaredDependency>juxt:crux-rocksdb:*</ignoredUnusedDeclaredDependency>

‎connector/src/main/java/org/odpi/egeria/connectors/juxt/crux/repositoryconnector/CruxOMRSMetadataCollection.java

+52-69
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,13 @@
22
/* Copyright Contributors to the ODPi Egeria project. */
33
package org.odpi.egeria.connectors.juxt.crux.repositoryconnector;
44

5+
import clojure.lang.IPersistentMap;
56
import crux.api.tx.Transaction;
67
import org.odpi.egeria.connectors.juxt.crux.auditlog.CruxOMRSAuditCode;
78
import org.odpi.egeria.connectors.juxt.crux.auditlog.CruxOMRSErrorCode;
8-
import org.odpi.egeria.connectors.juxt.crux.mapping.Constants;
9+
import org.odpi.egeria.connectors.juxt.crux.mapping.EntitySummaryMapping;
10+
import org.odpi.egeria.connectors.juxt.crux.mapping.RelationshipMapping;
11+
import org.odpi.egeria.connectors.juxt.crux.model.search.CruxQuery;
912
import org.odpi.openmetadata.frameworks.auditlog.AuditLog;
1013
import org.odpi.openmetadata.repositoryservices.connectors.stores.metadatacollectionstore.OMRSDynamicTypeMetadataCollectionBase;
1114
import org.odpi.openmetadata.repositoryservices.connectors.stores.metadatacollectionstore.properties.HistorySequencingOrder;
@@ -149,8 +152,7 @@ public List<EntityDetail> getEntityDetailHistory(String userId,
149152
HistorySequencingOrder sequencingOrder) throws
150153
InvalidParameterException,
151154
RepositoryErrorException,
152-
EntityNotKnownException,
153-
EntityProxyOnlyException {
155+
EntityNotKnownException {
154156
final String methodName = "getEntityDetailHistory";
155157
super.getInstanceHistoryParameterValidation(userId, guid, fromTime, toTime, methodName);
156158
return cruxRepositoryConnector.getPreviousVersionsOfEntity(guid, fromTime, toTime, startFromElement, pageSize, sequencingOrder);
@@ -259,11 +261,6 @@ public List<EntityDetail> findEntities(String userId,
259261
PagingErrorException {
260262

261263
this.findEntitiesParameterValidation(userId, entityTypeGUID, entitySubtypeGUIDs, matchProperties, fromEntityElement, limitResultsByStatus, matchClassifications, asOfTime, sequencingProperty, sequencingOrder, pageSize);
262-
263-
// TODO: rework this a bit, so that we retrieve the list of GUIDs and then loop through this
264-
// list making calls to getEntityDetail here in the MetadataCollection (?) (Then all of its additional
265-
// validations will be applied to each entity as well...)
266-
267264
return cruxRepositoryConnector.findEntities(entityTypeGUID,
268265
entitySubtypeGUIDs,
269266
matchProperties,
@@ -351,9 +348,7 @@ public List<EntityDetail> findEntitiesByPropertyValue(String userId,
351348
UserNotAuthorizedException {
352349

353350
super.findEntitiesByPropertyValueParameterValidation(userId, entityTypeGUID, searchCriteria, fromEntityElement, limitResultsByStatus, limitResultsByClassification, asOfTime, sequencingProperty, sequencingOrder, pageSize);
354-
355351
SearchClassifications searchClassifications = repositoryHelper.getSearchClassificationsFromList(limitResultsByClassification);
356-
357352
return cruxRepositoryConnector.findEntitiesByText(entityTypeGUID,
358353
searchCriteria,
359354
fromEntityElement,
@@ -460,11 +455,6 @@ public List<Relationship> findRelationships(String userId,
460455
PagingErrorException {
461456

462457
super.findRelationshipsParameterValidation(userId, relationshipTypeGUID, relationshipSubtypeGUIDs, matchProperties, fromRelationshipElement, limitResultsByStatus, asOfTime, sequencingProperty, sequencingOrder, pageSize);
463-
464-
// TODO: rework this a bit, so that we retrieve the list of GUIDs and then loop through this
465-
// list making calls to getEntityDetail here in the MetadataCollection (?) (Then all of its additional
466-
// validations will be applied to each entity as well...)
467-
468458
return cruxRepositoryConnector.findRelationships(relationshipTypeGUID,
469459
relationshipSubtypeGUIDs,
470460
matchProperties,
@@ -500,9 +490,7 @@ public List<Relationship> findRelationshipsByProperty(String userId,
500490
UserNotAuthorizedException {
501491

502492
super.findRelationshipsByPropertyParameterValidation(userId, relationshipTypeGUID, matchProperties, matchCriteria, fromRelationshipElement, limitResultsByStatus, asOfTime, sequencingProperty, sequencingOrder, pageSize);
503-
504493
SearchProperties searchProperties = repositoryHelper.getSearchPropertiesFromInstanceProperties(repositoryName, matchProperties, matchCriteria);
505-
506494
return findRelationships(userId,
507495
relationshipTypeGUID,
508496
null,
@@ -537,7 +525,6 @@ public List<Relationship> findRelationshipsByPropertyValue(String userId,
537525
UserNotAuthorizedException {
538526

539527
super.findRelationshipsByPropertyValueParameterValidation(userId, relationshipTypeGUID, searchCriteria, fromRelationshipElement, limitResultsByStatus, asOfTime, sequencingProperty, sequencingOrder, pageSize);
540-
541528
return cruxRepositoryConnector.findRelationshipsByText(relationshipTypeGUID,
542529
searchCriteria,
543530
fromRelationshipElement,
@@ -747,15 +734,13 @@ public EntityDetail addEntity(String userId,
747734
// (If not, then why not, and where should this validation be done instead?)
748735

749736
TypeDef typeDef = super.addEntityParameterValidation(userId, entityTypeGUID, initialProperties, initialClassifications, initialStatus, methodName);
750-
751737
EntityDetail newEntity = repositoryHelper.getNewEntity(repositoryName,
752738
metadataCollectionId,
753739
InstanceProvenanceType.LOCAL_COHORT,
754740
userId,
755741
typeDef.getName(),
756742
initialProperties,
757743
initialClassifications);
758-
759744
newEntity.setMetadataCollectionName(metadataCollectionName);
760745
// TODO: should we not add the calling user to the 'maintainedBy' list?
761746

@@ -789,15 +774,13 @@ public EntityDetail addExternalEntity(String userId,
789774
final String methodName = "addExternalEntity";
790775

791776
TypeDef typeDef = super.addExternalEntityParameterValidation(userId, entityTypeGUID, externalSourceGUID, initialProperties, initialClassifications, initialStatus, methodName);
792-
793777
EntityDetail newEntity = repositoryHelper.getNewEntity(repositoryName,
794778
externalSourceGUID,
795779
InstanceProvenanceType.EXTERNAL_SOURCE,
796780
userId,
797781
typeDef.getName(),
798782
initialProperties,
799783
initialClassifications);
800-
801784
newEntity.setMetadataCollectionName(externalSourceName);
802785
newEntity.setReplicatedBy(metadataCollectionId);
803786

@@ -966,40 +949,36 @@ public EntityDetail deleteEntity(String userId,
966949
repositoryValidator.validateTypeForInstanceDelete(repositoryName, typeDefGUID, typeDefName, entity, methodName);
967950
repositoryValidator.validateInstanceStatusForDelete(repositoryName, entity, methodName);
968951

952+
Transaction.Builder tx = Transaction.builder();
953+
954+
// 1. Soft-delete every HOMED relationship in which the entity is involved (note that we are only allowed to
955+
// do this against relationships that are homed in this repository, as a soft-delete can only be done by the
956+
// home repository: it changes the version, modification times, etc)
969957
try {
970-
// TODO: optimise this query since we do not really need the entire relationship to delete it?
971-
List<Relationship> relationships = this.getRelationshipsForEntity(userId,
972-
obsoleteEntityGUID,
973-
null,
974-
0,
975-
null,
976-
null,
977-
null,
978-
null,
979-
Constants.CASCADE_DELETES_PAGE_SIZE);
958+
List<Relationship> relationships = cruxRepositoryConnector.findHomedRelationshipsForEntity(entity, userId);
980959
if (relationships != null) {
981960
for (Relationship relationship : relationships) {
982961
if (relationship != null) {
983-
InstanceType type = relationship.getType();
984-
if (type != null) {
985-
this.deleteRelationship(userId,
986-
type.getTypeDefGUID(),
987-
type.getTypeDefName(),
988-
relationship.getGUID());
989-
}
962+
Relationship toDelete = getDeletedRelationshipRepresentation(relationship, userId);
963+
cruxRepositoryConnector.addUpdateRelationshipStatements(tx, toDelete);
990964
}
991965
}
992966
}
993967
} catch (Exception e) {
994968
auditLog.logException(methodName, CruxOMRSAuditCode.FAILED_RELATIONSHIP_DELETE_CASCADE.getMessageDefinition(obsoleteEntityGUID), e);
995969
}
996970

971+
// 2. Update the entity itself
997972
EntityDetail updatedEntity = new EntityDetail(entity);
998973
updatedEntity.setStatusOnDelete(entity.getStatus());
999974
updatedEntity.setStatus(InstanceStatus.DELETED);
1000975
updatedEntity = repositoryHelper.incrementVersion(userId, entity, updatedEntity);
976+
cruxRepositoryConnector.addUpdateEntityStatements(tx, updatedEntity);
1001977

1002-
return cruxRepositoryConnector.updateEntity(updatedEntity);
978+
// 3. Commit the transaction containing all of these write operations together
979+
cruxRepositoryConnector.runTx(tx.build());
980+
981+
return updatedEntity;
1003982

1004983
}
1005984

@@ -1033,37 +1012,42 @@ public void purgeEntity(String userId,
10331012
repositoryValidator.validateTypeForInstanceDelete(repositoryName, typeDefGUID, typeDefName, entity, methodName);
10341013
repositoryValidator.validateEntityIsDeleted(repositoryName, entity, methodName);
10351014

1036-
// TODO: the InMemoryOMRSMetadataCollection makes this call and catches all exceptions (with no logging), but
1037-
// actually the call itself will ALWAYS give an EntityNotKnownException because we are validating just above
1038-
// that this entity is deleted, while the getRelationshipsForEntity call is validating that the entity is NOT
1039-
// deleted (and if it is, will throw an EntityNotKnownException). So it ALWAYS throws an EntityNotKnownException.
1040-
// We can therefore either remove this code entirely (as it does nothing but add Exception processing overhead),
1041-
// or we need to change the logic across the various repositories.
1042-
/*try {
1043-
List<Relationship> relationships = this.getRelationshipsForEntity(userId,
1044-
deletedEntityGUID,
1015+
Transaction.Builder tx = Transaction.builder();
1016+
1017+
// 1. Purge EVERY SINGLE relationship in which the entity is involved (note that we should be able to do this
1018+
// against both homed and reference copy relationships since purge operations are allowed against both)
1019+
try {
1020+
CruxQuery query = new CruxQuery();
1021+
query.addRelationshipEndpointConditions(EntitySummaryMapping.getReference(deletedEntityGUID));
1022+
cruxRepositoryConnector.updateQuery(query,
1023+
TypeDefCategory.RELATIONSHIP_DEF,
10451024
null,
1046-
0,
10471025
null,
10481026
null,
10491027
null,
10501028
null,
1051-
Constants.CASCADE_DELETES_PAGE_SIZE);
1052-
if (relationships != null) {
1053-
for (Relationship relationship : relationships) {
1054-
if (relationship != null) {
1055-
// Note: we will not call this.purgeRelationship() directly to avoid re-retrieving all of these
1056-
// relationships one-by-one prior to purging
1057-
cruxRepositoryConnector.purgeRelationship(relationship.getGUID());
1058-
}
1059-
}
1029+
null,
1030+
null,
1031+
null,
1032+
userId);
1033+
IPersistentMap q = query.getQuery();
1034+
log.debug("Querying with: {}", q);
1035+
Collection<List<?>> results = cruxRepositoryConnector.getCruxAPI().db().query(q);
1036+
Collection<List<?>> relationshipRefs = cruxRepositoryConnector.deduplicate(results);
1037+
for (List<?> relationshipRef : relationshipRefs) {
1038+
String docRef = (String) relationshipRef.get(0);
1039+
String guid = RelationshipMapping.trimGuidFromReference(docRef);
1040+
cruxRepositoryConnector.addPurgeRelationshipStatements(tx, guid);
10601041
}
10611042
} catch (Exception e) {
1062-
log.error("Exception was thrown in purgeEntity.", e);
10631043
auditLog.logException(methodName, CruxOMRSAuditCode.FAILED_RELATIONSHIP_DELETE_CASCADE.getMessageDefinition(deletedEntityGUID), e);
1064-
}*/
1044+
}
1045+
1046+
// 2. Purge the entity itself
1047+
cruxRepositoryConnector.addPurgeEntityStatements(tx, entity.getGUID());
10651048

1066-
cruxRepositoryConnector.purgeEntity(entity.getGUID());
1049+
// 3. Commit the transaction containing all of these write operations together
1050+
cruxRepositoryConnector.runTx(tx.build());
10671051

10681052
}
10691053

@@ -1501,18 +1485,18 @@ public Relationship deleteRelationship(String userId,
15011485
final String parameterName = "obsoleteRelationshipGUID";
15021486

15031487
this.manageInstanceParameterValidation(userId, typeDefGUID, typeDefName, obsoleteRelationshipGUID, parameterName, methodName);
1504-
15051488
Relationship relationship = this.getRelationship(userId, obsoleteRelationshipGUID);
1506-
15071489
repositoryValidator.validateTypeForInstanceDelete(repositoryName, typeDefGUID, typeDefName, relationship, methodName);
1490+
Relationship updatedRelationship = getDeletedRelationshipRepresentation(relationship, userId);
1491+
return cruxRepositoryConnector.updateRelationship(updatedRelationship);
15081492

1493+
}
1494+
1495+
private Relationship getDeletedRelationshipRepresentation(Relationship relationship, String userId) {
15091496
Relationship updatedRelationship = new Relationship(relationship);
15101497
updatedRelationship.setStatusOnDelete(relationship.getStatus());
15111498
updatedRelationship.setStatus(InstanceStatus.DELETED);
1512-
updatedRelationship = repositoryHelper.incrementVersion(userId, relationship, updatedRelationship);
1513-
1514-
return cruxRepositoryConnector.updateRelationship(updatedRelationship);
1515-
1499+
return repositoryHelper.incrementVersion(userId, relationship, updatedRelationship);
15161500
}
15171501

15181502
/**
@@ -2125,7 +2109,6 @@ public void saveRelationshipReferenceCopy(String userId,
21252109
}
21262110

21272111
// Only create entity proxies if the above retrievals indicated that they do not yet exist
2128-
// TODO: there may be a more optimal way of achieving this directly through the Crux statements -- future optimisation?
21292112
Transaction.Builder tx = Transaction.builder();
21302113
if (one == null) {
21312114
cruxRepositoryConnector.addCreateEntityProxyStatements(tx, relationship.getEntityOneProxy());

‎connector/src/main/java/org/odpi/egeria/connectors/juxt/crux/repositoryconnector/CruxOMRSRepositoryConnector.java

+21-12
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,15 @@ public void addCreateEntityStatements(Transaction.Builder tx, EntityDetail entit
256256
put(tx, edm.toCrux());
257257
}
258258

259+
/**
260+
* Retrieve the statements that need to be executed against Crux to update (persist) the entity provided.
261+
* @param tx the transaction through which to update the entity
262+
* @param entity to be updated
263+
*/
264+
public void addUpdateEntityStatements(Transaction.Builder tx, EntityDetail entity) {
265+
addCreateEntityStatements(tx, entity);
266+
}
267+
259268
/**
260269
* Permanently delete the entity (and all of its history) from the Crux repository.
261270
* Note that this operation is NOT reversible!
@@ -2218,17 +2227,17 @@ public Collection<List<?>> findDirectNeighbors(ICruxDatasource db,
22182227
* @param userId of the user running the query
22192228
* @throws TypeErrorException if a requested type for searching is not known to the repository
22202229
*/
2221-
private void updateQuery(CruxQuery query,
2222-
TypeDefCategory category,
2223-
String typeGuid,
2224-
List<String> subtypeGuids,
2225-
SearchProperties matchProperties,
2226-
List<InstanceStatus> limitResultsByStatus,
2227-
SearchClassifications matchClassifications,
2228-
String sequencingProperty,
2229-
SequencingOrder sequencingOrder,
2230-
String namespace,
2231-
String userId) throws TypeErrorException {
2230+
void updateQuery(CruxQuery query,
2231+
TypeDefCategory category,
2232+
String typeGuid,
2233+
List<String> subtypeGuids,
2234+
SearchProperties matchProperties,
2235+
List<InstanceStatus> limitResultsByStatus,
2236+
SearchClassifications matchClassifications,
2237+
String sequencingProperty,
2238+
SequencingOrder sequencingOrder,
2239+
String namespace,
2240+
String userId) throws TypeErrorException {
22322241
// Note that we will put the property search criteria first to optimise the search, which can more than double
22332242
// the speed for very broad scenarios (where no type limiter is specified, or only Referenceable)
22342243
Set<String> completeTypeSet = getCompleteSetOfTypeNamesForSearch(userId, typeGuid, subtypeGuids, namespace);
@@ -2379,7 +2388,7 @@ private Collection<List<?>> deduplicateAndPage(Collection<List<?>> results, int
23792388
* @param results from a Crux query
23802389
* @return {@code Collection<List<?>>} of all unique results
23812390
*/
2382-
private Collection<List<?>> deduplicate(Collection<List<?>> results) {
2391+
Collection<List<?>> deduplicate(Collection<List<?>> results) {
23832392
if (results == null || results.isEmpty()) {
23842393
return results;
23852394
} else {

‎connector/src/test/resources/logback-test.xml

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
3+
<!-- SPDX-License-Identifier: Apache-2.0 -->
4+
<!-- Copyright Contributors to the ODPi Egeria project. -->
5+
6+
<configuration>
7+
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
8+
<encoder>
9+
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
10+
</encoder>
11+
</appender>
12+
<root level="ERROR">
13+
<appender-ref ref="STDOUT" />
14+
</root>
15+
</configuration>

0 commit comments

Comments
 (0)