Skip to content

Commit 89c9579

Browse files
hmottestadhmottestad
authored
GH-5115 Fix for performance issues with ShaclSail after fix for compressed validation tuples (#5119)
2 parents 5c956e1 + 10eb88e commit 89c9579

File tree

99 files changed

+3861
-542
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

99 files changed

+3861
-542
lines changed

‎core/sail/shacl/src/main/java/org/eclipse/rdf4j/sail/shacl/ShaclSailConnection.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -524,8 +524,9 @@ private ValidationReport performValidation(List<ContextWithShape> shapes, boolea
524524
sail.isPerformanceLogging())),
525525
sail.isGlobalLogValidationExecution(), sail.isLogValidationViolations(),
526526
sail.getEffectiveValidationResultsLimitPerConstraint(), sail.isPerformanceLogging(),
527-
logger
528-
))
527+
sail.isLogValidationPlans(),
528+
logger,
529+
connectionsGroup))
529530

530531
.filter(ShapeValidationContainer::hasPlanNode)
531532
.map(validationContainer -> validationContainer::performValidation);

‎core/sail/shacl/src/main/java/org/eclipse/rdf4j/sail/shacl/ShaclValidator.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,8 +115,8 @@ private static ValidationReport performValidation(List<ContextWithShape> shapes,
115115
() -> contextWithShape.getShape()
116116
.generatePlans(connectionsGroup,
117117
new ValidationSettings(contextWithShape.getDataGraph(), false, true, false)),
118-
false, false, 1000, false, logger
119-
)
118+
false, false, 1000, false, false, logger,
119+
connectionsGroup)
120120
)
121121
.filter(ShapeValidationContainer::hasPlanNode)
122122
.map(ShapeValidationContainer::performValidation)

‎core/sail/shacl/src/main/java/org/eclipse/rdf4j/sail/shacl/ShapeValidationContainer.java

Lines changed: 55 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,18 +11,22 @@
1111

1212
package org.eclipse.rdf4j.sail.shacl;
1313

14+
import java.util.Arrays;
1415
import java.util.List;
1516
import java.util.function.Supplier;
1617
import java.util.stream.Collectors;
1718

19+
import org.apache.commons.text.StringEscapeUtils;
1820
import org.eclipse.rdf4j.common.iteration.CloseableIteration;
21+
import org.eclipse.rdf4j.sail.SailConnection;
1922
import org.eclipse.rdf4j.sail.SailException;
2023
import org.eclipse.rdf4j.sail.shacl.ast.Shape;
2124
import org.eclipse.rdf4j.sail.shacl.ast.planNodes.PlanNode;
2225
import org.eclipse.rdf4j.sail.shacl.ast.planNodes.SingleCloseablePlanNode;
2326
import org.eclipse.rdf4j.sail.shacl.ast.planNodes.ValidationExecutionLogger;
2427
import org.eclipse.rdf4j.sail.shacl.ast.planNodes.ValidationTuple;
2528
import org.eclipse.rdf4j.sail.shacl.results.lazy.ValidationResultIterator;
29+
import org.eclipse.rdf4j.sail.shacl.wrapper.data.ConnectionsGroup;
2630
import org.slf4j.Logger;
2731

2832
class ShapeValidationContainer {
@@ -36,14 +40,64 @@ class ShapeValidationContainer {
3640

3741
public ShapeValidationContainer(Shape shape, Supplier<PlanNode> planNodeSupplier, boolean logValidationExecution,
3842
boolean logValidationViolations, long effectiveValidationResultsLimitPerConstraint,
39-
boolean performanceLogging, Logger logger) {
43+
boolean performanceLogging, boolean logValidationPlans, Logger logger, ConnectionsGroup connectionsGroup) {
4044
this.shape = shape;
4145
this.logValidationViolations = logValidationViolations;
4246
this.effectiveValidationResultsLimitPerConstraint = effectiveValidationResultsLimitPerConstraint;
4347
this.performanceLogging = performanceLogging;
4448
this.logger = logger;
4549
try {
4650
PlanNode planNode = planNodeSupplier.get();
51+
52+
if (logValidationPlans) {
53+
54+
StringBuilder planAsGraphvizDot = new StringBuilder();
55+
56+
planAsGraphvizDot.append(
57+
"rank1 [style=invisible];\n" +
58+
"rank2 [style=invisible];\n" +
59+
"\n" +
60+
"rank1 -> rank2 [color=white];\n");
61+
62+
planAsGraphvizDot.append("{\n")
63+
.append("\trank = same;\n")
64+
.append("\trank2 -> ")
65+
.append(System.identityHashCode(connectionsGroup.getBaseConnection()))
66+
.append(" -> ")
67+
.append(System.identityHashCode(connectionsGroup.getAddedStatements()))
68+
.append(" -> ")
69+
.append(System.identityHashCode(connectionsGroup.getRemovedStatements()))
70+
.append(" [ style=invis ];\n")
71+
.append("\trankdir = LR;\n")
72+
.append("}\n");
73+
74+
planAsGraphvizDot.append(System.identityHashCode(connectionsGroup.getBaseConnection()))
75+
.append(" [label=\"")
76+
.append("BaseConnection")
77+
.append("\" fillcolor=\"#CACADB\", style=filled];")
78+
.append("\n");
79+
80+
planAsGraphvizDot.append(System.identityHashCode(connectionsGroup.getAddedStatements()))
81+
.append(" [label=\"")
82+
.append("AddedStatements")
83+
.append("\" fillcolor=\"#CEDBCA\", style=filled];")
84+
.append("\n");
85+
86+
planAsGraphvizDot.append(System.identityHashCode(connectionsGroup.getRemovedStatements()))
87+
.append(" [label=\"")
88+
.append("RemovedStatements")
89+
.append("\" fillcolor=\"#DBCFC9r\", style=filled];")
90+
.append("\n");
91+
92+
planNode.getPlanAsGraphvizDot(planAsGraphvizDot);
93+
94+
String[] split = planAsGraphvizDot.toString().split("\n");
95+
planAsGraphvizDot = new StringBuilder();
96+
Arrays.stream(split).map(s -> "\t" + s + "\n").forEach(planAsGraphvizDot::append);
97+
98+
logger.info("Plan as Graphviz dot:\ndigraph G {\n{}}", planAsGraphvizDot);
99+
}
100+
47101
this.validationExecutionLogger = ValidationExecutionLogger
48102
.getInstance(logValidationExecution);
49103
if (!(planNode.isGuaranteedEmpty())) {
@@ -113,7 +167,6 @@ private void handlePostLogging(long before, ValidationResultIterator validationR
113167
}
114168

115169
if (validationResults != null) {
116-
117170
if (performanceLogging) {
118171
long after = System.currentTimeMillis();
119172
logger.info("Execution of plan took {} ms for:\n{}\n",

‎core/sail/shacl/src/main/java/org/eclipse/rdf4j/sail/shacl/ast/NodeShape.java

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -173,14 +173,15 @@ public PlanNode generateTransactionalValidationPlan(ConnectionsGroup connections
173173
return new ValidationResult(t.getActiveTarget(), t.getActiveTarget(), this,
174174
constraintComponent, getSeverity(), t.getScope(), t.getContexts(),
175175
getContexts());
176-
});
176+
}, connectionsGroup);
177177
}
178178

179179
if (scope == Scope.propertyShape) {
180-
validationPlanNode = Unique.getInstance(new ShiftToPropertyShape(validationPlanNode), true);
180+
validationPlanNode = Unique.getInstance(new ShiftToPropertyShape(validationPlanNode, connectionsGroup),
181+
true, connectionsGroup);
181182
}
182183

183-
union = UnionNode.getInstance(union, validationPlanNode);
184+
union = UnionNode.getInstance(connectionsGroup, union, validationPlanNode);
184185
}
185186

186187
return union;
@@ -207,7 +208,7 @@ public PlanNode getAllTargetsPlan(ConnectionsGroup connectionsGroup, Resource[]
207208
.map(c -> c.getAllTargetsPlan(connectionsGroup, dataGraph, Scope.nodeShape,
208209
new StatementMatcher.StableRandomVariableProvider()))
209210
.distinct()
210-
.reduce(UnionNode::getInstanceDedupe)
211+
.reduce((nodes, nodes2) -> UnionNode.getInstanceDedupe(connectionsGroup, nodes, nodes2))
211212
.orElse(EmptyNode.getInstance());
212213

213214
if (connectionsGroup.getStats().hasRemoved()) {
@@ -216,14 +217,14 @@ public PlanNode getAllTargetsPlan(ConnectionsGroup connectionsGroup, Resource[]
216217
stableRandomVariableProvider)
217218
.getPlanNode(connectionsGroup, dataGraph, Scope.nodeShape, true, null);
218219

219-
planNode = UnionNode.getInstanceDedupe(planNode, planNodeEffectiveTarget);
220+
planNode = UnionNode.getInstanceDedupe(connectionsGroup, planNode, planNodeEffectiveTarget);
220221
}
221222

222223
if (scope == Scope.propertyShape) {
223-
planNode = Unique.getInstance(new ShiftToPropertyShape(planNode), true);
224+
planNode = Unique.getInstance(new ShiftToPropertyShape(planNode, connectionsGroup), true, connectionsGroup);
224225
}
225226

226-
planNode = Unique.getInstance(planNode, false);
227+
planNode = Unique.getInstance(planNode, false, connectionsGroup);
227228

228229
return planNode;
229230
}

‎core/sail/shacl/src/main/java/org/eclipse/rdf4j/sail/shacl/ast/PropertyShape.java

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -236,16 +236,18 @@ public PlanNode generateTransactionalValidationPlan(ConnectionsGroup connections
236236
return new ValidationResult(t.getActiveTarget(), t.getValue(), this,
237237
constraintComponent, getSeverity(), t.getScope(), t.getContexts(),
238238
getContexts());
239-
});
239+
}, connectionsGroup);
240240
}
241241

242242
if (scope == Scope.propertyShape) {
243-
validationPlanNode = Unique.getInstance(new TargetChainPopper(validationPlanNode), true);
243+
validationPlanNode = Unique.getInstance(new TargetChainPopper(validationPlanNode, connectionsGroup),
244+
true, connectionsGroup);
244245
} else {
245-
validationPlanNode = Unique.getInstance(new ShiftToNodeShape(validationPlanNode), true);
246+
validationPlanNode = Unique.getInstance(new ShiftToNodeShape(validationPlanNode, connectionsGroup),
247+
true, connectionsGroup);
246248
}
247249

248-
union = UnionNode.getInstance(union, validationPlanNode);
250+
union = UnionNode.getInstance(connectionsGroup, union, validationPlanNode);
249251
}
250252

251253
return union;
@@ -258,7 +260,7 @@ public PlanNode getAllTargetsPlan(ConnectionsGroup connectionsGroup, Resource[]
258260
.map(c -> c.getAllTargetsPlan(connectionsGroup, dataGraph, Scope.propertyShape,
259261
new StatementMatcher.StableRandomVariableProvider()))
260262
.distinct()
261-
.reduce(UnionNode::getInstanceDedupe)
263+
.reduce((nodes, nodes2) -> UnionNode.getInstanceDedupe(connectionsGroup, nodes, nodes2))
262264
.orElse(EmptyNode.getInstance());
263265

264266
if (connectionsGroup.getStats().hasRemoved()) {
@@ -267,16 +269,16 @@ public PlanNode getAllTargetsPlan(ConnectionsGroup connectionsGroup, Resource[]
267269
stableRandomVariableProvider)
268270
.getPlanNode(connectionsGroup, dataGraph, Scope.propertyShape, true, null);
269271

270-
planNode = UnionNode.getInstanceDedupe(planNode, planNodeEffectiveTarget);
272+
planNode = UnionNode.getInstanceDedupe(connectionsGroup, planNode, planNodeEffectiveTarget);
271273
}
272274

273275
if (scope == Scope.propertyShape) {
274-
planNode = Unique.getInstance(new TargetChainPopper(planNode), true);
276+
planNode = Unique.getInstance(new TargetChainPopper(planNode, connectionsGroup), true, connectionsGroup);
275277
} else {
276-
planNode = new ShiftToNodeShape(planNode);
278+
planNode = new ShiftToNodeShape(planNode, connectionsGroup);
277279
}
278280

279-
planNode = Unique.getInstance(planNode, false);
281+
planNode = Unique.getInstance(planNode, false, connectionsGroup);
280282

281283
return planNode;
282284
}

‎core/sail/shacl/src/main/java/org/eclipse/rdf4j/sail/shacl/ast/Shape.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -422,7 +422,7 @@ public PlanNode generatePlans(ConnectionsGroup connectionsGroup, ValidationSetti
422422
Scope.none)
423423
.getValidationPlan(connectionsGroup.getBaseConnection(), validationSettings.getDataGraph(),
424424
getContexts()),
425-
this);
425+
this, connectionsGroup);
426426
} else {
427427
logger.debug("Use fall back validation approach for bulk validation instead of SPARQL for shape {}",
428428
this);
@@ -436,7 +436,7 @@ public PlanNode generatePlans(ConnectionsGroup connectionsGroup, ValidationSetti
436436
.getAllTargets(connectionsGroup,
437437
validationSettings.getDataGraph(),
438438
this instanceof NodeShape ? Scope.nodeShape : Scope.propertyShape),
439-
Scope.none), this);
439+
Scope.none), this, connectionsGroup);
440440
}
441441

442442
} else if (validationApproach == ValidationApproach.Transactional) {
@@ -447,7 +447,7 @@ public PlanNode generatePlans(ConnectionsGroup connectionsGroup, ValidationSetti
447447
return new SingleCloseablePlanNode(
448448
Shape.this.generateTransactionalValidationPlan(connectionsGroup, validationSettings, null,
449449
Scope.none),
450-
this);
450+
this, connectionsGroup);
451451
} else {
452452
return EmptyNode.getInstance();
453453
}

‎core/sail/shacl/src/main/java/org/eclipse/rdf4j/sail/shacl/ast/constraintcomponents/AbstractConstraintComponent.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -155,22 +155,22 @@ static PlanNode getAllTargetsIncludingThoseAddedByPath(ConnectionsGroup connecti
155155
ValidationSettings validationSettings, Scope scope, EffectiveTarget effectiveTarget, Path path,
156156
boolean includeTargetsAffectedByRemoval) {
157157
PlanNode allTargets;
158-
BufferedSplitter addedTargets = new BufferedSplitter(
158+
BufferedSplitter addedTargets = BufferedSplitter.getInstance(
159159
effectiveTarget.getPlanNode(connectionsGroup, validationSettings.getDataGraph(),
160160
scope, includeTargetsAffectedByRemoval, null));
161161

162162
PlanNode addedByPath = path.getAllAdded(connectionsGroup, validationSettings.getDataGraph(), null);
163163

164-
addedByPath = Unique.getInstance(new TrimToTarget(addedByPath), false);
164+
addedByPath = Unique.getInstance(new TrimToTarget(addedByPath, connectionsGroup), false, connectionsGroup);
165165

166-
addedByPath = new ReduceTargets(addedByPath, addedTargets.getPlanNode());
166+
addedByPath = new ReduceTargets(addedByPath, addedTargets.getPlanNode(), connectionsGroup);
167167

168168
addedByPath = effectiveTarget.extend(addedByPath, connectionsGroup, validationSettings.getDataGraph(),
169169
scope, EffectiveTarget.Extend.left,
170170
false,
171171
null);
172172

173-
allTargets = UnionNode.getInstance(addedTargets.getPlanNode(), addedByPath);
173+
allTargets = UnionNode.getInstance(connectionsGroup, addedTargets.getPlanNode(), addedByPath);
174174
return allTargets;
175175
}
176176

‎core/sail/shacl/src/main/java/org/eclipse/rdf4j/sail/shacl/ast/constraintcomponents/AbstractPairwiseConstraintComponent.java

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,9 @@ public PlanNode generateTransactionalValidationPlan(ConnectionsGroup connections
8989
validationSettings,
9090
effectiveTarget);
9191

92-
allTargets = Unique.getInstance(UnionNode.getInstance(allTargets, allTargetsBasedOnPredicate), false);
92+
allTargets = Unique.getInstance(
93+
UnionNode.getInstance(connectionsGroup, allTargets, allTargetsBasedOnPredicate), false,
94+
connectionsGroup);
9395

9496
} else {
9597
allTargets = effectiveTarget.getPlanNode(connectionsGroup, validationSettings.getDataGraph(), scope,
@@ -99,7 +101,9 @@ public PlanNode generateTransactionalValidationPlan(ConnectionsGroup connections
99101
validationSettings,
100102
effectiveTarget);
101103

102-
allTargets = Unique.getInstance(UnionNode.getInstance(allTargets, allTargetsBasedOnPredicate), false);
104+
allTargets = Unique.getInstance(
105+
UnionNode.getInstance(connectionsGroup, allTargets, allTargetsBasedOnPredicate), false,
106+
connectionsGroup);
103107
}
104108

105109
}
@@ -151,7 +155,8 @@ private PlanNode getAllTargetsBasedOnPredicate(ConnectionsGroup connectionsGroup
151155
PlanNode targetFilter2 = effectiveTarget.getTargetFilter(connectionsGroup, validationSettings.getDataGraph(),
152156
removedByPredicate);
153157

154-
return Unique.getInstance(UnionNode.getInstance(targetFilter1, targetFilter2), false);
158+
return Unique.getInstance(UnionNode.getInstance(connectionsGroup, targetFilter1, targetFilter2), false,
159+
connectionsGroup);
155160
}
156161

157162
@Override
@@ -163,7 +168,7 @@ public PlanNode getAllTargetsPlan(ConnectionsGroup connectionsGroup, Resource[]
163168
stableRandomVariableProvider)
164169
.getPlanNode(connectionsGroup, dataGraph, Scope.nodeShape, true, null);
165170

166-
allTargetsPlan = new ShiftToPropertyShape(allTargetsPlan);
171+
allTargetsPlan = new ShiftToPropertyShape(allTargetsPlan, connectionsGroup);
167172

168173
// removed statements that match predicate could affect sh:or
169174
if (connectionsGroup.getStats().hasRemoved()) {
@@ -181,7 +186,7 @@ public PlanNode getAllTargetsPlan(ConnectionsGroup connectionsGroup, Resource[]
181186
EffectiveTarget.Extend.left,
182187
false,
183188
null);
184-
allTargetsPlan = UnionNode.getInstanceDedupe(allTargetsPlan, deletedPredicates);
189+
allTargetsPlan = UnionNode.getInstanceDedupe(connectionsGroup, allTargetsPlan, deletedPredicates);
185190
}
186191

187192
// added statements that match predicate could affect sh:not
@@ -200,10 +205,10 @@ public PlanNode getAllTargetsPlan(ConnectionsGroup connectionsGroup, Resource[]
200205
EffectiveTarget.Extend.left,
201206
false,
202207
null);
203-
allTargetsPlan = UnionNode.getInstanceDedupe(allTargetsPlan, addedPredicates);
208+
allTargetsPlan = UnionNode.getInstanceDedupe(connectionsGroup, allTargetsPlan, addedPredicates);
204209
}
205210

206-
return Unique.getInstance(new TrimToTarget(allTargetsPlan), false);
211+
return Unique.getInstance(new TrimToTarget(allTargetsPlan, connectionsGroup), false, connectionsGroup);
207212
} else {
208213
assert scope == Scope.nodeShape;
209214

@@ -224,7 +229,7 @@ public PlanNode getAllTargetsPlan(ConnectionsGroup connectionsGroup, Resource[]
224229
.extend(deletedPredicates, connectionsGroup, dataGraph, Scope.nodeShape,
225230
EffectiveTarget.Extend.left,
226231
false, null);
227-
allTargetsPlan = UnionNode.getInstanceDedupe(allTargetsPlan, deletedPredicates);
232+
allTargetsPlan = UnionNode.getInstanceDedupe(connectionsGroup, allTargetsPlan, deletedPredicates);
228233

229234
}
230235

@@ -243,11 +248,11 @@ public PlanNode getAllTargetsPlan(ConnectionsGroup connectionsGroup, Resource[]
243248
.extend(addedPredicates, connectionsGroup, dataGraph, Scope.nodeShape,
244249
EffectiveTarget.Extend.left,
245250
false, null);
246-
allTargetsPlan = UnionNode.getInstanceDedupe(allTargetsPlan, addedPredicates);
251+
allTargetsPlan = UnionNode.getInstanceDedupe(connectionsGroup, allTargetsPlan, addedPredicates);
247252

248253
}
249254

250-
return Unique.getInstance(allTargetsPlan, false);
255+
return Unique.getInstance(allTargetsPlan, false, connectionsGroup);
251256

252257
}
253258

0 commit comments

Comments
 (0)