From d0328e39c5c2e2f1b1f9510323c07631c79c5096 Mon Sep 17 00:00:00 2001 From: "Amir M. Mir" Date: Thu, 15 Sep 2022 13:26:28 +0200 Subject: [PATCH 1/7] Build URIHierarchies in parallel in `OPALPartialCallGraphConstructor` --- .../data/OPALPartialCallGraphConstructor.java | 44 +++++++------ .../data/analysis/OPALClassHierarchy.java | 62 ++++++++++++++----- 2 files changed, 66 insertions(+), 40 deletions(-) diff --git a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructor.java b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructor.java index 843c6368d..eae2d7820 100644 --- a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructor.java +++ b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructor.java @@ -18,18 +18,16 @@ package eu.fasten.analyzer.javacgopal.data; +import com.google.common.collect.Lists; +import eu.fasten.analyzer.javacgopal.data.analysis.OPALClassHierarchy; +import eu.fasten.analyzer.javacgopal.data.analysis.OPALMethod; +import eu.fasten.analyzer.javacgopal.data.analysis.OPALType; +import eu.fasten.core.data.Constants; +import eu.fasten.core.data.JavaGraph; import eu.fasten.core.data.PartialJavaCallGraph; -import java.io.File; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - +import eu.fasten.core.data.opal.MavenArtifactDownloader; +import eu.fasten.core.data.opal.MavenCoordinate; +import eu.fasten.core.data.opal.exceptions.OPALException; import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.text.StringEscapeUtils; import org.opalj.br.Annotation; @@ -45,20 +43,20 @@ import org.opalj.value.ValueInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - -import com.google.common.collect.Lists; - -import eu.fasten.analyzer.javacgopal.data.analysis.OPALClassHierarchy; -import eu.fasten.analyzer.javacgopal.data.analysis.OPALMethod; -import eu.fasten.analyzer.javacgopal.data.analysis.OPALType; -import eu.fasten.core.data.Constants; -import eu.fasten.core.data.JavaGraph; -import eu.fasten.core.data.opal.MavenArtifactDownloader; -import eu.fasten.core.data.opal.MavenCoordinate; -import eu.fasten.core.data.opal.exceptions.OPALException; import scala.Function1; import scala.collection.JavaConverters; +import java.io.File; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + /** * Call graphs that are not still fully resolved. i.e. isolated call graphs which within-artifact * calls (edges) are known as internal calls and Cross-artifact calls are known as external calls. @@ -85,7 +83,7 @@ public OPALPartialCallGraph construct(OPALCallGraph ocg, CallPreservationStrateg createGraphWithExternalCHA(ocg, cha, callSiteOnly); pcg.nodeCount = cha.getNodeCount(); - pcg.classHierarchy = cha.asURIHierarchy(ocg.project.classHierarchy()); + pcg.classHierarchy = cha.asURIHierarchyParallel(ocg.project.classHierarchy()); } catch (Exception e) { if (e.getStackTrace().length > 0) { var stackTrace = e.getStackTrace()[0]; diff --git a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/analysis/OPALClassHierarchy.java b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/analysis/OPALClassHierarchy.java index 23b951528..811a37e61 100644 --- a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/analysis/OPALClassHierarchy.java +++ b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/analysis/OPALClassHierarchy.java @@ -18,16 +18,12 @@ package eu.fasten.analyzer.javacgopal.data.analysis; -import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.EnumMap; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - +import eu.fasten.analyzer.javacgopal.data.CallPreservationStrategy; +import eu.fasten.core.data.FastenURI; +import eu.fasten.core.data.JavaGraph; +import eu.fasten.core.data.JavaScope; +import eu.fasten.core.data.JavaType; +import it.unimi.dsi.fastutil.ints.IntIntPair; import org.opalj.br.ClassHierarchy; import org.opalj.br.DeclaredMethod; import org.opalj.br.Method; @@ -38,17 +34,21 @@ import org.opalj.tac.Stmt; import org.opalj.tac.UVar; import org.opalj.value.ValueInformation; - -import eu.fasten.analyzer.javacgopal.data.CallPreservationStrategy; -import eu.fasten.core.data.FastenURI; -import eu.fasten.core.data.JavaGraph; -import eu.fasten.core.data.JavaScope; -import eu.fasten.core.data.JavaType; -import it.unimi.dsi.fastutil.ints.IntIntPair; import scala.Tuple2; import scala.collection.Iterator; import scala.collection.JavaConverters; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + /** * Class hierarchy class containing two types of CHA - internal and external CHA * and also keeping track of node count. @@ -111,6 +111,34 @@ public EnumMap> asURIHierarchy(ClassHierarchy p JavaScope.resolvedTypes, new HashMap<>())); } + /** + * An optimized version of `asURIHierarchy`. + * Converts all of the members of the classHierarchy to {@link FastenURI}. + * + * @param projectHierarchy OPAL class hierarchy + * @return A {@link Map} of {@link FastenURI} and {@link JavaType} + */ + public EnumMap> asURIHierarchyParallel(ClassHierarchy projectHierarchy) { + + final Map internalResult = new ConcurrentHashMap<>(); + final Map externalResult = new ConcurrentHashMap<>(); + final var internals = this.getInternalCHA(); + + internals.keySet().stream().parallel().forEach(aClass -> { + final var klass = OPALType.getType(internals.get(aClass), aClass); + internalResult.put(klass.getLeft(), klass.getRight()); + }); + + final var externals = this.getExternalCHA(); + externals.keySet().stream().parallel().forEach(aClass -> { + externalResult + .putAll(OPALType.getType(projectHierarchy, externals.get(aClass), aClass)); + }); + + return new EnumMap<>(Map.of(JavaScope.internalTypes, internalResult, JavaScope.externalTypes, externalResult, + JavaScope.resolvedTypes, new HashMap<>())); + } + /** * Adds a method to the external CHA if the method doesn't already exist. * Otherwise returns and ID of the existing method. From c509621a5082656e82d037ba8ed143830cbcc184 Mon Sep 17 00:00:00 2001 From: "Amir M. Mir" Date: Thu, 15 Sep 2022 15:30:05 +0200 Subject: [PATCH 2/7] Make `createGraphWithExternalCHA` use parallel stream to speed up partial call graph construction --- .../data/OPALPartialCallGraphConstructor.java | 22 +++++++++---------- .../data/analysis/OPALClassHierarchy.java | 19 ++++++++-------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructor.java b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructor.java index eae2d7820..c54e227fa 100644 --- a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructor.java +++ b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructor.java @@ -223,37 +223,35 @@ private Map getMethodsMap(final int keyStartsFrom, private void createGraphWithExternalCHA(final OPALCallGraph ocg, final OPALClassHierarchy cha, CallPreservationStrategy callSiteOnly) { // TODO instead of relying on pcg field, use parameter - final var cg = ocg.callGraph; + final var cg = ocg.callGraph; final var tac = ocg.project.get(ComputeTACAIKey$.MODULE$); - - for (final var sourceDeclaration : JavaConverters - .asJavaIterable(cg.reachableMethods().toIterable())) { + + JavaConverters.asJavaCollection(cg.reachableMethods().toIterable()).parallelStream().forEach(sourceDeclaration -> { final List incompeletes = new ArrayList<>(); if (cg.incompleteCallSitesOf(sourceDeclaration) != null) { JavaConverters.asJavaIterator(cg.incompleteCallSitesOf(sourceDeclaration)) - .forEachRemaining(pc -> incompeletes.add((int) pc)); + .forEachRemaining(pc -> incompeletes.add((int) pc)); } final Set visitedPCs = new HashSet<>(); if (sourceDeclaration.hasMultipleDefinedMethods()) { for (final var source : JavaConverters - .asJavaIterable(sourceDeclaration.definedMethods())) { + .asJavaIterable(sourceDeclaration.definedMethods())) { cha.appendGraph(source, cg.calleesOf(sourceDeclaration), - getStmts(tac, sourceDeclaration.definedMethod()), pcg.graph, incompeletes, - visitedPCs, callSiteOnly); + getStmts(tac, sourceDeclaration.definedMethod()), pcg.graph, incompeletes, + visitedPCs, callSiteOnly); } } else if (sourceDeclaration.hasSingleDefinedMethod()) { final var definedMethod = sourceDeclaration.definedMethod(); cha.appendGraph(definedMethod, cg.calleesOf(sourceDeclaration), - getStmts(tac, definedMethod), pcg.graph, incompeletes, visitedPCs, callSiteOnly); + getStmts(tac, definedMethod), pcg.graph, incompeletes, visitedPCs, callSiteOnly); } else if (sourceDeclaration.isVirtualOrHasSingleDefinedMethod()) { cha.appendGraph(sourceDeclaration, cg.calleesOf(sourceDeclaration), getStmts(tac, - null), pcg.graph, incompeletes, visitedPCs, callSiteOnly); + null), pcg.graph, incompeletes, visitedPCs, callSiteOnly); } - - } + }); } private Stmt>[] getStmts(Function1> tac, diff --git a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/analysis/OPALClassHierarchy.java b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/analysis/OPALClassHierarchy.java index 811a37e61..b049df0f4 100644 --- a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/analysis/OPALClassHierarchy.java +++ b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/analysis/OPALClassHierarchy.java @@ -262,17 +262,18 @@ public Map getInternalMetadata(final Map, Map>> targets, - final Stmt>[] stmts, - final JavaGraph resultGraph, List incompeletes, - final Set visitedPCs, CallPreservationStrategy callSiteOnly) { + public synchronized void appendGraph(final Object source, + final Iterator>> targets, + final Stmt>[] stmts, + final JavaGraph resultGraph, List incompeletes, + final Set visitedPCs, CallPreservationStrategy callSiteOnly) { final var edges = this.getSubGraph(source, targets, stmts, incompeletes, visitedPCs, callSiteOnly); resultGraph.append(edges); } From e01e37cf90656799ca6359fc1f1a267d1ff95287 Mon Sep 17 00:00:00 2001 From: Mehdi Keshani Date: Wed, 21 Sep 2022 13:02:54 +0200 Subject: [PATCH 3/7] fix the name of test file --- ...structorTest.java => OPALPartialCallGraphConstructorTest.java} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/{OPALOPALPartialCallGraphConstructorTest.java => OPALPartialCallGraphConstructorTest.java} (100%) diff --git a/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/OPALOPALPartialCallGraphConstructorTest.java b/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructorTest.java similarity index 100% rename from analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/OPALOPALPartialCallGraphConstructorTest.java rename to analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructorTest.java From d1df8bc7ceac8dad5392c82986647e233e080156 Mon Sep 17 00:00:00 2001 From: Mehdi Keshani Date: Wed, 21 Sep 2022 13:05:06 +0200 Subject: [PATCH 4/7] add some helper methods to JavaGraph for easier putting and getting calls --- .../java/eu/fasten/core/data/JavaGraph.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/eu/fasten/core/data/JavaGraph.java b/core/src/main/java/eu/fasten/core/data/JavaGraph.java index 5e02a6058..4769c89d3 100644 --- a/core/src/main/java/eu/fasten/core/data/JavaGraph.java +++ b/core/src/main/java/eu/fasten/core/data/JavaGraph.java @@ -19,10 +19,11 @@ package eu.fasten.core.data; import it.unimi.dsi.fastutil.ints.IntIntPair; -import org.json.JSONArray; -import org.json.JSONObject; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.json.JSONArray; +import org.json.JSONObject; public class JavaGraph { @@ -40,6 +41,9 @@ public JavaGraph(final Map> callSites) { this.callSites = callSites; } + public void put(IntIntPair key, Map value){ + this.callSites.put(key, value); + } /** * Creates {@link JavaGraph} for the given JSONObject. * @@ -62,7 +66,7 @@ public JavaGraph(final HashMap> callSites) { * Creates {@link JavaGraph} with all fields empty. */ public JavaGraph() { - this.callSites = new HashMap<>(); + this.callSites = new ConcurrentHashMap<>(); } @@ -159,4 +163,12 @@ public boolean equals(Object o) { public int hashCode() { return callSites != null ? callSites.hashCode() : 0; } + + public Map getOrDefault(IntIntPair call, Map metadata) { + return this.callSites.getOrDefault(call, metadata); + } + + public Map get(IntIntPair key) { + return this.callSites.get(key); + } } From c8afc0e2b0b9682fbb80cea84b2d06a30192862e Mon Sep 17 00:00:00 2001 From: Mehdi Keshani Date: Wed, 21 Sep 2022 13:09:18 +0200 Subject: [PATCH 5/7] use atomic int for thread safety --- .../javacgopal/data/OPALPartialCallGraph.java | 14 +- .../OPALPartialCallGraphConstructorTest.java | 4 +- .../data/analysis/OPALClassHierarchyTest.java | 210 ++++++++---------- 3 files changed, 108 insertions(+), 120 deletions(-) diff --git a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraph.java b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraph.java index 3576687b1..7c1d52b96 100644 --- a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraph.java +++ b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraph.java @@ -15,17 +15,23 @@ */ package eu.fasten.analyzer.javacgopal.data; -import java.util.EnumMap; -import java.util.Map; - import eu.fasten.core.data.JavaGraph; import eu.fasten.core.data.JavaScope; import eu.fasten.core.data.JavaType; +import java.util.EnumMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; public class OPALPartialCallGraph { public JavaGraph graph; - public int nodeCount; + public AtomicInteger nodeCount; public EnumMap> classHierarchy; + public OPALPartialCallGraph(EnumMap> ch, JavaGraph graph, + AtomicInteger nodeCount) { + this.graph = graph; + this.nodeCount = nodeCount; + this.classHierarchy = ch; + } } \ No newline at end of file diff --git a/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructorTest.java b/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructorTest.java index ca541927e..85208dd1e 100644 --- a/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructorTest.java +++ b/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructorTest.java @@ -63,7 +63,7 @@ import scala.collection.Iterator; import scala.collection.mutable.HashSet; -class OPALOPALPartialCallGraphConstructorTest { +class OPALPartialCallGraphConstructorTest { private static OPALPartialCallGraph singleCallCG; @@ -155,7 +155,7 @@ void getGraph() { @Test void getNodeCount() { - assertEquals(4, singleCallCG.nodeCount); + assertEquals(4, singleCallCG.nodeCount.get()); } @Test diff --git a/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/analysis/OPALClassHierarchyTest.java b/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/analysis/OPALClassHierarchyTest.java index 9230d6b26..2b6e9cb0e 100644 --- a/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/analysis/OPALClassHierarchyTest.java +++ b/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/analysis/OPALClassHierarchyTest.java @@ -22,13 +22,15 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import eu.fasten.analyzer.javacgopal.data.CallPreservationStrategy; +import eu.fasten.core.data.JavaGraph; +import eu.fasten.core.data.JavaScope; +import it.unimi.dsi.fastutil.ints.IntIntPair; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedList; -import java.util.List; import java.util.Map; import java.util.Set; - import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -49,10 +51,6 @@ import org.opalj.collection.immutable.UIDSet; import org.opalj.collection.immutable.UIDSet1; import org.opalj.tac.Stmt; - -import eu.fasten.analyzer.javacgopal.data.CallPreservationStrategy; -import eu.fasten.core.data.JavaGraph; -import eu.fasten.core.data.JavaScope; import scala.Option; import scala.Tuple2; import scala.collection.Iterator; @@ -225,9 +223,8 @@ void getInternalCallKeys() { var internalKeys = classHierarchy.getInternalCallKeys(source, target); - assertEquals(2, internalKeys.size()); - assertEquals(123, internalKeys.get(0).intValue()); - assertEquals(234, internalKeys.get(1).intValue()); + assertEquals(123, internalKeys.firstInt()); + assertEquals(234, internalKeys.secondInt()); } @Test @@ -254,11 +251,12 @@ void getExternalCallKeysSourceMethodTargetDeclaredMethod() { var classHierarchy = new OPALClassHierarchy(internal, new HashMap<>(), 5); + Mockito.when(target.declaringClassType()).thenReturn(thisType); + var externalKeys = classHierarchy.getExternalCallKeys(source, target); - assertEquals(2, externalKeys.size()); - assertEquals(123, externalKeys.get(0).intValue()); - assertEquals(5, externalKeys.get(1).intValue()); + assertEquals(123, externalKeys.firstInt()); + assertEquals(5, externalKeys.secondInt()); } @Test @@ -273,6 +271,7 @@ void getExternalCallKeysSourceDeclaredMethodTargetMethod() { var source = Mockito.mock(DeclaredMethod.class); var target = Mockito.mock(Method.class); + Mockito.when(source.declaringClassType()).thenReturn(thisType); Mockito.when(target.declaringClassFile()).thenReturn(classFile); var methods = new HashMap(); @@ -287,9 +286,8 @@ void getExternalCallKeysSourceDeclaredMethodTargetMethod() { var externalKeys = classHierarchy.getExternalCallKeys(source, target); - assertEquals(2, externalKeys.size()); - assertEquals(5, externalKeys.get(0).intValue()); - assertEquals(123, externalKeys.get(1).intValue()); + assertEquals(5, externalKeys.firstInt()); + assertEquals(123, externalKeys.secondInt()); } @Test @@ -299,18 +297,22 @@ void getExternalCallKeysSourceDeclaredMethodTargetDeclaredMethod() { var classHierarchy = new OPALClassHierarchy(new HashMap<>(), new HashMap<>(), 5); + var thisType = Mockito.mock(ObjectType.class); + Mockito.when(source.declaringClassType()).thenReturn(thisType); + Mockito.when(target.declaringClassType()).thenReturn(thisType); + var externalKeys = classHierarchy.getExternalCallKeys(source, target); - assertEquals(2, externalKeys.size()); - assertEquals(5, externalKeys.get(0).intValue()); - assertEquals(6, externalKeys.get(1).intValue()); + assertEquals(5, externalKeys.firstInt()); + assertEquals(6, externalKeys.secondInt()); } @Test void getExternalCallKeysWrongTypes() { var classHierarchy = new OPALClassHierarchy(new HashMap<>(), new HashMap<>(), 5); - assertEquals(0, classHierarchy.getExternalCallKeys(new Object(), new Object()).size()); + assertEquals(IntIntPair.of(-1,-1), classHierarchy.getExternalCallKeys(new Object(), + new Object())); } @Test @@ -319,7 +321,8 @@ void getExternalCallKeysSourceMethodTargetWrongType() { var classHierarchy = new OPALClassHierarchy(new HashMap<>(), new HashMap<>(), 5); - assertEquals(0, classHierarchy.getExternalCallKeys(source, new Object()).size()); + assertEquals(IntIntPair.of(-1,-1), classHierarchy.getExternalCallKeys(source, + new Object())); } @Test @@ -345,25 +348,22 @@ void putCallsSourceMethod() { var internal = new HashMap(); internal.put(objectType, type); - - var classHierarchy = new OPALClassHierarchy(internal, new HashMap<>(), 5); - - var internalCalls = new HashMap, Map>(); - var internalCallKeys = classHierarchy.getInternalCallKeys(source, target); - internalCalls.put(internalCallKeys, new HashMap<>()); - - var externalCalls = new HashMap, Map>(); + var calls = new JavaGraph(); var newMetadata = new HashMap<>(); newMetadata.put(10, "newMetadata"); - assertEquals(0, internalCalls.get(internalCallKeys).size()); + var classHierarchy = new OPALClassHierarchy(internal, new HashMap<>(), 5, calls); - classHierarchy.putCalls(source, internalCalls, externalCalls, null, newMetadata, target - ); + var internalCallKeys = classHierarchy.getInternalCallKeys(source, target); + calls.put(internalCallKeys, new HashMap<>()); - assertEquals(1, internalCalls.get(internalCallKeys).size()); - assertEquals("newMetadata", internalCalls.get(internalCallKeys).get(10)); + assertEquals(0, calls.get(internalCallKeys).size()); + + classHierarchy.putCall(source, target, newMetadata); + + assertEquals(1, calls.get(internalCallKeys).size()); + assertEquals("newMetadata", calls.get(internalCallKeys).get(10)); } @Test @@ -378,30 +378,33 @@ void putCallsSourceDeclaredMethod() { var source = Mockito.mock(DeclaredMethod.class); var target = Mockito.mock(Method.class); + Mockito.when(source.declaringClassType()).thenReturn(thisType); Mockito.when(target.declaringClassFile()).thenReturn(classFile); - var type = new OPALType(new HashMap<>(), new LinkedList<>(), new ArrayList<>(), "source.java", "", false, new HashMap<>()); + var type = new OPALType(Map.of(target, 6), new LinkedList<>(), new ArrayList<>(), "source" + + ".java", "", false, new HashMap<>()); var internal = new HashMap(); internal.put(objectType, type); - var classHierarchy = new OPALClassHierarchy(internal, new HashMap<>(), 5); + var external = new HashMap>(); + external.put(thisType, Map.of(source, 5)); + var calls = new JavaGraph(); + calls.put(IntIntPair.of(5, 6), new HashMap<>()); - var externalCalls = new HashMap, Map>(); - externalCalls.put(List.of(5, 6), new HashMap<>()); - - var internalCalls = new HashMap, Map>(); + var classHierarchy = new OPALClassHierarchy(internal, external, 5, calls); var newMetadata = new HashMap<>(); newMetadata.put(10, "newMetadata"); - assertEquals(0, externalCalls.get(List.of(5, 6)).size()); + assertEquals(0, calls.get(IntIntPair.of(5, 6)).size()); + final var targetDeclaration = Mockito.mock(DeclaredMethod.class); + Mockito.when(targetDeclaration.declaringClassType()).thenReturn(thisType); - classHierarchy.putCalls(source, internalCalls, externalCalls, - Mockito.mock(DeclaredMethod.class), newMetadata, target); + classHierarchy.putCall(source, target, newMetadata); - assertEquals(1, externalCalls.get(List.of(5, 6)).size()); - assertEquals("newMetadata", externalCalls.get(List.of(5, 6)).get(10)); + assertEquals(1, calls.get(IntIntPair.of(5, 6)).size()); + assertEquals("newMetadata", calls.get(IntIntPair.of(5, 6)).get(10)); } @Test @@ -414,37 +417,35 @@ void putCallsTargetConstructor() { var classFile = Mockito.mock(ClassFile.class); Mockito.when(classFile.thisType()).thenReturn(thisType); - var source = Mockito.mock(DeclaredMethod.class); + var source = Mockito.mock(Method.class); var target = Mockito.mock(Method.class); + Mockito.when(source.declaringClassFile()).thenReturn(classFile); Mockito.when(target.declaringClassFile()).thenReturn(classFile); Mockito.when(target.isConstructor()).thenReturn(true); var methods = new HashMap(); + methods.put(source, 5); methods.put(target, 6); var type = new OPALType(methods, new LinkedList<>(), new ArrayList<>(), "source.java", "", false, new HashMap<>()); var internal = new HashMap(); internal.put(objectType, type); + var calls = new JavaGraph(); + calls.put(IntIntPair.of(5, 6), new HashMap<>()); - var classHierarchy = new OPALClassHierarchy(internal, new HashMap<>(), 5); - - var externalCalls = new HashMap, Map>(); - externalCalls.put(List.of(5, 6), new HashMap<>()); - - var internalCalls = new HashMap, Map>(); + var classHierarchy = new OPALClassHierarchy(internal, new HashMap<>(), 5, calls); var newMetadata = new HashMap<>(); newMetadata.put(10, "newMetadata"); - assertEquals(0, externalCalls.get(List.of(5, 6)).size()); + assertEquals(0, calls.get(IntIntPair.of(5, 6)).size()); - classHierarchy.putCalls(source, internalCalls, externalCalls, - Mockito.mock(DeclaredMethod.class), newMetadata, target); + classHierarchy.putCall(source, target, newMetadata); - assertEquals(1, externalCalls.size()); - assertEquals("newMetadata", externalCalls.get(List.of(5, 6)).get(10)); - assertNull(externalCalls.get(List.of(6, 6))); + assertEquals(1, calls.size()); + assertEquals("newMetadata", calls.get(IntIntPair.of(5, 6)).get(10)); + assertNull(calls.get(IntIntPair.of(6, 6))); } @Test @@ -452,13 +453,17 @@ void putExternalCall() { var source = Mockito.mock(DeclaredMethod.class); var target = Mockito.mock(DeclaredMethod.class); - var classHierarchy = new OPALClassHierarchy(new HashMap<>(), new HashMap<>(), 5); + var calls = new JavaGraph(); + var classHierarchy = new OPALClassHierarchy(new HashMap<>(), new HashMap<>(), 5, calls); assertEquals(0, classHierarchy.getExternalCHA().size()); + var thisType = Mockito.mock(ObjectType.class); + + Mockito.when(source.declaringClassType()).thenReturn(thisType); + Mockito.when(target.declaringClassType()).thenReturn(thisType); var externalKeys = classHierarchy.getExternalCallKeys(source, target); - var calls = new HashMap, Map>(); - classHierarchy.putExternalCall(source, calls, target, new HashMap<>()); + classHierarchy.putExternalCall(source, target, new HashMap<>()); assertEquals(1, classHierarchy.getExternalCHA().size()); assertNotNull(calls.get(externalKeys)); @@ -466,23 +471,21 @@ void putExternalCall() { @Test void getInternalMetadata() { - var callKeys = new ArrayList(); - callKeys.add(1); - callKeys.add(2); + var callKeys = IntIntPair.of(1,2); var internalMetadata = new HashMap<>(); - var internalCalls = new HashMap, Map>(); - internalCalls.put(callKeys, internalMetadata); + var calls = new JavaGraph(); + calls.put(callKeys, internalMetadata); var metadata = new HashMap<>(); metadata.put(123, "testMetadata"); - var classHierarchy = new OPALClassHierarchy(new HashMap<>(), new HashMap<>(), 5); + var classHierarchy = new OPALClassHierarchy(new HashMap<>(), new HashMap<>(), 5, calls); assertEquals(0, internalMetadata.size()); + var internalMetadataUpdated = classHierarchy.getInternalMetadata(metadata, callKeys); - var internalMetadataUpdated = classHierarchy.getInternalMetadata(internalCalls, metadata, callKeys); assertEquals(1, internalMetadata.size()); assertEquals(internalMetadata, internalMetadataUpdated); @@ -494,24 +497,19 @@ void getInternalMetadata() { void appendGraph() { OPALClassHierarchy classHierarchy = Mockito.spy(new OPALClassHierarchy(new HashMap<>(), new HashMap<>(), 5)); - var newGraph = Mockito.mock(JavaGraph.class); var existingGraph = Mockito.mock(JavaGraph.class); final var incompeletes = new ArrayList(); final Set visitedPCs = new java.util.HashSet<>(); - Mockito.doReturn(newGraph).when(classHierarchy).getSubGraph(Mockito.any(), Mockito.any(), - Mockito.any(), Mockito.any(), Mockito.any(), Mockito.eq(CallPreservationStrategy.INCLUDING_ALL_SUBTYPES)); - var source = Mockito.mock(Method.class); var targets = new HashSet>>().iterator(); var stmts = new Stmt[] { Mockito.mock(Stmt.class) }; - classHierarchy.appendGraph(source, targets, stmts, existingGraph, incompeletes, visitedPCs, - CallPreservationStrategy.INCLUDING_ALL_SUBTYPES); + classHierarchy.getSubGraph(source, targets, stmts, incompeletes, visitedPCs, + CallPreservationStrategy.INCLUDING_ALL_SUBTYPES); Mockito.verify(classHierarchy, Mockito.times(1)).getSubGraph(source, targets, stmts, incompeletes, visitedPCs, CallPreservationStrategy.INCLUDING_ALL_SUBTYPES); - Mockito.verify(existingGraph, Mockito.times(1)).append(newGraph); } @Test @@ -519,12 +517,10 @@ void getSubGraphTargetDeclarationNoDefinition() { OPALClassHierarchy classHierarchy = Mockito.spy(new OPALClassHierarchy(new HashMap<>(), new HashMap<>(), 5)); + Mockito.doNothing().when(classHierarchy).putCall(Mockito.any(), + Mockito.any(), Mockito.any()); Mockito.doNothing().when(classHierarchy) - .putCalls(Mockito.any(), Mockito.any(), Mockito.any(), - Mockito.any(), Mockito.any(), Mockito.any()); - Mockito.doNothing().when(classHierarchy) - .putExternalCall(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any() - ); + .putExternalCall(Mockito.any(), Mockito.any(), Mockito.any()); var callSite = new HashMap(); callSite.put("line", 20); @@ -551,11 +547,10 @@ void getSubGraphTargetDeclarationNoDefinition() { CallPreservationStrategy.INCLUDING_ALL_SUBTYPES); Mockito.verify(classHierarchy, Mockito.times(1)).getCallSite(source, 1, stmts); - Mockito.verify(classHierarchy, Mockito.never()).putCalls(Mockito.any(), Mockito.any(), - Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + Mockito.verify(classHierarchy, Mockito.never()).putCall(Mockito.any(), + Mockito.any(), Mockito.any()); Mockito.verify(classHierarchy, Mockito.never()) - .putExternalCall(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any() - ); + .putExternalCall(Mockito.any(), Mockito.any(), Mockito.any()); } @Test @@ -564,10 +559,9 @@ void getSubGraphTargetDeclarationMultipleDefinitions() { Mockito.spy(new OPALClassHierarchy(new HashMap<>(), new HashMap<>(), 5)); Mockito.doNothing().when(classHierarchy) - .putCalls(Mockito.any(), Mockito.any(), Mockito.any(), - Mockito.any(), Mockito.any(), Mockito.any()); + .putCall(Mockito.any(), Mockito.any(), Mockito.any()); Mockito.doNothing().when(classHierarchy) - .putExternalCall(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any() + .putExternalCall(Mockito.any(), Mockito.any(), Mockito.any() ); var callSite = new HashMap(); var stmts = new Stmt[] { Mockito.mock(Stmt.class) }; @@ -599,11 +593,10 @@ void getSubGraphTargetDeclarationMultipleDefinitions() { CallPreservationStrategy.INCLUDING_ALL_SUBTYPES); Mockito.verify(classHierarchy, Mockito.times(1)).getCallSite(source, 1, stmts); - Mockito.verify(classHierarchy, Mockito.times(2)).putCalls(Mockito.any(), Mockito.any(), - Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + Mockito.verify(classHierarchy, Mockito.times(2)).putCall(Mockito.any(), + Mockito.any(), Mockito.any()); Mockito.verify(classHierarchy, Mockito.never()) - .putExternalCall(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any() - ); + .putExternalCall(Mockito.any(), Mockito.any(), Mockito.any()); } @Test @@ -612,11 +605,9 @@ void getSubGraphTargetDeclarationSingleDefinition() { Mockito.spy(new OPALClassHierarchy(new HashMap<>(), new HashMap<>(), 5)); Mockito.doNothing().when(classHierarchy) - .putCalls(Mockito.any(), Mockito.any(), Mockito.any(), - Mockito.any(), Mockito.any(), Mockito.any()); + .putCall(Mockito.any(), Mockito.any(), Mockito.any()); Mockito.doNothing().when(classHierarchy) - .putExternalCall(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any() - ); + .putExternalCall(Mockito.any(), Mockito.any(), Mockito.any()); var callSite = new HashMap(); var stmts = new Stmt[] { Mockito.mock(Stmt.class) }; @@ -647,11 +638,10 @@ void getSubGraphTargetDeclarationSingleDefinition() { CallPreservationStrategy.INCLUDING_ALL_SUBTYPES); Mockito.verify(classHierarchy, Mockito.times(1)).getCallSite(source, 1, stmts); - Mockito.verify(classHierarchy, Mockito.times(1)).putCalls(Mockito.any(), Mockito.any(), - Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + Mockito.verify(classHierarchy, Mockito.times(1)).putCall(Mockito.any(), Mockito.any(), + Mockito.any()); Mockito.verify(classHierarchy, Mockito.never()) - .putExternalCall(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any() - ); + .putExternalCall(Mockito.any(), Mockito.any(), Mockito.any()); } @Test @@ -660,11 +650,9 @@ void getSubGraphTargetDeclarationVirtual() { Mockito.spy(new OPALClassHierarchy(new HashMap<>(), new HashMap<>(), 5)); Mockito.doNothing().when(classHierarchy) - .putCalls(Mockito.any(), Mockito.any(), Mockito.any(), - Mockito.any(), Mockito.any(), Mockito.any()); + .putCall(Mockito.any(), Mockito.any(), Mockito.any()); Mockito.doNothing().when(classHierarchy) - .putExternalCall(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any() - ); + .putExternalCall(Mockito.any(), Mockito.any(), Mockito.any()); var callSite = new HashMap(); var stmts = new Stmt[] { Mockito.mock(Stmt.class) }; @@ -692,11 +680,9 @@ void getSubGraphTargetDeclarationVirtual() { CallPreservationStrategy.INCLUDING_ALL_SUBTYPES); Mockito.verify(classHierarchy, Mockito.times(1)).getCallSite(source, 1, stmts); - Mockito.verify(classHierarchy, Mockito.times(0)).putCalls(Mockito.any(), Mockito.any(), - Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + Mockito.verify(classHierarchy, Mockito.times(0)).putCall(Mockito.any(), Mockito.any(), Mockito.any()); Mockito.verify(classHierarchy, Mockito.times(1)) - .putExternalCall(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any() - ); + .putExternalCall(Mockito.any(), Mockito.any(), Mockito.any()); } @Test @@ -705,11 +691,9 @@ void getSubGraphSourceWrongType() { Mockito.spy(new OPALClassHierarchy(new HashMap<>(), new HashMap<>(), 5)); Mockito.doNothing().when(classHierarchy) - .putCalls(Mockito.any(), Mockito.any(), Mockito.any(), - Mockito.any(), Mockito.any(), Mockito.any()); + .putCall(Mockito.any(), Mockito.any(),Mockito.any()); Mockito.doNothing().when(classHierarchy) - .putExternalCall(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any() - ); + .putExternalCall(Mockito.any(), Mockito.any(), Mockito.any()); var stmts = new Stmt[] { Mockito.mock(Stmt.class) }; var callSite = new HashMap(); @@ -737,11 +721,9 @@ void getSubGraphSourceWrongType() { Mockito.verify(classHierarchy, Mockito.times(0)).getCallSite(Mockito.any(), Mockito.any(), Mockito.eq(stmts)); - Mockito.verify(classHierarchy, Mockito.never()).putCalls(Mockito.any(), Mockito.any(), - Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + Mockito.verify(classHierarchy, Mockito.never()).putCall(Mockito.any(), Mockito.any(), Mockito.any()); Mockito.verify(classHierarchy, Mockito.never()) - .putExternalCall(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any() - ); + .putExternalCall(Mockito.any(), Mockito.any(), Mockito.any()); } @Test From 76f355e7ab8e42899812bf1e585b6ac68a7f957d Mon Sep 17 00:00:00 2001 From: Mehdi Keshani Date: Wed, 21 Sep 2022 13:29:00 +0200 Subject: [PATCH 6/7] make OPCGConstructor and OpalClassHierarchy thread safe --- .../data/OPALPartialCallGraphConstructor.java | 74 +++--- .../data/analysis/OPALClassHierarchy.java | 234 ++++++++---------- 2 files changed, 139 insertions(+), 169 deletions(-) diff --git a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructor.java b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructor.java index c54e227fa..0ea4fe1ab 100644 --- a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructor.java +++ b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructor.java @@ -23,17 +23,28 @@ import eu.fasten.analyzer.javacgopal.data.analysis.OPALMethod; import eu.fasten.analyzer.javacgopal.data.analysis.OPALType; import eu.fasten.core.data.Constants; -import eu.fasten.core.data.JavaGraph; +import eu.fasten.core.data.JavaScope; +import eu.fasten.core.data.JavaType; import eu.fasten.core.data.PartialJavaCallGraph; import eu.fasten.core.data.opal.MavenArtifactDownloader; import eu.fasten.core.data.opal.MavenCoordinate; import eu.fasten.core.data.opal.exceptions.OPALException; +import java.io.File; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.text.StringEscapeUtils; import org.opalj.br.Annotation; import org.opalj.br.ElementValuePair; import org.opalj.br.Method; -import org.opalj.br.ObjectType; import org.opalj.br.analyses.Project; import org.opalj.tac.AITACode; import org.opalj.tac.ComputeTACAIKey$; @@ -46,17 +57,6 @@ import scala.Function1; import scala.collection.JavaConverters; -import java.io.File; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - /** * Call graphs that are not still fully resolved. i.e. isolated call graphs which within-artifact * calls (edges) are known as internal calls and Cross-artifact calls are known as external calls. @@ -65,7 +65,11 @@ public class OPALPartialCallGraphConstructor { private static final Logger logger = LoggerFactory.getLogger(OPALPartialCallGraph.class); - private OPALPartialCallGraph pcg; + private final OPALClassHierarchy opalCha; + + public OPALPartialCallGraphConstructor() { + opalCha = new OPALClassHierarchy(); + } /** * Given a file, algorithm and main class (in case of application package) @@ -74,16 +78,13 @@ public class OPALPartialCallGraphConstructor { * @param ocg call graph constructor */ public OPALPartialCallGraph construct(OPALCallGraph ocg, CallPreservationStrategy callSiteOnly) { - pcg = new OPALPartialCallGraph(); - pcg.graph = new JavaGraph(); + EnumMap> ch; try { - final var cha = createInternalCHA(ocg.project); - - createGraphWithExternalCHA(ocg, cha, callSiteOnly); + createInternalCHA(ocg.project); + createGraphWithExternalCHA(ocg, callSiteOnly); + ch = opalCha.asURIHierarchyParallel(ocg.project.classHierarchy()); - pcg.nodeCount = cha.getNodeCount(); - pcg.classHierarchy = cha.asURIHierarchyParallel(ocg.project.classHierarchy()); } catch (Exception e) { if (e.getStackTrace().length > 0) { var stackTrace = e.getStackTrace()[0]; @@ -94,7 +95,7 @@ public OPALPartialCallGraph construct(OPALCallGraph ocg, CallPreservationStrateg throw e; } - return pcg; + return new OPALPartialCallGraph(ch, opalCha.graph, opalCha.getNodeCount()); } /** @@ -139,9 +140,7 @@ public static PartialJavaCallGraph createPartialJavaCG( * @return class hierarchy for a given package * @implNote Inside {@link OPALType} all of the methods are indexed. */ - private OPALClassHierarchy createInternalCHA(final Project project) { - final Map result = new HashMap<>(); - final AtomicInteger methodNum = new AtomicInteger(); + private void createInternalCHA(final Project project) { final var objs = Lists.newArrayList(JavaConverters.asJavaIterable(project.allClassFiles())); objs.sort(Comparator.comparing(Object::toString)); @@ -174,7 +173,7 @@ private OPALClassHierarchy createInternalCHA(final Project project) { } } final var currentClass = classFile.thisType(); - final var methods = getMethodsMap(methodNum.get(), + final var methods = getMethodsMap(opalCha.nodeCount.get(), JavaConverters.asJavaIterable(classFile.methods())); var namespace = OPALMethod.getPackageName(classFile.thisType()); var filepath = namespace != null ? namespace.replace(".", "/") : ""; @@ -187,10 +186,9 @@ private OPALClassHierarchy createInternalCHA(final Project project) { classFile.isPublic() ? "public" : "packagePrivate", classFile.isFinal(), opalAnnotations); - result.put(currentClass, type); - methodNum.addAndGet(methods.size()); + opalCha.internalCHA.put(currentClass, type); + opalCha.nodeCount.addAndGet(methods.size()); } - return new OPALClassHierarchy(result, new HashMap<>(), methodNum.get()); } /** @@ -217,11 +215,9 @@ private Map getMethodsMap(final int keyStartsFrom, * declared in the package that call external methods and add them to externalCHA of * a call hierarchy. Build a graph for both internal and external calls in parallel. * @param ocg call graph from OPAL generator - * @param cha class hierarchy - * @param callSiteOnly */ private void createGraphWithExternalCHA(final OPALCallGraph ocg, - final OPALClassHierarchy cha, CallPreservationStrategy callSiteOnly) { + CallPreservationStrategy callSiteOnly) { // TODO instead of relying on pcg field, use parameter final var cg = ocg.callGraph; final var tac = ocg.project.get(ComputeTACAIKey$.MODULE$); @@ -237,19 +233,19 @@ private void createGraphWithExternalCHA(final OPALCallGraph ocg, if (sourceDeclaration.hasMultipleDefinedMethods()) { for (final var source : JavaConverters .asJavaIterable(sourceDeclaration.definedMethods())) { - cha.appendGraph(source, cg.calleesOf(sourceDeclaration), - getStmts(tac, sourceDeclaration.definedMethod()), pcg.graph, incompeletes, - visitedPCs, callSiteOnly); + opalCha.getSubGraph(source, cg.calleesOf(sourceDeclaration), + getStmts(tac, sourceDeclaration.definedMethod()), incompeletes, visitedPCs, + callSiteOnly); } } else if (sourceDeclaration.hasSingleDefinedMethod()) { final var definedMethod = sourceDeclaration.definedMethod(); - cha.appendGraph(definedMethod, cg.calleesOf(sourceDeclaration), - getStmts(tac, definedMethod), pcg.graph, incompeletes, visitedPCs, callSiteOnly); + opalCha.getSubGraph(definedMethod, cg.calleesOf(sourceDeclaration), + getStmts(tac, definedMethod), incompeletes, visitedPCs, callSiteOnly); } else if (sourceDeclaration.isVirtualOrHasSingleDefinedMethod()) { - cha.appendGraph(sourceDeclaration, cg.calleesOf(sourceDeclaration), getStmts(tac, - null), pcg.graph, incompeletes, visitedPCs, callSiteOnly); + opalCha.getSubGraph(sourceDeclaration, cg.calleesOf(sourceDeclaration), getStmts(tac, + null), incompeletes, visitedPCs, callSiteOnly); } }); } diff --git a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/analysis/OPALClassHierarchy.java b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/analysis/OPALClassHierarchy.java index b049df0f4..14e9a37ab 100644 --- a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/analysis/OPALClassHierarchy.java +++ b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/analysis/OPALClassHierarchy.java @@ -24,6 +24,16 @@ import eu.fasten.core.data.JavaScope; import eu.fasten.core.data.JavaType; import it.unimi.dsi.fastutil.ints.IntIntPair; +import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; +import java.lang.reflect.InvocationTargetException; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import org.opalj.br.ClassHierarchy; import org.opalj.br.DeclaredMethod; import org.opalj.br.Method; @@ -38,26 +48,16 @@ import scala.collection.Iterator; import scala.collection.JavaConverters; -import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.EnumMap; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - /** * Class hierarchy class containing two types of CHA - internal and external CHA * and also keeping track of node count. */ public class OPALClassHierarchy { - private final Map internalCHA; - private final Map> externalCHA; - private int nodeCount; + public final Map internalCHA; + public final Map> externalCHA; + public final AtomicInteger nodeCount; + public final JavaGraph graph; /** * Class hierarchy constructor. @@ -66,12 +66,29 @@ public class OPALClassHierarchy { * @param externalCHA class hierarchy containing entries from outside the project scope * @param nodeCount number of nodes */ + public OPALClassHierarchy(Map internalCHA, + Map> externalCHA, + int nodeCount, JavaGraph graph) { + this.internalCHA = new ConcurrentHashMap<>(internalCHA); + this.externalCHA = new ConcurrentHashMap<>(externalCHA); + this.nodeCount = new AtomicInteger(nodeCount); + this.graph = graph; + } + public OPALClassHierarchy(Map internalCHA, Map> externalCHA, int nodeCount) { - this.internalCHA = internalCHA; - this.externalCHA = externalCHA; - this.nodeCount = nodeCount; + this.internalCHA = new ConcurrentHashMap<>(internalCHA); + this.externalCHA = new ConcurrentHashMap<>(externalCHA); + this.nodeCount = new AtomicInteger(nodeCount); + this.graph = new JavaGraph(); + } + + public OPALClassHierarchy() { + this.graph = new JavaGraph(); + this.internalCHA = new ConcurrentHashMap<>(); + this.externalCHA = new ConcurrentHashMap<>(); + this.nodeCount = new AtomicInteger(); } public Map getInternalCHA() { @@ -82,7 +99,7 @@ public Map> getExternalCHA() { return externalCHA; } - public int getNodeCount() { + public AtomicInteger getNodeCount() { return nodeCount; } @@ -147,17 +164,21 @@ public EnumMap> asURIHierarchyParallel(ClassHie * @return ID corresponding to the method */ public int addMethodToExternals(DeclaredMethod method) { - final var typeMethods = this.externalCHA - .getOrDefault(method.declaringClassType(), new HashMap<>()); - - if (typeMethods.containsKey(method)) { - return typeMethods.get(method); - } else { - typeMethods.put(method, this.nodeCount); - this.externalCHA.put(method.declaringClassType(), typeMethods); - this.nodeCount++; - return this.nodeCount - 1; + synchronized (nodeCount) { + final var klass = method.declaringClassType(); + if (!this.externalCHA.containsKey(klass)) { + this.externalCHA.put(klass, new Object2ObjectOpenHashMap<>()); + } + if (this.externalCHA.get(klass).containsKey(method)) { + return this.externalCHA.get(klass).get(method); + } else { + final var key = this.nodeCount.get(); + this.externalCHA.get(klass).put(method, key); + this.nodeCount.incrementAndGet(); + return key; + } } + } /** @@ -167,12 +188,12 @@ public int addMethodToExternals(DeclaredMethod method) { * @param target target method * @return list of call ids */ - public List getInternalCallKeys(final Method source, final Method target) { - return Arrays.asList( - this.internalCHA.get(source.declaringClassFile().thisType().asObjectType()) - .getMethods().get(source), - this.internalCHA.get(target.declaringClassFile().thisType().asObjectType()) - .getMethods().get(target)); + public IntIntPair getInternalCallKeys(final Method source, final Method target) { + return IntIntPair.of( + this.internalCHA.get(source.declaringClassFile().thisType().asObjectType()) + .getMethods().get(source), + this.internalCHA.get(target.declaringClassFile().thisType().asObjectType()) + .getMethods().get(target)); } /** @@ -182,47 +203,42 @@ public List getInternalCallKeys(final Method source, final Method targe * @param target target method * @return list of call ids */ - public List getExternalCallKeys(final Object source, final Object target) { + public IntIntPair getExternalCallKeys(final Object source, final Object target) { if (source instanceof Method && target instanceof DeclaredMethod) { - return Arrays.asList( - this.internalCHA - .get(((Method) source).declaringClassFile().thisType().asObjectType()) - .getMethods().get(source), - this.addMethodToExternals((DeclaredMethod) target)); + return IntIntPair.of( + this.internalCHA + .get(((Method) source).declaringClassFile().thisType().asObjectType()) + .getMethods().get(source), + this.addMethodToExternals((DeclaredMethod) target)); } else if (source instanceof DeclaredMethod && target instanceof Method) { - return Arrays.asList( - this.addMethodToExternals((DeclaredMethod) source), - this.internalCHA - .get(((Method) target).declaringClassFile().thisType().asObjectType()) - .getMethods().get(target)); + return IntIntPair.of( + this.addMethodToExternals((DeclaredMethod) source), + this.internalCHA + .get(((Method) target).declaringClassFile().thisType().asObjectType()) + .getMethods().get(target)); } else if (source instanceof DeclaredMethod) { - return Arrays.asList(this.addMethodToExternals((DeclaredMethod) source), - this.addMethodToExternals((DeclaredMethod) target)); + return IntIntPair.of(this.addMethodToExternals((DeclaredMethod) source), + this.addMethodToExternals((DeclaredMethod) target)); } else { - return new ArrayList<>(); + return IntIntPair.of(-1, -1); } } /** * Put calls to either internal or external maps of calls. * - * @param source source method - * @param internalCalls map of internal calls - * @param externalCalls map of external calls - * @param targetDeclaration target method declaration - * @param metadata metadata to put along the call - * @param target target method + * @param source source method + * @param target target method + * @param metadata metadata to put along the call */ - public void putCalls(final Object source, - final HashMap, Map> internalCalls, - final HashMap, Map> externalCalls, - final DeclaredMethod targetDeclaration, Map metadata, - final Method target) { - if (source instanceof Method) { - final var call = this.getInternalCallKeys((Method) source, target); - internalCalls.put(call, getInternalMetadata(internalCalls, metadata, call)); + public void putCall(final E source, + final T target, + final Map metadata) { + if (source instanceof Method && target instanceof Method) { + final var call = this.getInternalCallKeys((Method) source, (Method) target); + this.graph.put(call, getInternalMetadata(metadata, call)); } else { - putExternalCall(source, externalCalls, targetDeclaration, metadata); + putExternalCall(source, target, metadata); } } @@ -230,69 +246,44 @@ public void putCalls(final Object source, /** * Put external call to the list of calls. * - * @param source source method - * @param externalCalls map of external calls - * @param targetDeclaration target method declaration - * @param metadata metadata to put along the call + * @param source source method + * @param target target method declaration + * @param metadata metadata to put along the call */ - public void putExternalCall(final Object source, - final HashMap, Map> externalCalls, - final DeclaredMethod targetDeclaration, - final Map metadata) { - final var call = this.getExternalCallKeys(source, targetDeclaration); - final var externalMetadata = externalCalls.getOrDefault(call, new HashMap<>()); + public void putExternalCall(final E source, + final T target, + final Map metadata) { + final var call = this.getExternalCallKeys(source, target); + final var externalMetadata = this.graph.getOrDefault(call, new ConcurrentHashMap<>()); externalMetadata.putAll(metadata); - externalCalls.put(call, externalMetadata); + this.graph.put(call, externalMetadata); } /** * Get metadata of internal calls. * - * @param ic map of internal calls * @param metadata new metadata to add * @param call call to add metadata to * @return internal metadata */ - public Map getInternalMetadata(final Map, Map> ic, - final Map metadata, - final List call) { - final var internalMetadata = ic.getOrDefault(call, new HashMap<>()); + public Map getInternalMetadata(final Map metadata, + final IntIntPair call) { + final var internalMetadata = this.graph.getOrDefault(call, new ConcurrentHashMap<>()); internalMetadata.putAll(metadata); return internalMetadata; } - /** - * Append a sub-graph to already existing PartialJavaCallGraph. It is thread-safe. - * - * @param source source method - * @param targets list of targets - * @param resultGraph already existing PartialJavaCallGraph - * @param callSiteOnly - */ - public synchronized void appendGraph(final Object source, - final Iterator>> targets, - final Stmt>[] stmts, - final JavaGraph resultGraph, List incompeletes, - final Set visitedPCs, CallPreservationStrategy callSiteOnly) { - final var edges = this.getSubGraph(source, targets, stmts, incompeletes, visitedPCs, callSiteOnly); - resultGraph.append(edges); - } - /** * Given a source method and a list of targets return a sub-graph of PartialJavaCallGraph. * * @param source source method * @param targets list of targets - * @param callSiteOnly - * @return PartialJavaCallGraph sub-graph */ - public JavaGraph getSubGraph(final Object source, - final Iterator>> targets, - final Stmt>[] stmts, - final List incompeletes, - final Set visitedPCs, CallPreservationStrategy callSiteOnly) { - - final var callSites = new HashMap, Map>(); + public void getSubGraph(final Object source, + final Iterator>> targets, + final Stmt>[] stmts, + final List incompeletes, + final Set visitedPCs, CallPreservationStrategy callSiteOnly) { if (targets != null) { for (final var opalCallSite : JavaConverters.asJavaIterable(targets.toIterable())) { @@ -303,60 +294,43 @@ public JavaGraph getSubGraph(final Object source, incompeletes.remove(pc); if (callSiteOnly == CallPreservationStrategy.ONLY_STATIC_CALLSITES) { if (!visitedPCs.contains(pc)) { - processPC(source, stmts, visitedPCs, callSites, callSites, + processPC(source, stmts, visitedPCs, opalCallSite, targetDeclaration, pc); } } else { - processPC(source, stmts, visitedPCs, callSites, callSites, - opalCallSite, targetDeclaration, pc); + processPC(source, stmts, visitedPCs, + opalCallSite, targetDeclaration, pc); } } } } - return new JavaGraph(convert(callSites)); } private void processPC(final Object source, final Stmt>[] stmts, final Set visitedPCs, - final HashMap, Map> internalCalls, - final HashMap, Map> externalCalls, final Tuple2> opalCallSite, final DeclaredMethod targetDeclaration, final Integer pc) { visitedPCs.add(pc); Map metadata = new HashMap<>(); if (source instanceof Method) { - metadata = getCallSite((Method) source, (Integer) opalCallSite._1(), - stmts); + metadata = getCallSite((Method) source, (Integer) opalCallSite._1(), stmts); } if (targetDeclaration.hasMultipleDefinedMethods()) { for (final var target : JavaConverters .asJavaIterable(targetDeclaration.definedMethods())) { - this.putCalls(source, internalCalls, externalCalls, - targetDeclaration, - metadata, target); + this.putCall(source, target, metadata); } } else if (targetDeclaration.hasSingleDefinedMethod()) { - this.putCalls(source, internalCalls, externalCalls, targetDeclaration, - metadata, targetDeclaration.definedMethod()); + this.putCall(source, targetDeclaration.definedMethod(), metadata); } else if (targetDeclaration.isVirtualOrHasSingleDefinedMethod()) { - this.putExternalCall(source, externalCalls, targetDeclaration, - metadata); + this.putExternalCall(source, targetDeclaration, metadata); } } - // Conversion from List to IntIntPair - private HashMap> convert(final HashMap, Map> externalCalls) { - final HashMap> result = new HashMap<>(); - for (final var e : externalCalls.entrySet()) { - final List key = e.getKey(); - result.put(IntIntPair.of(key.get(0), key.get(1)), e.getValue()); - } - return result; - } - /** + /** * Get call site for a method. * * @param source source method From cefe630b58a2ac6d0d44caccf5e2038bb146729d Mon Sep 17 00:00:00 2001 From: Mehdi Keshani Date: Wed, 21 Sep 2022 13:38:27 +0200 Subject: [PATCH 7/7] parallelize internal class hierarchy --- .../data/OPALPartialCallGraphConstructor.java | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructor.java b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructor.java index 0ea4fe1ab..a628473e9 100644 --- a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructor.java +++ b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructor.java @@ -146,7 +146,7 @@ private void createInternalCHA(final Project project) { objs.sort(Comparator.comparing(Object::toString)); var opalAnnotations = new HashMap>>(); - for (final var classFile : objs) { + objs.parallelStream().forEach(classFile -> { var annotations = JavaConverters.asJavaIterable(classFile.annotations()); if (annotations != null) { for (Annotation annotation : annotations) { @@ -173,22 +173,25 @@ private void createInternalCHA(final Project project) { } } final var currentClass = classFile.thisType(); - final var methods = getMethodsMap(opalCha.nodeCount.get(), + synchronized (opalCha.nodeCount) { + final var methods = getMethodsMap(opalCha.nodeCount.get(), JavaConverters.asJavaIterable(classFile.methods())); - var namespace = OPALMethod.getPackageName(classFile.thisType()); - var filepath = namespace != null ? namespace.replace(".", "/") : ""; - final var type = new OPALType(methods, + var namespace = OPALMethod.getPackageName(classFile.thisType()); + var filepath = namespace != null ? namespace.replace(".", "/") : ""; + final var type = new OPALType(methods, OPALType.extractSuperClasses(project.classHierarchy(), currentClass), OPALType.extractSuperInterfaces(project.classHierarchy(), currentClass), classFile.sourceFile().isDefined() - ? filepath + "/" + classFile.sourceFile().get() - : "NotFound", + ? filepath + "/" + classFile.sourceFile().get() + : "NotFound", classFile.isPublic() ? "public" : "packagePrivate", classFile.isFinal(), opalAnnotations); - opalCha.internalCHA.put(currentClass, type); - opalCha.nodeCount.addAndGet(methods.size()); - } + opalCha.internalCHA.put(currentClass, type); + opalCha.nodeCount.addAndGet(methods.size()); + } + }); + } /**