Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Generate Java call graphs from local JARs in OPALPlugin #459

Open
wants to merge 6 commits into
base: develop
Choose a base branch
from
+282 −195
Open

Generate Java call graphs from local JARs in OPALPlugin #459

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
076ba74
Change the `DataWriter` interface to `DataRW` to read/write from/to t…
mir-am May 21, 2022
c2b6d0a
Add the `toPath` method to `MavenCoordinate`
mir-am May 21, 2022
05016df
In `OPALPlugin`, generate a call graph from a local JAR file if avail…
mir-am May 21, 2022
3b12da6
Make `toPath()` cross-platform and return `Path` in `MavenCoordinate`
mir-am Sep 9, 2022
b79f313
Extend `MavenArtifactDownloader` to download Maven artifacts into a g…
mir-am Sep 9, 2022
2c4d825
Adapt `OPALPartialCallGraphConstructor`'s `createPartialJavaCG` to us…
mir-am Sep 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 36 additions & 19 deletions analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/OPALPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,6 @@

package eu.fasten.analyzer.javacgopal;

import static eu.fasten.analyzer.javacgopal.data.CGAlgorithm.CHA;
import static eu.fasten.analyzer.javacgopal.data.CallPreservationStrategy.ONLY_STATIC_CALLSITES;
import static eu.fasten.core.maven.utils.MavenUtilities.MAVEN_CENTRAL_REPO;
import static java.lang.System.currentTimeMillis;

import java.io.File;
import java.util.Optional;

import org.json.JSONObject;
import org.pf4j.Extension;
import org.pf4j.Plugin;
import org.pf4j.PluginWrapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import eu.fasten.analyzer.javacgopal.data.OPALPartialCallGraphConstructor;
import eu.fasten.core.data.Constants;
import eu.fasten.core.data.JSONUtils;
Expand All @@ -42,6 +27,23 @@
import eu.fasten.core.data.opal.exceptions.MissingArtifactException;
import eu.fasten.core.data.opal.exceptions.OPALException;
import eu.fasten.core.plugins.AbstractKafkaPlugin;
import eu.fasten.core.plugins.DataRW;
import org.json.JSONObject;
import org.pf4j.Extension;
import org.pf4j.Plugin;
import org.pf4j.PluginWrapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileNotFoundException;
import java.nio.file.Paths;
import java.util.Optional;

import static eu.fasten.analyzer.javacgopal.data.CGAlgorithm.CHA;
import static eu.fasten.analyzer.javacgopal.data.CallPreservationStrategy.ONLY_STATIC_CALLSITES;
import static eu.fasten.core.maven.utils.MavenUtilities.MAVEN_CENTRAL_REPO;
import static java.lang.System.currentTimeMillis;

public class OPALPlugin extends Plugin {

Expand All @@ -50,12 +52,13 @@ public OPALPlugin(PluginWrapper wrapper) {
}

@Extension
public static class OPAL extends AbstractKafkaPlugin {
public static class OPAL extends AbstractKafkaPlugin implements DataRW {

private final Logger logger = LoggerFactory.getLogger(getClass());

private PartialJavaCallGraph graph;
private String outputPath;
private static String baseDir;

@Override
public void consume(String kafkaRecord, ProcessingLane l) {
Expand All @@ -76,9 +79,18 @@ public void consume(String kafkaRecord, ProcessingLane l) {
// Generate CG and measure construction duration.
logger.info("[CG-GENERATION] [UNPROCESSED] [-1] [" + mavenCoordinate.getCoordinate() + "] [NONE] ");
long date = json.optLong("releaseDate", -1);
this.graph = OPALPartialCallGraphConstructor.createPartialJavaCG(mavenCoordinate,
CHA, date, artifactRepository, ONLY_STATIC_CALLSITES);
long duration = currentTimeMillis() - startTime;
try {
this.graph = OPALPartialCallGraphConstructor.createPCGFromLocalJar(mavenCoordinate, date,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do not use exception handling for programming logic! Concrete proposal: There is no need to have two separate methods, one createPartialCG method should be enough, which just takes a path to the jar to be analyzed. The only difference is where the path comes from: either mavenCoord.getPath() exists or the .jar is downloaded and a path to the downloaded file is provided. As the jar should always exist locally in our setup, I would be fine with just throwing an error when the file is missing.

Paths.get(baseDir, mavenCoordinate.toPath()).toString(),
CHA, ONLY_STATIC_CALLSITES);
} catch (FileNotFoundException e) {
logger.warn(e.getMessage());
mir-am marked this conversation as resolved.
Show resolved Hide resolved
// Downloads the JAR file from Maven central
this.graph = OPALPartialCallGraphConstructor.createPartialJavaCG(mavenCoordinate,
CHA, date, artifactRepository, ONLY_STATIC_CALLSITES);
}

long duration = currentTimeMillis() - startTime;

if (this.graph.isCallGraphEmpty()) {
throw new EmptyCallGraphException();
Expand Down Expand Up @@ -149,5 +161,10 @@ public long getMaxConsumeTimeout() {
public long getSessionTimeout() {
return 1800000; // Due to static membership we also want to tune the session timeout to 30 minutes.
}

@Override
public void setBaseDir(String baseDir) {
OPAL.baseDir = baseDir;
}
}
}
72 changes: 49 additions & 23 deletions ...pal/src/main/java/eu/fasten/analyzer/javacgopal/data/OPALPartialCallGraphConstructor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,16 @@

package eu.fasten.analyzer.javacgopal.data;

import com.google.common.collect.Lists;
import eu.fasten.analyzer.javacgopal.data.analysis.OPALClassHierarchy;
import eu.fasten.analyzer.javacgopal.data.analysis.OPALMethod;
import eu.fasten.analyzer.javacgopal.data.analysis.OPALType;
import eu.fasten.core.data.Constants;
import eu.fasten.core.data.JavaGraph;
import eu.fasten.core.data.PartialJavaCallGraph;
import java.io.File;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;

import eu.fasten.core.data.opal.MavenArtifactDownloader;
import eu.fasten.core.data.opal.MavenCoordinate;
import eu.fasten.core.data.opal.exceptions.OPALException;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.text.StringEscapeUtils;
import org.opalj.br.Annotation;
Expand All @@ -45,20 +43,21 @@
import org.opalj.value.ValueInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.collect.Lists;

import eu.fasten.analyzer.javacgopal.data.analysis.OPALClassHierarchy;
import eu.fasten.analyzer.javacgopal.data.analysis.OPALMethod;
import eu.fasten.analyzer.javacgopal.data.analysis.OPALType;
import eu.fasten.core.data.Constants;
import eu.fasten.core.data.JavaGraph;
import eu.fasten.core.data.opal.MavenArtifactDownloader;
import eu.fasten.core.data.opal.MavenCoordinate;
import eu.fasten.core.data.opal.exceptions.OPALException;
import scala.Function1;
import scala.collection.JavaConverters;

import java.io.File;
import java.io.FileNotFoundException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;

/**
* Call graphs that are not still fully resolved. i.e. isolated call graphs which within-artifact
* calls (edges) are known as internal calls and Cross-artifact calls are known as external calls.
Expand Down Expand Up @@ -127,12 +126,39 @@ public static PartialJavaCallGraph createPartialJavaCG(
partialCallGraph.graph);
} finally {
if (file != null) {
// TODO use apache commons FileUtils instead
// TODO use apache commons FileUtils instead
file.delete();
}
}
}

/**
* Creates RevisionCallGraph from a local JAR given a Maven coordinate.
*
* @param coordinate maven coordinate of the revision to be processed
* @param timestamp timestamp of the revision release
* @return RevisionCallGraph of the given coordinate.
* @throws FileNotFoundException
*/
public static PartialJavaCallGraph createPCGFromLocalJar(final MavenCoordinate coordinate,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please merge the two createPCG methods into one. There is NO need for a case distinction, both should do exactly the same...

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Addressed in 2c4d825

final long timestamp, String jarFilePath,
CGAlgorithm algorithm, CallPreservationStrategy callSiteOnly) throws FileNotFoundException {

var file = new File(jarFilePath);
if (!file.exists()) {
throw new FileNotFoundException("Couldn't find the local JAR on filesystem for " + coordinate.toString());
}
final var opalCG = new OPALCallGraphConstructor().construct(file, algorithm);

final var partialCallGraph = new OPALPartialCallGraphConstructor().construct(opalCG, callSiteOnly);

return new PartialJavaCallGraph(Constants.mvnForge, coordinate.getProduct(),
coordinate.getVersionConstraint(), timestamp,
Constants.opalGenerator,
partialCallGraph.classHierarchy,
partialCallGraph.graph);
}

/**
* Creates a class hierarchy for the given call graph's artifact with entries
* only in internalCHA. ExternalCHA to be added at a later stage.
Expand Down
33 changes: 16 additions & 17 deletions ...lelvulnerabilitycacheinvalidationplugin/ParallelVulnerabilityCacheInvalidationPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,15 @@

package eu.fasten.analyzer.parallelvulnerabilitycacheinvalidationplugin;

import java.io.File;
import java.sql.Timestamp;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Optional;

import eu.fasten.core.maven.data.Revision;
import eu.fasten.core.maven.resolution.IMavenResolver;
import eu.fasten.core.maven.resolution.MavenResolver;
import eu.fasten.core.maven.resolution.MavenResolverIO;
import eu.fasten.core.maven.utils.MavenUtilities;
import eu.fasten.core.plugins.DataRW;
import eu.fasten.core.plugins.DependencyGraphUser;
import eu.fasten.core.plugins.KafkaPlugin;
import it.unimi.dsi.fastutil.objects.ObjectLinkedOpenHashSet;
import org.jooq.DSLContext;
import org.json.JSONArray;
import org.json.JSONObject;
Expand All @@ -34,23 +36,20 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import eu.fasten.core.maven.data.Revision;
import eu.fasten.core.maven.resolution.IMavenResolver;
import eu.fasten.core.maven.resolution.MavenResolver;
import eu.fasten.core.maven.resolution.MavenResolverIO;
import eu.fasten.core.maven.utils.MavenUtilities;
import eu.fasten.core.plugins.DataWriter;
import eu.fasten.core.plugins.DependencyGraphUser;
import eu.fasten.core.plugins.KafkaPlugin;
import it.unimi.dsi.fastutil.objects.ObjectLinkedOpenHashSet;
import java.io.File;
import java.sql.Timestamp;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Optional;

public class ParallelVulnerabilityCacheInvalidationPlugin extends Plugin {
public ParallelVulnerabilityCacheInvalidationPlugin(PluginWrapper wrapper) {
super(wrapper);
}

@Extension
public static class ParallelVulnerabilityCacheInvalidationExtension implements KafkaPlugin, DependencyGraphUser, DataWriter {
public static class ParallelVulnerabilityCacheInvalidationExtension implements KafkaPlugin, DependencyGraphUser, DataRW {

private final Logger logger = LoggerFactory.getLogger(ParallelVulnerabilityCacheInvalidationExtension.class.getName());

Expand Down
4 changes: 2 additions & 2 deletions ...epo-cloner-plugin/src/main/java/eu/fasten/analyzer/repoclonerplugin/RepoClonerPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
import eu.fasten.analyzer.repoclonerplugin.utils.SvnCloner;
import eu.fasten.core.data.Constants;
import eu.fasten.core.plugins.AbstractKafkaPlugin;
import eu.fasten.core.plugins.DataWriter;
import eu.fasten.core.plugins.DataRW;
import org.apache.commons.lang3.tuple.ImmutableTriple;
import org.apache.commons.lang3.tuple.Triple;
import org.apache.commons.math3.util.Pair;
Expand All @@ -46,7 +46,7 @@ public RepoClonerPlugin(PluginWrapper wrapper) {
}

@Extension
public static class RepoCloner extends AbstractKafkaPlugin implements DataWriter {
public static class RepoCloner extends AbstractKafkaPlugin implements DataRW {

private final Logger logger = LoggerFactory.getLogger(RepoCloner.class.getName());
private String repoPath = null;
Expand Down
39 changes: 19 additions & 20 deletions ...n/analyzer/vulnerabilitycacheinvalidationplugin/VulnerabilityCacheInvalidationPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,24 +18,6 @@

package eu.fasten.analyzer.vulnerabilitycacheinvalidationplugin;

import java.io.File;
import java.sql.Timestamp;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Optional;
import java.util.Set;

import org.jooq.DSLContext;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.pf4j.Extension;
import org.pf4j.Plugin;
import org.pf4j.PluginWrapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import eu.fasten.core.data.Constants;
import eu.fasten.core.maven.data.ResolvedRevision;
import eu.fasten.core.maven.data.Revision;
Expand All @@ -44,17 +26,34 @@
import eu.fasten.core.maven.resolution.MavenResolver;
import eu.fasten.core.maven.resolution.MavenResolverIO;
import eu.fasten.core.maven.utils.MavenUtilities;
import eu.fasten.core.plugins.DataWriter;
import eu.fasten.core.plugins.DataRW;
import eu.fasten.core.plugins.DependencyGraphUser;
import eu.fasten.core.plugins.KafkaPlugin;
import org.jooq.DSLContext;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.pf4j.Extension;
import org.pf4j.Plugin;
import org.pf4j.PluginWrapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.sql.Timestamp;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Optional;
import java.util.Set;

public class VulnerabilityCacheInvalidationPlugin extends Plugin {
public VulnerabilityCacheInvalidationPlugin(PluginWrapper wrapper) {
super(wrapper);
}

@Extension
public static class VulnerabilityCacheInvalidationExtension implements KafkaPlugin, DependencyGraphUser, DataWriter {
public static class VulnerabilityCacheInvalidationExtension implements KafkaPlugin, DependencyGraphUser, DataRW {

private final Logger logger = LoggerFactory.getLogger(VulnerabilityCacheInvalidationExtension.class.getName());
private List<String> consumeTopics = new LinkedList<>(Collections.singletonList("fasten.CallableIndexExtension.out"));
Expand Down
22 changes: 14 additions & 8 deletions core/src/main/java/eu/fasten/core/data/opal/MavenCoordinate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,16 @@
*/
package eu.fasten.core.data.opal;

import static eu.fasten.core.maven.utils.MavenUtilities.MAVEN_CENTRAL_REPO;
import eu.fasten.core.data.Constants;
import eu.fasten.core.maven.utils.MavenUtilities;
import org.json.JSONException;
import org.json.JSONObject;

import java.nio.file.Paths;
import java.util.LinkedList;
import java.util.List;

import org.json.JSONException;
import org.json.JSONObject;

import eu.fasten.core.data.Constants;
import eu.fasten.core.maven.utils.MavenUtilities;
import static eu.fasten.core.maven.utils.MavenUtilities.MAVEN_CENTRAL_REPO;

/**
* Maven coordinate as g:a:v e.g. "com.google.guava:guava:jar:28.1-jre".
Expand Down Expand Up @@ -156,9 +156,15 @@ public String toProductUrl() {
var repo = mavenRepos.get(0);
return this.toURL(repo) + "/" + artifactID + "-" + versionConstraint + "." + packaging;
mir-am marked this conversation as resolved.
Show resolved Hide resolved
}

@Override
public String toString() {
return getCoordinate();
return getCoordinate();
}

public String toPath() {
return Paths.get(groupID.replace('.', '/'), artifactID, versionConstraint,
mir-am marked this conversation as resolved.
Show resolved Hide resolved
artifactID + "-" + versionConstraint + "." + packaging).toString();
}

}
6 changes: 3 additions & 3 deletions ...va/eu/fasten/core/plugins/DataWriter.java → ...n/java/eu/fasten/core/plugins/DataRW.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,12 @@
package eu.fasten.core.plugins;

/**
* A plug-in that needs to write to disk some data should implement this interface.
* A plug-in that needs to read/write from/to disk some data should implement this interface.
*/
public interface DataWriter extends FastenPlugin {
public interface DataRW extends FastenPlugin {

/**
* Sets base directory into which the data will be written.
* Sets base directory into which the data will be read/written.
*
* @param baseDir Path to base directory
*/
Expand Down
Loading