Skip to content

Commit

Permalink
Merge pull request #6 from sse-labs/development
Browse files Browse the repository at this point in the history 
Development
  • Loading branch information
@jachiaram
jachiaram authored Aug 2, 2024
2 parents 32b34db + 7229abf commit eeaa517
Show file tree
Hide file tree
Showing 14 changed files with 370 additions and 67 deletions.
5 changes: 4 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,7 @@ target
.idea

# LastIndexProcessed
lastIndexProcessed
lastIndexProcessed

#maven resolution files
maven-resolution-files
Binary file added src/.DS_Store
View file
Binary file not shown.
Binary file added src/main/.DS_Store
View file
Binary file not shown.
20 changes: 20 additions & 0 deletions src/main/java/org/tudo/sse/CliInformation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ public class CliInformation {
private Path name;
private Path toCoordinates;
private Path toIndexPos;
private boolean output;
private Path toOutputDirectory;
private boolean multi;
private int threads;

Expand All @@ -26,6 +28,8 @@ public CliInformation() {
name = Paths.get("lastIndexProcessed");
toCoordinates = null;
toIndexPos = null;
toOutputDirectory = null;
output = false;
multi = false;
}

Expand Down Expand Up @@ -100,4 +104,20 @@ public int getThreads() {
public void setThreads(int threads) {
this.threads = threads;
}

public boolean isOutput() {
return output;
}

public void setOutput(boolean output) {
this.output = output;
}

public Path getToOutputDirectory() {
return toOutputDirectory;
}

public void setToOutputDirectory(Path toOutputDirectory) {
this.toOutputDirectory = toOutputDirectory;
}
}
6 changes: 5 additions & 1 deletion src/main/java/org/tudo/sse/Main.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,13 @@

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.tudo.sse.model.Artifact;
import org.tudo.sse.resolution.PomResolutionException;
import org.tudo.sse.utils.GAUtils;

import java.io.IOException;
import java.net.URISyntaxException;
import java.util.List;

public class Main {

Expand All @@ -16,7 +20,7 @@ public static void main(String[] args) throws URISyntaxException {
System.exit(1);
}

OwnImplementation imp = new OwnImplementation();
OwnImplementation imp = new OwnImplementation(false, true, false, true);
try {
imp.runAnalysis(args);
} catch (IOException e) {
Expand Down
74 changes: 48 additions & 26 deletions src/main/java/org/tudo/sse/MavenCentralAnalysis.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,10 @@ public void parseCmdLine(String[] args) {
setupInfo.setMulti(true);
setupInfo.setThreads(parseInt(args, i));
break;
case "--output":
setupInfo.setOutput(true);
setupInfo.setToOutputDirectory(parsePathName(args, i));
break;
default:
throw new CLIException(args[i]);
}
Expand Down Expand Up @@ -157,24 +161,11 @@ private int parseInt(String[] args, int i) throws CLIException {
}
}

private boolean parseBoolean(String[] args, int i) throws CLIException {
if(i + 1 < args.length) {
String toParse = args[i + 1];
if(toParse.toLowerCase().charAt(0) == 't') {
return true;
} else if(toParse.toLowerCase().charAt(0) == 'f') {
return false;
} else {
throw new CLIException(args[i]);
}
} else {
throw new CLIException(args[i]);
}
}

private Path parsePathName(String[] args, int i) throws CLIException {
if(i + 1 < args.length) {
if(Files.exists(Paths.get(args[i + 1])) || args[i].equals("--name")) {
if(Files.isRegularFile(Paths.get(args[i + 1])) || args[i].equals("--name")) {
return Paths.get(args[i + 1]);
} else if(args[i].equals("--output") && Files.isDirectory(Paths.get(args[i + 1]))) {
return Paths.get(args[i + 1]);
} else {
throw new CLIException(args[i], "Invalid path");
Expand All @@ -195,10 +186,12 @@ private Path parsePathName(String[] args, int i) throws CLIException {
* @throws IOException when there is an issue opening a file
*/
public Map<ArtifactIdent, Artifact> runAnalysis(String[] args) throws URISyntaxException, IOException {

//set up config from cli
parseCmdLine(args);
resolverFactory = new ResolverFactory(processTransitives);
if(setupInfo.isOutput()) {
resolverFactory = new ResolverFactory(setupInfo.isOutput(), setupInfo.getToOutputDirectory(), processTransitives);
} else {
resolverFactory = new ResolverFactory(processTransitives);
}

if(setupInfo.isMulti()) {
ActorSystem system = ActorSystem.create("my-system");
Expand All @@ -215,7 +208,6 @@ public Map<ArtifactIdent, Artifact> runAnalysis(String[] args) throws URISyntaxE
} catch (Exception e) {
e.printStackTrace();
}

} else {
if(setupInfo.getToCoordinates() == null) {
indexProcessor();
Expand Down Expand Up @@ -290,15 +282,15 @@ public List<Artifact> walkAllIndexes(IndexIterator indexIterator) throws IOExcep
while(indexIterator.hasNext()) {
Artifact current = ArtifactFactory.createArtifact(indexIterator.next());
artifacts.add(current);
processIndex(current);
if(artifacts.size() % 500000 == 0) {
log.info("{} artifacts have been parsed.", artifacts.size());
if(setupInfo.isOutput() && !resolvePom && !resolveJar) {
Path filePath = setupInfo.getToOutputDirectory().resolve(current.getIdent().getGroupID() + "-" + current.getIdent().getArtifactID() + "-" + current.getIdent().getVersion() + ".txt");
if(!Files.exists(filePath)) {
Files.createFile(filePath);
}
}
processIndex(current);
}

log.info("{} artifacts collected.", artifacts.get(artifacts.size() - 1).getIndexInformation().getIndex());
log.info("{} unique Identifiers.", artifacts.size());

if(setupInfo.isMulti()) {
queueActorRef.tell(new IndexProcessingMessage("Finished"), ActorRef.noSender());
}
Expand All @@ -323,6 +315,12 @@ public List<Artifact> walkPaginated(long take, IndexIterator indexIterator) thro
take += indexIterator.getIndex();
while(indexIterator.hasNext() && indexIterator.getIndex() < take) {
Artifact current = ArtifactFactory.createArtifact(indexIterator.next());
if(setupInfo.isOutput() && !resolvePom && !resolveJar) {
Path filePath = setupInfo.getToOutputDirectory().resolve(current.getIdent().getGroupID() + "-" + current.getIdent().getArtifactID() + "-" + current.getIdent().getVersion() + ".txt");
if(!Files.exists(filePath)) {
Files.createFile(filePath);
}
}
artifacts.add(current);
processIndex(current);
}
Expand Down Expand Up @@ -356,6 +354,12 @@ public List<Artifact> walkDates(long since, long until, IndexIterator indexItera
currentToSince = temp.getLastModified();
if(currentToSince >= since && currentToSince < until) {
Artifact current = ArtifactFactory.createArtifact(indexIterator.next());
if(setupInfo.isOutput() && !resolvePom && !resolveJar) {
Path filePath = setupInfo.getToOutputDirectory().resolve(current.getIdent().getGroupID() + "-" + current.getIdent().getArtifactID() + "-" + current.getIdent().getVersion() + ".txt");
if(!Files.exists(filePath)) {
Files.createFile(filePath);
}
}
artifacts.add(current);
processIndex(current);
}
Expand Down Expand Up @@ -392,6 +396,12 @@ public List<ArtifactIdent> lazyWalkAllIndexes(IndexIterator indexIterator) throw
while(indexIterator.hasNext()) {
ArtifactIdent ident = indexIterator.next().getIdent();
idents.add(ident);
if(setupInfo.isOutput() && !resolvePom && !resolveJar) {
Path filePath = setupInfo.getToOutputDirectory().resolve(ident.getGroupID() + "-" + ident.getArtifactID() + "-" + ident.getVersion() + ".txt");
if(!Files.exists(filePath)) {
Files.createFile(filePath);
}
}
processIndexIdentifier(ident);
}

Expand All @@ -418,6 +428,12 @@ public List<ArtifactIdent> lazyWalkPaginated(long take, IndexIterator indexItera
while(indexIterator.hasNext() && indexIterator.getIndex() < take) {
ArtifactIdent ident = indexIterator.next().getIdent();
idents.add(ident);
if(setupInfo.isOutput() && !resolvePom && !resolveJar) {
Path filePath = setupInfo.getToOutputDirectory().resolve(ident.getGroupID() + "-" + ident.getArtifactID() + "-" + ident.getVersion() + ".txt");
if(!Files.exists(filePath)) {
Files.createFile(filePath);
}
}
processIndexIdentifier(ident);
}

Expand Down Expand Up @@ -449,6 +465,12 @@ public List<ArtifactIdent> lazyWalkDates(long since, long until, IndexIterator i
if(currentToSince >= since && currentToSince < until) {
ArtifactIdent ident = temp.getIdent();
idents.add(ident);
if(setupInfo.isOutput() && !resolvePom && !resolveJar) {
Path filePath = setupInfo.getToOutputDirectory().resolve(ident.getGroupID() + "-" + ident.getArtifactID() + "-" + ident.getVersion() + ".txt");
if(!Files.exists(filePath)) {
Files.createFile(filePath);
}
}
processIndexIdentifier(ident);
}
}
Expand Down
29 changes: 26 additions & 3 deletions src/main/java/org/tudo/sse/resolution/JarResolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,11 @@
import java.io.DataInputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.jar.JarInputStream;

Expand All @@ -36,10 +39,26 @@
* @see JarInformation
*/
public class JarResolver {
private final Path pathToDirectory;
private boolean output;
private final Java16LibraryFramework cfReader = Project$.MODULE$.JavaClassFileReader(GlobalLogContext$.MODULE$, package$.MODULE$.BaseConfig());
private static final MavenCentralRepository MavenRepo = MavenCentralRepository.getInstance();
private static final Logger log = LogManager.getLogger(JarResolver.class);

public JarResolver() {
output = false;
pathToDirectory = null;
}

public JarResolver(boolean output, Path pathToDirectory) {
this.output = output;
this.pathToDirectory = pathToDirectory;
}

public void setOutput(boolean output) {
this.output = output;
}

/**
* This method resolves jar artifacts from a given list of artifact identifiers.
*
Expand Down Expand Up @@ -80,10 +99,8 @@ public Artifact parseJar(ArtifactIdent identifier) throws JarResolutionException
}

try {
boolean output = true;
URL jarURL = identifier.getMavenCentralJarUri().toURL();
InputStream jarInput = MavenRepo.openJarFileInputStream(identifier);
//TODO: move output to mavenCentralAnalysis and into jarResolver constructor
if(output) {
var baos = new ByteArrayOutputStream();
var buffer = new byte[32 * 1024];
Expand All @@ -98,7 +115,13 @@ public Artifact parseJar(ArtifactIdent identifier) throws JarResolutionException
}

byte[] jarBytes = baos.toByteArray();
//TODO: Write to a file

Path filePath = pathToDirectory.resolve(identifier.getGroupID() + "-" + identifier.getArtifactID() + "-" + identifier.getVersion() + ".jar");
if(!Files.exists(filePath)) {
Files.createFile(filePath);
Files.write(filePath, jarBytes);
}

jarInput = new ByteArrayInputStream(jarBytes);
}

Expand Down
Loading

0 comments on commit eeaa517

Please sign in to comment.