Skip to content

Commit

Permalink
Merge pull request #13 from sse-labs/feature/performance-improve
Browse files Browse the repository at this point in the history 
Improvements to performance and stability
  • Loading branch information
@johannesduesing
johannesduesing authored Sep 24, 2024
2 parents 0df04ac + 9ef389a commit 58b3187
Show file tree
Hide file tree
Showing 4 changed files with 138 additions and 43 deletions.
36 changes: 26 additions & 10 deletions src/main/java/org/tudo/sse/model/ArtifactIdent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
* As well as methods for retrieving different file types from the maven central repository.
*/
public class ArtifactIdent {

public static final String CENTRAL_REPOSITORY_URL = "https://repo1.maven.org/maven2/";
/**
* The group section of the identifier.
*/
Expand All @@ -27,31 +29,33 @@ public class ArtifactIdent {
*/
private String GA;

/**
* The GAV triple of this artifact
*/
private String GAV;

/**
* The version of the artifact.
*/
private String version;
/**
* The repository where this artifact can be found.
* The repository where this artifact can be found - if different from the central Repo
*/
private String repository;
private String customRepository;

private static final Logger log = LogManager.getLogger(ArtifactIdent.class);

public ArtifactIdent(String groupID, String artifactID, String version) {
this.artifactID = artifactID;
this.groupID = groupID;
this.GA = groupID + ":" + artifactID;
this.version = version;
this.repository = "https://repo1.maven.org/maven2/";
}

public ArtifactIdent(ArtifactIdent toCopy) {
this.groupID = toCopy.groupID;
this.artifactID = toCopy.artifactID;
this.version = toCopy.version;
this.repository = toCopy.repository;
this.GA = toCopy.getGA();
this.customRepository = toCopy.customRepository;
}

/**
Expand All @@ -69,6 +73,7 @@ public String getGroupID() {
public void setGroupID(String groupID) {
this.groupID = groupID;
this.GA = this.groupID + ":" + this.artifactID;
this.GAV = groupID + ":" + artifactID + ":" + version;
}

/**
Expand All @@ -86,13 +91,18 @@ public String getArtifactID() {
public void setArtifactID(String artifactID) {
this.artifactID = artifactID;
this.GA = this.groupID + ":" + this.artifactID;
this.GAV = groupID + ":" + artifactID + ":" + version;
}

/**
* Gets GA tuple for this artifact.
* @return GA tuple separated by colon
*/
public String getGA(){
if(this.GA == null){
this.GA = this.groupID + ":" + this.artifactID;
}

return this.GA;
}
/**
Expand All @@ -109,30 +119,36 @@ public String getVersion() {
*/
public void setVersion(String version) {
this.version = version;
this.GAV = groupID + ":" + artifactID + ":" + version;
}

/**
* Gets the repository where this artifact can be found.
* @return repository
*/
public String getRepository() {
return repository;
if(this.customRepository != null) return this.customRepository;
else return CENTRAL_REPOSITORY_URL;
}

/**
* Sets the repository for where this artifact can be found.
* @param repository new repository value
*/
public void setRepository(String repository) {
this.repository = repository;
this.customRepository = repository;
}

/**
* Gets the coordinates, the full built identifier for a maven artifact.
* @return full g:a:v value
*/
public String getCoordinates() {
return groupID + ":" + artifactID + ":" + version;

if(this.GAV == null){
this.GAV = groupID + ":" + artifactID + ":" + version;
}
return this.GAV;
}

/**
Expand All @@ -141,7 +157,7 @@ public String getCoordinates() {
*/
public URI getMavenCentralPomUri() {
try {
if(repository.equals("https://repo1.maven.org/maven2/")) {
if(customRepository == null) {
return MavenCentralRepository.buildPomFileURI(this);
} else {
return MavenCentralRepository.buildSecondaryPomFileURI(this, getRepository());
Expand Down
14 changes: 14 additions & 0 deletions src/main/java/org/tudo/sse/utils/GAUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,20 @@ public static List<Artifact> retrieveAllVersions(String groupId, String artifact
}
}

public static List<String> getReleasesFromMetadata(ArtifactIdent identifier){
try{
Metadata meta = getVersions(identifier.getGroupID(), identifier.getArtifactID());

if(meta.getVersioning() == null){
throw new RuntimeException("Invalid versioning in metadata: null");
}

return meta.getVersioning().getVersions();
} catch(FileNotFoundException | IOException | XmlPullParserException x){
throw new RuntimeException(x);
}
}

public static Artifact getLastModifiedVersion(String groupId, String artifactId) throws PomResolutionException {
try {
Metadata meta = getVersions(groupId, artifactId);
Expand Down
127 changes: 96 additions & 31 deletions src/main/java/org/tudo/sse/utils/IndexIterator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import org.tudo.sse.model.index.Package;
import org.tudo.sse.model.index.IndexInformation;

import javax.net.ssl.SSLException;
import java.io.IOException;
import java.net.URI;
import java.util.Iterator;
Expand All @@ -22,8 +23,10 @@

public class IndexIterator implements Iterator<IndexInformation> {
private long index;
private final IndexReader ir;
private final Iterator<Map<String, String>> cr;

private final URI baseUri;
private IndexReader ir;
private Iterator<Map<String, String>> cr;
private IndexInformation currentArtifact;
private IndexInformation nextArtifact;
private boolean prevHasNext;
Expand All @@ -32,6 +35,7 @@ public class IndexIterator implements Iterator<IndexInformation> {


public IndexIterator(URI base) throws IOException {
baseUri = base;
ir = new IndexReader(null, new HttpResourceHandler(base.resolve(".index/")));
cr = ir.iterator().next().iterator();
index = 0;
Expand All @@ -40,22 +44,36 @@ public IndexIterator(URI base) throws IOException {
}

public IndexIterator(URI base, long startingIndex) throws IOException {
ir = new IndexReader(null, new HttpResourceHandler(base.resolve(".index/")));
cr = ir.iterator().next().iterator();
index = 0;
this(base);

while(cr.hasNext() && index != startingIndex) {
cr.next();
index++;
}
currentArtifact = null;
nextArtifact = null;
}

public void closeReader() throws IOException {
ir.close();
}

private void recoverConnectionReset() throws IOException{
long indexPos = getIndex();
log.info("Recovering from connection reset at index {}", indexPos);


ir = new IndexReader(null, new HttpResourceHandler(baseUri.resolve(".index/")));
cr = ir.iterator().next().iterator();
index = 0;

while(cr.hasNext() && index < indexPos){
cr.next();
index++;
if(index % 1000000 == 0) log.debug("Skipping indices for recovery, {} processed so far ...", index);
}

log.info("Recovery successful, reset chunk reader to index {}.", indexPos);
}

/**
* This method takes in a string of the gav tuple and creates an artifactIdent from it.
* @param gav string version of an artifact identifier "g:a:v"
Expand Down Expand Up @@ -93,28 +111,37 @@ public Package processPackage(String information, String checksum) {
* @see Package
*/
public IndexInformation processIndex(Map<String, String> item) {
String uVal = item.get("u");
//process the G:A:V tuple
if(item.get("u") != null) {
ArtifactIdent temp = processArtifactIdent(item.get("u"));
if(uVal != null) {
ArtifactIdent temp = processArtifactIdent(uVal);

//Create an artifact using the values found in the 'i' and '1' tags
if(item.get("i") != null) {
String[] parts = item.get("i").split(IndexWalker.splitPattern);
return processIndex(item, temp);
}
return null;
}

Package tmpPackage = new Package(parts[0], Long.parseLong(parts[1]), Long.parseLong(parts[2]), Integer.parseInt(parts[3]), Integer.parseInt(parts[4]), Integer.parseInt(parts[5]), item.get("1"));
private IndexInformation processIndex(Map<String, String> item, ArtifactIdent ident){
String iVal = item.get("i");

IndexInformation t = new IndexInformation(temp, tmpPackage);
t.setName(item.get("n"));
t.setIndex(index);
index++;
//Create an artifact using the values found in the 'i' and '1' tags
if(iVal != null) {
String[] parts = iVal.split(IndexWalker.splitPattern);

if(index != 0 && index % 500000 == 0){
log.info("{} indexes have been processed.", index);
}
Package tmpPackage = new Package(parts[0], Long.parseLong(parts[1]), Long.parseLong(parts[2]), Integer.parseInt(parts[3]), Integer.parseInt(parts[4]), Integer.parseInt(parts[5]), item.get("1"));

IndexInformation t = new IndexInformation(ident, tmpPackage);
t.setName(item.get("n"));
t.setIndex(index);
index++;

return t;
if(index != 0 && index % 500000 == 0){
log.info("{} indexes have been processed.", index);
}

return t;
}

return null;
}

Expand All @@ -139,8 +166,34 @@ public boolean hasNext() {

//pass nextArtifact to currentArtifact
if(nextArtifact == null) {
while (cr.hasNext() && currentArtifact == null) {
currentArtifact = processIndex(cr.next());
while (currentArtifact == null && cr.hasNext()) {
try {
// This may fail with an SSL connection reset exception...
currentArtifact = processIndex(cr.next());
} catch(RuntimeException rx){

// Try to find out if this read error was caused by an SSL connection reset.
boolean causedBySsl = false;
Throwable current = rx;
while(!causedBySsl && current.getCause() != null){
current = current.getCause();
causedBySsl = current instanceof SSLException;
}

if(rx.getMessage().contains("read error") && causedBySsl){
// If so, try to recover by re-initializing the reader (and skipping to the right position)
try {
recoverConnectionReset();
} catch(Exception x){
log.error("Recovery unsuccessful: " + x.getMessage());
throw new RuntimeException(x);
}
} else {
// Don't try to handle other exceptions with recovery
throw rx;
}
}

}
} else {
currentArtifact = nextArtifact;
Expand All @@ -149,21 +202,33 @@ public boolean hasNext() {

//keep iterating the indexReader until the gav is different from the one in currentArtifact
while (cr.hasNext()) {
Map<String, String> curInfo = cr.next();
Map<String, String> currentEntry;

if (curInfo.get("u") == null) {
break;
try {
currentEntry = cr.next();
} catch(RuntimeException rx){
log.error("Failed to get entry from index: " + rx.getMessage());
currentEntry = null;
}

if (!(currentArtifact.getIdent().getCoordinates().equals(processArtifactIdent(curInfo.get("u")).getCoordinates()))) {
if (currentEntry == null) break;

String currentUVal = currentEntry.get("u");

//store into additional variable
nextArtifact = processIndex(curInfo);
if(currentUVal == null) break;

final String currentArtifactGAV = currentArtifact.getIdent().getCoordinates();
final ArtifactIdent currentEntryIdent = processArtifactIdent(currentUVal);

if(!currentArtifactGAV.equals(currentEntryIdent.getCoordinates())){
nextArtifact = processIndex(currentEntry, currentEntryIdent);
break;
}

if(curInfo.get("i") != null) {
currentArtifact.addAPackage(processPackage(curInfo.get("i"), curInfo.get("1")));
String currentIVal = currentEntry.get("i");

if(currentIVal != null) {
currentArtifact.addAPackage(processPackage(currentIVal, currentEntry.get("1")));
index++;
}
}
Expand Down
4 changes: 2 additions & 2 deletions src/test/resources/PomInputs.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -362,9 +362,9 @@
"org.springframework.boot:spring-boot-starter-web:2.5.0:compile",
"com.fasterxml.jackson.core:jackson-databind:2.10.0:compile",
"org.hibernate:hibernate-core:5.3.0.CR2:compile",
"com.google.guava:guava:33.3.0-jre:compile",
"com.google.guava:guava:33.3.1-jre:compile",
"org.apache.httpcomponents:httpclient:4.5.13:compile",
"commons-io:commons-io:2.16.1:compile"
"commons-io:commons-io:2.17.0:compile"
]
],

Expand Down

0 comments on commit 58b3187

Please sign in to comment.