Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 43 additions & 22 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<junit.jupiter.version>5.14.1</junit.jupiter.version>
</properties>

<repositories>
Expand All @@ -36,15 +37,19 @@
</repository>

<repository>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
<id>epi-public-repo</id>
<name>Epimorphics Public Repository</name>
<url>https://repository.epimorphics.com</url>
<id>epi-public-s3-release</id>
<name>Epimorphics S3 release repository</name>
<url>https://epi-repository.s3-eu-west-1.amazonaws.com/release</url>
<releases><enabled>true</enabled></releases>
<snapshots><enabled>false</enabled></snapshots>
</repository>

<repository>
<id>epi-public-s3-snapshot</id>
<name>Epimorphics S3 snapshot repository</name>
<url>https://epi-repository.s3-eu-west-1.amazonaws.com/snapshot</url>
<releases><enabled>false</enabled></releases>
<snapshots><enabled>true</enabled></snapshots>
</repository>

</repositories>
Expand All @@ -70,15 +75,26 @@

<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13.1</version>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>${junit.jupiter.version}</version>
<scope>test</scope>
</dependency>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>${junit.jupiter.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.epimorphics</groupId>
<artifactId>appbase</artifactId>
<version>3.1.14</version>
<version>4.0.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.20.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
Expand All @@ -88,31 +104,36 @@
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.3</version>
<version>2.14.0</version>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>2.4</version>
<version>2.5</version>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.5.4</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.3</version>
<version>3.14.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<source>21</source>
<target>21</target>
</configuration>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-scm-plugin</artifactId>
<version>1.9.4</version>
<version>2.2.1</version>
<configuration>
<connectionType>developerConnection</connectionType>
</configuration>
Expand All @@ -121,13 +142,13 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-release-plugin</artifactId>
<version>2.5.2</version>
<version>3.3.0</version>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.1</version>
<version>3.6.1</version>
<configuration>
<filters>
<filter>
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/epimorphics/dclib/Main.java
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ public Boolean call() {
String filebasename = NameUtils.removeExtension(filename);
dc.getGlobalEnv().put(ConverterProcess.FILE_NAME, filename);
dc.getGlobalEnv().put(ConverterProcess.FILE_BASE_NAME, filebasename);
InputStream is = new BOMInputStream( new FileInputStream(dataFileF) );
InputStream is = BOMInputStream.builder().setInputStream( new FileInputStream(dataFileF) ).get();

ConverterProcess process = new ConverterProcess(dc, is);
process.setDebug( args.isDebug() );
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import java.io.InputStream;
import java.util.Map.Entry;

import com.opencsv.exceptions.CsvValidationException;
import org.apache.commons.collections.map.LRUMap;
import org.apache.jena.riot.RDFDataMgr;
import org.apache.jena.riot.system.StreamRDF;
Expand Down Expand Up @@ -221,6 +222,8 @@ public boolean process() {

} catch (IOException e) {
messageReporter.reportError("Problem reading next line of source");
} catch (CsvValidationException csvve) {
messageReporter.reportError("The CSV content was invalid: " + csvve.getMessage());
} finally {
current.set(null);
}
Expand Down Expand Up @@ -291,7 +294,7 @@ public BindingEnv peekRow() {
}
}

protected void preprocess() throws IOException {
protected void preprocess() throws IOException, CsvValidationException {
Node dataset = NodeFactory.createBlankNode();

Object baseURI = env.get(BASE_OBJECT_NAME);
Expand Down Expand Up @@ -332,7 +335,7 @@ protected void preprocess() throws IOException {
try {
Node prop = new Pattern(peekRow[1], dataContext).evaluateAsURINode(env, this, -1);
Node value = new Pattern(peekRow[2], dataContext).evaluateAsNode(env, this, -1);
getOutputStream().triple( new Triple(dataset, prop, value) );
getOutputStream().triple(Triple.create(dataset, prop, value));
} catch (Exception e) {
messageReporter.report("Failed to process metadata row: " + e, dataSource.getLineNumber());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import java.io.InputStream;

import org.apache.commons.io.input.BOMInputStream;
import org.apache.jena.riot.RDFDataMgr;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -50,7 +51,7 @@ public class ConverterService extends ComponentBase {
public ConverterService(String defaultPrefixes) {
dc = new DataContext();
dc.getGlobalEnv().put(ConverterProcess.BASE_OBJECT_NAME, DEFAULT_BASE_URI);
PrefixMapping prefixes = FileManager.get().loadModel(defaultPrefixes);
PrefixMapping prefixes = RDFDataMgr.loadModel(defaultPrefixes);
if (prefixes != null) {
dc.setPrefixes(prefixes);
} else {
Expand All @@ -64,7 +65,7 @@ public ConverterService(String defaultPrefixes) {
* all qnames in templates.
*/
public void setPrefixFile(String prefixes) {
PrefixMapping pref = FileManager.get().loadModel(prefixes);
PrefixMapping pref = RDFDataMgr.loadModel(prefixes);
dc.setPrefixes(pref);
}

Expand Down Expand Up @@ -147,7 +148,7 @@ public Model simpleConvert(String templateFile, String dataFile, ProgressMonitor
String filebasename = NameUtils.removeExtension(filename);
put(ConverterProcess.FILE_NAME, filename);
put(ConverterProcess.FILE_BASE_NAME, filebasename);
InputStream is = new BOMInputStream( new FileInputStream(dataFileF) );
InputStream is = BOMInputStream.builder().setInputStream( new FileInputStream(dataFileF) ).get();

ConverterProcess process = new ConverterProcess(dc, is);
process.setDebug(debug);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
package com.epimorphics.dclib.framework ;

import org.apache.jena.atlas.web.TypedInputStream ;
import org.apache.jena.riot.web.HttpOp ;
import org.apache.jena.http.HttpOp;
import org.slf4j.Logger ;
import org.slf4j.LoggerFactory ;

Expand All @@ -47,7 +47,7 @@ public LocatorHTTP() {
@Override
public TypedInputStream performOpen(String uri) {
if ( uri.startsWith("http://") || uri.startsWith("https://") )
return HttpOp.execHttpGet(uri, acceptHeader) ;
return HttpOp.httpGet(uri, acceptHeader) ;
return null ;
}

Expand Down
8 changes: 5 additions & 3 deletions src/main/java/com/epimorphics/dclib/framework/Pattern.java
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,13 @@
import java.util.ArrayList;
import java.util.List;

import javax.ws.rs.NotSupportedException;
import jakarta.ws.rs.NotSupportedException;

import org.apache.commons.jexl2.Expression;
import org.apache.commons.jexl2.JexlEngine;
import org.apache.commons.jexl2.Script;
import org.apache.jena.datatypes.xsd.XSDDatatype;
import org.apache.jena.vocabulary.XSD;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -225,11 +227,11 @@ public Node asNode(Object result) {
} else if (result instanceof RDFNodeWrapper) {
return ((RDFNodeWrapper)result).asRDFNode().asNode();
} else if (result instanceof String) {
return NodeFactory.createLiteral( (String)result );
return NodeFactory.createLiteralString( (String)result );
} else if (result instanceof Number) {
return ValueNumber.nodeFromNumber( (Number)result );
} else if (result instanceof Boolean) {
return NodeFactory.createLiteral( LiteralLabelFactory.createTypedLiteral(result) );
return NodeFactory.createLiteralDT(result.toString(), XSDDatatype.XSDboolean);
} else if (result instanceof Value) {
return ((Value)result).asNode();
} else {
Expand Down
22 changes: 12 additions & 10 deletions src/main/java/com/epimorphics/dclib/sources/CSVInput.java
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,13 @@
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;

import com.opencsv.CSVParser;
import com.opencsv.CSVParserBuilder;
import com.opencsv.CSVReader;
import com.opencsv.CSVReaderBuilder;
import com.opencsv.exceptions.CsvValidationException;
import org.apache.commons.io.input.BOMInputStream;

import au.com.bytecode.opencsv.CSVParser;
import au.com.bytecode.opencsv.CSVReader;

import com.epimorphics.dclib.framework.BindingEnv;
import com.epimorphics.util.EpiException;
import com.epimorphics.util.NameUtils;
Expand All @@ -35,12 +37,12 @@ public class CSVInput {
protected boolean hasPreamble = false;
protected String[] peekRow;

public CSVInput(String filename) throws IOException {
this( new BOMInputStream( new FileInputStream(filename) ) );
public CSVInput(String filename) throws IOException, CsvValidationException {
this(BOMInputStream.builder().setInputStream( new FileInputStream(filename) ).get());
}

public CSVInput(InputStream ins) throws IOException {
in = new CSVReader( new InputStreamReader(ins, StandardCharsets.UTF_8), CSVParser.DEFAULT_SEPARATOR, CSVParser.DEFAULT_QUOTE_CHARACTER, '\0' );
public CSVInput(InputStream ins) throws IOException, CsvValidationException {
in = new CSVReaderBuilder(new InputStreamReader(ins, StandardCharsets.UTF_8)).build();

String[] headerLine = in.readNext();
if (headerLine == null) {
Expand Down Expand Up @@ -72,7 +74,7 @@ private String safeColName(String col) {
* Return a look ahead to the next row.
* Repeat calls do not advance to further rows,
*/
public String[] getPeekRow() throws IOException {
public String[] getPeekRow() throws IOException, CsvValidationException {
if (peekRow == null) {
peekRow = in.readNext();
}
Expand All @@ -83,7 +85,7 @@ public String[] getPeekRow() throws IOException {
* Advances to the next row after a prior peek.
* Returns true if a new peek was available.
*/
public boolean advancePeek() throws IOException {
public boolean advancePeek() throws IOException, CsvValidationException {
peekRow = in.readNext();
lineNumber++;
return peekRow != null;
Expand All @@ -96,7 +98,7 @@ public boolean advancePeek() throws IOException {
* If there have been any peek rows then returns an env based
* on the last peeked row.
*/
public BindingEnv nextRow() throws IOException {
public BindingEnv nextRow() throws IOException, CsvValidationException {
if (in != null) {
String[] rowValues = (peekRow != null) ? peekRow : in.readNext();
lineNumber++;
Expand Down
5 changes: 3 additions & 2 deletions src/main/java/com/epimorphics/dclib/sources/CSVMapSource.java
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

import java.io.IOException;

import com.opencsv.exceptions.CsvValidationException;
import org.apache.jena.atlas.json.JsonObject;

import com.epimorphics.dclib.framework.BindingEnv;
Expand Down Expand Up @@ -39,7 +40,7 @@ public static boolean isSpec(JsonObject spec) {
return false;
}

public CSVMapSource(JsonObject spec, ConverterProcess config) throws IOException {
public CSVMapSource(JsonObject spec, ConverterProcess config) throws IOException, CsvValidationException {
super(spec);
String keyCol = getField(JSONConstants.KEY, "key");
String valueCol = getField(JSONConstants.VALUE, "value");
Expand Down Expand Up @@ -76,7 +77,7 @@ private Node asNode(boolean isURI, String value) {
if (isURI) {
return NodeFactory.createURI(value);
} else {
return NodeFactory.createLiteral(value);
return NodeFactory.createLiteralString(value);
}
}
}
6 changes: 4 additions & 2 deletions src/main/java/com/epimorphics/dclib/sources/LexIndex.java
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import java.util.regex.Pattern;

import org.apache.commons.lang3.StringUtils;
import org.apache.commons.text.similarity.LevenshteinDistance;
import org.apache.jena.util.OneToManyMap;

/**
Expand All @@ -25,6 +26,7 @@
* @author <a href="mailto:dave@epimorphics.com">Dave Reynolds</a>
*/
public class LexIndex<T> {
private static final LevenshteinDistance levDist = LevenshteinDistance.getDefaultInstance();
protected OneToManyMap<String, Record> table = new OneToManyMap<>();

/**
Expand All @@ -48,11 +50,11 @@ public T lookup(String key) {
// There is a result but is there more than one
if (i.hasNext()) {
// Yes, so search for best
int bestDist = StringUtils.getLevenshteinDistance(key, match.key);
int bestDist = levDist.apply(key, match.key);
Record best = match;
while (i.hasNext()) {
match = i.next();
int dist = StringUtils.getLevenshteinDistance(key, match.key);
int dist = levDist.apply(key, match.key);
if (dist < bestDist) {
bestDist = dist;
best = match;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,15 @@

import java.io.IOException;

import com.opencsv.exceptions.CsvValidationException;
import org.apache.jena.atlas.json.JsonObject;

import com.epimorphics.dclib.framework.ConverterProcess;
import com.epimorphics.dclib.framework.MapSource;

public class MapSourceFactory {

public static MapSource sourceFrom(JsonObject spec, ConverterProcess proc) throws IOException {
public static MapSource sourceFrom(JsonObject spec, ConverterProcess proc) throws IOException, CsvValidationException {
if (CSVMapSource.isSpec(spec)) {
return new CSVMapSource(spec, proc);
} else if (RDFMapSource.isSpec(spec)) {
Expand Down
Loading
Loading