Skip to content

Commit

Permalink
jdbf 2.0: add ability to read DBF files with MEMO fields
Browse files Browse the repository at this point in the history
  • Loading branch information
iryndin committed Jul 15, 2014
1 parent 622abb6 commit c5a2b72
Show file tree
Hide file tree
Showing 17 changed files with 392 additions and 58 deletions.
3 changes: 3 additions & 0 deletions .idea/dictionaries/user.xml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 1 addition & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,4 @@ jdbf

Java utility to read/write DBF files

Please see for examples:

http://iryndin.net/projects/jdbf
Version 2.0 - add ability to read MEMO files (tested with Visual FoxPro DBFs)
35 changes: 22 additions & 13 deletions pom.xml
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>net.iryndin</groupId>
<artifactId>jdbf</artifactId>
<version>1.1</version>
<version>2.0</version>
<packaging>jar</packaging>

<name>jdbf</name>
<url>http://github.com/iryndin/jdbf</url>
<description>jdbf - Java library to write/read DBF files</description>
<url>http://github.com/iryndin/jdbf</url>
<description>jdbf - Java library to read/write DBF files</description>



<scm>
<url>https://github.com/iryndin/jdbf</url>
</scm>
Expand All @@ -23,9 +23,17 @@
<url>http://apache.org/licenses/LICENSE-2.0.txt</url>
</license>
</licenses>

<dependencies/>


<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
</dependencies>


<developers>
<developer>
<name>Ivan Ryndin</name>
Expand All @@ -48,15 +56,16 @@
<name>bugy</name>
</contributor>
</contributors>

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.7</maven.compiler.source>
<maven.compiler.target>1.7</maven.compiler.target>

<maven.compiler.version>3.1</maven.compiler.version>
<junit.version>4.11</junit.version>
</properties>

<build>
<plugins>
<plugin>
Expand All @@ -66,5 +75,5 @@
</plugin>
</plugins>
</build>

</project>
9 changes: 7 additions & 2 deletions src/main/java/net/iryndin/jdbf/core/DbfMetadata.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package net.iryndin.jdbf.core;

import java.text.SimpleDateFormat;
import java.util.Collection;
import java.util.Date;
import java.util.LinkedHashMap;
Expand Down Expand Up @@ -70,7 +71,7 @@ public void setFields(List<DbfField> fields) {
}

private void processFields(List<DbfField> fields) {
fieldMap = new LinkedHashMap<String,DbfField>(fields.size()*2);
fieldMap = new LinkedHashMap<>(fields.size()*2);
int offset = 1;
for (DbfField f : fields) {
// 1. count offset
Expand All @@ -97,10 +98,14 @@ public String getFieldsStringRepresentation() {
}
return sb.toString();
}

private String formatUpdateDate() {
return new SimpleDateFormat("yyyy-MM-dd").format(updateDate);
}
@Override
public String toString() {
return "DbfMetadata [\n type=" + type + ", \n updateDate="
+ updateDate + ", \n recordsQty=" + recordsQty
+ formatUpdateDate() + ", \n recordsQty=" + recordsQty
+ ", \n fullHeaderLength=" + fullHeaderLength
+ ", \n oneRecordLength=" + oneRecordLength
+ ", \n uncompletedTxFlag=" + uncompletedTxFlag
Expand Down
71 changes: 62 additions & 9 deletions src/main/java/net/iryndin/jdbf/core/DbfRecord.java
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package net.iryndin.jdbf.core;

import net.iryndin.jdbf.reader.MemoReader;
import net.iryndin.jdbf.util.BitUtils;
import net.iryndin.jdbf.util.JdbfUtils;

import java.io.IOException;
import java.math.BigDecimal;
import java.nio.charset.Charset;
import java.text.ParseException;
Expand All @@ -17,12 +19,14 @@ public class DbfRecord {

private byte[] bytes;
private DbfMetadata metadata;
private MemoReader memoReader;
private Charset stringCharset;

public DbfRecord(byte[] source, DbfMetadata metadata) {
public DbfRecord(byte[] source, DbfMetadata metadata, MemoReader memoReader) {
this.bytes = new byte[source.length];
System.arraycopy(source, 0, this.bytes, 0, source.length);
this.metadata = metadata;
this.memoReader = memoReader;
}

public DbfRecord(DbfMetadata metadata) {
Expand All @@ -48,10 +52,11 @@ public byte[] getBytes() {
}

public String getString(String fieldName) {
if (stringCharset == null) {
stringCharset = Charset.defaultCharset();
Charset charset = this.stringCharset;
if (charset == null) {
charset = Charset.defaultCharset();
}
return getString(fieldName, stringCharset);
return getString(fieldName, charset);
}

public String getString(String fieldName, String charsetName) {
Expand All @@ -60,18 +65,22 @@ public String getString(String fieldName, String charsetName) {

public String getString(String fieldName, Charset charset) {
DbfField f = getField(fieldName);
if (f.getType() == DbfFieldTypeEnum.Memo) {
return getMemoAsString(f, charset);
}
int actualOffset = f.getOffset();
int actualLength = f.getLength();

byte[] fieldBytes = new byte[actualLength];
System.arraycopy(bytes, actualOffset, fieldBytes, 0, actualLength);

// check for empty strings
while ((actualLength > 0)
&& (bytes[actualOffset] == JdbfUtils.EMPTY)) {
while ((actualLength > 0) && (bytes[actualOffset] == JdbfUtils.EMPTY)) {
actualOffset++;
actualLength--;
}

while ((actualLength > 0)
&& (bytes[actualOffset + actualLength - 1] == JdbfUtils.EMPTY)) {
while ((actualLength > 0) && (bytes[actualOffset + actualLength - 1] == JdbfUtils.EMPTY)) {
actualLength--;
}

Expand Down Expand Up @@ -101,6 +110,40 @@ public String getString(String fieldName, Charset charset) {
*/
}

public byte[] getMemoAsBytes(String fieldName) throws IOException {
DbfField f = getField(fieldName);
if (f.getType() != DbfFieldTypeEnum.Memo) {
throw new IllegalArgumentException("Field '" + fieldName + "' is not MEMO field!");
}
byte[] dbfFieldBytes = new byte[f.getLength()];
System.arraycopy(bytes, f.getOffset(), dbfFieldBytes, 0, f.getLength());
int offsetInBlocks = BitUtils.makeInt(dbfFieldBytes[0],dbfFieldBytes[1],dbfFieldBytes[2],dbfFieldBytes[3]);
return memoReader.read(offsetInBlocks).getValue();
}

public String getMemoAsString(String fieldName, Charset charset) throws IOException {
DbfField f = getField(fieldName);
if (f.getType() != DbfFieldTypeEnum.Memo) {
throw new IllegalArgumentException("Field '" + fieldName + "' is not MEMO field!");
}
byte[] dbfFieldBytes = new byte[f.getLength()];
System.arraycopy(bytes, f.getOffset(), dbfFieldBytes, 0, f.getLength());
int offsetInBlocks = BitUtils.makeInt(dbfFieldBytes[0],dbfFieldBytes[1],dbfFieldBytes[2],dbfFieldBytes[3]);
return memoReader.read(offsetInBlocks).getValueAsString(charset);
}

public String getMemoAsString(String fieldName) throws IOException {
Charset charset = getStringCharset();
if (charset == null) {
charset = Charset.defaultCharset();
}
return getMemoAsString(fieldName, charset);
}

private String getMemoAsString(DbfField field, Charset charset) {
return null;
}

public Date getDate(String fieldName) throws ParseException {
String s = getString(fieldName);
if (s == null) {
Expand Down Expand Up @@ -181,7 +224,8 @@ public String getStringRepresentation() throws Exception {
sb.append(f.getName()).append("=");
switch (f.getType()) {
case Character: {
String s = getString(f.getName(), "Cp866");
//String s = getString(f.getName(), "Cp866");
String s = getString(f.getName());
//System.out.println(f.getName()+"="+s);
sb.append(s);
break;
Expand Down Expand Up @@ -242,4 +286,13 @@ public Map<String, Object> toMap() throws ParseException {

return map;
}

public String getMemo(String fieldName) {
Charset charset = this.stringCharset;
if (charset == null) {
charset = Charset.defaultCharset();
}
DbfField f = getField(fieldName);
return f.toString();
}
}
44 changes: 44 additions & 0 deletions src/main/java/net/iryndin/jdbf/core/MemoFileHeader.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package net.iryndin.jdbf.core;

import net.iryndin.jdbf.util.BitUtils;

/**
* See http://msdn.microsoft.com/en-US/library/8599s21w(v=vs.80).aspx
*/
public class MemoFileHeader {
private byte[] headerBytes;
private int nextFreeBlockLocation;
private int blockSize;

public static MemoFileHeader create(byte[] headerBytes) {
MemoFileHeader h = new MemoFileHeader();
h.setHeaderBytes(headerBytes);
h.calculateHeaderFields();
return h;
}

private void calculateHeaderFields() {
this.nextFreeBlockLocation = BitUtils.makeInt(headerBytes[3],headerBytes[2],headerBytes[1],headerBytes[0]);
this.blockSize = BitUtils.makeInt(headerBytes[7],headerBytes[6]);
}

private void setHeaderBytes(byte[] headerBytes) {
this.headerBytes = headerBytes;
}

public int getNextFreeBlockLocation() {
return nextFreeBlockLocation;
}

public int getBlockSize() {
return blockSize;
}

@Override
public String toString() {
return "MemoFileHeader{" +
"nextFreeBlockLocation=" + nextFreeBlockLocation +
", blockSize=" + blockSize +
'}';
}
}
54 changes: 54 additions & 0 deletions src/main/java/net/iryndin/jdbf/core/MemoRecord.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package net.iryndin.jdbf.core;

import net.iryndin.jdbf.util.BitUtils;

import java.nio.charset.Charset;

public class MemoRecord {
private final int blockSize;
private final int offsetInBlocks;
private byte[] value;
private int length;
private MemoRecordTypeEnum memoType;

public MemoRecord(byte[] header, byte[] value, int blockSize, int offsetInBlocks) {
this.value = value;
calculateFields(header);
this.blockSize = blockSize;
this.offsetInBlocks = offsetInBlocks;
}

private void calculateFields(byte[] bytes) {
int type = BitUtils.makeInt(bytes[3],bytes[2],bytes[1],bytes[0]);
this.memoType = MemoRecordTypeEnum.fromInt(type);
this.length = BitUtils.makeInt(bytes[7],bytes[6],bytes[5],bytes[4]);
}

public byte[] getValue() {
return value;
}

public String getValueAsString(Charset charset) {
return new String(value, charset);
}

/**
* Memo record length in bytes
* @return
*/
public int getLength() {
return length;
}

public MemoRecordTypeEnum getMemoType() {
return memoType;
}

public int getBlockSize() {
return blockSize;
}

public int getOffsetInBlocks() {
return offsetInBlocks;
}
}
21 changes: 21 additions & 0 deletions src/main/java/net/iryndin/jdbf/core/MemoRecordTypeEnum.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package net.iryndin.jdbf.core;

public enum MemoRecordTypeEnum {
IMAGE(0x0),
TEXT(0x1);

final int type;

MemoRecordTypeEnum(int type) {
this.type= type;
}

public static MemoRecordTypeEnum fromInt(int type) {
for (MemoRecordTypeEnum e : MemoRecordTypeEnum.values()) {
if (e.type == type) {
return e;
}
}
return null;
}
}
Loading

0 comments on commit c5a2b72

Please sign in to comment.