Skip to content

Commit

Permalink
Merge pull request #35 from trocco-io/fix/null-handling
Browse files Browse the repository at this point in the history 
Fix null handling
  • Loading branch information
@t3t5u
t3t5u authored Feb 24, 2023
2 parents b665af4 + 602f88a commit 5493826
Show file tree
Hide file tree
Showing 4 changed files with 214 additions and 35 deletions.
2 changes: 1 addition & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ repositories {
}

group = "io.trocco"
version = "0.1.7"
version = "0.1.8"

sourceCompatibility = 1.8
targetCompatibility = 1.8
Expand Down
51 changes: 34 additions & 17 deletions src/main/java/org/embulk/input/kintone/KintoneAccessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,11 @@
import com.kintone.client.model.record.TimeFieldValue;
import com.kintone.client.model.record.UserSelectFieldValue;

import java.math.BigDecimal;
import java.time.LocalDate;
import java.time.LocalTime;
import java.util.List;
import java.util.Objects;
import java.util.function.BiFunction;
import java.util.function.Function;

Expand All @@ -55,31 +59,35 @@ public String get(final String name)

private String getAsString(final String fieldCode)
{
switch (getFieldType(fieldCode)) {
final FieldType fieldType = getFieldType(fieldCode);
if (fieldType == null) {
return null;
}
switch (fieldType) {
case RECORD_NUMBER:
return record.getRecordNumberFieldValue();
case __ID__:
return String.valueOf(record.getId());
return toString(record.getId(), Objects::toString);
case __REVISION__:
return String.valueOf(record.getRevision());
return toString(record.getRevision(), Objects::toString);
case CREATOR:
return record.getCreatorFieldValue().getCode();
return toString(record.getCreatorFieldValue(), User::getCode);
case CREATED_TIME:
return String.valueOf(record.getCreatedTimeFieldValue().toInstant());
return toString(record.getCreatedTimeFieldValue(), (value) -> value.toInstant().toString());
case MODIFIER:
return record.getModifierFieldValue().getCode();
return toString(record.getModifierFieldValue(), User::getCode);
case UPDATED_TIME:
return String.valueOf(record.getUpdatedTimeFieldValue().toInstant());
return toString(record.getUpdatedTimeFieldValue(), (value) -> value.toInstant().toString());
case SINGLE_LINE_TEXT:
return record.getSingleLineTextFieldValue(fieldCode);
case MULTI_LINE_TEXT:
return record.getMultiLineTextFieldValue(fieldCode);
case RICH_TEXT:
return record.getRichTextFieldValue(fieldCode);
case NUMBER:
return String.valueOf(record.getNumberFieldValue(fieldCode));
return toString(record.getNumberFieldValue(fieldCode), BigDecimal::toString);
case CALC:
return String.valueOf(record.getCalcFieldValue(fieldCode));
return toString(record.getCalcFieldValue(fieldCode), BigDecimal::toString);
case CHECK_BOX:
return toString(record.getCheckBoxFieldValue(fieldCode));
case RADIO_BUTTON:
Expand All @@ -95,11 +103,11 @@ private String getAsString(final String fieldCode)
case GROUP_SELECT:
return toString(record.getGroupSelectFieldValue(fieldCode), Group::getCode);
case DATE:
return String.valueOf(record.getDateFieldValue(fieldCode));
return toString(record.getDateFieldValue(fieldCode), LocalDate::toString);
case TIME:
return String.valueOf(record.getTimeFieldValue(fieldCode));
return toString(record.getTimeFieldValue(fieldCode), LocalTime::toString);
case DATETIME:
return String.valueOf(record.getDateTimeFieldValue(fieldCode).toInstant());
return toString(record.getDateTimeFieldValue(fieldCode), (value) -> value.toInstant().toString());
case LINK:
return record.getLinkFieldValue(fieldCode);
case FILE:
Expand All @@ -119,7 +127,7 @@ private String getAsString(final String fieldCode)
case HR:
case GROUP:
default:
return "";
return null;
}
}

Expand All @@ -135,16 +143,25 @@ private FieldType getFieldType(final String fieldCode)
return fieldType;
}

private <T> String toString(final T value, final Function<T, String> mapper)
{
return value == null ? null : mapper.apply(value);
}

private String toString(final List<String> list)
{
return list.stream()
.filter(Objects::nonNull)
.reduce((accum, value) -> accum + delimiter + value)
.orElse("");
}

private <T> String toString(final List<T> list, final Function<T, String> mapper)
{
return list.stream().map(mapper)
return list.stream()
.filter(Objects::nonNull)
.map(mapper)
.filter(Objects::nonNull)
.reduce((accum, value) -> accum + delimiter + value)
.orElse("");
}
Expand Down Expand Up @@ -261,7 +278,7 @@ private JsonElement serialize(final TimeFieldValue src, final JsonSerializationC

private JsonElement serialize(final DateTimeFieldValue src, final JsonSerializationContext context)
{
return serialize(src, src.getValue().toInstant());
return serialize(src, toString(src.getValue(), (value) -> value.toInstant().toString()));
}

private JsonElement serialize(final LinkFieldValue src, final JsonSerializationContext context)
Expand All @@ -280,15 +297,15 @@ private JsonElement serialize(final FileBody src, final JsonSerializationContext
object.addProperty("contentType", src.getContentType());
object.addProperty("fileKey", src.getFileKey());
object.addProperty("name", src.getName());
object.addProperty("size", String.valueOf(src.getSize()));
object.addProperty("size", toString(src.getSize(), Objects::toString));
return object;
}

private <T> JsonElement serialize(final FieldValue src, final T value)
{
final JsonObject object = new JsonObject();
object.addProperty("type", src.getType().name());
object.addProperty("value", String.valueOf(value));
object.addProperty("value", toString(value, T::toString));
return object;
}

Expand Down
52 changes: 36 additions & 16 deletions src/main/java/org/embulk/input/kintone/KintoneInputColumnVisitor.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package org.embulk.input.kintone;

import com.google.gson.JsonElement;
import org.embulk.spi.Column;
import org.embulk.spi.ColumnVisitor;
import org.embulk.spi.PageBuilder;
Expand All @@ -12,7 +11,6 @@
import org.slf4j.LoggerFactory;

import java.time.Instant;
import java.util.List;
import java.util.Objects;

public class KintoneInputColumnVisitor implements ColumnVisitor
Expand All @@ -35,13 +33,12 @@ public KintoneInputColumnVisitor(final KintoneAccessor accessor, final PageBuild
public void stringColumn(Column column)
{
try {
String data = accessor.get(column.getName());
final String data = accessor.get(column.getName());
if (Objects.isNull(data)) {
pageBuilder.setNull(column);
return;
}
else {
pageBuilder.setString(column, data);
}
pageBuilder.setString(column, data);
}
catch (Exception e) {
logger.warn(String.format("Invalid string column: %s", column.getName()), e);
Expand All @@ -53,7 +50,11 @@ public void stringColumn(Column column)
public void booleanColumn(Column column)
{
try {
String data = accessor.get(column.getName());
final String data = accessor.get(column.getName());
if (Objects.isNull(data)) {
pageBuilder.setNull(column);
return;
}
pageBuilder.setBoolean(column, Boolean.parseBoolean(data));
}
catch (Exception e) {
Expand All @@ -66,7 +67,11 @@ public void booleanColumn(Column column)
public void longColumn(Column column)
{
try {
String data = accessor.get(column.getName());
final String data = accessor.get(column.getName());
if (Objects.isNull(data)) {
pageBuilder.setNull(column);
return;
}
pageBuilder.setLong(column, Long.parseLong(data));
}
catch (Exception e) {
Expand All @@ -79,7 +84,11 @@ public void longColumn(Column column)
public void doubleColumn(Column column)
{
try {
String data = accessor.get(column.getName());
final String data = accessor.get(column.getName());
if (Objects.isNull(data)) {
pageBuilder.setNull(column);
return;
}
pageBuilder.setDouble(column, Double.parseDouble(data));
}
catch (Exception e) {
Expand All @@ -92,9 +101,13 @@ public void doubleColumn(Column column)
public void timestampColumn(Column column)
{
try {
List<ColumnConfig> columnConfigs = pluginTask.getFields().getColumns();
final String data = accessor.get(column.getName());
if (Objects.isNull(data)) {
pageBuilder.setNull(column);
return;
}
String pattern = DEFAULT_TIMESTAMP_PATTERN;
for (ColumnConfig config : columnConfigs) {
for (ColumnConfig config : pluginTask.getFields().getColumns()) {
if (config.getName().equals(column.getName())
&& config.getOption() != null
&& config.getFormat() != null) {
Expand All @@ -103,7 +116,7 @@ public void timestampColumn(Column column)
}
}
final TimestampFormatter formatter = TimestampFormatter.builder("ruby:" + pattern).build();
Instant instant = formatter.parse(accessor.get(column.getName()));
final Instant instant = formatter.parse(data);
pageBuilder.setTimestamp(column, Timestamp.ofInstant(instant));
}
catch (Exception e) {
Expand All @@ -116,13 +129,20 @@ public void timestampColumn(Column column)
public void jsonColumn(Column column)
{
try {
JsonElement data = com.google.gson.JsonParser.parseString(accessor.get(column.getName()));
if (data.isJsonNull() || data.isJsonPrimitive()) {
final String data = accessor.get(column.getName());
if (Objects.isNull(data)) {
pageBuilder.setNull(column);
return;
}
final com.google.gson.JsonElement gson = com.google.gson.JsonParser.parseString(data);
if (gson.isJsonNull()) {
pageBuilder.setNull(column);
return;
}
else {
pageBuilder.setJson(column, new JsonParser().parse(data.toString()));
if (gson.isJsonPrimitive()) {
throw new Exception(String.format("'%s' is json primitive", gson));
}
pageBuilder.setJson(column, new JsonParser().parse(gson.toString()));
}
catch (Exception e) {
logger.warn(String.format("Invalid json column: %s", column.getName()), e);
Expand Down
Loading

0 comments on commit 5493826

Please sign in to comment.