From 738985d66f22171a9b40573dd4171a06118f4059 Mon Sep 17 00:00:00 2001 From: Tetsuro Sano Date: Sat, 13 Apr 2024 01:02:08 +0900 Subject: [PATCH 1/4] Refactoring as preparation --- .../embulk/output/kintone/KintoneClient.java | 96 +++++++++++++ .../output/kintone/KintoneColumnType.java | 17 ++- .../embulk/output/kintone/KintoneMode.java | 52 +++++++- .../output/kintone/KintoneOutputPlugin.java | 17 --- .../output/kintone/KintonePageOutput.java | 126 +++++------------- .../org/embulk/output/kintone/util/Lazy.java | 20 +++ .../output/kintone/KintoneClientTest.java | 122 +++++++++++++++++ .../kintone/KintonePageOutputVerifier.java | 120 +++++++++-------- .../org/embulk/output/kintone/MockClient.java | 112 ++++++++++++++++ .../kintone/TestKintoneOutputPlugin.java | 41 +++++- .../embulk/output/kintone/client/config.yml | 1 + ... => insert_ignore_nulls_add_records.jsonl} | 0 ... => insert_prefer_nulls_add_records.jsonl} | 0 ... update_ignore_nulls_update_records.jsonl} | 0 ... update_prefer_nulls_update_records.jsonl} | 0 ...rt_double_single_line_text_add_values.json | 1 + ...psert_double_single_line_text_values.json} | 0 ... => upsert_ignore_nulls_add_records.jsonl} | 0 ...nulls_double_single_line_text_values.json} | 0 ... upsert_ignore_nulls_update_records.jsonl} | 0 ... => upsert_prefer_nulls_add_records.jsonl} | 0 ...nulls_double_single_line_text_values.json} | 0 ... upsert_prefer_nulls_update_records.jsonl} | 0 ... => insert_ignore_nulls_add_records.jsonl} | 0 ... => insert_prefer_nulls_add_records.jsonl} | 0 ... update_ignore_nulls_update_records.jsonl} | 0 ... update_prefer_nulls_update_records.jsonl} | 0 ...rt_double_single_line_text_add_values.json | 1 + ...psert_double_single_line_text_values.json} | 0 ... => upsert_ignore_nulls_add_records.jsonl} | 0 ...nulls_double_single_line_text_values.json} | 0 ... upsert_ignore_nulls_update_records.jsonl} | 0 ... => upsert_prefer_nulls_add_records.jsonl} | 0 ...nulls_double_single_line_text_values.json} | 0 ... upsert_prefer_nulls_update_records.jsonl} | 0 ... => insert_ignore_nulls_add_records.jsonl} | 0 ... => insert_prefer_nulls_add_records.jsonl} | 0 ... update_ignore_nulls_update_records.jsonl} | 0 ... update_prefer_nulls_update_records.jsonl} | 0 .../reduce_subtable/upsert_add_records.jsonl | 0 ...psert_double_single_line_text_values.json} | 0 ... => upsert_ignore_nulls_add_records.jsonl} | 0 ...nulls_double_single_line_text_values.json} | 0 ... upsert_ignore_nulls_update_records.jsonl} | 0 ... => upsert_prefer_nulls_add_records.jsonl} | 0 ...nulls_double_single_line_text_values.json} | 0 ... upsert_prefer_nulls_update_records.jsonl} | 0 47 files changed, 551 insertions(+), 175 deletions(-) create mode 100644 src/main/java/org/embulk/output/kintone/KintoneClient.java create mode 100644 src/main/java/org/embulk/output/kintone/util/Lazy.java create mode 100644 src/test/java/org/embulk/output/kintone/KintoneClientTest.java create mode 100644 src/test/java/org/embulk/output/kintone/MockClient.java create mode 100644 src/test/resources/org/embulk/output/kintone/client/config.yml rename src/test/resources/org/embulk/output/kintone/task/mode/{insert_add_ignore_nulls_records.jsonl => insert_ignore_nulls_add_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/mode/{insert_add_prefer_nulls_records.jsonl => insert_prefer_nulls_add_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/mode/{update_update_ignore_nulls_records.jsonl => update_ignore_nulls_update_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/mode/{update_update_prefer_nulls_records.jsonl => update_prefer_nulls_update_records.jsonl} (100%) create mode 100644 src/test/resources/org/embulk/output/kintone/task/mode/upsert_double_single_line_text_add_values.json rename src/test/resources/org/embulk/output/kintone/task/mode/{values.json => upsert_double_single_line_text_values.json} (100%) rename src/test/resources/org/embulk/output/kintone/task/mode/{upsert_add_ignore_nulls_records.jsonl => upsert_ignore_nulls_add_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/mode/{values_ignore_nulls.json => upsert_ignore_nulls_double_single_line_text_values.json} (100%) rename src/test/resources/org/embulk/output/kintone/task/mode/{upsert_update_ignore_nulls_records.jsonl => upsert_ignore_nulls_update_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/mode/{upsert_add_prefer_nulls_records.jsonl => upsert_prefer_nulls_add_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/mode/{values_prefer_nulls.json => upsert_prefer_nulls_double_single_line_text_values.json} (100%) rename src/test/resources/org/embulk/output/kintone/task/mode/{upsert_update_prefer_nulls_records.jsonl => upsert_prefer_nulls_update_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce/{insert_add_ignore_nulls_records.jsonl => insert_ignore_nulls_add_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce/{insert_add_prefer_nulls_records.jsonl => insert_prefer_nulls_add_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce/{update_update_ignore_nulls_records.jsonl => update_ignore_nulls_update_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce/{update_update_prefer_nulls_records.jsonl => update_prefer_nulls_update_records.jsonl} (100%) create mode 100644 src/test/resources/org/embulk/output/kintone/task/reduce/upsert_double_single_line_text_add_values.json rename src/test/resources/org/embulk/output/kintone/task/reduce/{values.json => upsert_double_single_line_text_values.json} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce/{upsert_add_ignore_nulls_records.jsonl => upsert_ignore_nulls_add_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce/{values_ignore_nulls.json => upsert_ignore_nulls_double_single_line_text_values.json} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce/{upsert_update_ignore_nulls_records.jsonl => upsert_ignore_nulls_update_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce/{upsert_add_prefer_nulls_records.jsonl => upsert_prefer_nulls_add_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce/{values_prefer_nulls.json => upsert_prefer_nulls_double_single_line_text_values.json} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce/{upsert_update_prefer_nulls_records.jsonl => upsert_prefer_nulls_update_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce_subtable/{insert_add_ignore_nulls_records.jsonl => insert_ignore_nulls_add_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce_subtable/{insert_add_prefer_nulls_records.jsonl => insert_prefer_nulls_add_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce_subtable/{update_update_ignore_nulls_records.jsonl => update_ignore_nulls_update_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce_subtable/{update_update_prefer_nulls_records.jsonl => update_prefer_nulls_update_records.jsonl} (100%) delete mode 100644 src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_add_records.jsonl rename src/test/resources/org/embulk/output/kintone/task/reduce_subtable/{values.json => upsert_double_single_line_text_values.json} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce_subtable/{upsert_add_ignore_nulls_records.jsonl => upsert_ignore_nulls_add_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce_subtable/{values_ignore_nulls.json => upsert_ignore_nulls_double_single_line_text_values.json} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce_subtable/{upsert_update_ignore_nulls_records.jsonl => upsert_ignore_nulls_update_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce_subtable/{upsert_add_prefer_nulls_records.jsonl => upsert_prefer_nulls_add_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce_subtable/{values_prefer_nulls.json => upsert_prefer_nulls_double_single_line_text_values.json} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce_subtable/{upsert_update_prefer_nulls_records.jsonl => upsert_prefer_nulls_update_records.jsonl} (100%) diff --git a/src/main/java/org/embulk/output/kintone/KintoneClient.java b/src/main/java/org/embulk/output/kintone/KintoneClient.java new file mode 100644 index 0000000..a2c24f3 --- /dev/null +++ b/src/main/java/org/embulk/output/kintone/KintoneClient.java @@ -0,0 +1,96 @@ +package org.embulk.output.kintone; + +import com.kintone.client.KintoneClientBuilder; +import com.kintone.client.RecordClient; +import com.kintone.client.model.app.field.FieldProperty; +import com.kintone.client.model.record.FieldType; +import java.io.IOException; +import java.util.Map; +import java.util.function.Supplier; +import org.embulk.config.ConfigException; +import org.embulk.output.kintone.util.Lazy; +import org.embulk.spi.Column; +import org.embulk.spi.Schema; + +public class KintoneClient implements AutoCloseable { + private final PluginTask task; + private final Schema schema; + private final com.kintone.client.KintoneClient client; + private final Map fields; + + public static Lazy lazy(Supplier task, Schema schema) { + return new Lazy() { + @Override + protected KintoneClient initialValue() { + return new KintoneClient(task.get(), schema); + } + }; + } + + private KintoneClient(PluginTask task, Schema schema) { + this.task = task; + this.schema = schema; + KintoneClientBuilder builder = KintoneClientBuilder.create("https://" + task.getDomain()); + if (task.getGuestSpaceId().isPresent()) { + builder.setGuestSpaceId(task.getGuestSpaceId().get()); + } + if (task.getBasicAuthUsername().isPresent() && task.getBasicAuthPassword().isPresent()) { + builder.withBasicAuth(task.getBasicAuthUsername().get(), task.getBasicAuthPassword().get()); + } + if (task.getUsername().isPresent() && task.getPassword().isPresent()) { + builder.authByPassword(task.getUsername().get(), task.getPassword().get()); + } else if (task.getToken().isPresent()) { + builder.authByApiToken(task.getToken().get()); + } else { + throw new ConfigException("Username and password or token must be configured."); + } + client = builder.build(); + fields = client.app().getFormFields(task.getAppId()); + KintoneMode.of(task).validate(task, this); + } + + public void validateUpdateKey(String columnName) { + Column column = getColumn(columnName); + if (column == null) { + throw new ConfigException("The column '" + columnName + "' for update does not exist."); + } + String fieldCode = getFieldCode(column); + FieldType fieldType = getFieldType(fieldCode); + if (fieldType == null) { + throw new ConfigException("The field '" + fieldCode + "' for update does not exist."); + } + if (fieldType != FieldType.SINGLE_LINE_TEXT && fieldType != FieldType.NUMBER) { + throw new ConfigException("The update_key must be 'SINGLE_LINE_TEXT' or 'NUMBER'."); + } + } + + public Column getColumn(String columnName) { + return schema.getColumns().stream() + .filter(column -> column.getName().equals(columnName)) + .findFirst() + .orElse(null); + } + + public FieldType getFieldType(String fieldCode) { + FieldProperty field = fields.get(fieldCode); + return field == null ? null : field.getType(); + } + + public RecordClient record() { + return client.record(); + } + + @Override + public void close() { + try { + client.close(); + } catch (IOException e) { + throw new RuntimeException("kintone throw exception", e); + } + } + + private String getFieldCode(Column column) { + KintoneColumnOption option = task.getColumnOptions().get(column.getName()); + return option != null ? option.getFieldCode() : column.getName(); + } +} diff --git a/src/main/java/org/embulk/output/kintone/KintoneColumnType.java b/src/main/java/org/embulk/output/kintone/KintoneColumnType.java index d2460fc..cb6ce43 100644 --- a/src/main/java/org/embulk/output/kintone/KintoneColumnType.java +++ b/src/main/java/org/embulk/output/kintone/KintoneColumnType.java @@ -13,6 +13,7 @@ import com.kintone.client.model.record.NumberFieldValue; import com.kintone.client.model.record.OrganizationSelectFieldValue; import com.kintone.client.model.record.RadioButtonFieldValue; +import com.kintone.client.model.record.Record; import com.kintone.client.model.record.RichTextFieldValue; import com.kintone.client.model.record.SingleLineTextFieldValue; import com.kintone.client.model.record.SubtableFieldValue; @@ -49,6 +50,11 @@ public SingleLineTextFieldValue getFieldValue(String value, KintoneColumnOption return new SingleLineTextFieldValue(value); } + @Override + public Object getValue(Record record, String fieldCode) { + return record.getSingleLineTextFieldValue(fieldCode); + } + @Override public void setUpdateKey(UpdateKey updateKey, String field) { updateKey.setField(field); @@ -132,6 +138,11 @@ public NumberFieldValue getFieldValue(Timestamp value, KintoneColumnOption optio return (NumberFieldValue) getFieldValue(value.getEpochSecond(), option); } + @Override + public Object getValue(Record record, String fieldCode) { + return record.getNumberFieldValue(fieldCode); + } + @Override public void setUpdateKey(UpdateKey updateKey, String field) { updateKey.setField(field); @@ -144,7 +155,7 @@ public void setUpdateKey(UpdateKey updateKey, String field, FieldValue value) { @Override public Value asValue(FieldValue value) { - return ValueFactory.newString(((NumberFieldValue) value).getValue().toString()); + return ValueFactory.newString(((NumberFieldValue) value).getValue().toPlainString()); } @Override @@ -540,6 +551,10 @@ public FieldValue getFieldValue(Value value, KintoneColumnOption option) { } } + public Object getValue(Record Record, String fieldCode) { + throw new UnsupportedOperationException(); + } + public void setUpdateKey(UpdateKey updateKey, String field) { throw new UnsupportedOperationException(); } diff --git a/src/main/java/org/embulk/output/kintone/KintoneMode.java b/src/main/java/org/embulk/output/kintone/KintoneMode.java index 635f001..5fa7a49 100644 --- a/src/main/java/org/embulk/output/kintone/KintoneMode.java +++ b/src/main/java/org/embulk/output/kintone/KintoneMode.java @@ -1,23 +1,67 @@ package org.embulk.output.kintone; import org.embulk.config.ConfigException; +import org.embulk.spi.Page; public enum KintoneMode { - INSERT("insert"), - UPDATE("update"), - UPSERT("upsert"); + INSERT("insert") { + @Override + public void validate(PluginTask task, KintoneClient client) { + if (task.getUpdateKeyName().isPresent()) { + throw new ConfigException("When mode is insert, require no update_key."); + } + } + + @Override + public void add(Page page, KintonePageOutput output) { + output.insertPage(page); + } + }, + UPDATE("update") { + @Override + public void validate(PluginTask task, KintoneClient client) { + if (!task.getUpdateKeyName().isPresent()) { + throw new ConfigException("When mode is update, require update_key."); + } + client.validateUpdateKey(task.getUpdateKeyName().orElse(null)); + } + + @Override + public void add(Page page, KintonePageOutput output) { + output.updatePage(page); + } + }, + UPSERT("upsert") { + @Override + public void validate(PluginTask task, KintoneClient client) { + if (!task.getUpdateKeyName().isPresent()) { + throw new ConfigException("When mode is upsert, require update_key."); + } + client.validateUpdateKey(task.getUpdateKeyName().orElse(null)); + } + + @Override + public void add(Page page, KintonePageOutput output) { + output.upsertPage(page); + } + }; private final String value; KintoneMode(String value) { this.value = value; } + public abstract void validate(PluginTask task, KintoneClient client); + + public abstract void add(Page page, KintonePageOutput output); + @Override public String toString() { return value; } - public static KintoneMode getKintoneModeByValue(String value) { + public static KintoneMode of(PluginTask task) { + String value = task.getMode(); for (KintoneMode mode : values()) { if (mode.toString().equals(value)) { return mode; diff --git a/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java b/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java index 0c0b35d..503c644 100644 --- a/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java +++ b/src/main/java/org/embulk/output/kintone/KintoneOutputPlugin.java @@ -3,7 +3,6 @@ import java.util.Collections; import java.util.List; import org.embulk.config.ConfigDiff; -import org.embulk.config.ConfigException; import org.embulk.config.ConfigSource; import org.embulk.config.TaskReport; import org.embulk.config.TaskSource; @@ -40,22 +39,6 @@ public void cleanup( @Override public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex) { PluginTask task = taskSource.loadTask(PluginTask.class); - KintoneMode mode = KintoneMode.getKintoneModeByValue(task.getMode()); - switch (mode) { - case INSERT: - if (task.getUpdateKeyName().isPresent()) { - throw new ConfigException("when mode is insert, require no update_key."); - } - break; - case UPDATE: - case UPSERT: - if (!task.getUpdateKeyName().isPresent()) { - throw new ConfigException("when mode is update or upsert, require update_key."); - } - break; - default: - throw new ConfigException(String.format("Unknown mode '%s'", task.getMode())); - } return task.getReduceKeyName().isPresent() ? new ReducedPageOutput(schema, taskIndex) : new KintonePageOutput(task, schema); diff --git a/src/main/java/org/embulk/output/kintone/KintonePageOutput.java b/src/main/java/org/embulk/output/kintone/KintonePageOutput.java index dc480d6..12d9927 100644 --- a/src/main/java/org/embulk/output/kintone/KintonePageOutput.java +++ b/src/main/java/org/embulk/output/kintone/KintonePageOutput.java @@ -5,11 +5,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Maps; -import com.kintone.client.KintoneClient; -import com.kintone.client.KintoneClientBuilder; import com.kintone.client.api.record.GetRecordsByCursorResponseBody; import com.kintone.client.exception.KintoneApiRuntimeException; -import com.kintone.client.model.app.field.FieldProperty; import com.kintone.client.model.record.FieldType; import com.kintone.client.model.record.Record; import com.kintone.client.model.record.RecordForUpdate; @@ -28,8 +25,8 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Pair; -import org.embulk.config.ConfigException; import org.embulk.config.TaskReport; +import org.embulk.output.kintone.util.Lazy; import org.embulk.spi.Exec; import org.embulk.spi.Page; import org.embulk.spi.PageReader; @@ -53,30 +50,17 @@ public class KintonePageOutput implements TransactionalPageOutput { private final Map> wrongTypeFields = new TreeMap<>(); private final PluginTask task; private final PageReader reader; - private KintoneClient client; - private Map formFields; + private final Lazy client; public KintonePageOutput(PluginTask task, Schema schema) { this.task = task; reader = new PageReader(schema); + client = KintoneClient.lazy(() -> task, schema); } @Override public void add(Page page) { - KintoneMode mode = KintoneMode.getKintoneModeByValue(task.getMode()); - switch (mode) { - case INSERT: - insertPage(page); - break; - case UPDATE: - updatePage(page); - break; - case UPSERT: - upsertPage(page); - break; - default: - throw new UnsupportedOperationException(String.format("Unknown mode '%s'", task.getMode())); - } + KintoneMode.of(task).add(page, this); } @Override @@ -86,14 +70,7 @@ public void finish() { @Override public void close() { - if (client == null) { - return; // Not connected - } - try { - client.close(); - } catch (Exception e) { - throw new RuntimeException("kintone throw exception", e); - } + client.get().close(); } @Override @@ -112,38 +89,15 @@ public TaskReport commit() { return Exec.newTaskReport(); } - private void connectIfNeeded() { - if (client != null) { - return; // Already connected - } - KintoneClientBuilder builder = KintoneClientBuilder.create("https://" + task.getDomain()); - if (task.getGuestSpaceId().isPresent()) { - builder.setGuestSpaceId(task.getGuestSpaceId().get()); - } - if (task.getBasicAuthUsername().isPresent() && task.getBasicAuthPassword().isPresent()) { - builder.withBasicAuth(task.getBasicAuthUsername().get(), task.getBasicAuthPassword().get()); - } - if (task.getUsername().isPresent() && task.getPassword().isPresent()) { - builder.authByPassword(task.getUsername().get(), task.getPassword().get()); - } else if (task.getToken().isPresent()) { - builder.authByApiToken(task.getToken().get()); - } else { - throw new ConfigException("Username and password or token must be configured."); - } - client = builder.build(); - formFields = client.app().getFormFields(task.getAppId()); - } - private void insert(List records) { - executeWithRetry(() -> client.record().addRecords(task.getAppId(), records)); + executeWithRetry(() -> client.get().record().addRecords(task.getAppId(), records)); } private void update(List records) { - executeWithRetry(() -> client.record().updateRecords(task.getAppId(), records)); + executeWithRetry(() -> client.get().record().updateRecords(task.getAppId(), records)); } private T executeWithRetry(Supplier operation) { - connectIfNeeded(); KintoneRetryOption retryOption = task.getRetryOptions(); try { return retryExecutor() @@ -153,7 +107,7 @@ private T executeWithRetry(Supplier operation) { .runInterruptible( new Retryable() { @Override - public T call() throws Exception { + public T call() { return operation.get(); } @@ -175,8 +129,7 @@ public boolean isRetryableException(Exception exception) { @Override public void onRetry( - Exception exception, int retryCount, int retryLimit, int retryWait) - throws RetryGiveupException { + Exception exception, int retryCount, int retryLimit, int retryWait) { String message = String.format( "Retrying %d/%d after %d seconds. Message: %s", @@ -189,15 +142,14 @@ public void onRetry( } @Override - public void onGiveup(Exception firstException, Exception lastException) - throws RetryGiveupException {} + public void onGiveup(Exception firstException, Exception lastException) {} }); } catch (RetryGiveupException | InterruptedException e) { throw new RuntimeException("kintone throw exception", e); } } - private void insertPage(Page page) { + public void insertPage(Page page) { List records = new ArrayList<>(); reader.setPage(page); KintoneColumnVisitor visitor = @@ -224,7 +176,7 @@ private void insertPage(Page page) { } } - private void updatePage(Page page) { + public void updatePage(Page page) { List records = new ArrayList<>(); reader.setPage(page); KintoneColumnVisitor visitor = @@ -235,8 +187,7 @@ private void updatePage(Page page) { task.getPreferNulls(), task.getIgnoreNulls(), task.getReduceKeyName().orElse(null), - task.getUpdateKeyName() - .orElseThrow(() -> new RuntimeException("unreachable"))); // Already validated + task.getUpdateKeyName().orElse(null)); while (reader.nextRecord()) { Record record = new Record(); UpdateKey updateKey = new UpdateKey(); @@ -259,7 +210,7 @@ private void updatePage(Page page) { } } - private void upsertPage(Page page) { + public void upsertPage(Page page) { List records = new ArrayList<>(); List updateKeys = new ArrayList<>(); reader.setPage(page); @@ -271,8 +222,7 @@ private void upsertPage(Page page) { task.getPreferNulls(), task.getIgnoreNulls(), task.getReduceKeyName().orElse(null), - task.getUpdateKeyName() - .orElseThrow(() -> new RuntimeException("unreachable"))); // Already validated + task.getUpdateKeyName().orElse(null)); while (reader.nextRecord()) { Record record = new Record(); UpdateKey updateKey = new UpdateKey(); @@ -325,24 +275,6 @@ private void upsert(List records, List updateKeys) { } private List getExistingValuesByUpdateKey(List updateKeys) { - String fieldCode = - updateKeys.stream() - .map(UpdateKey::getField) - .filter(Objects::nonNull) - .findFirst() - .orElse(null); - if (fieldCode == null) { - return Collections.emptyList(); - } - Function fieldValueAsString; - FieldType fieldType = getFieldType(fieldCode); - if (fieldType == FieldType.SINGLE_LINE_TEXT) { - fieldValueAsString = record -> record.getSingleLineTextFieldValue(fieldCode); - } else if (fieldType == FieldType.NUMBER) { - fieldValueAsString = record -> toString(record.getNumberFieldValue(fieldCode)); - } else { - throw new ConfigException("The update_key must be 'SINGLE_LINE_TEXT' or 'NUMBER'."); - } List queryValues = updateKeys.stream() .filter(k -> k.getValue() != null && !k.getValue().toString().isEmpty()) @@ -351,24 +283,34 @@ private List getExistingValuesByUpdateKey(List updateKeys) { if (queryValues.isEmpty()) { return Collections.emptyList(); } + String columnName = task.getUpdateKeyName().orElse(null); + KintoneColumnOption option = task.getColumnOptions().get(columnName); + String fieldCode = option != null ? option.getFieldCode() : columnName; + KintoneColumnType type = KintoneColumnType.valueOf(getFieldType(fieldCode).name()); + return getExistingValues(fieldCode, record -> type.getValue(record, fieldCode), queryValues); + } + + private List getExistingValues( + String fieldCode, Function toValue, List queryValues) { String cursorId = client + .get() .record() .createCursor( task.getAppId(), Collections.singletonList(fieldCode), fieldCode + " in (" + String.join(",", queryValues) + ")"); - List allRecords = new ArrayList<>(); + List records = new ArrayList<>(); while (true) { - GetRecordsByCursorResponseBody resp = client.record().getRecordsByCursor(cursorId); - List records = resp.getRecords(); - allRecords.addAll(records); - if (!resp.hasNext()) { + GetRecordsByCursorResponseBody cursor = client.get().record().getRecordsByCursor(cursorId); + records.addAll(cursor.getRecords()); + if (!cursor.hasNext()) { break; } } - return allRecords.stream() - .map(fieldValueAsString) + return records.stream() + .map(toValue) + .map(KintonePageOutput::toString) .filter(Objects::nonNull) .collect(Collectors.toList()); } @@ -384,9 +326,7 @@ private void putWrongTypeFields(Record record) { } private FieldType getFieldType(String fieldCode) { - connectIfNeeded(); - FieldProperty field = formFields.get(fieldCode); - return field == null ? null : field.getType(); + return client.get().getFieldType(fieldCode); } private static boolean existsRecord(List existingValues, UpdateKey updateKey) { diff --git a/src/main/java/org/embulk/output/kintone/util/Lazy.java b/src/main/java/org/embulk/output/kintone/util/Lazy.java new file mode 100644 index 0000000..5b27f56 --- /dev/null +++ b/src/main/java/org/embulk/output/kintone/util/Lazy.java @@ -0,0 +1,20 @@ +package org.embulk.output.kintone.util; + +public abstract class Lazy implements AutoCloseable { + private T value; + + public T get() { + if (value == null) { + value = initialValue(); + } + return value; + } + + public void close() throws Exception { + if (value != null) { + value.close(); + } + } + + protected abstract T initialValue(); +} diff --git a/src/test/java/org/embulk/output/kintone/KintoneClientTest.java b/src/test/java/org/embulk/output/kintone/KintoneClientTest.java new file mode 100644 index 0000000..e3f2398 --- /dev/null +++ b/src/test/java/org/embulk/output/kintone/KintoneClientTest.java @@ -0,0 +1,122 @@ +package org.embulk.output.kintone; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThrows; + +import java.util.Collections; +import org.embulk.config.ConfigException; +import org.embulk.config.ConfigSource; +import org.embulk.output.kintone.util.Lazy; +import org.embulk.spi.Schema; +import org.embulk.spi.type.Types; +import org.junit.Before; +import org.junit.Test; + +public class KintoneClientTest extends TestKintoneOutputPlugin { + private ConfigSource config; + + @Before + public void before() { + config = loadConfigYaml("client/config.yml"); + } + + @Test + public void testInsert() { + merge(config("mode: insert")); + merge(config("update_key: null")); + runWithMockClient(Lazy::get); + merge(config("update_key: long_number")); + assertConfigException("When mode is insert, require no update_key."); + merge(config("update_key: string_single_line_text")); + assertConfigException("When mode is insert, require no update_key."); + } + + @Test + public void testUpdate() { + merge(config("mode: update")); + merge(config("update_key: null")); + assertConfigException("When mode is update, require update_key."); + merge(config("update_key: non_existing_column")); + assertConfigException("The column 'non_existing_column' for update does not exist."); + merge(config("update_key: non_existing_field")); + assertConfigException("The field 'non_existing_field' for update does not exist."); + merge(config("update_key: invalid_type_field_multi_line_text")); + assertConfigException("The update_key must be 'SINGLE_LINE_TEXT' or 'NUMBER'."); + merge(config("update_key: long_number")); + runWithMockClient(Lazy::get); + merge(config("update_key: string_single_line_text")); + runWithMockClient(Lazy::get); + } + + @Test + public void testUpsert() { + merge(config("mode: upsert")); + merge(config("update_key: null")); + assertConfigException("When mode is upsert, require update_key."); + merge(config("update_key: non_existing_column")); + assertConfigException("The column 'non_existing_column' for update does not exist."); + merge(config("update_key: non_existing_field")); + assertConfigException("The field 'non_existing_field' for update does not exist."); + merge(config("update_key: invalid_type_field_multi_line_text")); + assertConfigException("The update_key must be 'SINGLE_LINE_TEXT' or 'NUMBER'."); + merge(config("update_key: long_number")); + runWithMockClient(Lazy::get); + merge(config("update_key: string_single_line_text")); + runWithMockClient(Lazy::get); + } + + private void assertConfigException(String message) { + assertConfigException(message, builder()); + } + + private void assertConfigException(String message, Schema.Builder builder) { + runWithMockClient( + client -> + assertThat(assertThrows(ConfigException.class, client::get).getMessage(), is(message)), + builder); + } + + private void runWithMockClient(Consumer> consumer) { + runWithMockClient(consumer, builder()); + } + + private void runWithMockClient(Consumer> consumer, Schema.Builder builder) { + MockClient mockClient = + new MockClient( + config.get(String.class, "domain"), + Collections.emptyList(), + Collections.emptyList(), + ""); + try (Lazy client = KintoneClient.lazy(this::task, schema(builder))) { + mockClient.run(() -> consumer.accept(client)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void merge(ConfigSource config) { + this.config.merge(config); + } + + private PluginTask task() { + return config.loadConfig(PluginTask.class); + } + + private static Schema schema(Schema.Builder builder) { + return builder + .add("non_existing_field", Types.LONG) + .add("invalid_type_field_multi_line_text", Types.STRING) + .add("long_number", Types.LONG) + .add("string_single_line_text", Types.STRING) + .build(); + } + + private static Schema.Builder builder() { + return Schema.builder(); + } + + private interface Consumer { + void accept(T t) throws Exception; + } +} diff --git a/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java b/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java index 5104390..bde0c7b 100644 --- a/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java +++ b/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java @@ -3,24 +3,13 @@ import static org.embulk.output.kintone.deserializer.DeserializerTest.assertTableRows; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; -import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.matches; import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.atLeast; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import com.kintone.client.AppClient; -import com.kintone.client.KintoneClient; -import com.kintone.client.KintoneClientBuilder; import com.kintone.client.RecordClient; -import com.kintone.client.api.record.GetRecordsByCursorResponseBody; -import com.kintone.client.model.app.field.FieldProperty; -import com.kintone.client.model.app.field.NumberFieldProperty; -import com.kintone.client.model.app.field.SingleLineTextFieldProperty; import com.kintone.client.model.record.FieldType; import com.kintone.client.model.record.FieldValue; import com.kintone.client.model.record.NumberFieldValue; @@ -33,7 +22,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.embulk.config.TaskReport; @@ -44,12 +33,15 @@ import org.embulk.spi.TransactionalPageOutput; import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; +import org.msgpack.value.StringValue; +import org.msgpack.value.ValueFactory; public class KintonePageOutputVerifier implements TransactionalPageOutput { private final TransactionalPageOutput transactionalPageOutput; private final String domain; private final String field; private final List values; + private final List addValues; private final List addRecords; private final List updateRecords; @@ -57,9 +49,10 @@ public KintonePageOutputVerifier( String domain, String field, List values, + List addValues, List addRecords, List updateRecords) { - this(null, domain, field, values, addRecords, updateRecords); + this(null, domain, field, values, addValues, addRecords, updateRecords); } public KintonePageOutputVerifier( @@ -67,12 +60,14 @@ public KintonePageOutputVerifier( String domain, String field, List values, + List addValues, List addRecords, List updateRecords) { this.transactionalPageOutput = transactionalPageOutput; this.domain = domain; this.field = field; this.values = values; + this.addValues = addValues; this.addRecords = addRecords; this.updateRecords = updateRecords; } @@ -105,7 +100,7 @@ public TaskReport commit() { return transactionalPageOutput.commit(); } - public void runWithMock(Runnable runnable) { + public void runWithMock(MockClient.Runnable runnable) { try { runWithMockExec(runnable); } catch (Exception e) { @@ -113,7 +108,7 @@ public void runWithMock(Runnable runnable) { } } - private void runWithMockExec(Runnable runnable) throws Exception { + private void runWithMockExec(MockClient.Runnable runnable) throws Exception { BufferAllocator bufferAllocator = PooledBufferAllocator.create(1024 * 1024); try (MockedStatic mocked = mockStatic(Exec.class, CALLS_REAL_METHODS)) { mocked.when(Exec::getBufferAllocator).thenReturn(bufferAllocator); @@ -121,37 +116,11 @@ private void runWithMockExec(Runnable runnable) throws Exception { } } - private void runWithMockClient(Runnable runnable) throws Exception { - @SuppressWarnings("unchecked") - Map mockFormFields = mock(Map.class); - when(mockFormFields.get(matches("^.*_single_line_text$"))) - .thenReturn(new SingleLineTextFieldProperty()); - when(mockFormFields.get(matches("^.*_number$"))).thenReturn(new NumberFieldProperty()); - AppClient mockAppClient = mock(AppClient.class); - when(mockAppClient.getFormFields(eq(0L))).thenReturn(mockFormFields); - GetRecordsByCursorResponseBody mockGetRecordsByCursorResponseBody = - mock(GetRecordsByCursorResponseBody.class); - when(mockGetRecordsByCursorResponseBody.getRecords()) - .thenReturn(updateRecords.stream().map(this::getRecord).collect(Collectors.toList())); - when(mockGetRecordsByCursorResponseBody.hasNext()).thenReturn(false); - RecordClient mockRecordClient = mock(RecordClient.class); - when(mockRecordClient.createCursor(eq(0L), eq(getFields()), eq(getQuery()))).thenReturn("id"); - when(mockRecordClient.getRecordsByCursor(eq("id"))) - .thenReturn(mockGetRecordsByCursorResponseBody); - when(mockRecordClient.addRecords(eq(0L), anyList())).thenReturn(Collections.emptyList()); - when(mockRecordClient.updateRecords(eq(0L), anyList())).thenReturn(Collections.emptyList()); - KintoneClient mockKintoneClient = mock(KintoneClient.class); - when(mockKintoneClient.app()).thenReturn(mockAppClient); - when(mockKintoneClient.record()).thenReturn(mockRecordClient); - KintoneClientBuilder mockKintoneClientBuilder = mock(KintoneClientBuilder.class); - when(mockKintoneClientBuilder.authByApiToken(eq("token"))).thenReturn(mockKintoneClientBuilder); - when(mockKintoneClientBuilder.build()).thenReturn(mockKintoneClient); - try (MockedStatic mocked = mockStatic(KintoneClientBuilder.class)) { - mocked - .when(() -> KintoneClientBuilder.create(String.format("https://%s", domain))) - .thenReturn(mockKintoneClientBuilder); - runnable.run(); - } + private void runWithMockClient(MockClient.Runnable runnable) throws Exception { + assertValues(domain, getValues(), sorted(values)); + MockClient mockClient = new MockClient(domain, getRecords(), getFields(), getQuery()); + RecordClient mockRecordClient = mockClient.getMockRecordClient(); + mockClient.run(runnable); @SuppressWarnings("unchecked") ArgumentCaptor> addRecordsArgumentCaptor = ArgumentCaptor.forClass(List.class); verify(mockRecordClient, atLeast(0)).addRecords(eq(0L), addRecordsArgumentCaptor.capture()); @@ -174,6 +143,27 @@ private void runWithMockClient(Runnable runnable) throws Exception { updateRecords); } + private List getValues() { + if (values.isEmpty() || updateRecords.isEmpty()) { + return values; + } + Function toValue = + value -> + field.matches("^.*_number$") + ? ValueFactory.newString(((NumberFieldValue) value).getValue().toPlainString()) + : ValueFactory.newString(((SingleLineTextFieldValue) value).getValue()); + List values = + getRecords().stream() + .map(record -> toValue.apply(record.getFieldValue(field)).toJson()) + .collect(Collectors.toList()); + values.addAll(addValues); + return sorted(values); + } + + private List getRecords() { + return updateRecords.stream().map(this::getRecord).collect(Collectors.toList()); + } + private Record getRecord(RecordForUpdate updateRecord) { UpdateKey key = updateRecord.getUpdateKey(); String field = key.getField(); @@ -194,6 +184,18 @@ private String getQuery() { return String.format("%s in (%s)", field, String.join(",", values)); } + private static void assertValues(String domain, List actual, List expected) { + assertThat(domain, actual.size(), is(expected.size())); + // spotless:off + IntStream.range(0, actual.size()).forEach(index -> assertValue(domain, index, actual.get(index), expected.get(index))); + // spotless:on + } + + private static void assertValue(String domain, int index, String actual, String expected) { + String reason = String.format("%s:%d", domain, index); + assertThat(reason, actual, is(expected)); + } + private static void assertRecords(String domain, List actual, List expected) { assertThat(domain, actual.size(), is(expected.size())); // spotless:off @@ -245,20 +247,32 @@ private static void assertRecordForUpdates( private static void assertRecordForUpdate( String domain, int index, RecordForUpdate actual, RecordForUpdate expected) { String reason = String.format("%s:%d", domain, index); - assertThat(reason, actual.getId(), is(expected.getId())); - assertUpdateKey(domain, index, actual.getUpdateKey(), expected.getUpdateKey()); + assertId(domain, index, actual, expected); + assertUpdateKey(domain, index, actual, expected); assertRecord(domain, index, actual.getRecord(), expected.getRecord()); assertThat(reason, actual.getRevision(), is(expected.getRevision())); } + private static void assertId( + String domain, int index, RecordForUpdate actual, RecordForUpdate expected) { + if (actual.getUpdateKey() != null && expected.getUpdateKey() != null) { + return; + } + String reason = String.format("%s:%d", domain, index); + assertThat(reason, actual.getId(), is(expected.getId())); + } + private static void assertUpdateKey( - String domain, int index, UpdateKey actual, UpdateKey expected) { + String domain, int index, RecordForUpdate actual, RecordForUpdate expected) { + if (actual.getId() != null && expected.getId() != null) { + return; + } String reason = String.format("%s:%d", domain, index); - assertThat(reason, actual.getField(), is(expected.getField())); - assertThat(reason, actual.getValue(), is(expected.getValue())); + assertThat(reason, actual.getUpdateKey().getField(), is(expected.getUpdateKey().getField())); + assertThat(reason, actual.getUpdateKey().getValue(), is(expected.getUpdateKey().getValue())); } - public interface Runnable { - void run() throws Exception; + private static List sorted(List list) { + return list.stream().sorted().collect(Collectors.toList()); } } diff --git a/src/test/java/org/embulk/output/kintone/MockClient.java b/src/test/java/org/embulk/output/kintone/MockClient.java new file mode 100644 index 0000000..c763da9 --- /dev/null +++ b/src/test/java/org/embulk/output/kintone/MockClient.java @@ -0,0 +1,112 @@ +package org.embulk.output.kintone; + +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.matches; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +import com.kintone.client.AppClient; +import com.kintone.client.KintoneClient; +import com.kintone.client.KintoneClientBuilder; +import com.kintone.client.RecordClient; +import com.kintone.client.api.record.GetRecordsByCursorResponseBody; +import com.kintone.client.model.app.field.CheckBoxFieldProperty; +import com.kintone.client.model.app.field.DateFieldProperty; +import com.kintone.client.model.app.field.DateTimeFieldProperty; +import com.kintone.client.model.app.field.DropDownFieldProperty; +import com.kintone.client.model.app.field.FieldProperty; +import com.kintone.client.model.app.field.FileFieldProperty; +import com.kintone.client.model.app.field.GroupSelectFieldProperty; +import com.kintone.client.model.app.field.LinkFieldProperty; +import com.kintone.client.model.app.field.MultiLineTextFieldProperty; +import com.kintone.client.model.app.field.MultiSelectFieldProperty; +import com.kintone.client.model.app.field.NumberFieldProperty; +import com.kintone.client.model.app.field.OrganizationSelectFieldProperty; +import com.kintone.client.model.app.field.RadioButtonFieldProperty; +import com.kintone.client.model.app.field.RichTextFieldProperty; +import com.kintone.client.model.app.field.SingleLineTextFieldProperty; +import com.kintone.client.model.app.field.SubtableFieldProperty; +import com.kintone.client.model.app.field.TimeFieldProperty; +import com.kintone.client.model.app.field.UserSelectFieldProperty; +import com.kintone.client.model.record.Record; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.mockito.MockedStatic; + +public class MockClient { + private final String domain; + private final List records; + private final List fields; + private final String query; + private final RecordClient mockRecordClient; + + public MockClient(String domain, List records, List fields, String query) { + this.domain = domain; + this.records = records; + this.fields = fields; + this.query = query; + mockRecordClient = mock(RecordClient.class); + } + + public RecordClient getMockRecordClient() { + return mockRecordClient; + } + + public void run(Runnable runnable) throws Exception { + @SuppressWarnings("unchecked") + Map mockFormFields = mock(Map.class); + when(mockFormFields.get(matches("^.*_single_line_text$"))) + .thenReturn(new SingleLineTextFieldProperty()); + when(mockFormFields.get(matches("^.*_multi_line_text$"))) + .thenReturn(new MultiLineTextFieldProperty()); + when(mockFormFields.get(matches("^.*_rich_text$"))).thenReturn(new RichTextFieldProperty()); + when(mockFormFields.get(matches("^.*_number$"))).thenReturn(new NumberFieldProperty()); + when(mockFormFields.get(matches("^.*_check_box$"))).thenReturn(new CheckBoxFieldProperty()); + when(mockFormFields.get(matches("^.*_radio_button$"))) + .thenReturn(new RadioButtonFieldProperty()); + when(mockFormFields.get(matches("^.*_multi_select$"))) + .thenReturn(new MultiSelectFieldProperty()); + when(mockFormFields.get(matches("^.*_drop_down$"))).thenReturn(new DropDownFieldProperty()); + when(mockFormFields.get(matches("^.*_user_select$"))).thenReturn(new UserSelectFieldProperty()); + when(mockFormFields.get(matches("^.*_organization_select$"))) + .thenReturn(new OrganizationSelectFieldProperty()); + when(mockFormFields.get(matches("^.*_group_select$"))) + .thenReturn(new GroupSelectFieldProperty()); + when(mockFormFields.get(matches("^.*_date$"))).thenReturn(new DateFieldProperty()); + when(mockFormFields.get(matches("^.*_time$"))).thenReturn(new TimeFieldProperty()); + when(mockFormFields.get(matches("^.*_datetime$"))).thenReturn(new DateTimeFieldProperty()); + when(mockFormFields.get(matches("^.*_link$"))).thenReturn(new LinkFieldProperty()); + when(mockFormFields.get(matches("^.*_file$"))).thenReturn(new FileFieldProperty()); + when(mockFormFields.get(matches("^.*_subtable$"))).thenReturn(new SubtableFieldProperty()); + AppClient mockAppClient = mock(AppClient.class); + when(mockAppClient.getFormFields(eq(0L))).thenReturn(mockFormFields); + GetRecordsByCursorResponseBody mockGetRecordsByCursorResponseBody = + mock(GetRecordsByCursorResponseBody.class); + when(mockGetRecordsByCursorResponseBody.getRecords()).thenReturn(records); + when(mockGetRecordsByCursorResponseBody.hasNext()).thenReturn(false); + when(mockRecordClient.createCursor(eq(0L), eq(fields), eq(query))).thenReturn("id"); + when(mockRecordClient.getRecordsByCursor(eq("id"))) + .thenReturn(mockGetRecordsByCursorResponseBody); + when(mockRecordClient.addRecords(eq(0L), anyList())).thenReturn(Collections.emptyList()); + when(mockRecordClient.updateRecords(eq(0L), anyList())).thenReturn(Collections.emptyList()); + com.kintone.client.KintoneClient mockKintoneClient = mock(KintoneClient.class); + when(mockKintoneClient.app()).thenReturn(mockAppClient); + when(mockKintoneClient.record()).thenReturn(mockRecordClient); + KintoneClientBuilder mockKintoneClientBuilder = mock(KintoneClientBuilder.class); + when(mockKintoneClientBuilder.authByApiToken(eq("token"))).thenReturn(mockKintoneClientBuilder); + when(mockKintoneClientBuilder.build()).thenReturn(mockKintoneClient); + try (MockedStatic mocked = mockStatic(KintoneClientBuilder.class)) { + mocked + .when(() -> KintoneClientBuilder.create(String.format("https://%s", domain))) + .thenReturn(mockKintoneClientBuilder); + runnable.run(); + } + } + + public interface Runnable { + void run() throws Exception; + } +} diff --git a/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java b/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java index 6c31b01..fd4151a 100644 --- a/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java +++ b/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java @@ -137,7 +137,8 @@ private KintonePageOutputVerifier verifier(ConfigSource config) { return new KintonePageOutputVerifier( test, field, - getValues(test, preferNulls, ignoreNulls), + getValues(test, mode, preferNulls, ignoreNulls, field), + getAddValues(test, mode, preferNulls, ignoreNulls, field), getAddRecords(test, mode, preferNulls, ignoreNulls), getUpdateRecords(test, mode, preferNulls, ignoreNulls, field)); } @@ -153,7 +154,8 @@ private TransactionalPageOutput openWithVerifier( super.open(taskSource, schema, taskIndex), test, field, - getValues(test, preferNulls, ignoreNulls), + getValues(test, mode, preferNulls, ignoreNulls, field), + getAddValues(test, mode, preferNulls, ignoreNulls, field), getAddRecords(test, mode, preferNulls, ignoreNulls), getUpdateRecords(test, mode, preferNulls, ignoreNulls, field)); } @@ -168,11 +170,32 @@ private static Set getDerivedColumns(String test) { .collect(Collectors.toSet()); } - private static List getValues(String test, boolean preferNulls, boolean ignoreNulls) { + private static List getValues( + String test, String mode, boolean preferNulls, boolean ignoreNulls, String field) { + String name = + String.format( + "%s/%s%s%s_values.json", + test, + mode, + ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : "", + format(field)); + String json = existsResource(name) ? readResource(name) : null; + return json == null || json.isEmpty() + ? Collections.emptyList() + : PARSER.parse(json).asArrayValue().list().stream() + .map(Value::toJson) + .collect(Collectors.toList()); + } + + private static List getAddValues( + String test, String mode, boolean preferNulls, boolean ignoreNulls, String field) { String name = String.format( - "%s/values%s.json", - test, ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : ""); + "%s/%s%s%s_add_values.json", + test, + mode, + ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : "", + format(field)); String json = existsResource(name) ? readResource(name) : null; return json == null || json.isEmpty() ? Collections.emptyList() @@ -185,7 +208,7 @@ private static List getAddRecords( String test, String mode, boolean preferNulls, boolean ignoreNulls) { String name = String.format( - "%s/%s_add%s_records.jsonl", + "%s/%s%s_add_records.jsonl", test, mode, ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : ""); String jsonl = existsResource(name) ? readResource(name) : null; return jsonl == null || jsonl.isEmpty() @@ -200,7 +223,7 @@ private static List getUpdateRecords( Function key = getKey(field); String name = String.format( - "%s/%s_update%s_records.jsonl", + "%s/%s%s_update_records.jsonl", test, mode, ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : ""); String jsonl = existsResource(name) ? readResource(name) : null; return jsonl == null || jsonl.isEmpty() @@ -211,6 +234,10 @@ private static List getUpdateRecords( .collect(Collectors.toList()); } + private static String format(String string) { + return string == null ? "" : String.format("_%s", string.replace('$', '_')); + } + private static Function getKey(String field) { return field == null ? record -> null diff --git a/src/test/resources/org/embulk/output/kintone/client/config.yml b/src/test/resources/org/embulk/output/kintone/client/config.yml new file mode 100644 index 0000000..8f60655 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/client/config.yml @@ -0,0 +1 @@ +domain: client diff --git a/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_ignore_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/mode/insert_ignore_nulls_add_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/mode/insert_add_ignore_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/mode/insert_ignore_nulls_add_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/mode/insert_add_prefer_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/mode/insert_prefer_nulls_add_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/mode/insert_add_prefer_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/mode/insert_prefer_nulls_add_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/mode/update_update_ignore_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/mode/update_ignore_nulls_update_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/mode/update_update_ignore_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/mode/update_ignore_nulls_update_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/mode/update_update_prefer_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/mode/update_prefer_nulls_update_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/mode/update_update_prefer_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/mode/update_prefer_nulls_update_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/mode/upsert_double_single_line_text_add_values.json b/src/test/resources/org/embulk/output/kintone/task/mode/upsert_double_single_line_text_add_values.json new file mode 100644 index 0000000..2e0108c --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/mode/upsert_double_single_line_text_add_values.json @@ -0,0 +1 @@ +["1.0","123.0"] diff --git a/src/test/resources/org/embulk/output/kintone/task/mode/values.json b/src/test/resources/org/embulk/output/kintone/task/mode/upsert_double_single_line_text_values.json similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/mode/values.json rename to src/test/resources/org/embulk/output/kintone/task/mode/upsert_double_single_line_text_values.json diff --git a/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_ignore_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/mode/upsert_ignore_nulls_add_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_ignore_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/mode/upsert_ignore_nulls_add_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/mode/values_ignore_nulls.json b/src/test/resources/org/embulk/output/kintone/task/mode/upsert_ignore_nulls_double_single_line_text_values.json similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/mode/values_ignore_nulls.json rename to src/test/resources/org/embulk/output/kintone/task/mode/upsert_ignore_nulls_double_single_line_text_values.json diff --git a/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_ignore_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/mode/upsert_ignore_nulls_update_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_ignore_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/mode/upsert_ignore_nulls_update_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_prefer_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/mode/upsert_prefer_nulls_add_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_prefer_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/mode/upsert_prefer_nulls_add_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/mode/values_prefer_nulls.json b/src/test/resources/org/embulk/output/kintone/task/mode/upsert_prefer_nulls_double_single_line_text_values.json similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/mode/values_prefer_nulls.json rename to src/test/resources/org/embulk/output/kintone/task/mode/upsert_prefer_nulls_double_single_line_text_values.json diff --git a/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_prefer_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/mode/upsert_prefer_nulls_update_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_prefer_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/mode/upsert_prefer_nulls_update_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce/insert_add_ignore_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce/insert_ignore_nulls_add_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce/insert_add_ignore_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce/insert_ignore_nulls_add_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce/insert_add_prefer_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce/insert_prefer_nulls_add_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce/insert_add_prefer_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce/insert_prefer_nulls_add_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce/update_update_ignore_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce/update_ignore_nulls_update_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce/update_update_ignore_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce/update_ignore_nulls_update_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce/update_update_prefer_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce/update_prefer_nulls_update_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce/update_update_prefer_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce/update_prefer_nulls_update_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_double_single_line_text_add_values.json b/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_double_single_line_text_add_values.json new file mode 100644 index 0000000..2e0108c --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_double_single_line_text_add_values.json @@ -0,0 +1 @@ +["1.0","123.0"] diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce/values.json b/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_double_single_line_text_values.json similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce/values.json rename to src/test/resources/org/embulk/output/kintone/task/reduce/upsert_double_single_line_text_values.json diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_add_ignore_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_ignore_nulls_add_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce/upsert_add_ignore_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce/upsert_ignore_nulls_add_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce/values_ignore_nulls.json b/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_ignore_nulls_double_single_line_text_values.json similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce/values_ignore_nulls.json rename to src/test/resources/org/embulk/output/kintone/task/reduce/upsert_ignore_nulls_double_single_line_text_values.json diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_update_ignore_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_ignore_nulls_update_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce/upsert_update_ignore_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce/upsert_ignore_nulls_update_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_add_prefer_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_prefer_nulls_add_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce/upsert_add_prefer_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce/upsert_prefer_nulls_add_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce/values_prefer_nulls.json b/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_prefer_nulls_double_single_line_text_values.json similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce/values_prefer_nulls.json rename to src/test/resources/org/embulk/output/kintone/task/reduce/upsert_prefer_nulls_double_single_line_text_values.json diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_update_prefer_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_prefer_nulls_update_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce/upsert_update_prefer_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce/upsert_prefer_nulls_update_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_add_ignore_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_ignore_nulls_add_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_add_ignore_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_ignore_nulls_add_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_add_prefer_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_prefer_nulls_add_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_add_prefer_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce_subtable/insert_prefer_nulls_add_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_update_ignore_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_ignore_nulls_update_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_update_ignore_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_ignore_nulls_update_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_update_prefer_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_prefer_nulls_update_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_update_prefer_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce_subtable/update_prefer_nulls_update_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_add_records.jsonl deleted file mode 100644 index e69de29..0000000 diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/values.json b/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_double_single_line_text_values.json similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce_subtable/values.json rename to src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_double_single_line_text_values.json diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_add_ignore_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_ignore_nulls_add_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_add_ignore_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_ignore_nulls_add_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/values_ignore_nulls.json b/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_ignore_nulls_double_single_line_text_values.json similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce_subtable/values_ignore_nulls.json rename to src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_ignore_nulls_double_single_line_text_values.json diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_update_ignore_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_ignore_nulls_update_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_update_ignore_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_ignore_nulls_update_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_add_prefer_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_prefer_nulls_add_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_add_prefer_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_prefer_nulls_add_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/values_prefer_nulls.json b/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_prefer_nulls_double_single_line_text_values.json similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce_subtable/values_prefer_nulls.json rename to src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_prefer_nulls_double_single_line_text_values.json diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_update_prefer_nulls_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_prefer_nulls_update_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_update_prefer_nulls_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_prefer_nulls_update_records.jsonl From fa77644a5c04bfb85f9b714f55b9426bd18223ca Mon Sep 17 00:00:00 2001 From: Tetsuro Sano Date: Sun, 21 Apr 2024 00:29:37 +0900 Subject: [PATCH 2/4] Add support for update by id --- README.md | 1 + .../embulk/output/kintone/KintoneClient.java | 39 +++++-- .../output/kintone/KintoneColumnVisitor.java | 58 +++++++++- .../embulk/output/kintone/KintoneMode.java | 26 +++-- .../output/kintone/KintonePageOutput.java | 106 +++++++++++++----- .../org/embulk/output/kintone/PluginTask.java | 5 + .../org/embulk/output/kintone/record/Id.java | 18 +++ .../output/kintone/record/IdOrUpdateKey.java | 45 ++++++++ .../embulk/output/kintone/record/Skip.java | 14 +++ .../output/kintone/KintoneClientTest.java | 21 +++- .../kintone/KintoneColumnVisitorTest.java | 66 +++++------ .../kintone/KintoneColumnVisitorVerifier.java | 12 +- .../kintone/KintonePageOutputVerifier.java | 20 +++- .../kintone/TestKintoneOutputPlugin.java | 59 ++++++---- .../embulk/output/kintone/TestTaskMode.java | 1 + .../embulk/output/kintone/TestTaskReduce.java | 1 + .../kintone/TestTaskReduceSubtable.java | 1 + .../embulk/output/kintone/TestTaskSkip.java | 80 +++++++++++++ .../embulk/output/kintone/TestTaskSkipId.java | 80 +++++++++++++ .../org/embulk/output/kintone/task/config.yml | 1 + ...nl => upsert_never_skip_add_records.jsonl} | 0 ...p_double_single_line_text_add_values.json} | 0 ..._skip_double_single_line_text_values.json} | 0 ...never_skip_ignore_nulls_add_records.jsonl} | 0 ...nulls_double_single_line_text_values.json} | 0 ...er_skip_ignore_nulls_update_records.jsonl} | 0 ...never_skip_prefer_nulls_add_records.jsonl} | 0 ...nulls_double_single_line_text_values.json} | 0 ...er_skip_prefer_nulls_update_records.jsonl} | 0 ...=> upsert_never_skip_update_records.jsonl} | 0 ...nl => upsert_never_skip_add_records.jsonl} | 0 ...p_double_single_line_text_add_values.json} | 0 ..._skip_double_single_line_text_values.json} | 0 ...never_skip_ignore_nulls_add_records.jsonl} | 0 ...nulls_double_single_line_text_values.json} | 0 ...er_skip_ignore_nulls_update_records.jsonl} | 0 ...never_skip_prefer_nulls_add_records.jsonl} | 0 ...nulls_double_single_line_text_values.json} | 0 ...er_skip_prefer_nulls_update_records.jsonl} | 0 ...=> upsert_never_skip_update_records.jsonl} | 0 ..._skip_double_single_line_text_values.json} | 0 ...never_skip_ignore_nulls_add_records.jsonl} | 0 ...nulls_double_single_line_text_values.json} | 0 ...er_skip_ignore_nulls_update_records.jsonl} | 0 ...never_skip_prefer_nulls_add_records.jsonl} | 0 ...nulls_double_single_line_text_values.json} | 0 ...er_skip_prefer_nulls_update_records.jsonl} | 0 ...=> upsert_never_skip_update_records.jsonl} | 0 .../output/kintone/task/skip/config.yml | 1 + .../embulk/output/kintone/task/skip/input.csv | 7 ++ .../task/skip/insert_add_records.jsonl | 6 + .../skip/insert_always_skip_add_records.jsonl | 6 + ...always_skip_prefer_nulls_add_records.jsonl | 6 + .../skip/insert_never_skip_add_records.jsonl | 6 + ..._never_skip_prefer_nulls_add_records.jsonl | 6 + .../insert_prefer_nulls_add_records.jsonl | 6 + ...te_always_skip_long_number_add_values.json | 1 + ...update_always_skip_long_number_values.json | 1 + ...p_prefer_nulls_long_number_add_values.json | 1 + ..._skip_prefer_nulls_long_number_values.json | 1 + ...ays_skip_prefer_nulls_update_records.jsonl | 2 + .../update_always_skip_update_records.jsonl | 5 + .../update_never_skip_update_records.jsonl | 6 + .../update_prefer_nulls_update_records.jsonl | 3 + .../task/skip/update_update_records.jsonl | 6 + .../task/skip/upsert_add_records.jsonl | 1 + ...ls_string_single_line_text_add_values.json | 1 + ..._nulls_string_single_line_text_values.json | 1 + ...ays_skip_prefer_nulls_update_records.jsonl | 2 + ...ip_string_single_line_text_add_values.json | 1 + ...s_skip_string_single_line_text_values.json | 1 + .../upsert_always_skip_update_records.jsonl | 2 + .../skip/upsert_never_skip_add_records.jsonl | 4 + ..._never_skip_prefer_nulls_add_records.jsonl | 4 + ...ls_string_single_line_text_add_values.json | 1 + ..._nulls_string_single_line_text_values.json | 1 + ...ver_skip_prefer_nulls_update_records.jsonl | 2 + ...ip_string_single_line_text_add_values.json | 1 + ...r_skip_string_single_line_text_values.json | 1 + .../upsert_never_skip_update_records.jsonl | 2 + .../upsert_prefer_nulls_add_records.jsonl | 1 + ...ls_string_single_line_text_add_values.json | 1 + ..._nulls_string_single_line_text_values.json | 1 + .../upsert_prefer_nulls_update_records.jsonl | 2 + ...rt_string_single_line_text_add_values.json | 1 + ...upsert_string_single_line_text_values.json | 1 + .../task/skip/upsert_update_records.jsonl | 2 + .../output/kintone/task/skip_id/config.yml | 1 + .../output/kintone/task/skip_id/input.csv | 7 ++ .../task/skip_id/insert_add_records.jsonl | 6 + .../insert_always_skip_add_records.jsonl | 6 + ...always_skip_prefer_nulls_add_records.jsonl | 6 + .../insert_never_skip_add_records.jsonl | 6 + ..._never_skip_prefer_nulls_add_records.jsonl | 6 + .../insert_prefer_nulls_add_records.jsonl | 6 + .../update_always_skip__id_add_values.json | 1 + .../update_always_skip__id_values.json | 1 + ...ways_skip_prefer_nulls__id_add_values.json | 1 + ...e_always_skip_prefer_nulls__id_values.json | 1 + ...ays_skip_prefer_nulls_update_records.jsonl | 2 + .../update_always_skip_update_records.jsonl | 5 + .../update_never_skip_update_records.jsonl | 6 + .../update_prefer_nulls_update_records.jsonl | 3 + .../task/skip_id/update_update_records.jsonl | 6 + .../task/skip_id/upsert__id_add_values.json | 1 + .../task/skip_id/upsert__id_values.json | 1 + .../upsert_always_skip__id_add_values.json | 1 + .../upsert_always_skip__id_values.json | 1 + ...ways_skip_prefer_nulls__id_add_values.json | 1 + ...t_always_skip_prefer_nulls__id_values.json | 1 + ...ays_skip_prefer_nulls_update_records.jsonl | 2 + .../upsert_always_skip_update_records.jsonl | 5 + .../upsert_never_skip__id_add_values.json | 1 + .../skip_id/upsert_never_skip__id_values.json | 1 + .../upsert_never_skip_add_records.jsonl | 1 + ...ever_skip_prefer_nulls__id_add_values.json | 1 + ...rt_never_skip_prefer_nulls__id_values.json | 1 + ..._never_skip_prefer_nulls_add_records.jsonl | 4 + ...ver_skip_prefer_nulls_update_records.jsonl | 2 + .../upsert_never_skip_update_records.jsonl | 5 + .../upsert_prefer_nulls__id_add_values.json | 1 + .../upsert_prefer_nulls__id_values.json | 1 + .../upsert_prefer_nulls_add_records.jsonl | 3 + .../upsert_prefer_nulls_update_records.jsonl | 2 + .../task/skip_id/upsert_update_records.jsonl | 5 + 125 files changed, 752 insertions(+), 117 deletions(-) create mode 100644 src/main/java/org/embulk/output/kintone/record/Id.java create mode 100644 src/main/java/org/embulk/output/kintone/record/IdOrUpdateKey.java create mode 100644 src/main/java/org/embulk/output/kintone/record/Skip.java create mode 100644 src/test/java/org/embulk/output/kintone/TestTaskSkip.java create mode 100644 src/test/java/org/embulk/output/kintone/TestTaskSkipId.java rename src/test/resources/org/embulk/output/kintone/task/mode/{upsert_add_records.jsonl => upsert_never_skip_add_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/mode/{upsert_double_single_line_text_add_values.json => upsert_never_skip_double_single_line_text_add_values.json} (100%) rename src/test/resources/org/embulk/output/kintone/task/mode/{upsert_double_single_line_text_values.json => upsert_never_skip_double_single_line_text_values.json} (100%) rename src/test/resources/org/embulk/output/kintone/task/mode/{upsert_ignore_nulls_add_records.jsonl => upsert_never_skip_ignore_nulls_add_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/mode/{upsert_ignore_nulls_double_single_line_text_values.json => upsert_never_skip_ignore_nulls_double_single_line_text_values.json} (100%) rename src/test/resources/org/embulk/output/kintone/task/mode/{upsert_ignore_nulls_update_records.jsonl => upsert_never_skip_ignore_nulls_update_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/mode/{upsert_prefer_nulls_add_records.jsonl => upsert_never_skip_prefer_nulls_add_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/mode/{upsert_prefer_nulls_double_single_line_text_values.json => upsert_never_skip_prefer_nulls_double_single_line_text_values.json} (100%) rename src/test/resources/org/embulk/output/kintone/task/mode/{upsert_prefer_nulls_update_records.jsonl => upsert_never_skip_prefer_nulls_update_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/mode/{upsert_update_records.jsonl => upsert_never_skip_update_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce/{upsert_add_records.jsonl => upsert_never_skip_add_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce/{upsert_double_single_line_text_add_values.json => upsert_never_skip_double_single_line_text_add_values.json} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce/{upsert_double_single_line_text_values.json => upsert_never_skip_double_single_line_text_values.json} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce/{upsert_ignore_nulls_add_records.jsonl => upsert_never_skip_ignore_nulls_add_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce/{upsert_ignore_nulls_double_single_line_text_values.json => upsert_never_skip_ignore_nulls_double_single_line_text_values.json} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce/{upsert_ignore_nulls_update_records.jsonl => upsert_never_skip_ignore_nulls_update_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce/{upsert_prefer_nulls_add_records.jsonl => upsert_never_skip_prefer_nulls_add_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce/{upsert_prefer_nulls_double_single_line_text_values.json => upsert_never_skip_prefer_nulls_double_single_line_text_values.json} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce/{upsert_prefer_nulls_update_records.jsonl => upsert_never_skip_prefer_nulls_update_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce/{upsert_update_records.jsonl => upsert_never_skip_update_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce_subtable/{upsert_double_single_line_text_values.json => upsert_never_skip_double_single_line_text_values.json} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce_subtable/{upsert_ignore_nulls_add_records.jsonl => upsert_never_skip_ignore_nulls_add_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce_subtable/{upsert_ignore_nulls_double_single_line_text_values.json => upsert_never_skip_ignore_nulls_double_single_line_text_values.json} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce_subtable/{upsert_ignore_nulls_update_records.jsonl => upsert_never_skip_ignore_nulls_update_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce_subtable/{upsert_prefer_nulls_add_records.jsonl => upsert_never_skip_prefer_nulls_add_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce_subtable/{upsert_prefer_nulls_double_single_line_text_values.json => upsert_never_skip_prefer_nulls_double_single_line_text_values.json} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce_subtable/{upsert_prefer_nulls_update_records.jsonl => upsert_never_skip_prefer_nulls_update_records.jsonl} (100%) rename src/test/resources/org/embulk/output/kintone/task/reduce_subtable/{upsert_update_records.jsonl => upsert_never_skip_update_records.jsonl} (100%) create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/config.yml create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/input.csv create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/insert_add_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/insert_always_skip_add_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/insert_always_skip_prefer_nulls_add_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/insert_never_skip_add_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/insert_never_skip_prefer_nulls_add_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/insert_prefer_nulls_add_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_long_number_add_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_long_number_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_prefer_nulls_long_number_add_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_prefer_nulls_long_number_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_prefer_nulls_update_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_update_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/update_never_skip_update_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/update_prefer_nulls_update_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/update_update_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/upsert_add_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_prefer_nulls_string_single_line_text_add_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_prefer_nulls_string_single_line_text_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_prefer_nulls_update_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_string_single_line_text_add_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_string_single_line_text_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_update_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_add_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_prefer_nulls_add_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_prefer_nulls_string_single_line_text_add_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_prefer_nulls_string_single_line_text_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_prefer_nulls_update_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_string_single_line_text_add_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_string_single_line_text_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_update_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/upsert_prefer_nulls_add_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/upsert_prefer_nulls_string_single_line_text_add_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/upsert_prefer_nulls_string_single_line_text_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/upsert_prefer_nulls_update_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/upsert_string_single_line_text_add_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/upsert_string_single_line_text_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip/upsert_update_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/config.yml create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/input.csv create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/insert_add_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/insert_always_skip_add_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/insert_always_skip_prefer_nulls_add_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/insert_never_skip_add_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/insert_never_skip_prefer_nulls_add_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/insert_prefer_nulls_add_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip__id_add_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip__id_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip_prefer_nulls__id_add_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip_prefer_nulls__id_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip_prefer_nulls_update_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip_update_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/update_never_skip_update_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/update_prefer_nulls_update_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/update_update_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/upsert__id_add_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/upsert__id_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip__id_add_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip__id_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip_prefer_nulls__id_add_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip_prefer_nulls__id_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip_prefer_nulls_update_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip_update_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip__id_add_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip__id_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_add_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_prefer_nulls__id_add_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_prefer_nulls__id_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_prefer_nulls_add_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_prefer_nulls_update_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_update_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_prefer_nulls__id_add_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_prefer_nulls__id_values.json create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_prefer_nulls_add_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_prefer_nulls_update_records.jsonl create mode 100644 src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_update_records.jsonl diff --git a/README.md b/README.md index 9e47ddd..9e9d090 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ kintone output plugin for Embulk stores app records from kintone. - **max_sort_memory**: Maximum memory usage for sorting input records (bytes in long, default is the estimated available memory, which is the approximate value of the JVM's current free memory) - **prefer_nulls**: Whether to set fields to null instead of default value of type when column is null (boolean, default is `false`) - **ignore_nulls**: Whether to completely ignore fields when column is null (boolean, default is `false`) +- **skip_if_non_existing_id_or_update_key**: Skip policy if id or update key of record does not exist (string `auto`, `never` or `always`, default is `auto`) - **column_options** advanced: a key-value pairs where key is a column name and value is options for the column. - **field_code**: field code (string, required) - **type**: field type (string, required). See [this page](https://cybozu.dev/ja/kintone/docs/overview/field-types/#field-type-update) for list of available types. However, following types are not yet supported diff --git a/src/main/java/org/embulk/output/kintone/KintoneClient.java b/src/main/java/org/embulk/output/kintone/KintoneClient.java index a2c24f3..d2f9b19 100644 --- a/src/main/java/org/embulk/output/kintone/KintoneClient.java +++ b/src/main/java/org/embulk/output/kintone/KintoneClient.java @@ -8,9 +8,12 @@ import java.util.Map; import java.util.function.Supplier; import org.embulk.config.ConfigException; +import org.embulk.output.kintone.record.Id; import org.embulk.output.kintone.util.Lazy; import org.embulk.spi.Column; import org.embulk.spi.Schema; +import org.embulk.spi.type.Type; +import org.embulk.spi.type.Types; public class KintoneClient implements AutoCloseable { private final PluginTask task; @@ -49,19 +52,13 @@ private KintoneClient(PluginTask task, Schema schema) { KintoneMode.of(task).validate(task, this); } - public void validateUpdateKey(String columnName) { + public void validateIdOrUpdateKey(String columnName) { Column column = getColumn(columnName); if (column == null) { throw new ConfigException("The column '" + columnName + "' for update does not exist."); } - String fieldCode = getFieldCode(column); - FieldType fieldType = getFieldType(fieldCode); - if (fieldType == null) { - throw new ConfigException("The field '" + fieldCode + "' for update does not exist."); - } - if (fieldType != FieldType.SINGLE_LINE_TEXT && fieldType != FieldType.NUMBER) { - throw new ConfigException("The update_key must be 'SINGLE_LINE_TEXT' or 'NUMBER'."); - } + validateId(column); + validateUpdateKey(column); } public Column getColumn(String columnName) { @@ -89,6 +86,30 @@ public void close() { } } + private void validateId(Column column) { + if (!column.getName().equals(Id.FIELD)) { + return; + } + Type type = column.getType(); + if (!type.equals(Types.LONG)) { + throw new ConfigException("The id column must be 'long'."); + } + } + + private void validateUpdateKey(Column column) { + if (column.getName().equals(Id.FIELD)) { + return; + } + String fieldCode = getFieldCode(column); + FieldType fieldType = getFieldType(fieldCode); + if (fieldType == null) { + throw new ConfigException("The field '" + fieldCode + "' for update does not exist."); + } + if (fieldType != FieldType.SINGLE_LINE_TEXT && fieldType != FieldType.NUMBER) { + throw new ConfigException("The update_key must be 'SINGLE_LINE_TEXT' or 'NUMBER'."); + } + } + private String getFieldCode(Column column) { KintoneColumnOption option = task.getColumnOptions().get(column.getName()); return option != null ? option.getFieldCode() : column.getName(); diff --git a/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java b/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java index 3f890b2..d58037c 100644 --- a/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java +++ b/src/main/java/org/embulk/output/kintone/KintoneColumnVisitor.java @@ -4,8 +4,12 @@ import com.kintone.client.model.record.Record; import com.kintone.client.model.record.UpdateKey; import java.time.Instant; +import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Set; +import org.embulk.output.kintone.record.Id; +import org.embulk.output.kintone.record.IdOrUpdateKey; import org.embulk.spi.Column; import org.embulk.spi.ColumnVisitor; import org.embulk.spi.PageReader; @@ -14,6 +18,7 @@ import org.msgpack.value.ValueFactory; public class KintoneColumnVisitor implements ColumnVisitor { + private static final List BUILTIN_FIELD_CODES = Arrays.asList(Id.FIELD, "$revision"); private final PageReader reader; private final Set derived; private final Map options; @@ -22,7 +27,7 @@ public class KintoneColumnVisitor implements ColumnVisitor { private final String reduceKeyName; private final String updateKeyName; private Record record; - private UpdateKey updateKey; + private IdOrUpdateKey idOrUpdateKey; public KintoneColumnVisitor( PageReader reader, @@ -55,8 +60,8 @@ public void setRecord(Record record) { this.record = record; } - public void setUpdateKey(UpdateKey updateKey) { - this.updateKey = updateKey; + public void setIdOrUpdateKey(IdOrUpdateKey idOrUpdateKey) { + this.idOrUpdateKey = idOrUpdateKey; } @Override @@ -68,6 +73,9 @@ public void booleanColumn(Column column) { UpdateKey updateKey = getUpdateKey(column); KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.NUMBER); String fieldCode = getFieldCode(column); + if (isBuiltin(fieldCode)) { + return; + } if (isPreferNull(column)) { setNull(type, fieldCode, updateKey); } else if (reader.isNull(column)) { @@ -82,10 +90,18 @@ public void longColumn(Column column) { if (isReduced(column) || isIgnoreNull(column)) { return; } + Id id = getId(column); + if (id != null) { + setIdValue(id, column); + return; + } KintoneColumnOption option = getOption(column); UpdateKey updateKey = getUpdateKey(column); KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.NUMBER); String fieldCode = getFieldCode(column); + if (isBuiltin(fieldCode)) { + return; + } if (isPreferNull(column)) { setNull(type, fieldCode, updateKey); } else if (reader.isNull(column)) { @@ -104,6 +120,9 @@ public void doubleColumn(Column column) { UpdateKey updateKey = getUpdateKey(column); KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.NUMBER); String fieldCode = getFieldCode(column); + if (isBuiltin(fieldCode)) { + return; + } if (isPreferNull(column)) { setNull(type, fieldCode, updateKey); } else if (reader.isNull(column)) { @@ -124,6 +143,9 @@ public void stringColumn(Column column) { updateKey != null ? KintoneColumnType.SINGLE_LINE_TEXT : KintoneColumnType.MULTI_LINE_TEXT; KintoneColumnType type = KintoneColumnType.getType(option, defaultType); String fieldCode = getFieldCode(column); + if (isBuiltin(fieldCode)) { + return; + } if (isPreferNull(column)) { setNull(type, fieldCode, updateKey); } else if (reader.isNull(column)) { @@ -142,6 +164,9 @@ public void timestampColumn(Column column) { UpdateKey updateKey = getUpdateKey(column); KintoneColumnType type = KintoneColumnType.getType(option, KintoneColumnType.DATETIME); String fieldCode = getFieldCode(column); + if (isBuiltin(fieldCode)) { + return; + } if (isPreferNull(column)) { setNull(type, fieldCode, updateKey); } else if (reader.isNull(column)) { @@ -162,6 +187,9 @@ public void jsonColumn(Column column) { isDerived(column) ? KintoneColumnType.SUBTABLE : KintoneColumnType.MULTI_LINE_TEXT; KintoneColumnType type = KintoneColumnType.getType(option, defaultType); String fieldCode = getFieldCode(column); + if (isBuiltin(fieldCode)) { + return; + } if (isPreferNull(column)) { setNull(type, fieldCode, updateKey); } else if (reader.isNull(column)) { @@ -171,6 +199,16 @@ public void jsonColumn(Column column) { } } + private void setIdValue(Id id, Column column) { + if (isPreferNull(column)) { + id.setValue(null); + } else if (reader.isNull(column)) { + id.setValue(0L); + } else { + id.setValue(reader.getLong(column)); + } + } + private void setNull(KintoneColumnType type, String fieldCode, UpdateKey updateKey) { if (updateKey != null) { type.setUpdateKey(updateKey, fieldCode); @@ -265,8 +303,16 @@ private KintoneColumnOption getOption(Column column) { return options.get(column.getName()); } + private Id getId(Column column) { + return column.getName().equals(updateKeyName) && updateKeyName.equals(Id.FIELD) + ? idOrUpdateKey.getId() + : null; + } + private UpdateKey getUpdateKey(Column column) { - return updateKeyName != null && updateKeyName.equals(column.getName()) ? updateKey : null; + return column.getName().equals(updateKeyName) && !updateKeyName.equals(Id.FIELD) + ? idOrUpdateKey.getUpdateKey() + : null; } private boolean isReduced(Column column) { @@ -284,4 +330,8 @@ private boolean isIgnoreNull(Column column) { private boolean isPreferNull(Column column) { return preferNulls && reader.isNull(column); } + + private static boolean isBuiltin(String fieldCode) { + return BUILTIN_FIELD_CODES.contains(fieldCode); + } } diff --git a/src/main/java/org/embulk/output/kintone/KintoneMode.java b/src/main/java/org/embulk/output/kintone/KintoneMode.java index 5fa7a49..0211426 100644 --- a/src/main/java/org/embulk/output/kintone/KintoneMode.java +++ b/src/main/java/org/embulk/output/kintone/KintoneMode.java @@ -1,6 +1,9 @@ package org.embulk.output.kintone; +import java.util.function.Consumer; import org.embulk.config.ConfigException; +import org.embulk.output.kintone.record.Id; +import org.embulk.output.kintone.record.Skip; import org.embulk.spi.Page; public enum KintoneMode { @@ -13,35 +16,36 @@ public void validate(PluginTask task, KintoneClient client) { } @Override - public void add(Page page, KintonePageOutput output) { + public void add(Page page, Skip skip, KintonePageOutput output) { output.insertPage(page); } }, UPDATE("update") { @Override public void validate(PluginTask task, KintoneClient client) { - if (!task.getUpdateKeyName().isPresent()) { - throw new ConfigException("When mode is update, require update_key."); + if (!task.getUpdateKeyName().isPresent() && client.getColumn(Id.FIELD) == null) { + throw new ConfigException("When mode is update, require update_key or id column."); } - client.validateUpdateKey(task.getUpdateKeyName().orElse(null)); + client.validateIdOrUpdateKey(task.getUpdateKeyName().orElse(Id.FIELD)); } @Override - public void add(Page page, KintonePageOutput output) { - output.updatePage(page); + public void add(Page page, Skip skip, KintonePageOutput output) { + Consumer consumer = skip == Skip.ALWAYS ? output::upsertPage : output::updatePage; + consumer.accept(page); } }, UPSERT("upsert") { @Override public void validate(PluginTask task, KintoneClient client) { - if (!task.getUpdateKeyName().isPresent()) { - throw new ConfigException("When mode is upsert, require update_key."); + if (!task.getUpdateKeyName().isPresent() && client.getColumn(Id.FIELD) == null) { + throw new ConfigException("When mode is upsert, require update_key or id column."); } - client.validateUpdateKey(task.getUpdateKeyName().orElse(null)); + client.validateIdOrUpdateKey(task.getUpdateKeyName().orElse(Id.FIELD)); } @Override - public void add(Page page, KintonePageOutput output) { + public void add(Page page, Skip skip, KintonePageOutput output) { output.upsertPage(page); } }; @@ -53,7 +57,7 @@ public void add(Page page, KintonePageOutput output) { public abstract void validate(PluginTask task, KintoneClient client); - public abstract void add(Page page, KintonePageOutput output); + public abstract void add(Page page, Skip skip, KintonePageOutput output); @Override public String toString() { diff --git a/src/main/java/org/embulk/output/kintone/KintonePageOutput.java b/src/main/java/org/embulk/output/kintone/KintonePageOutput.java index 12d9927..78e627b 100644 --- a/src/main/java/org/embulk/output/kintone/KintonePageOutput.java +++ b/src/main/java/org/embulk/output/kintone/KintonePageOutput.java @@ -10,7 +10,6 @@ import com.kintone.client.model.record.FieldType; import com.kintone.client.model.record.Record; import com.kintone.client.model.record.RecordForUpdate; -import com.kintone.client.model.record.UpdateKey; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.math.BigDecimal; @@ -26,6 +25,9 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Pair; import org.embulk.config.TaskReport; +import org.embulk.output.kintone.record.Id; +import org.embulk.output.kintone.record.IdOrUpdateKey; +import org.embulk.output.kintone.record.Skip; import org.embulk.output.kintone.util.Lazy; import org.embulk.spi.Exec; import org.embulk.spi.Page; @@ -60,7 +62,7 @@ public KintonePageOutput(PluginTask task, Schema schema) { @Override public void add(Page page) { - KintoneMode.of(task).add(page, this); + KintoneMode.of(task).add(page, task.getSkipIfNonExistingIdOrUpdateKey(), this); } @Override @@ -177,6 +179,7 @@ public void insertPage(Page page) { } public void updatePage(Page page) { + Skip skip = task.getSkipIfNonExistingIdOrUpdateKey(); List records = new ArrayList<>(); reader.setPage(page); KintoneColumnVisitor visitor = @@ -187,19 +190,21 @@ public void updatePage(Page page) { task.getPreferNulls(), task.getIgnoreNulls(), task.getReduceKeyName().orElse(null), - task.getUpdateKeyName().orElse(null)); + task.getUpdateKeyName().orElse(Id.FIELD)); while (reader.nextRecord()) { Record record = new Record(); - UpdateKey updateKey = new UpdateKey(); + IdOrUpdateKey idOrUpdateKey = new IdOrUpdateKey(); visitor.setRecord(record); - visitor.setUpdateKey(updateKey); + visitor.setIdOrUpdateKey(idOrUpdateKey); reader.getSchema().visitColumns(visitor); putWrongTypeFields(record); - if (updateKey.getValue() == null || updateKey.getValue().toString().isEmpty()) { - LOGGER.warn("Record skipped because no update key value was specified"); + if (skip == Skip.NEVER && !idOrUpdateKey.isPresent()) { + throw new RuntimeException("No id or update key value was specified"); + } else if (!idOrUpdateKey.isPresent()) { + LOGGER.warn("Record skipped because no id or update key value was specified"); continue; } - records.add(new RecordForUpdate(updateKey, record.removeField(updateKey.getField()))); + records.add(idOrUpdateKey.forUpdate(record)); if (records.size() == task.getChunkSize()) { update(records); records.clear(); @@ -212,7 +217,7 @@ public void updatePage(Page page) { public void upsertPage(Page page) { List records = new ArrayList<>(); - List updateKeys = new ArrayList<>(); + List idOrUpdateKeys = new ArrayList<>(); reader.setPage(page); KintoneColumnVisitor visitor = new KintoneColumnVisitor( @@ -222,39 +227,72 @@ public void upsertPage(Page page) { task.getPreferNulls(), task.getIgnoreNulls(), task.getReduceKeyName().orElse(null), - task.getUpdateKeyName().orElse(null)); + task.getUpdateKeyName().orElse(Id.FIELD)); while (reader.nextRecord()) { Record record = new Record(); - UpdateKey updateKey = new UpdateKey(); + IdOrUpdateKey idOrUpdateKey = new IdOrUpdateKey(); visitor.setRecord(record); - visitor.setUpdateKey(updateKey); + visitor.setIdOrUpdateKey(idOrUpdateKey); reader.getSchema().visitColumns(visitor); putWrongTypeFields(record); records.add(record); - updateKeys.add(updateKey); + idOrUpdateKeys.add(idOrUpdateKey); if (records.size() == UPSERT_BATCH_SIZE) { - upsert(records, updateKeys); + upsert(records, idOrUpdateKeys); records.clear(); - updateKeys.clear(); + idOrUpdateKeys.clear(); } } if (!records.isEmpty()) { - upsert(records, updateKeys); + upsert(records, idOrUpdateKeys); } } - private void upsert(List records, List updateKeys) { - if (records.size() != updateKeys.size()) { - throw new RuntimeException("records.size() != updateKeys.size()"); + private void upsert(List records, List idOrUpdateKeys) { + if (records.size() != idOrUpdateKeys.size()) { + throw new RuntimeException("records.size() != idOrUpdateKeys.size()"); } - List existingValues = executeWithRetry(() -> getExistingValuesByUpdateKey(updateKeys)); + Skip skip = task.getSkipIfNonExistingIdOrUpdateKey(); + String columnName = task.getUpdateKeyName().orElse(Id.FIELD); + boolean isId = columnName.equals(Id.FIELD); + List existingValues = + executeWithRetry(() -> getExistingValuesByIdOrUpdateKey(idOrUpdateKeys, columnName)); List insertRecords = new ArrayList<>(); List updateRecords = new ArrayList<>(); for (int i = 0; i < records.size(); i++) { + RecordForUpdate recordForUpdate = null; Record record = records.get(i); - UpdateKey updateKey = updateKeys.get(i); - if (existsRecord(existingValues, updateKey)) { - updateRecords.add(new RecordForUpdate(updateKey, record.removeField(updateKey.getField()))); + IdOrUpdateKey idOrUpdateKey = idOrUpdateKeys.get(i); + if (existsRecord(existingValues, idOrUpdateKey)) { + recordForUpdate = idOrUpdateKey.forUpdate(record); + } else if (skip == Skip.ALWAYS && idOrUpdateKey.isPresent()) { + LOGGER.warn( + "Record skipped because non existing id or update key '" + + idOrUpdateKey.getValue() + + "' was specified"); + continue; + } else if (skip == Skip.ALWAYS && !idOrUpdateKey.isPresent()) { + LOGGER.warn("Record skipped because no id or update key value was specified"); + continue; + } else if (skip == Skip.AUTO && idOrUpdateKey.isIdPresent()) { + LOGGER.warn( + "Record skipped because non existing id '" + + idOrUpdateKey.getValue() + + "' was specified"); + continue; + } else if (skip == Skip.AUTO && !isId && !idOrUpdateKey.isUpdateKeyPresent()) { + LOGGER.warn("Record skipped because no update key value was specified"); + continue; + } else if (idOrUpdateKey.isIdPresent()) { + LOGGER.warn( + "Record inserted though non existing id '" + + idOrUpdateKey.getValue() + + "' was specified"); + } else if (!isId && !idOrUpdateKey.isUpdateKeyPresent()) { + LOGGER.warn("Record inserted though no update key value was specified"); + } + if (recordForUpdate != null) { + updateRecords.add(recordForUpdate); } else { insertRecords.add(record); } @@ -274,16 +312,26 @@ private void upsert(List records, List updateKeys) { } } - private List getExistingValuesByUpdateKey(List updateKeys) { + private List getExistingValuesByIdOrUpdateKey( + List idOrUpdateKeys, String columnName) { List queryValues = - updateKeys.stream() - .filter(k -> k.getValue() != null && !k.getValue().toString().isEmpty()) + idOrUpdateKeys.stream() + .filter(IdOrUpdateKey::isPresent) .map(k -> "\"" + k.getValue() + "\"") .collect(Collectors.toList()); if (queryValues.isEmpty()) { return Collections.emptyList(); } - String columnName = task.getUpdateKeyName().orElse(null); + return columnName.equals(Id.FIELD) + ? getExistingValuesById(queryValues) + : getExistingValuesByUpdateKey(columnName, queryValues); + } + + private List getExistingValuesById(List queryValues) { + return getExistingValues(Id.FIELD, Record::getId, queryValues); + } + + private List getExistingValuesByUpdateKey(String columnName, List queryValues) { KintoneColumnOption option = task.getColumnOptions().get(columnName); String fieldCode = option != null ? option.getFieldCode() : columnName; KintoneColumnType type = KintoneColumnType.valueOf(getFieldType(fieldCode).name()); @@ -329,8 +377,8 @@ private FieldType getFieldType(String fieldCode) { return client.get().getFieldType(fieldCode); } - private static boolean existsRecord(List existingValues, UpdateKey updateKey) { - String value = toString(updateKey.getValue()); + private static boolean existsRecord(List existingValues, IdOrUpdateKey idOrUpdateKey) { + String value = toString(idOrUpdateKey.getValue()); return value != null && existingValues.stream().anyMatch(v -> v.equals(value)); } diff --git a/src/main/java/org/embulk/output/kintone/PluginTask.java b/src/main/java/org/embulk/output/kintone/PluginTask.java index dc41bb4..22631e3 100644 --- a/src/main/java/org/embulk/output/kintone/PluginTask.java +++ b/src/main/java/org/embulk/output/kintone/PluginTask.java @@ -7,6 +7,7 @@ import org.embulk.config.Config; import org.embulk.config.ConfigDefault; import org.embulk.config.Task; +import org.embulk.output.kintone.record.Skip; import org.embulk.spi.Column; public interface PluginTask extends Task { @@ -52,6 +53,10 @@ public interface PluginTask extends Task { @ConfigDefault("\"false\"") boolean getIgnoreNulls(); + @Config("skip_if_non_existing_id_or_update_key") + @ConfigDefault("\"auto\"") + Skip getSkipIfNonExistingIdOrUpdateKey(); + @Config("mode") @ConfigDefault("\"insert\"") String getMode(); diff --git a/src/main/java/org/embulk/output/kintone/record/Id.java b/src/main/java/org/embulk/output/kintone/record/Id.java new file mode 100644 index 0000000..9dcc723 --- /dev/null +++ b/src/main/java/org/embulk/output/kintone/record/Id.java @@ -0,0 +1,18 @@ +package org.embulk.output.kintone.record; + +public class Id { + public static final String FIELD = "$id"; + private Long value; + + public void setValue(Long value) { + this.value = value; + } + + public Long getValue() { + return value; + } + + public boolean isPresent() { + return value != null; + } +} diff --git a/src/main/java/org/embulk/output/kintone/record/IdOrUpdateKey.java b/src/main/java/org/embulk/output/kintone/record/IdOrUpdateKey.java new file mode 100644 index 0000000..8080bc6 --- /dev/null +++ b/src/main/java/org/embulk/output/kintone/record/IdOrUpdateKey.java @@ -0,0 +1,45 @@ +package org.embulk.output.kintone.record; + +import com.kintone.client.model.record.Record; +import com.kintone.client.model.record.RecordForUpdate; +import com.kintone.client.model.record.UpdateKey; + +public class IdOrUpdateKey { + private final Id id = new Id(); + private final UpdateKey updateKey = new UpdateKey(); + + public Id getId() { + return id; + } + + public UpdateKey getUpdateKey() { + return updateKey; + } + + public String getField() { + return isIdPresent() ? Id.FIELD : updateKey.getField(); + } + + public Object getValue() { + return isIdPresent() ? id.getValue() : updateKey.getValue(); + } + + public RecordForUpdate forUpdate(Record record) { + return isIdPresent() + ? new RecordForUpdate(id.getValue(), record) + : new RecordForUpdate(updateKey, record.removeField(updateKey.getField())); + } + + public boolean isPresent() { + return isIdPresent() || isUpdateKeyPresent(); + } + + public boolean isIdPresent() { + return id.isPresent(); + } + + public boolean isUpdateKeyPresent() { + Object value = updateKey.getValue(); + return value != null && !value.toString().isEmpty(); + } +} diff --git a/src/main/java/org/embulk/output/kintone/record/Skip.java b/src/main/java/org/embulk/output/kintone/record/Skip.java new file mode 100644 index 0000000..b1109ea --- /dev/null +++ b/src/main/java/org/embulk/output/kintone/record/Skip.java @@ -0,0 +1,14 @@ +package org.embulk.output.kintone.record; + +import com.fasterxml.jackson.annotation.JsonCreator; + +public enum Skip { + AUTO, + NEVER, + ALWAYS; + + @JsonCreator + public static Skip of(String name) { + return valueOf(name.toUpperCase()); + } +} diff --git a/src/test/java/org/embulk/output/kintone/KintoneClientTest.java b/src/test/java/org/embulk/output/kintone/KintoneClientTest.java index e3f2398..47ac133 100644 --- a/src/test/java/org/embulk/output/kintone/KintoneClientTest.java +++ b/src/test/java/org/embulk/output/kintone/KintoneClientTest.java @@ -9,6 +9,7 @@ import org.embulk.config.ConfigSource; import org.embulk.output.kintone.util.Lazy; import org.embulk.spi.Schema; +import org.embulk.spi.type.Type; import org.embulk.spi.type.Types; import org.junit.Before; import org.junit.Test; @@ -30,13 +31,17 @@ public void testInsert() { assertConfigException("When mode is insert, require no update_key."); merge(config("update_key: string_single_line_text")); assertConfigException("When mode is insert, require no update_key."); + merge(config("update_key: $id")); + assertConfigException("When mode is insert, require no update_key.", id(Types.LONG)); + merge(config("update_key: null")); + runWithMockClient(Lazy::get, id(Types.STRING)); } @Test public void testUpdate() { merge(config("mode: update")); merge(config("update_key: null")); - assertConfigException("When mode is update, require update_key."); + assertConfigException("When mode is update, require update_key or id column."); merge(config("update_key: non_existing_column")); assertConfigException("The column 'non_existing_column' for update does not exist."); merge(config("update_key: non_existing_field")); @@ -47,13 +52,17 @@ public void testUpdate() { runWithMockClient(Lazy::get); merge(config("update_key: string_single_line_text")); runWithMockClient(Lazy::get); + merge(config("update_key: $id")); + runWithMockClient(Lazy::get, id(Types.LONG)); + merge(config("update_key: null")); + assertConfigException("The id column must be 'long'.", id(Types.STRING)); } @Test public void testUpsert() { merge(config("mode: upsert")); merge(config("update_key: null")); - assertConfigException("When mode is upsert, require update_key."); + assertConfigException("When mode is upsert, require update_key or id column."); merge(config("update_key: non_existing_column")); assertConfigException("The column 'non_existing_column' for update does not exist."); merge(config("update_key: non_existing_field")); @@ -64,6 +73,10 @@ public void testUpsert() { runWithMockClient(Lazy::get); merge(config("update_key: string_single_line_text")); runWithMockClient(Lazy::get); + merge(config("update_key: $id")); + runWithMockClient(Lazy::get, id(Types.LONG)); + merge(config("update_key: null")); + assertConfigException("The id column must be 'long'.", id(Types.STRING)); } private void assertConfigException(String message) { @@ -112,6 +125,10 @@ private static Schema schema(Schema.Builder builder) { .build(); } + private static Schema.Builder id(Type type) { + return builder().add("$id", type); + } + private static Schema.Builder builder() { return Schema.builder(); } diff --git a/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorTest.java b/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorTest.java index 8254723..c42ac67 100644 --- a/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorTest.java +++ b/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorTest.java @@ -44,7 +44,7 @@ public class KintoneColumnVisitorTest { public void test() { KintoneColumnVisitorVerifier verifier = verifier(null, "STRING|SINGLE_LINE_TEXT"); verifier.verify( - (record, updateKey) -> { + (record, idOrUpdateKey) -> { assertThat(record.getSingleLineTextFieldValue("BOOLEAN|SINGLE_LINE_TEXT"), is("false")); assertThat(record.getNumberFieldValue("BOOLEAN"), is(number("0"))); assertThat(record.getSingleLineTextFieldValue("LONG|SINGLE_LINE_TEXT"), is("0")); @@ -104,11 +104,11 @@ public void test() { assertThat(record.getSubtableFieldValue("JSON|SUBTABLE"), is(list())); assertThat( record.getSingleLineTextFieldValue("JSON|SUBTABLE.SINGLE_LINE_TEXT"), is("\"\"")); - assertThat(updateKey.getField(), is("STRING|SINGLE_LINE_TEXT")); - assertThat(updateKey.getValue(), is("")); + assertThat(idOrUpdateKey.getField(), is("STRING|SINGLE_LINE_TEXT")); + assertThat(idOrUpdateKey.getValue(), is("")); }); verifier.verify( - (record, updateKey) -> { + (record, idOrUpdateKey) -> { assertThat(record.getSingleLineTextFieldValue("BOOLEAN|SINGLE_LINE_TEXT"), is("false")); assertThat(record.getNumberFieldValue("BOOLEAN"), is(number("0"))); assertThat(record.getSingleLineTextFieldValue("LONG|SINGLE_LINE_TEXT"), is("0")); @@ -168,11 +168,11 @@ public void test() { assertThat(record.getSubtableFieldValue("JSON|SUBTABLE"), is(list())); assertThat( record.getSingleLineTextFieldValue("JSON|SUBTABLE.SINGLE_LINE_TEXT"), is("\"\"")); - assertThat(updateKey.getField(), is("STRING|SINGLE_LINE_TEXT")); - assertThat(updateKey.getValue(), is("")); + assertThat(idOrUpdateKey.getField(), is("STRING|SINGLE_LINE_TEXT")); + assertThat(idOrUpdateKey.getValue(), is("")); }); verifier.verify( - (record, updateKey) -> { + (record, idOrUpdateKey) -> { assertThat(record.getSingleLineTextFieldValue("BOOLEAN|SINGLE_LINE_TEXT"), is("true")); assertThat(record.getNumberFieldValue("BOOLEAN"), is(number("1"))); assertThat(record.getSingleLineTextFieldValue("LONG|SINGLE_LINE_TEXT"), is("123")); @@ -233,11 +233,11 @@ public void test() { assertTableRows(record.getSubtableFieldValue("JSON|SUBTABLE"), rows(0L, 1L, 2L)); assertThat( record.getSingleLineTextFieldValue("JSON|SUBTABLE.SINGLE_LINE_TEXT"), is("\"abc\"")); - assertThat(updateKey.getField(), is("STRING|SINGLE_LINE_TEXT")); - assertThat(updateKey.getValue(), is("abc")); + assertThat(idOrUpdateKey.getField(), is("STRING|SINGLE_LINE_TEXT")); + assertThat(idOrUpdateKey.getValue(), is("abc")); }); verifier.verify( - (record, updateKey) -> { + (record, idOrUpdateKey) -> { assertThat(record.getSingleLineTextFieldValue("BOOLEAN|SINGLE_LINE_TEXT"), is("false")); assertThat(record.getNumberFieldValue("BOOLEAN"), is(number("0"))); assertThat(record.getSingleLineTextFieldValue("LONG|SINGLE_LINE_TEXT"), is("456")); @@ -298,8 +298,8 @@ public void test() { assertTableRows(record.getSubtableFieldValue("JSON|SUBTABLE"), rows(3L, 4L, 5L)); assertThat( record.getSingleLineTextFieldValue("JSON|SUBTABLE.SINGLE_LINE_TEXT"), is("\"def\"")); - assertThat(updateKey.getField(), is("STRING|SINGLE_LINE_TEXT")); - assertThat(updateKey.getValue(), is("def")); + assertThat(idOrUpdateKey.getField(), is("STRING|SINGLE_LINE_TEXT")); + assertThat(idOrUpdateKey.getValue(), is("def")); }); } @@ -307,7 +307,7 @@ public void test() { public void testPreferNulls() { KintoneColumnVisitorVerifier verifier = verifier(null, "LONG", true, false); verifier.verify( - (record, updateKey) -> { + (record, idOrUpdateKey) -> { assertThat( record.getFieldType("BOOLEAN|SINGLE_LINE_TEXT"), is(FieldType.SINGLE_LINE_TEXT)); assertThat(record.getFieldType("BOOLEAN"), is(FieldType.NUMBER)); @@ -415,8 +415,8 @@ public void testPreferNulls() { assertThat(record.getSubtableFieldValue("JSON|SUBTABLE"), is(list())); assertThat( record.getSingleLineTextFieldValue("JSON|SUBTABLE.SINGLE_LINE_TEXT"), nullValue()); - assertThat(updateKey.getField(), is("LONG")); - assertThat(updateKey.getValue(), nullValue()); + assertThat(idOrUpdateKey.getField(), is("LONG")); + assertThat(idOrUpdateKey.getValue(), nullValue()); }); } @@ -424,7 +424,7 @@ public void testPreferNulls() { public void testIgnoreNulls() { KintoneColumnVisitorVerifier verifier = verifier("JSON", "JSON", false, true); verifier.verify( - (record, updateKey) -> { + (record, idOrUpdateKey) -> { assertThat(record.getFieldValue("BOOLEAN|SINGLE_LINE_TEXT"), nullValue()); assertThat(record.getFieldValue("BOOLEAN"), nullValue()); assertThat(record.getFieldValue("LONG|SINGLE_LINE_TEXT"), nullValue()); @@ -475,8 +475,8 @@ public void testIgnoreNulls() { assertThat(record.getFieldValue("JSON"), nullValue()); assertThat(record.getFieldValue("JSON|SUBTABLE"), nullValue()); assertThat(record.getFieldValue("JSON|SUBTABLE.SINGLE_LINE_TEXT"), nullValue()); - assertThat(updateKey.getField(), nullValue()); - assertThat(updateKey.getValue(), nullValue()); + assertThat(idOrUpdateKey.getField(), nullValue()); + assertThat(idOrUpdateKey.getValue(), nullValue()); }, true); } @@ -485,7 +485,7 @@ public void testIgnoreNulls() { public void testReduceKey() { KintoneColumnVisitorVerifier verifier = verifier("JSON|SUBTABLE", null); verifier.verify( - (record, updateKey) -> { + (record, idOrUpdateKey) -> { assertThat(record.getSingleLineTextFieldValue("BOOLEAN|SINGLE_LINE_TEXT"), is("false")); assertThat(record.getNumberFieldValue("BOOLEAN"), is(number("0"))); assertThat(record.getSingleLineTextFieldValue("LONG|SINGLE_LINE_TEXT"), is("0")); @@ -544,8 +544,8 @@ public void testReduceKey() { assertThat(record.getMultiLineTextFieldValue("JSON"), is("\"\"")); assertThat(record.getSubtableFieldValue("JSON|SUBTABLE"), is(list())); assertThat(record.getFieldValue("JSON|SUBTABLE.SINGLE_LINE_TEXT"), nullValue()); - assertThat(updateKey.getField(), nullValue()); - assertThat(updateKey.getValue(), nullValue()); + assertThat(idOrUpdateKey.getField(), nullValue()); + assertThat(idOrUpdateKey.getValue(), nullValue()); }, true); } @@ -555,24 +555,24 @@ public void testUpdateKey() { assertThrows(UnsupportedOperationException.class, () -> verifier(null, "TIMESTAMP").verify()); KintoneColumnVisitorVerifier verifier = verifier(null, "TIMESTAMP|NUMBER"); verifier.verify( - (record, updateKey) -> { - assertThat(updateKey.getField(), is("TIMESTAMP|NUMBER")); - assertThat(updateKey.getValue(), is(number("0"))); + (record, idOrUpdateKey) -> { + assertThat(idOrUpdateKey.getField(), is("TIMESTAMP|NUMBER")); + assertThat(idOrUpdateKey.getValue(), is(number("0"))); }); verifier.verify( - (record, updateKey) -> { - assertThat(updateKey.getField(), is("TIMESTAMP|NUMBER")); - assertThat(updateKey.getValue(), is(number("0"))); + (record, idOrUpdateKey) -> { + assertThat(idOrUpdateKey.getField(), is("TIMESTAMP|NUMBER")); + assertThat(idOrUpdateKey.getValue(), is(number("0"))); }); verifier.verify( - (record, updateKey) -> { - assertThat(updateKey.getField(), is("TIMESTAMP|NUMBER")); - assertThat(updateKey.getValue(), is(number("946684799"))); + (record, idOrUpdateKey) -> { + assertThat(idOrUpdateKey.getField(), is("TIMESTAMP|NUMBER")); + assertThat(idOrUpdateKey.getValue(), is(number("946684799"))); }); verifier.verify( - (record, updateKey) -> { - assertThat(updateKey.getField(), is("TIMESTAMP|NUMBER")); - assertThat(updateKey.getValue(), is(number("946684800"))); + (record, idOrUpdateKey) -> { + assertThat(idOrUpdateKey.getField(), is("TIMESTAMP|NUMBER")); + assertThat(idOrUpdateKey.getValue(), is(number("946684800"))); }); } diff --git a/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorVerifier.java b/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorVerifier.java index 8ec59f5..f4765bd 100644 --- a/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorVerifier.java +++ b/src/test/java/org/embulk/output/kintone/KintoneColumnVisitorVerifier.java @@ -2,11 +2,11 @@ import com.kintone.client.model.record.FieldType; import com.kintone.client.model.record.Record; -import com.kintone.client.model.record.UpdateKey; import java.util.Map; import java.util.Set; import java.util.function.BiConsumer; import javax.validation.UnexpectedTypeException; +import org.embulk.output.kintone.record.IdOrUpdateKey; import org.embulk.spi.Column; import org.embulk.spi.Page; import org.embulk.spi.PageReader; @@ -50,21 +50,21 @@ public void verify() { verify((record, updateKey) -> {}); } - public void verify(BiConsumer consumer) { + public void verify(BiConsumer consumer) { verify(consumer, false); } - public void verify(BiConsumer consumer, boolean nullable) { + public void verify(BiConsumer consumer, boolean nullable) { if (!reader.nextRecord()) { throw new IllegalStateException(); } Record record = new Record(); visitor.setRecord(record); - UpdateKey updateKey = new UpdateKey(); - visitor.setUpdateKey(updateKey); + IdOrUpdateKey idOrUpdateKey = new IdOrUpdateKey(); + visitor.setIdOrUpdateKey(idOrUpdateKey); schema.visitColumns(visitor); schema.getColumns().forEach(column -> verify(record, column, nullable)); - consumer.accept(record, updateKey); + consumer.accept(record, idOrUpdateKey); } private void verify(Record record, Column column, boolean nullable) { diff --git a/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java b/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java index bde0c7b..b3e4591 100644 --- a/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java +++ b/src/test/java/org/embulk/output/kintone/KintonePageOutputVerifier.java @@ -27,6 +27,7 @@ import java.util.stream.IntStream; import org.embulk.config.TaskReport; import org.embulk.deps.buffer.PooledBufferAllocator; +import org.embulk.output.kintone.record.Id; import org.embulk.spi.BufferAllocator; import org.embulk.spi.Exec; import org.embulk.spi.Page; @@ -152,10 +153,12 @@ private List getValues() { field.matches("^.*_number$") ? ValueFactory.newString(((NumberFieldValue) value).getValue().toPlainString()) : ValueFactory.newString(((SingleLineTextFieldValue) value).getValue()); - List values = - getRecords().stream() - .map(record -> toValue.apply(record.getFieldValue(field)).toJson()) - .collect(Collectors.toList()); + Function toJson = + record -> + Id.FIELD.equals(field) + ? ValueFactory.newString(record.getId().toString()).toJson() + : toValue.apply(record.getFieldValue(field)).toJson(); + List values = getRecords().stream().map(toJson).collect(Collectors.toList()); values.addAll(addValues); return sorted(values); } @@ -165,7 +168,16 @@ private List getRecords() { } private Record getRecord(RecordForUpdate updateRecord) { + Long id = updateRecord.getId(); UpdateKey key = updateRecord.getUpdateKey(); + return id != null ? getRecord(id) : getRecord(key); + } + + private Record getRecord(Long id) { + return new Record(id, null); + } + + private Record getRecord(UpdateKey key) { String field = key.getField(); Object value = key.getValue(); return new Record() diff --git a/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java b/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java index fd4151a..77ab969 100644 --- a/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java +++ b/src/test/java/org/embulk/output/kintone/TestKintoneOutputPlugin.java @@ -26,6 +26,8 @@ import org.embulk.config.ConfigDiff; import org.embulk.config.ConfigSource; import org.embulk.config.TaskSource; +import org.embulk.output.kintone.record.Id; +import org.embulk.output.kintone.record.Skip; import org.embulk.spi.Column; import org.embulk.spi.Exec; import org.embulk.spi.OutputPlugin; @@ -132,15 +134,16 @@ private KintonePageOutputVerifier verifier(ConfigSource config) { String test = config.get(String.class, "domain"); String mode = config.get(String.class, "mode"); String field = config.get(String.class, "update_key", null); + Skip skip = config.get(Skip.class, "skip_if_non_existing_id_or_update_key"); boolean preferNulls = config.get(boolean.class, "prefer_nulls", false); boolean ignoreNulls = config.get(boolean.class, "ignore_nulls", false); return new KintonePageOutputVerifier( test, field, - getValues(test, mode, preferNulls, ignoreNulls, field), - getAddValues(test, mode, preferNulls, ignoreNulls, field), - getAddRecords(test, mode, preferNulls, ignoreNulls), - getUpdateRecords(test, mode, preferNulls, ignoreNulls, field)); + getValues(test, mode, skip, preferNulls, ignoreNulls, field), + getAddValues(test, mode, skip, preferNulls, ignoreNulls, field), + getAddRecords(test, mode, skip, preferNulls, ignoreNulls), + getUpdateRecords(test, mode, skip, preferNulls, ignoreNulls, field)); } private TransactionalPageOutput openWithVerifier( @@ -148,16 +151,17 @@ private TransactionalPageOutput openWithVerifier( String test = taskSource.get(String.class, "Domain"); String mode = taskSource.get(String.class, "Mode"); String field = taskSource.get(String.class, "UpdateKeyName"); + Skip skip = taskSource.get(Skip.class, "SkipIfNonExistingIdOrUpdateKey"); boolean preferNulls = taskSource.get(boolean.class, "PreferNulls"); boolean ignoreNulls = taskSource.get(boolean.class, "IgnoreNulls"); return new KintonePageOutputVerifier( super.open(taskSource, schema, taskIndex), test, field, - getValues(test, mode, preferNulls, ignoreNulls, field), - getAddValues(test, mode, preferNulls, ignoreNulls, field), - getAddRecords(test, mode, preferNulls, ignoreNulls), - getUpdateRecords(test, mode, preferNulls, ignoreNulls, field)); + getValues(test, mode, skip, preferNulls, ignoreNulls, field), + getAddValues(test, mode, skip, preferNulls, ignoreNulls, field), + getAddRecords(test, mode, skip, preferNulls, ignoreNulls), + getUpdateRecords(test, mode, skip, preferNulls, ignoreNulls, field)); } private static Set getDerivedColumns(String test) { @@ -171,12 +175,13 @@ private static Set getDerivedColumns(String test) { } private static List getValues( - String test, String mode, boolean preferNulls, boolean ignoreNulls, String field) { + String test, String mode, Skip skip, boolean preferNulls, boolean ignoreNulls, String field) { String name = String.format( - "%s/%s%s%s_values.json", + "%s/%s%s%s%s_values.json", test, mode, + format(skip), ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : "", format(field)); String json = existsResource(name) ? readResource(name) : null; @@ -188,12 +193,13 @@ private static List getValues( } private static List getAddValues( - String test, String mode, boolean preferNulls, boolean ignoreNulls, String field) { + String test, String mode, Skip skip, boolean preferNulls, boolean ignoreNulls, String field) { String name = String.format( - "%s/%s%s%s_add_values.json", + "%s/%s%s%s%s_add_values.json", test, mode, + format(skip), ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : "", format(field)); String json = existsResource(name) ? readResource(name) : null; @@ -205,11 +211,14 @@ private static List getAddValues( } private static List getAddRecords( - String test, String mode, boolean preferNulls, boolean ignoreNulls) { + String test, String mode, Skip skip, boolean preferNulls, boolean ignoreNulls) { String name = String.format( - "%s/%s%s_add_records.jsonl", - test, mode, ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : ""); + "%s/%s%s%s_add_records.jsonl", + test, + mode, + format(skip), + ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : ""); String jsonl = existsResource(name) ? readResource(name) : null; return jsonl == null || jsonl.isEmpty() ? Collections.emptyList() @@ -219,21 +228,33 @@ private static List getAddRecords( } private static List getUpdateRecords( - String test, String mode, boolean preferNulls, boolean ignoreNulls, String field) { + String test, String mode, Skip skip, boolean preferNulls, boolean ignoreNulls, String field) { Function key = getKey(field); + Function forUpdate = + record -> + Id.FIELD.equals(field) + ? new RecordForUpdate(record.getId(), Record.newFrom(record)) + : new RecordForUpdate(key.apply(record), record.removeField(field)); String name = String.format( - "%s/%s%s_update_records.jsonl", - test, mode, ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : ""); + "%s/%s%s%s_update_records.jsonl", + test, + mode, + format(skip), + ignoreNulls ? "_ignore_nulls" : preferNulls ? "_prefer_nulls" : ""); String jsonl = existsResource(name) ? readResource(name) : null; return jsonl == null || jsonl.isEmpty() ? Collections.emptyList() : Arrays.stream(jsonl.split("\\r?\\n|\\r")) .map(s -> Json.parse(s, Record.class)) - .map(record -> new RecordForUpdate(key.apply(record), record.removeField(field))) + .map(forUpdate) .collect(Collectors.toList()); } + private static String format(Skip skip) { + return skip == Skip.AUTO ? "" : String.format("_%s_skip", skip.name().toLowerCase()); + } + private static String format(String string) { return string == null ? "" : String.format("_%s", string.replace('$', '_')); } diff --git a/src/test/java/org/embulk/output/kintone/TestTaskMode.java b/src/test/java/org/embulk/output/kintone/TestTaskMode.java index 91b70c6..8cb61ab 100644 --- a/src/test/java/org/embulk/output/kintone/TestTaskMode.java +++ b/src/test/java/org/embulk/output/kintone/TestTaskMode.java @@ -34,6 +34,7 @@ public void testUpdate() throws Exception { @Test public void testUpsert() throws Exception { merge(config("mode: upsert", "update_key: double_single_line_text")); + merge(config("skip_if_non_existing_id_or_update_key: never")); runOutput(); merge(config("prefer_nulls: true")); runOutput(); diff --git a/src/test/java/org/embulk/output/kintone/TestTaskReduce.java b/src/test/java/org/embulk/output/kintone/TestTaskReduce.java index 9302128..370abd5 100644 --- a/src/test/java/org/embulk/output/kintone/TestTaskReduce.java +++ b/src/test/java/org/embulk/output/kintone/TestTaskReduce.java @@ -36,6 +36,7 @@ public void testUpdate() throws Exception { @Test public void testUpsert() throws Exception { merge(config("mode: upsert", "update_key: double_single_line_text")); + merge(config("skip_if_non_existing_id_or_update_key: never")); merge(config("reduce_key: string_number")); runOutput(); merge(config("prefer_nulls: true")); diff --git a/src/test/java/org/embulk/output/kintone/TestTaskReduceSubtable.java b/src/test/java/org/embulk/output/kintone/TestTaskReduceSubtable.java index 7ef36d4..03e13f4 100644 --- a/src/test/java/org/embulk/output/kintone/TestTaskReduceSubtable.java +++ b/src/test/java/org/embulk/output/kintone/TestTaskReduceSubtable.java @@ -36,6 +36,7 @@ public void testUpdate() throws Exception { @Test public void testUpsert() throws Exception { merge(config("mode: upsert", "update_key: double_single_line_text")); + merge(config("skip_if_non_existing_id_or_update_key: never")); merge(config("reduce_key: double_single_line_text")); runOutput(); merge(config("prefer_nulls: true")); diff --git a/src/test/java/org/embulk/output/kintone/TestTaskSkip.java b/src/test/java/org/embulk/output/kintone/TestTaskSkip.java new file mode 100644 index 0000000..77356bd --- /dev/null +++ b/src/test/java/org/embulk/output/kintone/TestTaskSkip.java @@ -0,0 +1,80 @@ +package org.embulk.output.kintone; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThrows; + +import net.jcip.annotations.NotThreadSafe; +import org.embulk.exec.PartialExecutionException; +import org.junit.Test; + +@NotThreadSafe +public class TestTaskSkip extends TestTask { + @Override + public void before() { + super.before(); + merge(config("domain: task/skip")); + } + + @Test + public void testInsert() throws Exception { + merge(config("mode: insert", "update_key: null")); + merge(config("skip_if_non_existing_id_or_update_key: auto")); + runOutput(); + merge(config("skip_if_non_existing_id_or_update_key: never")); + runOutput(); + merge(config("skip_if_non_existing_id_or_update_key: always")); + runOutput(); + merge(config("prefer_nulls: true")); + merge(config("skip_if_non_existing_id_or_update_key: auto")); + runOutput(); + merge(config("skip_if_non_existing_id_or_update_key: never")); + runOutput(); + merge(config("skip_if_non_existing_id_or_update_key: always")); + runOutput(); + } + + @Test + public void testUpdate() throws Exception { + merge(config("mode: update", "update_key: long_number")); + merge(config("skip_if_non_existing_id_or_update_key: auto")); + runOutput(); + merge(config("skip_if_non_existing_id_or_update_key: never")); + runOutput(); + merge(config("skip_if_non_existing_id_or_update_key: always")); + runOutput(); + merge(config("prefer_nulls: true")); + merge(config("skip_if_non_existing_id_or_update_key: auto")); + runOutput(); + merge(config("skip_if_non_existing_id_or_update_key: never")); + assertNoIdOrUpdateKeyValueWasSpecified(); + merge(config("skip_if_non_existing_id_or_update_key: always")); + runOutput(); + } + + @Test + public void testUpsert() throws Exception { + merge(config("mode: upsert", "update_key: string_single_line_text")); + merge(config("skip_if_non_existing_id_or_update_key: auto")); + runOutput(); + merge(config("skip_if_non_existing_id_or_update_key: never")); + runOutput(); + merge(config("skip_if_non_existing_id_or_update_key: always")); + runOutput(); + merge(config("prefer_nulls: true")); + merge(config("skip_if_non_existing_id_or_update_key: auto")); + runOutput(); + merge(config("skip_if_non_existing_id_or_update_key: never")); + runOutput(); + merge(config("skip_if_non_existing_id_or_update_key: always")); + runOutput(); + } + + private void assertNoIdOrUpdateKeyValueWasSpecified() { + Exception e = assertThrows(PartialExecutionException.class, this::runOutput); + assertThat(e.getCause(), is(instanceOf(RuntimeException.class))); + assertThat(e.getCause().getCause(), is(instanceOf(RuntimeException.class))); + assertThat(e.getCause().getCause().getMessage(), is("No id or update key value was specified")); + } +} diff --git a/src/test/java/org/embulk/output/kintone/TestTaskSkipId.java b/src/test/java/org/embulk/output/kintone/TestTaskSkipId.java new file mode 100644 index 0000000..1f215d0 --- /dev/null +++ b/src/test/java/org/embulk/output/kintone/TestTaskSkipId.java @@ -0,0 +1,80 @@ +package org.embulk.output.kintone; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThrows; + +import net.jcip.annotations.NotThreadSafe; +import org.embulk.exec.PartialExecutionException; +import org.junit.Test; + +@NotThreadSafe +public class TestTaskSkipId extends TestTask { + @Override + public void before() { + super.before(); + merge(config("domain: task/skip_id")); + } + + @Test + public void testInsert() throws Exception { + merge(config("mode: insert")); + merge(config("skip_if_non_existing_id_or_update_key: auto")); + runOutput(); + merge(config("skip_if_non_existing_id_or_update_key: never")); + runOutput(); + merge(config("skip_if_non_existing_id_or_update_key: always")); + runOutput(); + merge(config("prefer_nulls: true")); + merge(config("skip_if_non_existing_id_or_update_key: auto")); + runOutput(); + merge(config("skip_if_non_existing_id_or_update_key: never")); + runOutput(); + merge(config("skip_if_non_existing_id_or_update_key: always")); + runOutput(); + } + + @Test + public void testUpdate() throws Exception { + merge(config("mode: update", "update_key: $id")); + merge(config("skip_if_non_existing_id_or_update_key: auto")); + runOutput(); + merge(config("skip_if_non_existing_id_or_update_key: never")); + runOutput(); + merge(config("skip_if_non_existing_id_or_update_key: always")); + runOutput(); + merge(config("prefer_nulls: true")); + merge(config("skip_if_non_existing_id_or_update_key: auto")); + runOutput(); + merge(config("skip_if_non_existing_id_or_update_key: never")); + assertNoIdOrUpdateKeyValueWasSpecified(); + merge(config("skip_if_non_existing_id_or_update_key: always")); + runOutput(); + } + + @Test + public void testUpsert() throws Exception { + merge(config("mode: upsert", "update_key: $id")); + merge(config("skip_if_non_existing_id_or_update_key: auto")); + runOutput(); + merge(config("skip_if_non_existing_id_or_update_key: never")); + runOutput(); + merge(config("skip_if_non_existing_id_or_update_key: always")); + runOutput(); + merge(config("prefer_nulls: true")); + merge(config("skip_if_non_existing_id_or_update_key: auto")); + runOutput(); + merge(config("skip_if_non_existing_id_or_update_key: never")); + runOutput(); + merge(config("skip_if_non_existing_id_or_update_key: always")); + runOutput(); + } + + private void assertNoIdOrUpdateKeyValueWasSpecified() { + Exception e = assertThrows(PartialExecutionException.class, this::runOutput); + assertThat(e.getCause(), is(instanceOf(RuntimeException.class))); + assertThat(e.getCause().getCause(), is(instanceOf(RuntimeException.class))); + assertThat(e.getCause().getCause().getMessage(), is("No id or update key value was specified")); + } +} diff --git a/src/test/resources/org/embulk/output/kintone/task/config.yml b/src/test/resources/org/embulk/output/kintone/task/config.yml index 2bce816..6756f5b 100644 --- a/src/test/resources/org/embulk/output/kintone/task/config.yml +++ b/src/test/resources/org/embulk/output/kintone/task/config.yml @@ -1 +1,2 @@ max_sort_memory: 67108864 +skip_if_non_existing_id_or_update_key: auto diff --git a/src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/mode/upsert_never_skip_add_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/mode/upsert_add_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/mode/upsert_never_skip_add_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/mode/upsert_double_single_line_text_add_values.json b/src/test/resources/org/embulk/output/kintone/task/mode/upsert_never_skip_double_single_line_text_add_values.json similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/mode/upsert_double_single_line_text_add_values.json rename to src/test/resources/org/embulk/output/kintone/task/mode/upsert_never_skip_double_single_line_text_add_values.json diff --git a/src/test/resources/org/embulk/output/kintone/task/mode/upsert_double_single_line_text_values.json b/src/test/resources/org/embulk/output/kintone/task/mode/upsert_never_skip_double_single_line_text_values.json similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/mode/upsert_double_single_line_text_values.json rename to src/test/resources/org/embulk/output/kintone/task/mode/upsert_never_skip_double_single_line_text_values.json diff --git a/src/test/resources/org/embulk/output/kintone/task/mode/upsert_ignore_nulls_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/mode/upsert_never_skip_ignore_nulls_add_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/mode/upsert_ignore_nulls_add_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/mode/upsert_never_skip_ignore_nulls_add_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/mode/upsert_ignore_nulls_double_single_line_text_values.json b/src/test/resources/org/embulk/output/kintone/task/mode/upsert_never_skip_ignore_nulls_double_single_line_text_values.json similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/mode/upsert_ignore_nulls_double_single_line_text_values.json rename to src/test/resources/org/embulk/output/kintone/task/mode/upsert_never_skip_ignore_nulls_double_single_line_text_values.json diff --git a/src/test/resources/org/embulk/output/kintone/task/mode/upsert_ignore_nulls_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/mode/upsert_never_skip_ignore_nulls_update_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/mode/upsert_ignore_nulls_update_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/mode/upsert_never_skip_ignore_nulls_update_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/mode/upsert_prefer_nulls_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/mode/upsert_never_skip_prefer_nulls_add_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/mode/upsert_prefer_nulls_add_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/mode/upsert_never_skip_prefer_nulls_add_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/mode/upsert_prefer_nulls_double_single_line_text_values.json b/src/test/resources/org/embulk/output/kintone/task/mode/upsert_never_skip_prefer_nulls_double_single_line_text_values.json similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/mode/upsert_prefer_nulls_double_single_line_text_values.json rename to src/test/resources/org/embulk/output/kintone/task/mode/upsert_never_skip_prefer_nulls_double_single_line_text_values.json diff --git a/src/test/resources/org/embulk/output/kintone/task/mode/upsert_prefer_nulls_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/mode/upsert_never_skip_prefer_nulls_update_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/mode/upsert_prefer_nulls_update_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/mode/upsert_never_skip_prefer_nulls_update_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/mode/upsert_never_skip_update_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/mode/upsert_update_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/mode/upsert_never_skip_update_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_never_skip_add_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce/upsert_add_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce/upsert_never_skip_add_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_double_single_line_text_add_values.json b/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_never_skip_double_single_line_text_add_values.json similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce/upsert_double_single_line_text_add_values.json rename to src/test/resources/org/embulk/output/kintone/task/reduce/upsert_never_skip_double_single_line_text_add_values.json diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_double_single_line_text_values.json b/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_never_skip_double_single_line_text_values.json similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce/upsert_double_single_line_text_values.json rename to src/test/resources/org/embulk/output/kintone/task/reduce/upsert_never_skip_double_single_line_text_values.json diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_ignore_nulls_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_never_skip_ignore_nulls_add_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce/upsert_ignore_nulls_add_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce/upsert_never_skip_ignore_nulls_add_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_ignore_nulls_double_single_line_text_values.json b/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_never_skip_ignore_nulls_double_single_line_text_values.json similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce/upsert_ignore_nulls_double_single_line_text_values.json rename to src/test/resources/org/embulk/output/kintone/task/reduce/upsert_never_skip_ignore_nulls_double_single_line_text_values.json diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_ignore_nulls_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_never_skip_ignore_nulls_update_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce/upsert_ignore_nulls_update_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce/upsert_never_skip_ignore_nulls_update_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_prefer_nulls_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_never_skip_prefer_nulls_add_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce/upsert_prefer_nulls_add_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce/upsert_never_skip_prefer_nulls_add_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_prefer_nulls_double_single_line_text_values.json b/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_never_skip_prefer_nulls_double_single_line_text_values.json similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce/upsert_prefer_nulls_double_single_line_text_values.json rename to src/test/resources/org/embulk/output/kintone/task/reduce/upsert_never_skip_prefer_nulls_double_single_line_text_values.json diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_prefer_nulls_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_never_skip_prefer_nulls_update_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce/upsert_prefer_nulls_update_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce/upsert_never_skip_prefer_nulls_update_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce/upsert_never_skip_update_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce/upsert_update_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce/upsert_never_skip_update_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_double_single_line_text_values.json b/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_never_skip_double_single_line_text_values.json similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_double_single_line_text_values.json rename to src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_never_skip_double_single_line_text_values.json diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_ignore_nulls_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_never_skip_ignore_nulls_add_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_ignore_nulls_add_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_never_skip_ignore_nulls_add_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_ignore_nulls_double_single_line_text_values.json b/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_never_skip_ignore_nulls_double_single_line_text_values.json similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_ignore_nulls_double_single_line_text_values.json rename to src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_never_skip_ignore_nulls_double_single_line_text_values.json diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_ignore_nulls_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_never_skip_ignore_nulls_update_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_ignore_nulls_update_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_never_skip_ignore_nulls_update_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_prefer_nulls_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_never_skip_prefer_nulls_add_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_prefer_nulls_add_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_never_skip_prefer_nulls_add_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_prefer_nulls_double_single_line_text_values.json b/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_never_skip_prefer_nulls_double_single_line_text_values.json similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_prefer_nulls_double_single_line_text_values.json rename to src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_never_skip_prefer_nulls_double_single_line_text_values.json diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_prefer_nulls_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_never_skip_prefer_nulls_update_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_prefer_nulls_update_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_never_skip_prefer_nulls_update_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_never_skip_update_records.jsonl similarity index 100% rename from src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_update_records.jsonl rename to src/test/resources/org/embulk/output/kintone/task/reduce_subtable/upsert_never_skip_update_records.jsonl diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/config.yml b/src/test/resources/org/embulk/output/kintone/task/skip/config.yml new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/config.yml @@ -0,0 +1 @@ +{} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/input.csv b/src/test/resources/org/embulk/output/kintone/task/skip/input.csv new file mode 100644 index 0000000..a9e8d9e --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/input.csv @@ -0,0 +1,7 @@ +$id:long,long_number:long,string_single_line_text:string,boolean:boolean,long:long,double:double,string:string,timestamp:timestamp,json:json +123,123,abc,true,123,123,abc,1999-12-31 23:59:59.000 +0000,"""abc""" +,,,,,,,, +456,456,def,false,456,456,def,2000-01-01 00:00:00.000 +0000,"""def""" +,,,,,,,, +789,789,ghi,,789,789,ghi,,"""ghi""" +,,,,,,,, diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/insert_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip/insert_add_records.jsonl new file mode 100644 index 0000000..e6145bb --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/insert_add_records.jsonl @@ -0,0 +1,6 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/insert_always_skip_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip/insert_always_skip_add_records.jsonl new file mode 100644 index 0000000..e6145bb --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/insert_always_skip_add_records.jsonl @@ -0,0 +1,6 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/insert_always_skip_prefer_nulls_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip/insert_always_skip_prefer_nulls_add_records.jsonl new file mode 100644 index 0000000..92cfb06 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/insert_always_skip_prefer_nulls_add_records.jsonl @@ -0,0 +1,6 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} +{"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} +{"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/insert_never_skip_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip/insert_never_skip_add_records.jsonl new file mode 100644 index 0000000..e6145bb --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/insert_never_skip_add_records.jsonl @@ -0,0 +1,6 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/insert_never_skip_prefer_nulls_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip/insert_never_skip_prefer_nulls_add_records.jsonl new file mode 100644 index 0000000..92cfb06 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/insert_never_skip_prefer_nulls_add_records.jsonl @@ -0,0 +1,6 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} +{"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} +{"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/insert_prefer_nulls_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip/insert_prefer_nulls_add_records.jsonl new file mode 100644 index 0000000..92cfb06 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/insert_prefer_nulls_add_records.jsonl @@ -0,0 +1,6 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} +{"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} +{"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_long_number_add_values.json b/src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_long_number_add_values.json new file mode 100644 index 0000000..2f825cf --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_long_number_add_values.json @@ -0,0 +1 @@ +["789"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_long_number_values.json b/src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_long_number_values.json new file mode 100644 index 0000000..83ed89d --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_long_number_values.json @@ -0,0 +1 @@ +["123","0","456","0","789","0"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_prefer_nulls_long_number_add_values.json b/src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_prefer_nulls_long_number_add_values.json new file mode 100644 index 0000000..2f825cf --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_prefer_nulls_long_number_add_values.json @@ -0,0 +1 @@ +["789"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_prefer_nulls_long_number_values.json b/src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_prefer_nulls_long_number_values.json new file mode 100644 index 0000000..c4c83f0 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_prefer_nulls_long_number_values.json @@ -0,0 +1 @@ +["123","456","789"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_prefer_nulls_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_prefer_nulls_update_records.jsonl new file mode 100644 index 0000000..e845116 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_prefer_nulls_update_records.jsonl @@ -0,0 +1,2 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_update_records.jsonl new file mode 100644 index 0000000..a0ffd02 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/update_always_skip_update_records.jsonl @@ -0,0 +1,5 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/update_never_skip_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip/update_never_skip_update_records.jsonl new file mode 100644 index 0000000..e6145bb --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/update_never_skip_update_records.jsonl @@ -0,0 +1,6 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/update_prefer_nulls_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip/update_prefer_nulls_update_records.jsonl new file mode 100644 index 0000000..998ef19 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/update_prefer_nulls_update_records.jsonl @@ -0,0 +1,3 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/update_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip/update_update_records.jsonl new file mode 100644 index 0000000..e6145bb --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/update_update_records.jsonl @@ -0,0 +1,6 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/upsert_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_add_records.jsonl new file mode 100644 index 0000000..3d1d33d --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_add_records.jsonl @@ -0,0 +1 @@ +{"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"SINGLE_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_prefer_nulls_string_single_line_text_add_values.json b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_prefer_nulls_string_single_line_text_add_values.json new file mode 100644 index 0000000..11d928d --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_prefer_nulls_string_single_line_text_add_values.json @@ -0,0 +1 @@ +["ghi"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_prefer_nulls_string_single_line_text_values.json b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_prefer_nulls_string_single_line_text_values.json new file mode 100644 index 0000000..08a60b7 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_prefer_nulls_string_single_line_text_values.json @@ -0,0 +1 @@ +["abc","def","ghi"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_prefer_nulls_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_prefer_nulls_update_records.jsonl new file mode 100644 index 0000000..26f0fb6 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_prefer_nulls_update_records.jsonl @@ -0,0 +1,2 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"SINGLE_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"SINGLE_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_string_single_line_text_add_values.json b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_string_single_line_text_add_values.json new file mode 100644 index 0000000..11d928d --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_string_single_line_text_add_values.json @@ -0,0 +1 @@ +["ghi"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_string_single_line_text_values.json b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_string_single_line_text_values.json new file mode 100644 index 0000000..08a60b7 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_string_single_line_text_values.json @@ -0,0 +1 @@ +["abc","def","ghi"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_update_records.jsonl new file mode 100644 index 0000000..26f0fb6 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_always_skip_update_records.jsonl @@ -0,0 +1,2 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"SINGLE_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"SINGLE_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_add_records.jsonl new file mode 100644 index 0000000..71759e0 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_add_records.jsonl @@ -0,0 +1,4 @@ +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"SINGLE_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"SINGLE_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"SINGLE_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"SINGLE_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_prefer_nulls_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_prefer_nulls_add_records.jsonl new file mode 100644 index 0000000..267f9a8 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_prefer_nulls_add_records.jsonl @@ -0,0 +1,4 @@ +{"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"SINGLE_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} +{"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"SINGLE_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} +{"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"SINGLE_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} +{"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"SINGLE_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_prefer_nulls_string_single_line_text_add_values.json b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_prefer_nulls_string_single_line_text_add_values.json new file mode 100644 index 0000000..11d928d --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_prefer_nulls_string_single_line_text_add_values.json @@ -0,0 +1 @@ +["ghi"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_prefer_nulls_string_single_line_text_values.json b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_prefer_nulls_string_single_line_text_values.json new file mode 100644 index 0000000..08a60b7 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_prefer_nulls_string_single_line_text_values.json @@ -0,0 +1 @@ +["abc","def","ghi"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_prefer_nulls_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_prefer_nulls_update_records.jsonl new file mode 100644 index 0000000..26f0fb6 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_prefer_nulls_update_records.jsonl @@ -0,0 +1,2 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"SINGLE_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"SINGLE_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_string_single_line_text_add_values.json b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_string_single_line_text_add_values.json new file mode 100644 index 0000000..11d928d --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_string_single_line_text_add_values.json @@ -0,0 +1 @@ +["ghi"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_string_single_line_text_values.json b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_string_single_line_text_values.json new file mode 100644 index 0000000..08a60b7 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_string_single_line_text_values.json @@ -0,0 +1 @@ +["abc","def","ghi"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_update_records.jsonl new file mode 100644 index 0000000..26f0fb6 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_never_skip_update_records.jsonl @@ -0,0 +1,2 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"SINGLE_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"SINGLE_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/upsert_prefer_nulls_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_prefer_nulls_add_records.jsonl new file mode 100644 index 0000000..29e9d2c --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_prefer_nulls_add_records.jsonl @@ -0,0 +1 @@ +{"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"SINGLE_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/upsert_prefer_nulls_string_single_line_text_add_values.json b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_prefer_nulls_string_single_line_text_add_values.json new file mode 100644 index 0000000..11d928d --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_prefer_nulls_string_single_line_text_add_values.json @@ -0,0 +1 @@ +["ghi"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/upsert_prefer_nulls_string_single_line_text_values.json b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_prefer_nulls_string_single_line_text_values.json new file mode 100644 index 0000000..08a60b7 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_prefer_nulls_string_single_line_text_values.json @@ -0,0 +1 @@ +["abc","def","ghi"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/upsert_prefer_nulls_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_prefer_nulls_update_records.jsonl new file mode 100644 index 0000000..26f0fb6 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_prefer_nulls_update_records.jsonl @@ -0,0 +1,2 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"SINGLE_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"SINGLE_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/upsert_string_single_line_text_add_values.json b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_string_single_line_text_add_values.json new file mode 100644 index 0000000..11d928d --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_string_single_line_text_add_values.json @@ -0,0 +1 @@ +["ghi"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/upsert_string_single_line_text_values.json b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_string_single_line_text_values.json new file mode 100644 index 0000000..08a60b7 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_string_single_line_text_values.json @@ -0,0 +1 @@ +["abc","def","ghi"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip/upsert_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_update_records.jsonl new file mode 100644 index 0000000..26f0fb6 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip/upsert_update_records.jsonl @@ -0,0 +1,2 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"SINGLE_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"SINGLE_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/config.yml b/src/test/resources/org/embulk/output/kintone/task/skip_id/config.yml new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/config.yml @@ -0,0 +1 @@ +{} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/input.csv b/src/test/resources/org/embulk/output/kintone/task/skip_id/input.csv new file mode 100644 index 0000000..a9e8d9e --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/input.csv @@ -0,0 +1,7 @@ +$id:long,long_number:long,string_single_line_text:string,boolean:boolean,long:long,double:double,string:string,timestamp:timestamp,json:json +123,123,abc,true,123,123,abc,1999-12-31 23:59:59.000 +0000,"""abc""" +,,,,,,,, +456,456,def,false,456,456,def,2000-01-01 00:00:00.000 +0000,"""def""" +,,,,,,,, +789,789,ghi,,789,789,ghi,,"""ghi""" +,,,,,,,, diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/insert_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip_id/insert_add_records.jsonl new file mode 100644 index 0000000..e6145bb --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/insert_add_records.jsonl @@ -0,0 +1,6 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/insert_always_skip_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip_id/insert_always_skip_add_records.jsonl new file mode 100644 index 0000000..e6145bb --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/insert_always_skip_add_records.jsonl @@ -0,0 +1,6 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/insert_always_skip_prefer_nulls_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip_id/insert_always_skip_prefer_nulls_add_records.jsonl new file mode 100644 index 0000000..92cfb06 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/insert_always_skip_prefer_nulls_add_records.jsonl @@ -0,0 +1,6 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} +{"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} +{"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/insert_never_skip_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip_id/insert_never_skip_add_records.jsonl new file mode 100644 index 0000000..e6145bb --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/insert_never_skip_add_records.jsonl @@ -0,0 +1,6 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} +{"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/insert_never_skip_prefer_nulls_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip_id/insert_never_skip_prefer_nulls_add_records.jsonl new file mode 100644 index 0000000..92cfb06 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/insert_never_skip_prefer_nulls_add_records.jsonl @@ -0,0 +1,6 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} +{"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} +{"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/insert_prefer_nulls_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip_id/insert_prefer_nulls_add_records.jsonl new file mode 100644 index 0000000..92cfb06 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/insert_prefer_nulls_add_records.jsonl @@ -0,0 +1,6 @@ +{"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} +{"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} +{"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} +{"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip__id_add_values.json b/src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip__id_add_values.json new file mode 100644 index 0000000..2f825cf --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip__id_add_values.json @@ -0,0 +1 @@ +["789"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip__id_values.json b/src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip__id_values.json new file mode 100644 index 0000000..83ed89d --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip__id_values.json @@ -0,0 +1 @@ +["123","0","456","0","789","0"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip_prefer_nulls__id_add_values.json b/src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip_prefer_nulls__id_add_values.json new file mode 100644 index 0000000..2f825cf --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip_prefer_nulls__id_add_values.json @@ -0,0 +1 @@ +["789"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip_prefer_nulls__id_values.json b/src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip_prefer_nulls__id_values.json new file mode 100644 index 0000000..c4c83f0 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip_prefer_nulls__id_values.json @@ -0,0 +1 @@ +["123","456","789"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip_prefer_nulls_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip_prefer_nulls_update_records.jsonl new file mode 100644 index 0000000..f0881e1 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip_prefer_nulls_update_records.jsonl @@ -0,0 +1,2 @@ +{"$id":{"type":"__ID__","value":"123"},"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"$id":{"type":"__ID__","value":"456"},"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip_update_records.jsonl new file mode 100644 index 0000000..cedb6a3 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/update_always_skip_update_records.jsonl @@ -0,0 +1,5 @@ +{"$id":{"type":"__ID__","value":"123"},"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"$id":{"type":"__ID__","value":"0"},"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"$id":{"type":"__ID__","value":"456"},"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"$id":{"type":"__ID__","value":"0"},"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"$id":{"type":"__ID__","value":"0"},"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/update_never_skip_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip_id/update_never_skip_update_records.jsonl new file mode 100644 index 0000000..bbea5d5 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/update_never_skip_update_records.jsonl @@ -0,0 +1,6 @@ +{"$id":{"type":"__ID__","value":"123"},"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"$id":{"type":"__ID__","value":"0"},"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"$id":{"type":"__ID__","value":"456"},"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"$id":{"type":"__ID__","value":"0"},"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"$id":{"type":"__ID__","value":"789"},"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} +{"$id":{"type":"__ID__","value":"0"},"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/update_prefer_nulls_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip_id/update_prefer_nulls_update_records.jsonl new file mode 100644 index 0000000..6b0a3d9 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/update_prefer_nulls_update_records.jsonl @@ -0,0 +1,3 @@ +{"$id":{"type":"__ID__","value":"123"},"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"$id":{"type":"__ID__","value":"456"},"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"$id":{"type":"__ID__","value":"789"},"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/update_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip_id/update_update_records.jsonl new file mode 100644 index 0000000..bbea5d5 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/update_update_records.jsonl @@ -0,0 +1,6 @@ +{"$id":{"type":"__ID__","value":"123"},"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"$id":{"type":"__ID__","value":"0"},"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"$id":{"type":"__ID__","value":"456"},"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"$id":{"type":"__ID__","value":"0"},"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"$id":{"type":"__ID__","value":"789"},"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} +{"$id":{"type":"__ID__","value":"0"},"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert__id_add_values.json b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert__id_add_values.json new file mode 100644 index 0000000..2f825cf --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert__id_add_values.json @@ -0,0 +1 @@ +["789"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert__id_values.json b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert__id_values.json new file mode 100644 index 0000000..83ed89d --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert__id_values.json @@ -0,0 +1 @@ +["123","0","456","0","789","0"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip__id_add_values.json b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip__id_add_values.json new file mode 100644 index 0000000..2f825cf --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip__id_add_values.json @@ -0,0 +1 @@ +["789"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip__id_values.json b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip__id_values.json new file mode 100644 index 0000000..83ed89d --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip__id_values.json @@ -0,0 +1 @@ +["123","0","456","0","789","0"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip_prefer_nulls__id_add_values.json b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip_prefer_nulls__id_add_values.json new file mode 100644 index 0000000..2f825cf --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip_prefer_nulls__id_add_values.json @@ -0,0 +1 @@ +["789"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip_prefer_nulls__id_values.json b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip_prefer_nulls__id_values.json new file mode 100644 index 0000000..c4c83f0 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip_prefer_nulls__id_values.json @@ -0,0 +1 @@ +["123","456","789"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip_prefer_nulls_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip_prefer_nulls_update_records.jsonl new file mode 100644 index 0000000..f0881e1 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip_prefer_nulls_update_records.jsonl @@ -0,0 +1,2 @@ +{"$id":{"type":"__ID__","value":"123"},"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"$id":{"type":"__ID__","value":"456"},"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip_update_records.jsonl new file mode 100644 index 0000000..cedb6a3 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_always_skip_update_records.jsonl @@ -0,0 +1,5 @@ +{"$id":{"type":"__ID__","value":"123"},"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"$id":{"type":"__ID__","value":"0"},"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"$id":{"type":"__ID__","value":"456"},"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"$id":{"type":"__ID__","value":"0"},"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"$id":{"type":"__ID__","value":"0"},"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip__id_add_values.json b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip__id_add_values.json new file mode 100644 index 0000000..2f825cf --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip__id_add_values.json @@ -0,0 +1 @@ +["789"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip__id_values.json b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip__id_values.json new file mode 100644 index 0000000..83ed89d --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip__id_values.json @@ -0,0 +1 @@ +["123","0","456","0","789","0"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_add_records.jsonl new file mode 100644 index 0000000..d02b0e4 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_add_records.jsonl @@ -0,0 +1 @@ +{"$id":{"type":"__ID__","value":null},"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_prefer_nulls__id_add_values.json b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_prefer_nulls__id_add_values.json new file mode 100644 index 0000000..2f825cf --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_prefer_nulls__id_add_values.json @@ -0,0 +1 @@ +["789"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_prefer_nulls__id_values.json b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_prefer_nulls__id_values.json new file mode 100644 index 0000000..c4c83f0 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_prefer_nulls__id_values.json @@ -0,0 +1 @@ +["123","456","789"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_prefer_nulls_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_prefer_nulls_add_records.jsonl new file mode 100644 index 0000000..6a4f9cd --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_prefer_nulls_add_records.jsonl @@ -0,0 +1,4 @@ +{"$id":{"type":"__ID__","value":null},"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} +{"$id":{"type":"__ID__","value":null},"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} +{"$id":{"type":"__ID__","value":null},"long_number":{"type":"NUMBER","value":"789"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"ghi"},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":"789"},"double":{"type":"NUMBER","value":"789.0"},"string":{"type":"MULTI_LINE_TEXT","value":"ghi"},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":"\"ghi\""}} +{"$id":{"type":"__ID__","value":null},"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_prefer_nulls_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_prefer_nulls_update_records.jsonl new file mode 100644 index 0000000..f0881e1 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_prefer_nulls_update_records.jsonl @@ -0,0 +1,2 @@ +{"$id":{"type":"__ID__","value":"123"},"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"$id":{"type":"__ID__","value":"456"},"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_update_records.jsonl new file mode 100644 index 0000000..cedb6a3 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_never_skip_update_records.jsonl @@ -0,0 +1,5 @@ +{"$id":{"type":"__ID__","value":"123"},"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"$id":{"type":"__ID__","value":"0"},"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"$id":{"type":"__ID__","value":"456"},"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"$id":{"type":"__ID__","value":"0"},"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"$id":{"type":"__ID__","value":"0"},"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_prefer_nulls__id_add_values.json b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_prefer_nulls__id_add_values.json new file mode 100644 index 0000000..2f825cf --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_prefer_nulls__id_add_values.json @@ -0,0 +1 @@ +["789"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_prefer_nulls__id_values.json b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_prefer_nulls__id_values.json new file mode 100644 index 0000000..c4c83f0 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_prefer_nulls__id_values.json @@ -0,0 +1 @@ +["123","456","789"] diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_prefer_nulls_add_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_prefer_nulls_add_records.jsonl new file mode 100644 index 0000000..004e584 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_prefer_nulls_add_records.jsonl @@ -0,0 +1,3 @@ +{"$id":{"type":"__ID__","value":null},"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} +{"$id":{"type":"__ID__","value":null},"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} +{"$id":{"type":"__ID__","value":null},"long_number":{"type":"NUMBER","value":null},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":null},"boolean":{"type":"NUMBER","value":null},"long":{"type":"NUMBER","value":null},"double":{"type":"NUMBER","value":null},"string":{"type":"MULTI_LINE_TEXT","value":null},"timestamp":{"type":"DATETIME","value":null},"json":{"type":"MULTI_LINE_TEXT","value":null}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_prefer_nulls_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_prefer_nulls_update_records.jsonl new file mode 100644 index 0000000..f0881e1 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_prefer_nulls_update_records.jsonl @@ -0,0 +1,2 @@ +{"$id":{"type":"__ID__","value":"123"},"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"$id":{"type":"__ID__","value":"456"},"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} diff --git a/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_update_records.jsonl b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_update_records.jsonl new file mode 100644 index 0000000..cedb6a3 --- /dev/null +++ b/src/test/resources/org/embulk/output/kintone/task/skip_id/upsert_update_records.jsonl @@ -0,0 +1,5 @@ +{"$id":{"type":"__ID__","value":"123"},"long_number":{"type":"NUMBER","value":"123"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"abc"},"boolean":{"type":"NUMBER","value":"1"},"long":{"type":"NUMBER","value":"123"},"double":{"type":"NUMBER","value":"123.0"},"string":{"type":"MULTI_LINE_TEXT","value":"abc"},"timestamp":{"type":"DATETIME","value":"1999-12-31T23:59:59Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"abc\""}} +{"$id":{"type":"__ID__","value":"0"},"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"$id":{"type":"__ID__","value":"456"},"long_number":{"type":"NUMBER","value":"456"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":"def"},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"456"},"double":{"type":"NUMBER","value":"456.0"},"string":{"type":"MULTI_LINE_TEXT","value":"def"},"timestamp":{"type":"DATETIME","value":"2000-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"def\""}} +{"$id":{"type":"__ID__","value":"0"},"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} +{"$id":{"type":"__ID__","value":"0"},"long_number":{"type":"NUMBER","value":"0"},"string_single_line_text":{"type":"MULTI_LINE_TEXT","value":""},"boolean":{"type":"NUMBER","value":"0"},"long":{"type":"NUMBER","value":"0"},"double":{"type":"NUMBER","value":"0.0"},"string":{"type":"MULTI_LINE_TEXT","value":""},"timestamp":{"type":"DATETIME","value":"1970-01-01T00:00:00Z"},"json":{"type":"MULTI_LINE_TEXT","value":"\"\""}} From fd45f00813beee5a43c79d231569e3708fd4c630 Mon Sep 17 00:00:00 2001 From: Tetsuro Sano Date: Wed, 24 Apr 2024 15:46:31 +0900 Subject: [PATCH 3/4] Change return type to concrete type --- .../output/kintone/KintoneColumnType.java | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/embulk/output/kintone/KintoneColumnType.java b/src/main/java/org/embulk/output/kintone/KintoneColumnType.java index cb6ce43..b61f819 100644 --- a/src/main/java/org/embulk/output/kintone/KintoneColumnType.java +++ b/src/main/java/org/embulk/output/kintone/KintoneColumnType.java @@ -35,6 +35,8 @@ import org.embulk.spi.time.Timestamp; import org.embulk.spi.type.Type; import org.embulk.spi.type.Types; +import org.msgpack.value.ArrayValue; +import org.msgpack.value.StringValue; import org.msgpack.value.Value; import org.msgpack.value.ValueFactory; @@ -51,7 +53,7 @@ public SingleLineTextFieldValue getFieldValue(String value, KintoneColumnOption } @Override - public Object getValue(Record record, String fieldCode) { + public String getValue(Record record, String fieldCode) { return record.getSingleLineTextFieldValue(fieldCode); } @@ -66,7 +68,7 @@ public void setUpdateKey(UpdateKey updateKey, String field, FieldValue value) { } @Override - public Value asValue(FieldValue value) { + public StringValue asValue(FieldValue value) { return ValueFactory.newString(((SingleLineTextFieldValue) value).getValue()); } @@ -87,7 +89,7 @@ public MultiLineTextFieldValue getFieldValue(String value, KintoneColumnOption o } @Override - public Value asValue(FieldValue value) { + public StringValue asValue(FieldValue value) { return ValueFactory.newString(((MultiLineTextFieldValue) value).getValue()); } @@ -108,7 +110,7 @@ public RichTextFieldValue getFieldValue(String value, KintoneColumnOption option } @Override - public Value asValue(FieldValue value) { + public StringValue asValue(FieldValue value) { return ValueFactory.newString(((RichTextFieldValue) value).getValue()); } @@ -139,7 +141,7 @@ public NumberFieldValue getFieldValue(Timestamp value, KintoneColumnOption optio } @Override - public Object getValue(Record record, String fieldCode) { + public BigDecimal getValue(Record record, String fieldCode) { return record.getNumberFieldValue(fieldCode); } @@ -154,7 +156,7 @@ public void setUpdateKey(UpdateKey updateKey, String field, FieldValue value) { } @Override - public Value asValue(FieldValue value) { + public StringValue asValue(FieldValue value) { return ValueFactory.newString(((NumberFieldValue) value).getValue().toPlainString()); } @@ -175,7 +177,7 @@ public CheckBoxFieldValue getFieldValue(String value, KintoneColumnOption option } @Override - public Value asValue(FieldValue value) { + public ArrayValue asValue(FieldValue value) { return ValueFactory.newArray( ((CheckBoxFieldValue) value) .getValues().stream().map(ValueFactory::newString).collect(Collectors.toList())); @@ -198,7 +200,7 @@ public RadioButtonFieldValue getFieldValue(String value, KintoneColumnOption opt } @Override - public Value asValue(FieldValue value) { + public StringValue asValue(FieldValue value) { return ValueFactory.newString(((RadioButtonFieldValue) value).getValue()); } @@ -219,7 +221,7 @@ public MultiSelectFieldValue getFieldValue(String value, KintoneColumnOption opt } @Override - public Value asValue(FieldValue value) { + public ArrayValue asValue(FieldValue value) { return ValueFactory.newArray( ((MultiSelectFieldValue) value) .getValues().stream().map(ValueFactory::newString).collect(Collectors.toList())); @@ -242,7 +244,7 @@ public DropDownFieldValue getFieldValue(String value, KintoneColumnOption option } @Override - public Value asValue(FieldValue value) { + public StringValue asValue(FieldValue value) { return ValueFactory.newString(((DropDownFieldValue) value).getValue()); } @@ -347,7 +349,7 @@ public DateFieldValue getFieldValue(Timestamp value, KintoneColumnOption option) } @Override - public Value asValue(FieldValue value) { + public StringValue asValue(FieldValue value) { return ValueFactory.newString(((DateFieldValue) value).getValue().toString()); } @@ -389,7 +391,7 @@ public TimeFieldValue getFieldValue(Timestamp value, KintoneColumnOption option) } @Override - public Value asValue(FieldValue value) { + public StringValue asValue(FieldValue value) { return ValueFactory.newString(((TimeFieldValue) value).getValue().toString()); } @@ -427,7 +429,7 @@ public DateTimeFieldValue getFieldValue(Timestamp value, KintoneColumnOption opt } @Override - public Value asValue(FieldValue value) { + public StringValue asValue(FieldValue value) { return ValueFactory.newString(((DateTimeFieldValue) value).getValue().toString()); } @@ -448,7 +450,7 @@ public LinkFieldValue getFieldValue(String value, KintoneColumnOption option) { } @Override - public Value asValue(FieldValue value) { + public StringValue asValue(FieldValue value) { return ValueFactory.newString(((LinkFieldValue) value).getValue()); } From ee6bcf3fccf5caf03d16696f903a2b6b1fcdc063 Mon Sep 17 00:00:00 2001 From: Tetsuro Sano Date: Mon, 13 May 2024 08:51:03 +0900 Subject: [PATCH 4/4] Added description of the skip policy to README --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9e9d090..faba5fb 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,14 @@ kintone output plugin for Embulk stores app records from kintone. - **max_sort_memory**: Maximum memory usage for sorting input records (bytes in long, default is the estimated available memory, which is the approximate value of the JVM's current free memory) - **prefer_nulls**: Whether to set fields to null instead of default value of type when column is null (boolean, default is `false`) - **ignore_nulls**: Whether to completely ignore fields when column is null (boolean, default is `false`) -- **skip_if_non_existing_id_or_update_key**: Skip policy if id or update key of record does not exist (string `auto`, `never` or `always`, default is `auto`) +- **skip_if_non_existing_id_or_update_key**: The skip policy if the record corresponding to the id or update key does not exist (string `auto`, `never` or `always`, default is `auto`). No effect for insert mode. + - **auto**: + - **update mode**: Skip the record if no id or update key value is specified. + - **upsert mode**: Skip the record if corresponds to the id does not exist or no update key value is specified. + - **never**: Never skip the record even if corresponds to the id or update key does not exist. + - **update mode**: Throw exception if no id or update key value is specified. + - **upsert mode**: Insert the record if corresponds to the id or update key does not exist (also, if no id or update key value is specified). + - **always**: Always skip the record if corresponds to the id or update key does not exist (also, if no id or update key value is specified). update mode and upsert mode will the same behavior (only updated, never inserted). - **column_options** advanced: a key-value pairs where key is a column name and value is options for the column. - **field_code**: field code (string, required) - **type**: field type (string, required). See [this page](https://cybozu.dev/ja/kintone/docs/overview/field-types/#field-type-update) for list of available types. However, following types are not yet supported