Skip to content

Commit 85cef95

Browse files
d-hrsd-hrs
authored
Merge pull request #48 from trocco-io/feature/update_embulk_v10
 bump embulk to 0.10.0
2 parents b1cce88 + d11313d commit 85cef95

File tree

11 files changed

+229
-45
lines changed

11 files changed

+229
-45
lines changed

‎build.gradle

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -28,17 +28,23 @@ tasks.withType(JavaCompile) {
2828
options.compilerArgs << "-Xlint:deprecation" << "-Xlint:unchecked"
2929
}
3030

31+
def embulkVersion = '0.10.43'
32+
3133
dependencies {
32-
compileOnly "org.embulk:embulk-core:0.9.23"
33-
// compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
34+
compileOnly("org.embulk:embulk-api:$embulkVersion") { exclude group: "org.slf4j", module: "slf4j-api" }
35+
compileOnly("org.embulk:embulk-spi:$embulkVersion") { exclude group: "org.slf4j", module: "slf4j-api" }
36+
compile "org.embulk:embulk-util-timestamp:0.2.1"
37+
compile "org.embulk:embulk-util-config:0.3.3"
38+
compile "org.embulk:embulk-util-json:0.2.2"
39+
compile "com.google.guava:guava:31.1-jre"
40+
compile "org.slf4j:slf4j-log4j12:1.7.36"
3441
compile project(path: ":shadow-google-ads-helper", configuration: "shadow")
42+
3543
testCompile "junit:junit:4.13"
36-
testCompile 'org.embulk:embulk-standards:0.9.23'
37-
testCompile 'org.embulk:embulk-test:0.9.23'
3844
testCompile "org.mockito:mockito-core:1.10.19"
39-
testCompile "org.embulk:embulk-core:0.9.23:tests"
40-
testCompile "org.embulk:embulk-deps-buffer:0.9.23"
41-
testCompile "org.embulk:embulk-deps-config:0.9.23"
45+
testCompile("org.embulk:embulk-core:$embulkVersion") { exclude group: "org.slf4j", module: "slf4j-api" }
46+
testCompile("org.embulk:embulk-deps:$embulkVersion") { exclude group: "org.slf4j", module: "slf4j-api" }
47+
testCompile("org.embulk:embulk-junit4:$embulkVersion") { exclude group: "org.slf4j", module: "slf4j-api" }
4248
}
4349

4450
embulkPlugin {

‎gradle/dependency-locks/embulkPluginRuntime.lockfile

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,22 @@
11
# This is a Gradle generated file for dependency locking.
22
# Manual edits can break the build and are not advised.
33
# This file is expected to be part of source control.
4+
ch.qos.reload4j:reload4j:1.2.19
5+
com.fasterxml.jackson.core:jackson-annotations:2.6.7
6+
com.fasterxml.jackson.core:jackson-core:2.6.7
7+
com.fasterxml.jackson.core:jackson-databind:2.6.7.5
8+
com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7
9+
com.google.code.findbugs:jsr305:3.0.2
10+
com.google.errorprone:error_prone_annotations:2.11.0
11+
com.google.guava:failureaccess:1.0.1
12+
com.google.guava:guava:31.1-jre
13+
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
14+
com.google.j2objc:j2objc-annotations:1.3
15+
javax.validation:validation-api:1.1.0.Final
16+
org.checkerframework:checker-qual:3.12.0
17+
org.embulk:embulk-util-config:0.3.3
18+
org.embulk:embulk-util-json:0.2.2
19+
org.embulk:embulk-util-rubytime:0.3.2
20+
org.embulk:embulk-util-timestamp:0.2.1
21+
org.slf4j:slf4j-api:1.7.36
22+
org.slf4j:slf4j-reload4j:1.7.36

‎src/main/java/org/embulk/input/google_ads/GoogleAdsColumnVisitor.java

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,13 @@
22

33
import com.google.gson.JsonElement;
44
import org.embulk.spi.Column;
5-
import org.embulk.spi.ColumnConfig;
5+
66
import org.embulk.spi.ColumnVisitor;
77
import org.embulk.spi.PageBuilder;
8-
import org.embulk.spi.json.JsonParser;
9-
import org.embulk.spi.time.Timestamp;
10-
import org.embulk.spi.time.TimestampParser;
8+
9+
import org.embulk.util.config.units.ColumnConfig;
10+
import org.embulk.util.json.JsonParser;
11+
import org.embulk.util.timestamp.TimestampFormatter;
1112

1213
import java.util.List;
1314

@@ -69,6 +70,7 @@ public void doubleColumn(Column column)
6970
}
7071
}
7172

73+
@SuppressWarnings("deprecation") // After the end of embulk v0.9 support, replace Timestamp with Instant
7274
@Override
7375
public void timestampColumn(Column column)
7476
{
@@ -79,15 +81,13 @@ public void timestampColumn(Column column)
7981
String configColumnName = GoogleAdsUtil.escapeColumnName(config.getName(), task);
8082
if (configColumnName.equals(column.getName())
8183
&& config.getConfigSource() != null
82-
&& config.getConfigSource().getObjectNode() != null
83-
&& config.getConfigSource().getObjectNode().get("format") != null
84-
&& config.getConfigSource().getObjectNode().get("format").isTextual()) {
85-
pattern = config.getConfigSource().getObjectNode().get("format").asText();
84+
&& config.getConfigSource().get(String.class, "format", null) != null) {
85+
pattern = config.getConfigSource().get(String.class, "format", null);
8686
break;
8787
}
8888
}
89-
TimestampParser parser = TimestampParser.of(pattern, "UTC");
90-
Timestamp result = parser.parse(accessor.get(column.getName()));
89+
TimestampFormatter formatter = TimestampFormatter.builder(pattern, true).build();
90+
org.embulk.spi.time.Timestamp result = org.embulk.spi.time.Timestamp.ofInstant(formatter.parse(accessor.get(column.getName())));
9191
pageBuilder.setTimestamp(column, result);
9292
} catch (Exception e) {
9393
pageBuilder.setNull(column);

‎src/main/java/org/embulk/input/google_ads/GoogleAdsInputPlugin.java

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,15 @@
1010
import org.embulk.config.TaskSource;
1111

1212
import org.embulk.spi.Column;
13-
import org.embulk.spi.ColumnConfig;
1413
import org.embulk.spi.Exec;
1514
import org.embulk.spi.InputPlugin;
1615
import org.embulk.spi.PageBuilder;
1716
import org.embulk.spi.PageOutput;
1817
import org.embulk.spi.Schema;
19-
18+
import org.embulk.util.config.ConfigMapper;
19+
import org.embulk.util.config.ConfigMapperFactory;
20+
import org.embulk.util.config.TaskMapper;
21+
import org.embulk.util.config.units.ColumnConfig;
2022
import org.slf4j.Logger;
2123
import org.slf4j.LoggerFactory;
2224

@@ -27,18 +29,20 @@
2729
public class GoogleAdsInputPlugin
2830
implements InputPlugin
2931
{
32+
private static final ConfigMapperFactory CONFIG_MAPPER_FACTORY = ConfigMapperFactory.builder().addDefaultModules().build();
3033
private final Logger logger = LoggerFactory.getLogger(GoogleAdsInputPlugin.class);
3134

3235
@Override
3336
public ConfigDiff transaction(ConfigSource config,
3437
InputPlugin.Control control)
3538
{
36-
PluginTask task = config.loadConfig(PluginTask.class);
39+
final ConfigMapper configMapper = CONFIG_MAPPER_FACTORY.createConfigMapper();
40+
final PluginTask task = configMapper.map(config, PluginTask.class);
3741
Schema schema = buildSchema(task);
3842

3943
int taskCount = 1; // number of run() method calls
4044

41-
return resume(task.dump(), schema, taskCount, control);
45+
return resume(task.toTaskSource(), schema, taskCount, control);
4246
}
4347

4448
@Override
@@ -47,7 +51,7 @@ public ConfigDiff resume(TaskSource taskSource,
4751
InputPlugin.Control control)
4852
{
4953
control.run(taskSource, schema, taskCount);
50-
return Exec.newConfigDiff();
54+
return CONFIG_MAPPER_FACTORY.newConfigDiff();
5155
}
5256

5357
@Override
@@ -62,7 +66,9 @@ public TaskReport run(TaskSource taskSource,
6266
Schema schema, int taskIndex,
6367
PageOutput output)
6468
{
65-
PluginTask task = taskSource.loadTask(PluginTask.class);
69+
final TaskMapper taskMapper = CONFIG_MAPPER_FACTORY.createTaskMapper();
70+
final PluginTask task = taskMapper.map(taskSource, PluginTask.class);
71+
6672
GoogleAdsReporter reporter = new GoogleAdsReporter(task);
6773
reporter.connect();
6874
try {
@@ -87,13 +93,13 @@ public TaskReport run(TaskSource taskSource,
8793
throw e;
8894
}
8995

90-
return Exec.newTaskReport();
96+
return CONFIG_MAPPER_FACTORY.newTaskReport();
9197
}
9298

9399
@Override
94100
public ConfigDiff guess(ConfigSource config)
95101
{
96-
return Exec.newConfigDiff();
102+
return CONFIG_MAPPER_FACTORY.newConfigDiff();
97103
}
98104

99105
public GoogleAdsReporter getClient(PluginTask task)
@@ -112,6 +118,7 @@ public Schema buildSchema(PluginTask task)
112118
return new Schema(builder.build());
113119
}
114120

121+
@SuppressWarnings("deprecation") // After　the end of embulk v0.9 support, use Exec.getPageBuilder
115122
protected PageBuilder getPageBuilder(final Schema schema, final PageOutput output)
116123
{
117124
return new PageBuilder(Exec.getBufferAllocator(), schema, output);

‎src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@
2020
import com.google.protobuf.Descriptors;
2121
import com.google.protobuf.GeneratedMessageV3;
2222
import com.google.protobuf.util.JsonFormat;
23-
import org.embulk.spi.ColumnConfig;
2423

24+
import org.embulk.util.config.units.ColumnConfig;
2525
import org.msgpack.core.annotations.VisibleForTesting;
2626

2727
import org.slf4j.Logger;

‎src/main/java/org/embulk/input/google_ads/PluginTask.java

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
package org.embulk.input.google_ads;
22

3-
import org.embulk.config.Config;
4-
import org.embulk.config.ConfigDefault;
5-
import org.embulk.config.Task;
6-
import org.embulk.spi.SchemaConfig;
3+
import org.embulk.util.config.Config;
4+
import org.embulk.util.config.ConfigDefault;
5+
6+
import org.embulk.util.config.Task;
7+
import org.embulk.util.config.units.SchemaConfig;
78

89
import java.util.List;
910
import java.util.Optional;

‎src/test/java/org/embulk/input/google_ads/TestGoogleAdsAccessor.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import org.embulk.config.ConfigSource;
44
import org.embulk.spi.InputPlugin;
55
import org.embulk.test.TestingEmbulk;
6+
67
import org.junit.Assert;
78
import org.junit.Rule;
89
import org.junit.Test;
@@ -24,7 +25,7 @@ public void testAccessor()
2425
row.put("name2", "value2");
2526

2627
ConfigSource conf = TestHelper.getBaseConfig(embulk);
27-
GoogleAdsAccessor accessor = new GoogleAdsAccessor(conf.loadConfig(PluginTask.class), row);
28+
GoogleAdsAccessor accessor = new GoogleAdsAccessor(TestHelper.loadTask(conf), row);
2829
String result = accessor.get("name");
2930
String notExist = accessor.get("not_exist");
3031

‎src/test/java/org/embulk/input/google_ads/TestGoogleAdsColumnVisitor.java

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
package org.embulk.input.google_ads;
2+
3+
import com.google.common.collect.ImmutableList;
4+
import org.embulk.EmbulkTestRuntime;
5+
import org.embulk.config.ConfigSource;
6+
import org.embulk.spi.Column;
7+
import org.embulk.spi.InputPlugin;
8+
import org.embulk.spi.PageBuilder;
9+
import org.embulk.spi.PageBuilderImpl;
10+
import org.embulk.spi.PageReader;
11+
import org.embulk.spi.Schema;
12+
import org.embulk.spi.TestPageBuilderReader;
13+
import org.embulk.test.TestingEmbulk;
14+
import org.embulk.util.config.units.ColumnConfig;
15+
import org.junit.Assert;
16+
import org.junit.Before;
17+
import org.junit.Rule;
18+
import org.junit.Test;
19+
20+
import java.time.Instant;
21+
import java.util.HashMap;
22+
23+
public class TestGoogleAdsColumnVisitor
24+
{
25+
@Rule
26+
public TestingEmbulk embulk = TestingEmbulk.builder()
27+
.registerPlugin(InputPlugin.class, "google_ads", GoogleAdsInputPlugin.class)
28+
.build();
29+
30+
@Rule public EmbulkTestRuntime runtime;
31+
32+
@Before
33+
public void setUp()
34+
{
35+
runtime = new EmbulkTestRuntime();
36+
}
37+
38+
@Test
39+
public void testStringColumn()
40+
{
41+
PageReader pageReader = visitColumns("100", embulk.newConfig().set("name", "name").set("type", "string"));
42+
Assert.assertTrue(pageReader.nextRecord());
43+
Assert.assertEquals("100", pageReader.getString(0));
44+
Assert.assertFalse(pageReader.nextRecord());
45+
}
46+
47+
@Test
48+
public void testLongColumn()
49+
{
50+
PageReader pageReader = visitColumns("100", embulk.newConfig().set("name", "name").set("type", "long"));
51+
Assert.assertTrue(pageReader.nextRecord());
52+
Assert.assertEquals(100, pageReader.getLong(0));
53+
Assert.assertFalse(pageReader.nextRecord());
54+
}
55+
56+
@Test
57+
public void testBooleanColumnTrue()
58+
{
59+
PageReader pageReader = visitColumns("true", embulk.newConfig().set("name", "name").set("type", "boolean"));
60+
Assert.assertTrue(pageReader.nextRecord());
61+
Assert.assertTrue(pageReader.getBoolean(0));
62+
Assert.assertFalse(pageReader.nextRecord());
63+
}
64+
65+
@Test
66+
public void testBooleanColumnFalse()
67+
{
68+
PageReader pageReader = visitColumns("false", embulk.newConfig().set("name", "name").set("type", "boolean"));
69+
Assert.assertTrue(pageReader.nextRecord());
70+
Assert.assertFalse(pageReader.getBoolean(0));
71+
Assert.assertFalse(pageReader.nextRecord());
72+
}
73+
74+
@Test
75+
public void testDoubleColumn()
76+
{
77+
PageReader pageReader = visitColumns("1.23", embulk.newConfig().set("name", "name").set("type", "double"));
78+
Assert.assertTrue(pageReader.nextRecord());
79+
Assert.assertEquals(1.23, pageReader.getDouble(0), 0.01);
80+
Assert.assertFalse(pageReader.nextRecord());
81+
}
82+
83+
@Test
84+
public void testTimestampColumn()
85+
{
86+
PageReader pageReader = visitColumns("1970-01-01T09:00:10+09:00", embulk.newConfig().set("name", "name").set("type", "timestamp"));
87+
Assert.assertTrue(pageReader.nextRecord());
88+
Assert.assertEquals(Instant.ofEpochSecond(10), pageReader.getTimestampInstant(0));
89+
Assert.assertFalse(pageReader.nextRecord());
90+
}
91+
92+
@Test
93+
public void testTimestampColumnWithFormat()
94+
{
95+
PageReader pageReader = visitColumns("1970/01/01 00:00:10", embulk.newConfig().set("name", "name").set("type", "timestamp").set("format", "%Y/%m/%d %H:%M:%S"));
96+
Assert.assertTrue(pageReader.nextRecord());
97+
Assert.assertEquals(Instant.ofEpochSecond(10), pageReader.getTimestampInstant(0));
98+
Assert.assertFalse(pageReader.nextRecord());
99+
}
100+
101+
@Test
102+
public void testJsonColumn()
103+
{
104+
PageReader pageReader = visitColumns("{\"key0\":\"value0\"}", embulk.newConfig().set("name", "name").set("type", "json"));
105+
Assert.assertTrue(pageReader.nextRecord());
106+
Assert.assertEquals("{\"key0\":\"value0\"}", pageReader.getJsonValue(0).toJson());
107+
Assert.assertFalse(pageReader.nextRecord());
108+
}
109+
110+
private PageReader visitColumns(String value, ConfigSource configSource)
111+
{
112+
ConfigSource conf = TestHelper.getBaseConfigWithFields(embulk, configSource);
113+
114+
PluginTask task = TestHelper.loadTask(conf);
115+
Schema schema = buildSchema(task);
116+
HashMap<String, String> row = new HashMap<>();
117+
row.put("name", value);
118+
GoogleAdsAccessor accessor = new GoogleAdsAccessor(task, row);
119+
final TestPageBuilderReader.MockPageOutput output = new TestPageBuilderReader.MockPageOutput();
120+
final PageBuilder pageBuilder = new PageBuilderImpl(runtime.getBufferAllocator(), schema, output);
121+
GoogleAdsColumnVisitor columnVisitor = new GoogleAdsColumnVisitor(accessor, pageBuilder, task);
122+
schema.visitColumns(columnVisitor);
123+
pageBuilder.addRecord();
124+
pageBuilder.finish();
125+
126+
Assert.assertEquals(1, output.pages.size());
127+
128+
@SuppressWarnings("deprecation") // embulk v0.9-compatible
129+
PageReader pageReader = new PageReader(schema);
130+
pageReader.setPage(output.pages.get(0));
131+
132+
return pageReader;
133+
}
134+
135+
private Schema buildSchema(PluginTask task)
136+
{
137+
ImmutableList.Builder<Column> builder = ImmutableList.builder();
138+
for (int i = 0; i < task.getFields().size(); i++) {
139+
ColumnConfig columnConfig = task.getFields().getColumn(i);
140+
ColumnConfig escapedColumnConfig = new ColumnConfig(GoogleAdsUtil.escapeColumnName(columnConfig.getName(), task), columnConfig.getType(), columnConfig.getOption());
141+
builder.add(escapedColumnConfig.toColumn(i));
142+
}
143+
return new Schema(builder.build());
144+
}
145+
}

‎src/test/java/org/embulk/input/google_ads/TestGoogleAdsReporter.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ public void testTraverse()
3434
"}";
3535
ObjectMapper mapper = new ObjectMapper();
3636
ConfigSource conf = TestHelper.getBaseConfig(embulk);
37-
GoogleAdsReporter reporter = new GoogleAdsReporter(conf.loadConfig(PluginTask.class));
37+
GoogleAdsReporter reporter = new GoogleAdsReporter(TestHelper.loadTask(conf));
3838
try {
3939
JsonNode jsonNode = mapper.readTree(json);
4040
JsonNode resultJson = reporter.traverse(jsonNode);

0 commit comments

Comments
 (0)