Skip to content

Commit

Permalink
tmp
Browse files Browse the repository at this point in the history
  • Loading branch information
@d-hrs
d-hrs committed Jan 12, 2024
1 parent 1e37a05 commit e91b53b
Show file tree
Hide file tree
Showing 2 changed files with 71 additions and 48 deletions.
39 changes: 29 additions & 10 deletions src/main/java/org/embulk/input/google_ads/GoogleAdsInputPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,18 +69,37 @@ public TaskReport run(TaskSource taskSource,
reporter.connect();
try {
try (PageBuilder pageBuilder = getPageBuilder(schema, output)) {
for (GoogleAdsServiceClient.SearchPage page : reporter.getReportPage()) {
for (GoogleAdsRow row : page.getValues()) {
result = new HashMap<String, String>()
{
};
reporter.flattenResource(null, row.getAllFields(), result);
schema.visitColumns(new GoogleAdsColumnVisitor(new GoogleAdsAccessor(task, result), pageBuilder, task));
pageBuilder.addRecord();
}
Map<String, String> params = new HashMap<>();
reporter.search(params);
while(reporter.fetchResult(
pages -> {
pages.forEach(page -> {
for (GoogleAdsRow row : page.getValues()) {
Map<String, String> ret = new HashMap<>();
reporter.flattenResource(null, row.getAllFields(), ret);
schema.visitColumns(new GoogleAdsColumnVisitor(new GoogleAdsAccessor(task, ret), pageBuilder, task));
pageBuilder.addRecord();
}
});
return null;
}
)){
pageBuilder.flush();
}
};
pageBuilder.finish();

// for (GoogleAdsServiceClient.SearchPage page : reporter.getReportPage(params)) {
// for (GoogleAdsRow row : page.getValues()) {
// result = new HashMap<String, String>()
// {
// };
// reporter.flattenResource(null, row.getAllFields(), result);
// schema.visitColumns(new GoogleAdsColumnVisitor(new GoogleAdsAccessor(task, result), pageBuilder, task));
// pageBuilder.addRecord();
// }
// pageBuilder.flush();
// }
// pageBuilder.finish();
}
} catch (Exception e) {
logger.error(e.getMessage());
Expand Down
80 changes: 42 additions & 38 deletions src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;

Expand All @@ -38,6 +36,8 @@ public class GoogleAdsReporter
private GoogleAdsClient client;
private ObjectMapper mapper = new ObjectMapper();

private Iterable<GoogleAdsServiceClient.SearchPage> searchResult;

public GoogleAdsReporter(PluginTask task)
{
this.task = task;
Expand All @@ -53,40 +53,44 @@ private UserCredentials buildCredential(PluginTask task)
.build();
}

public Iterable<GoogleAdsServiceClient.SearchPage> getReportPage()
{
List<GoogleAdsServiceClient.SearchPage> pages = new ArrayList<GoogleAdsServiceClient.SearchPage>();

String startDateTime = null;
do {
String query = buildQuery(task, startDateTime);
logger.info(query);
SearchGoogleAdsRequest request = buildRequest(task, query);
GoogleAdsServiceClient googleAdsService = client.getVersion14().createGoogleAdsServiceClient();
GoogleAdsServiceClient.SearchPagedResponse response = googleAdsService.search(request);

if (response.getPage().getResponse().getResultsCount() == 0) {
return pages;
}
public void search(Map<String, String> params) {
String query = buildQuery(task, params);
logger.info(query);
SearchGoogleAdsRequest request = buildRequest(task, query);
GoogleAdsServiceClient googleAdsService = client.getVersion14().createGoogleAdsServiceClient();
GoogleAdsServiceClient.SearchPagedResponse response = googleAdsService.search(request);
this.searchResult = response.iteratePages();
}

response.iteratePages().iterator().forEachRemaining(pages::add);
public boolean fetchResult(Function<Iterable<GoogleAdsServiceClient.SearchPage>, Void> func) {
logger.info("fetchResult");
func.apply(this.searchResult);
Iterator< GoogleAdsServiceClient.SearchPage> pageItr = this.searchResult.iterator();

if (task.getResourceType().equals("change_event")) {
GoogleAdsServiceClient.SearchPage lastPage = pages.get(pages.size() - 1);
GoogleAdsRow lastRow = null;
for(GoogleAdsRow row : lastPage.getValues()) {
lastRow = row;
}
if (!task.getResourceType().equals("change_event")) return false;

if (lastRow == null) {
break;
} else {
startDateTime = lastRow.getChangeEvent().getChangeDateTime();
}
}
} while (startDateTime != null && !startDateTime.isEmpty());
// NOTE: reset iterator
this.searchResult.iterator();

return pages;
GoogleAdsServiceClient.SearchPage lastPage = null;
while(pageItr.hasNext()) {
lastPage = pageItr.next();
}
if (lastPage == null) return false;

Iterator<GoogleAdsRow> rowItr = lastPage.getValues().iterator();
GoogleAdsRow lastRow = null;
while(rowItr.hasNext()) {
lastRow = rowItr.next();
}
if (lastRow == null) return false;
lastRow.getChangeEvent().getChangeDateTime();

this.searchResult = null;
Map<String, String> params = new HashMap<>();
params.put("start_datetime", lastRow.getChangeEvent().getChangeDateTime());
search(params);
return true;
}

public void flattenResource(String resourceName, Map<Descriptors.FieldDescriptor, Object> fields, Map<String, String> result)
Expand Down Expand Up @@ -230,7 +234,7 @@ public SearchGoogleAdsRequest buildRequest(PluginTask task, String query)
.build();
}

public String buildQuery(PluginTask task, String startDateTime)
public String buildQuery(PluginTask task, Map<String, String> params)
{
StringBuilder sb = new StringBuilder();

Expand All @@ -240,7 +244,7 @@ public String buildQuery(PluginTask task, String startDateTime)
sb.append(" FROM ");
sb.append(task.getResourceType());

List<String> whereClause = buildWhereClauseConditions(task, startDateTime);
List<String> whereClause = buildWhereClauseConditions(task, params);
if (!whereClause.isEmpty()) {
sb.append(" WHERE ");
sb.append(String.join(" AND ", whereClause));
Expand All @@ -255,7 +259,7 @@ public String buildQuery(PluginTask task, String startDateTime)
}

@VisibleForTesting
public List<String> buildWhereClauseConditions(PluginTask task, String startDateTime)
public List<String> buildWhereClauseConditions(PluginTask task, Map<String, String> params)
{
List<String> whereConditions = new ArrayList<String>()
{
Expand All @@ -264,7 +268,7 @@ public List<String> buildWhereClauseConditions(PluginTask task, String startDate
if (task.getDateRange().isPresent()) {
StringBuilder dateSb = new StringBuilder();
if (task.getResourceType().equals("change_event")) {
dateSb.append(buildWhereClauseConditionsForChangeEvent(startDateTime));
dateSb.append(buildWhereClauseConditionsForChangeEvent(params.get("start_datetime")));
} else {
dateSb.append("segments.date BETWEEN '");
dateSb.append(task.getDateRange().get().getStartDate());
Expand Down

0 comments on commit e91b53b

Please sign in to comment.