Skip to content

Commit

Permalink
Merge pull request #50 from trocco-io/fixed
Browse files Browse the repository at this point in the history 
decrease memory utilization
  • Loading branch information
@d-hrs
d-hrs authored Jan 17, 2024
2 parents 1e37a05 + 003f153 commit 036925b
Show file tree
Hide file tree
Showing 2 changed files with 45 additions and 47 deletions.
26 changes: 13 additions & 13 deletions src/main/java/org/embulk/input/google_ads/GoogleAdsInputPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,23 +63,23 @@ public TaskReport run(TaskSource taskSource,
PageOutput output)
{
PluginTask task = taskSource.loadTask(PluginTask.class);
Map<String, String> result;

GoogleAdsReporter reporter = new GoogleAdsReporter(task);
reporter.connect();
try {
try (PageBuilder pageBuilder = getPageBuilder(schema, output)) {
for (GoogleAdsServiceClient.SearchPage page : reporter.getReportPage()) {
for (GoogleAdsRow row : page.getValues()) {
result = new HashMap<String, String>()
{
};
reporter.flattenResource(null, row.getAllFields(), result);
schema.visitColumns(new GoogleAdsColumnVisitor(new GoogleAdsAccessor(task, result), pageBuilder, task));
pageBuilder.addRecord();
}
pageBuilder.flush();
}
Map<String, String> params = new HashMap<>();
reporter.search(
searchPage -> {
for (GoogleAdsRow row : searchPage.getValues()) {
Map<String, String> result = new HashMap<>();
reporter.flattenResource(null, row.getAllFields(), result);
schema.visitColumns(new GoogleAdsColumnVisitor(new GoogleAdsAccessor(task, result), pageBuilder, task));
pageBuilder.addRecord();
}
pageBuilder.flush();
},
params
);
pageBuilder.finish();
}
} catch (Exception e) {
Expand Down
66 changes: 32 additions & 34 deletions src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.Stream;

Expand All @@ -38,6 +40,8 @@ public class GoogleAdsReporter
private GoogleAdsClient client;
private ObjectMapper mapper = new ObjectMapper();

private Iterable<GoogleAdsServiceClient.SearchPage> searchResult;

public GoogleAdsReporter(PluginTask task)
{
this.task = task;
Expand All @@ -53,40 +57,34 @@ private UserCredentials buildCredential(PluginTask task)
.build();
}

public Iterable<GoogleAdsServiceClient.SearchPage> getReportPage()
{
List<GoogleAdsServiceClient.SearchPage> pages = new ArrayList<GoogleAdsServiceClient.SearchPage>();

String startDateTime = null;
do {
String query = buildQuery(task, startDateTime);
logger.info(query);
SearchGoogleAdsRequest request = buildRequest(task, query);
GoogleAdsServiceClient googleAdsService = client.getVersion14().createGoogleAdsServiceClient();
GoogleAdsServiceClient.SearchPagedResponse response = googleAdsService.search(request);

if (response.getPage().getResponse().getResultsCount() == 0) {
return pages;
}

response.iteratePages().iterator().forEachRemaining(pages::add);
private Iterable<GoogleAdsServiceClient.SearchPage> search(Map<String, String> params) {
String query = buildQuery(task, params);
logger.info(query);
SearchGoogleAdsRequest request = buildRequest(task, query);
GoogleAdsServiceClient googleAdsService = client.getVersion14().createGoogleAdsServiceClient();
GoogleAdsServiceClient.SearchPagedResponse response = googleAdsService.search(request);
return response.iteratePages();
}

if (task.getResourceType().equals("change_event")) {
GoogleAdsServiceClient.SearchPage lastPage = pages.get(pages.size() - 1);
GoogleAdsRow lastRow = null;
for(GoogleAdsRow row : lastPage.getValues()) {
lastRow = row;
}
public void search(Consumer<GoogleAdsServiceClient.SearchPage> consumer, Map<String, String> params) {
GoogleAdsServiceClient.SearchPage lastPage = null;
for(GoogleAdsServiceClient.SearchPage page: search(params)) {
consumer.accept(page);
lastPage = page;
}

if (lastRow == null) {
break;
} else {
startDateTime = lastRow.getChangeEvent().getChangeDateTime();
}
if (task.getResourceType().equals("change_event")) {
if (lastPage == null) return ;
GoogleAdsRow lastRow = null;
for (GoogleAdsRow row: lastPage.getValues()) {
lastRow = row;
}
} while (startDateTime != null && !startDateTime.isEmpty());
if (lastRow == null) return ;

return pages;
Map<String, String> nextParams = new HashMap<>();
nextParams.put("start_datetime", lastRow.getChangeEvent().getChangeDateTime());
search(consumer, nextParams);
}
}

public void flattenResource(String resourceName, Map<Descriptors.FieldDescriptor, Object> fields, Map<String, String> result)
Expand Down Expand Up @@ -230,7 +228,7 @@ public SearchGoogleAdsRequest buildRequest(PluginTask task, String query)
.build();
}

public String buildQuery(PluginTask task, String startDateTime)
public String buildQuery(PluginTask task, Map<String, String> params)
{
StringBuilder sb = new StringBuilder();

Expand All @@ -240,7 +238,7 @@ public String buildQuery(PluginTask task, String startDateTime)
sb.append(" FROM ");
sb.append(task.getResourceType());

List<String> whereClause = buildWhereClauseConditions(task, startDateTime);
List<String> whereClause = buildWhereClauseConditions(task, params);
if (!whereClause.isEmpty()) {
sb.append(" WHERE ");
sb.append(String.join(" AND ", whereClause));
Expand All @@ -255,7 +253,7 @@ public String buildQuery(PluginTask task, String startDateTime)
}

@VisibleForTesting
public List<String> buildWhereClauseConditions(PluginTask task, String startDateTime)
public List<String> buildWhereClauseConditions(PluginTask task, Map<String, String> params)
{
List<String> whereConditions = new ArrayList<String>()
{
Expand All @@ -264,7 +262,7 @@ public List<String> buildWhereClauseConditions(PluginTask task, String startDate
if (task.getDateRange().isPresent()) {
StringBuilder dateSb = new StringBuilder();
if (task.getResourceType().equals("change_event")) {
dateSb.append(buildWhereClauseConditionsForChangeEvent(startDateTime));
dateSb.append(buildWhereClauseConditionsForChangeEvent(params.get("start_datetime")));
} else {
dateSb.append("segments.date BETWEEN '");
dateSb.append(task.getDateRange().get().getStartDate());
Expand Down

0 comments on commit 036925b

Please sign in to comment.