From 6b52ccb5d1b04829ccaacbf4a35e9ab37e5bc60e Mon Sep 17 00:00:00 2001 From: dcanar9 Date: Mon, 9 May 2022 12:59:53 -0700 Subject: [PATCH 1/7] added helper function to return db results in batches to avoid memory issues --- .../dashboard/rest/DashboardController.java | 2 +- .../dashboard/service/DashboardService.java | 4 +- .../service/DashboardServiceImpl.java | 54 ++++++++++++------- .../service/DashboardServiceTest.java | 2 +- 4 files changed, 40 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/capitalone/dashboard/rest/DashboardController.java b/src/main/java/com/capitalone/dashboard/rest/DashboardController.java index fd0da8b5..52125d5d 100644 --- a/src/main/java/com/capitalone/dashboard/rest/DashboardController.java +++ b/src/main/java/com/capitalone/dashboard/rest/DashboardController.java @@ -485,7 +485,7 @@ public ResponseEntity> myDashboardByTitlePage(@RequestParam(valu @RequestMapping(value = "/dashboard/removeWidgetDuplicates", method = DELETE) public ResponseEntity removeWidgetDuplicates(@RequestParam(value="title", required = false)String title, @RequestParam(value="dryRun", required = true) boolean dryRun){ - String message = dashboardService.removeWidgetDuplicates(title, dryRun); + String message = dashboardService.removeWidgetDuplicatesHelper(title, dryRun); return ResponseEntity.ok().body(message); } } diff --git a/src/main/java/com/capitalone/dashboard/service/DashboardService.java b/src/main/java/com/capitalone/dashboard/service/DashboardService.java index b3ce073a..0b270e95 100644 --- a/src/main/java/com/capitalone/dashboard/service/DashboardService.java +++ b/src/main/java/com/capitalone/dashboard/service/DashboardService.java @@ -244,7 +244,9 @@ public interface DashboardService { Iterable allTemplate(String template); - String removeWidgetDuplicates(String title, boolean dryRun); + void removeWidgetDuplicates(List dashboards, boolean dryRun); + + String removeWidgetDuplicatesHelper(String title, boolean dryRun); } diff --git a/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java b/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java index 770a28ea..f08a1728 100644 --- a/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java +++ b/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java @@ -37,6 +37,7 @@ import org.bson.types.ObjectId; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Sort; import org.springframework.stereotype.Service; @@ -1006,16 +1007,44 @@ public Dashboard updateScoreSettings(ObjectId dashboardId, boolean scoreEnabled, } @Override - public String removeWidgetDuplicates(String title, boolean dryRun) { + public String removeWidgetDuplicatesHelper(String title, boolean dryRun){ + // page results and clean until there are no more pages + if(StringUtils.isEmpty(title)){ + Pageable pageable = new PageRequest(0, 500); + Page page = findDashboardsByPage("", pageable); - List dashboards; - if (StringUtils.isEmpty(title)){ - dashboards = (List) all(); + while(page.hasNext()){ + pageable = pageable.next(); + removeWidgetDuplicates(page.getContent(), dryRun); + page = findDashboardsByPage("", pageable); + } } else{ - dashboards = dashboardRepository.findByTitle(title); + List dashboards = dashboardRepository.findByTitle(title); if (CollectionUtils.isEmpty(dashboards)){return "No dashboards with that title";} + removeWidgetDuplicates(dashboards, dryRun); + } + + // messages upon success + if(StringUtils.isEmpty(title)){ + if(dryRun){ + return "DRY_RUN: All Dashboard widgets cleaned"; + } else{ + return "All Dashboard widgets cleaned"; + } } + else { + if(dryRun){ + return "DRY_RUN: Cleaned widgets for dashboard " + title; + } else{ + return "Cleaned widgets for dashboard " + title; + } + } + + } + + @Override + public void removeWidgetDuplicates(List dashboards, boolean dryRun) { for (Dashboard dashboard : dashboards) { List nonDuplicates = new ArrayList(); @@ -1039,19 +1068,6 @@ public String removeWidgetDuplicates(String title, boolean dryRun) { " widgets simplified to " + nonDuplicates.stream().map(Widget::getName).collect(Collectors.toList())); } - if(StringUtils.isEmpty(title)){ - if(dryRun){ - return "DRY_RUN: All Dashboard widgets cleaned"; - } else{ - return "All Dashboard widgets cleaned"; - } - } - else { - if(dryRun){ - return "DRY_RUN: Cleaned widgets for dashboard " + title; - } else{ - return "Cleaned widgets for dashboard " + title; - } - } + } } diff --git a/src/test/java/com/capitalone/dashboard/service/DashboardServiceTest.java b/src/test/java/com/capitalone/dashboard/service/DashboardServiceTest.java index 3c921965..3fe8f8ed 100644 --- a/src/test/java/com/capitalone/dashboard/service/DashboardServiceTest.java +++ b/src/test/java/com/capitalone/dashboard/service/DashboardServiceTest.java @@ -956,7 +956,7 @@ public void removeWidgetDuplicates(){ assertThat(dashboard.getWidgets().size(), is(4)); when(dashboardRepository.findByTitle(dashboard.getTitle())).thenReturn(Collections.singletonList(dashboard)); - dashboardService.removeWidgetDuplicates(dashboard.getTitle(), false); + dashboardService.removeWidgetDuplicatesHelper(dashboard.getTitle(), false); assertThat(dashboard.getWidgets().size(), is(2)); } From 59d6065e0562bb51e92bf378deb92d32df7ac17a Mon Sep 17 00:00:00 2001 From: dcanar9 Date: Mon, 9 May 2022 13:12:46 -0700 Subject: [PATCH 2/7] Increment pom version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e666acfd..5fc79e13 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ api jar ${project.groupId}:${project.artifactId} - 3.4.35 + 3.4.36 Hygieia Rest API Layer https://github.com/Hygieia/api From 48730a0cce92385660ba4d0ac324396fe14000b4 Mon Sep 17 00:00:00 2001 From: dcanar9 Date: Mon, 9 May 2022 13:54:27 -0700 Subject: [PATCH 3/7] Update DashboardServiceImpl.java Increased batch size in removeWidgetsDuplicatesHelper --- .../com/capitalone/dashboard/service/DashboardServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java b/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java index f08a1728..78bd5d9a 100644 --- a/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java +++ b/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java @@ -1010,7 +1010,7 @@ public Dashboard updateScoreSettings(ObjectId dashboardId, boolean scoreEnabled, public String removeWidgetDuplicatesHelper(String title, boolean dryRun){ // page results and clean until there are no more pages if(StringUtils.isEmpty(title)){ - Pageable pageable = new PageRequest(0, 500); + Pageable pageable = new PageRequest(0, 1000); Page page = findDashboardsByPage("", pageable); while(page.hasNext()){ From 5c7043a732156fc48b7a8afe8b82dae027d2cddb Mon Sep 17 00:00:00 2001 From: dcanar9 Date: Tue, 10 May 2022 06:51:02 -0700 Subject: [PATCH 4/7] made page size a value from the properties file --- .../dashboard/service/DashboardServiceImpl.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java b/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java index 78bd5d9a..de3b507d 100644 --- a/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java +++ b/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java @@ -36,6 +36,7 @@ import org.apache.commons.logging.LogFactory; import org.bson.types.ObjectId; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; @@ -81,6 +82,8 @@ public class DashboardServiceImpl implements DashboardService { public static final String TEST = "test"; public static final String CODEANALYSIS = "codeanalysis"; public static final String INFRA_SCAN = "infrascan"; + @Value("${pageSize:500}") + private int pageSize; @Autowired private ApiSettings settings; @@ -1008,9 +1011,10 @@ public Dashboard updateScoreSettings(ObjectId dashboardId, boolean scoreEnabled, @Override public String removeWidgetDuplicatesHelper(String title, boolean dryRun){ - // page results and clean until there are no more pages + System.out.println("Pages of size: " + String.valueOf(pageSize)); + // get page and clean until there are no more pages if(StringUtils.isEmpty(title)){ - Pageable pageable = new PageRequest(0, 1000); + Pageable pageable = new PageRequest(0, pageSize); Page page = findDashboardsByPage("", pageable); while(page.hasNext()){ From 628d1d83df76c10b3f8000fe1948120d4af0fc30 Mon Sep 17 00:00:00 2001 From: dcanar9 Date: Tue, 10 May 2022 06:55:29 -0700 Subject: [PATCH 5/7] Update DashboardServiceImpl.java Removed extra print stmt --- .../com/capitalone/dashboard/service/DashboardServiceImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java b/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java index de3b507d..00dd8aa5 100644 --- a/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java +++ b/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java @@ -1011,7 +1011,6 @@ public Dashboard updateScoreSettings(ObjectId dashboardId, boolean scoreEnabled, @Override public String removeWidgetDuplicatesHelper(String title, boolean dryRun){ - System.out.println("Pages of size: " + String.valueOf(pageSize)); // get page and clean until there are no more pages if(StringUtils.isEmpty(title)){ Pageable pageable = new PageRequest(0, pageSize); From aa2f09f4c1088d429be3d3e728275e1ff3f81ed7 Mon Sep 17 00:00:00 2001 From: dcanar9 Date: Tue, 10 May 2022 08:04:47 -0700 Subject: [PATCH 6/7] moved properties var to Api settings --- .../dashboard/service/DashboardServiceImpl.java | 6 ++---- .../capitalone/dashboard/settings/ApiSettings.java | 11 +++++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java b/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java index 00dd8aa5..e8c76904 100644 --- a/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java +++ b/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java @@ -36,7 +36,6 @@ import org.apache.commons.logging.LogFactory; import org.bson.types.ObjectId; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; @@ -82,8 +81,6 @@ public class DashboardServiceImpl implements DashboardService { public static final String TEST = "test"; public static final String CODEANALYSIS = "codeanalysis"; public static final String INFRA_SCAN = "infrascan"; - @Value("${pageSize:500}") - private int pageSize; @Autowired private ApiSettings settings; @@ -1011,9 +1008,10 @@ public Dashboard updateScoreSettings(ObjectId dashboardId, boolean scoreEnabled, @Override public String removeWidgetDuplicatesHelper(String title, boolean dryRun){ + System.out.println(settings.getBatchSize()); // get page and clean until there are no more pages if(StringUtils.isEmpty(title)){ - Pageable pageable = new PageRequest(0, pageSize); + Pageable pageable = new PageRequest(0, settings.getBatchSize()); Page page = findDashboardsByPage("", pageable); while(page.hasNext()){ diff --git a/src/main/java/com/capitalone/dashboard/settings/ApiSettings.java b/src/main/java/com/capitalone/dashboard/settings/ApiSettings.java index a54e2f39..9da1f20a 100644 --- a/src/main/java/com/capitalone/dashboard/settings/ApiSettings.java +++ b/src/main/java/com/capitalone/dashboard/settings/ApiSettings.java @@ -70,6 +70,9 @@ public class ApiSettings { @Value("${optimizeUserCallsToGithub:true}") private boolean optimizeUserCallsToGithub; + @Value("${batchSize:500}") + private int batchSize; + private String hygieia_ui_url=""; public Map getFunctional() { @@ -267,4 +270,12 @@ public boolean isOptimizeUserCallsToGithub() { public void setOptimizeUserCallsToGithub(boolean optimizeUserCallsToGithub) { this.optimizeUserCallsToGithub = optimizeUserCallsToGithub; } + + public int getBatchSize() { + return batchSize; + } + + public void setBatchSize(int batchSize) { + this.batchSize = batchSize; + } } From e0a34f6c6f453e93beb44f641800025a10e231ac Mon Sep 17 00:00:00 2001 From: dcanar9 Date: Tue, 10 May 2022 08:30:47 -0700 Subject: [PATCH 7/7] removed unnecessary printing --- .../com/capitalone/dashboard/service/DashboardServiceImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java b/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java index e8c76904..e552ae86 100644 --- a/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java +++ b/src/main/java/com/capitalone/dashboard/service/DashboardServiceImpl.java @@ -1008,7 +1008,6 @@ public Dashboard updateScoreSettings(ObjectId dashboardId, boolean scoreEnabled, @Override public String removeWidgetDuplicatesHelper(String title, boolean dryRun){ - System.out.println(settings.getBatchSize()); // get page and clean until there are no more pages if(StringUtils.isEmpty(title)){ Pageable pageable = new PageRequest(0, settings.getBatchSize());