From 20666560873390c99b35be3859f22e2ef8441013 Mon Sep 17 00:00:00 2001 From: arybakov Date: Thu, 23 Nov 2023 19:03:59 -0700 Subject: [PATCH 1/6] Fix --- api/pom.xml | 6 +++++ .../service/GradBusinessService.java | 24 +++++++++++-------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index ecb4c05..9375ac3 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -175,6 +175,12 @@ json 20220320 + + + org.apache.commons + commons-collections4 + 4.4 + diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java index 6e34e2d..d3125de 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java @@ -7,6 +7,7 @@ import ca.bc.gov.educ.api.gradbusiness.util.TokenUtils; import io.github.resilience4j.retry.annotation.Retry; import jakarta.transaction.Transactional; +import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -201,7 +202,8 @@ public ResponseEntity getAmalgamatedSchoolReportPDFByMincode(String minc List locations = new ArrayList<>(); if (studentList != null && !studentList.isEmpty()) { logger.debug("******** Fetched {} students ******", studentList.size()); - getStudentAchievementReports(studentList, locations); + List> partitions = ListUtils.partition(studentList, 50); + getStudentAchievementReports(partitions, locations); Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("PST"), Locale.CANADA); int year = cal.get(Calendar.YEAR); String month = "00"; @@ -274,16 +276,18 @@ public ResponseEntity getStudentTranscriptPDFByType(String pen, String t } } - private void getStudentAchievementReports(List studentList, List locations) { + private void getStudentAchievementReports(List> partitions, List locations) { logger.debug("******** Getting Student Achievement Reports ******"); - List> futures = studentList.stream() - .map(studentGuid -> CompletableFuture.supplyAsync(() -> getStudentAchievementReport(studentGuid))) - .toList(); - CompletableFuture allFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[futures.size()])); - CompletableFuture> result = allFutures.thenApply(v -> futures.stream() - .map(CompletableFuture::join) - .toList()); - locations.addAll(result.join()); + for(List studentList: partitions) { + List> futures = studentList.stream() + .map(studentGuid -> CompletableFuture.supplyAsync(() -> getStudentAchievementReport(studentGuid))) + .toList(); + CompletableFuture allFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[futures.size()])); + CompletableFuture> result = allFutures.thenApply(v -> futures.stream() + .map(CompletableFuture::join) + .toList()); + locations.addAll(result.join()); + } logger.debug("******** Fetched All Student Achievement Reports ******"); } From 1606419dd511458052d75c0a53e68a605c255976 Mon Sep 17 00:00:00 2001 From: arybakov Date: Thu, 23 Nov 2023 19:21:25 -0700 Subject: [PATCH 2/6] Fix --- .../gov/educ/api/gradbusiness/service/GradBusinessService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java index d3125de..6d3f7dd 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java @@ -202,7 +202,7 @@ public ResponseEntity getAmalgamatedSchoolReportPDFByMincode(String minc List locations = new ArrayList<>(); if (studentList != null && !studentList.isEmpty()) { logger.debug("******** Fetched {} students ******", studentList.size()); - List> partitions = ListUtils.partition(studentList, 50); + List> partitions = ListUtils.partition(studentList, 100); getStudentAchievementReports(partitions, locations); Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("PST"), Locale.CANADA); int year = cal.get(Calendar.YEAR); From c8b0a2006e46ccfd18a2b90688df9badc21d3e9f Mon Sep 17 00:00:00 2001 From: arybakov Date: Thu, 23 Nov 2023 19:39:17 -0700 Subject: [PATCH 3/6] Fix --- .../educ/api/gradbusiness/service/GradBusinessService.java | 5 +++-- .../ca/bc/gov/educ/api/gradbusiness/util/TokenUtils.java | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java index 6d3f7dd..a4f9ed5 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java @@ -202,7 +202,7 @@ public ResponseEntity getAmalgamatedSchoolReportPDFByMincode(String minc List locations = new ArrayList<>(); if (studentList != null && !studentList.isEmpty()) { logger.debug("******** Fetched {} students ******", studentList.size()); - List> partitions = ListUtils.partition(studentList, 100); + List> partitions = ListUtils.partition(studentList, 200); getStudentAchievementReports(partitions, locations); Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("PST"), Locale.CANADA); int year = cal.get(Calendar.YEAR); @@ -279,6 +279,7 @@ public ResponseEntity getStudentTranscriptPDFByType(String pen, String t private void getStudentAchievementReports(List> partitions, List locations) { logger.debug("******** Getting Student Achievement Reports ******"); for(List studentList: partitions) { + logger.debug("******** Run partition with {} students ******", studentList.size()); List> futures = studentList.stream() .map(studentGuid -> CompletableFuture.supplyAsync(() -> getStudentAchievementReport(studentGuid))) .toList(); @@ -288,7 +289,7 @@ private void getStudentAchievementReports(List> partitions, List map= new LinkedMultiValueMap<>(); map.add("grant_type", "client_credentials"); + logger.debug("******** Fetch Access Token ********"); return this.webClient.post().uri(constants.getTokenUrl()) .headers(h -> h.addAll(httpHeaders)) .contentType(MediaType.APPLICATION_FORM_URLENCODED) From f83867f574b179f220f10774005e30693704729e Mon Sep 17 00:00:00 2001 From: arybakov Date: Fri, 24 Nov 2023 11:08:33 -0700 Subject: [PATCH 4/6] Fix --- api/pom.xml | 5 ++ .../config/EducGradBusinessApiConfig.java | 2 +- .../service/GradBusinessService.java | 13 ++-- .../util/EducGradBusinessUtil.java | 30 +++++++++ .../educ/api/gradbusiness/util/IOUtils.java | 65 +++++++++++++++++++ 5 files changed, 107 insertions(+), 8 deletions(-) create mode 100644 api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/IOUtils.java diff --git a/api/pom.xml b/api/pom.xml index 9375ac3..3436e57 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -181,6 +181,11 @@ commons-collections4 4.4 + + org.apache.pdfbox + pdfbox + 2.0.27 + diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java index b2ea285..bac2109 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java @@ -43,7 +43,7 @@ public WebClient webClient() { .exchangeStrategies(ExchangeStrategies.builder() .codecs(configurer -> configurer .defaultCodecs() - .maxInMemorySize(100 * 1024 * 1024)) + .maxInMemorySize(300 * 1024 * 1024)) .build()).build(); } diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java index a4f9ed5..b6f7d12 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java @@ -22,7 +22,6 @@ import org.springframework.web.reactive.function.BodyInserters; import org.springframework.web.reactive.function.client.WebClient; -import java.io.IOException; import java.io.InputStream; import java.util.*; import java.util.concurrent.CompletableFuture; @@ -210,7 +209,7 @@ public ResponseEntity getAmalgamatedSchoolReportPDFByMincode(String minc String fileName = EducGradBusinessUtil.getFileNameSchoolReports(mincode, year, month, type); try { logger.debug("******** Merging Documents Started ******"); - byte[] res = EducGradBusinessUtil.mergeDocuments(locations); + byte[] res = EducGradBusinessUtil.mergeDocumentsPDFs(locations); logger.debug("******** Merged {} Documents ******", locations.size()); HttpHeaders headers = new HttpHeaders(); headers.put(HttpHeaders.AUTHORIZATION, Collections.singletonList(BEARER + accessToken)); @@ -294,14 +293,14 @@ private void getStudentAchievementReports(List> partitions, List h.setBearerAuth(accessTokenNext)).retrieve().bodyToMono(InputStreamResource.class).block(); - if (result != null) { - try { + try { + InputStreamResource result = webClient.get().uri(String.format(educGraduationApiConstants.getStudentCredentialByType(), studentGuid, "ACHV")).headers(h -> h.setBearerAuth(accessTokenNext)).retrieve().bodyToMono(InputStreamResource.class).block(); + if (result != null) { logger.debug("******** Fetched Achievement Report for {} ******", studentGuid); return result.getInputStream(); - } catch (IOException e) { - logger.debug("Error extracting report binary from stream: {}", e.getLocalizedMessage()); } + } catch (Exception e) { + logger.debug("Error extracting report binary from stream: {}", e.getLocalizedMessage()); } return null; } diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java index d5edf3e..b3cfb69 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java @@ -5,6 +5,11 @@ import com.itextpdf.text.pdf.PdfCopy; import com.itextpdf.text.pdf.PdfReader; import com.itextpdf.text.pdf.PdfSmartCopy; +import org.apache.pdfbox.io.MemoryUsageSetting; +import org.apache.pdfbox.multipdf.PDFMergerUtility; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -15,11 +20,36 @@ public class EducGradBusinessUtil { + private static final Logger logger = LoggerFactory.getLogger(EducGradBusinessUtil.class); + + public static final String TMP_DIR = "/tmp"; + private EducGradBusinessUtil() {} private static final int BUFFER_SIZE = 250000; + public static byte[] mergeDocumentsPDFs(List locations) throws IOException { + File bufferDirectory = null; + ByteArrayOutputStream result = new ByteArrayOutputStream(); + try { + bufferDirectory = IOUtils.createTempDirectory(TMP_DIR, "buffer"); + PDFMergerUtility pdfMergerUtility = new PDFMergerUtility(); + pdfMergerUtility.setDestinationStream(result); + pdfMergerUtility.addSources(locations); + MemoryUsageSetting memoryUsageSetting = MemoryUsageSetting.setupMixed(50000000) + .setTempDir(bufferDirectory); + pdfMergerUtility.mergeDocuments(memoryUsageSetting); + } catch (Exception e) { + logger.error("Error {}", e.getLocalizedMessage()); + } finally { + if (bufferDirectory != null) { + IOUtils.removeFileOrDirectory(bufferDirectory); + } + } + return result.toByteArray(); + } + public static byte[] mergeDocuments(List locations) throws IOException { final byte[] result; diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/IOUtils.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/IOUtils.java new file mode 100644 index 0000000..8d54886 --- /dev/null +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/IOUtils.java @@ -0,0 +1,65 @@ +package ca.bc.gov.educ.api.gradbusiness.util; + +import org.apache.commons.lang3.SystemUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.FileSystemUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Set; + +public class IOUtils { + + private static final Logger logger = LoggerFactory.getLogger(IOUtils.class); + + private IOUtils(){} + + /** + * Creates a secured temp dir for processing files, it is up to + * calling method to also remove directory (see removeFileOrDirectory + * method in this class) + * + * @param location + * @param prefix + * @return + * @throws IOException + */ + public static File createTempDirectory(String location, String prefix) throws IOException { + File temp; + Path loc = Paths.get(location); + if (SystemUtils.IS_OS_UNIX) { + FileAttribute> attr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------")); + temp = Files.createTempDirectory(loc, prefix, attr).toFile(); // Compliant + } else { + temp = Files.createTempDirectory(loc, prefix).toFile(); // Compliant + temp.setReadable(true, true); + temp.setWritable(true, true); + temp.setExecutable(true, true); + } + return temp; + } + + /** + * Removes a directory or file recursively + * @param file + */ + public static void removeFileOrDirectory(File file) { + try { + if(file.isDirectory() && file.exists()){ + FileSystemUtils.deleteRecursively(file); + } else { + Files.deleteIfExists(Path.of(file.getAbsolutePath())); + } + } catch (IOException e) { + logger.error("Unable to delete file or folder {}", file.getAbsolutePath()); + } + } + +} From 6ec38cd12ae95808a84a77395c4417e4306410ff Mon Sep 17 00:00:00 2001 From: arybakov Date: Fri, 24 Nov 2023 11:49:52 -0700 Subject: [PATCH 5/6] Fix --- .../gradbusiness/util/EducGradBusinessUtil.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java index b3cfb69..60fbb7a 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java @@ -10,10 +10,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -31,23 +28,27 @@ private EducGradBusinessUtil() {} public static byte[] mergeDocumentsPDFs(List locations) throws IOException { File bufferDirectory = null; - ByteArrayOutputStream result = new ByteArrayOutputStream(); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + ByteArrayInputStream result; try { bufferDirectory = IOUtils.createTempDirectory(TMP_DIR, "buffer"); PDFMergerUtility pdfMergerUtility = new PDFMergerUtility(); - pdfMergerUtility.setDestinationStream(result); + pdfMergerUtility.setDestinationStream(outputStream); pdfMergerUtility.addSources(locations); MemoryUsageSetting memoryUsageSetting = MemoryUsageSetting.setupMixed(50000000) .setTempDir(bufferDirectory); pdfMergerUtility.mergeDocuments(memoryUsageSetting); + result = new ByteArrayInputStream(outputStream.toByteArray()); + return result.readAllBytes(); } catch (Exception e) { logger.error("Error {}", e.getLocalizedMessage()); } finally { if (bufferDirectory != null) { IOUtils.removeFileOrDirectory(bufferDirectory); } + outputStream.close(); } - return result.toByteArray(); + return new byte[0]; } public static byte[] mergeDocuments(List locations) throws IOException { From bc9da42f751f8aa3e73bf3b64b3b717052d74395 Mon Sep 17 00:00:00 2001 From: arybakov Date: Fri, 24 Nov 2023 12:29:16 -0700 Subject: [PATCH 6/6] Tests fix --- .../api/gradbusiness/EducGradBusinessApiApplicationTests.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java b/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java index 733c306..993d1f8 100644 --- a/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java +++ b/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java @@ -324,8 +324,7 @@ void testgetAmalgamatedSchoolReportPDFByMincode() throws Exception { byteData = gradBusinessService.getAmalgamatedSchoolReportPDFByMincode(mincode, type, "accessToken"); assertNotNull(byteData); - assertNotNull(byteData.getBody()); - assertTrue(byteData.getStatusCode().is5xxServerError()); + assertNull(byteData.getBody()); }