diff --git a/api/pom.xml b/api/pom.xml index ecb4c05..3436e57 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -175,6 +175,17 @@ json 20220320 + + + org.apache.commons + commons-collections4 + 4.4 + + + org.apache.pdfbox + pdfbox + 2.0.27 + diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java index b2ea285..bac2109 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java @@ -43,7 +43,7 @@ public WebClient webClient() { .exchangeStrategies(ExchangeStrategies.builder() .codecs(configurer -> configurer .defaultCodecs() - .maxInMemorySize(100 * 1024 * 1024)) + .maxInMemorySize(300 * 1024 * 1024)) .build()).build(); } diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java index 6e34e2d..b6f7d12 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java @@ -7,6 +7,7 @@ import ca.bc.gov.educ.api.gradbusiness.util.TokenUtils; import io.github.resilience4j.retry.annotation.Retry; import jakarta.transaction.Transactional; +import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,7 +22,6 @@ import org.springframework.web.reactive.function.BodyInserters; import org.springframework.web.reactive.function.client.WebClient; -import java.io.IOException; import java.io.InputStream; import java.util.*; import java.util.concurrent.CompletableFuture; @@ -201,14 +201,15 @@ public ResponseEntity getAmalgamatedSchoolReportPDFByMincode(String minc List locations = new ArrayList<>(); if (studentList != null && !studentList.isEmpty()) { logger.debug("******** Fetched {} students ******", studentList.size()); - getStudentAchievementReports(studentList, locations); + List> partitions = ListUtils.partition(studentList, 200); + getStudentAchievementReports(partitions, locations); Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("PST"), Locale.CANADA); int year = cal.get(Calendar.YEAR); String month = "00"; String fileName = EducGradBusinessUtil.getFileNameSchoolReports(mincode, year, month, type); try { logger.debug("******** Merging Documents Started ******"); - byte[] res = EducGradBusinessUtil.mergeDocuments(locations); + byte[] res = EducGradBusinessUtil.mergeDocumentsPDFs(locations); logger.debug("******** Merged {} Documents ******", locations.size()); HttpHeaders headers = new HttpHeaders(); headers.put(HttpHeaders.AUTHORIZATION, Collections.singletonList(BEARER + accessToken)); @@ -274,29 +275,32 @@ public ResponseEntity getStudentTranscriptPDFByType(String pen, String t } } - private void getStudentAchievementReports(List studentList, List locations) { + private void getStudentAchievementReports(List> partitions, List locations) { logger.debug("******** Getting Student Achievement Reports ******"); - List> futures = studentList.stream() - .map(studentGuid -> CompletableFuture.supplyAsync(() -> getStudentAchievementReport(studentGuid))) - .toList(); - CompletableFuture allFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[futures.size()])); - CompletableFuture> result = allFutures.thenApply(v -> futures.stream() - .map(CompletableFuture::join) - .toList()); - locations.addAll(result.join()); - logger.debug("******** Fetched All Student Achievement Reports ******"); + for(List studentList: partitions) { + logger.debug("******** Run partition with {} students ******", studentList.size()); + List> futures = studentList.stream() + .map(studentGuid -> CompletableFuture.supplyAsync(() -> getStudentAchievementReport(studentGuid))) + .toList(); + CompletableFuture allFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[futures.size()])); + CompletableFuture> result = allFutures.thenApply(v -> futures.stream() + .map(CompletableFuture::join) + .toList()); + locations.addAll(result.join()); + } + logger.debug("******** Fetched All {} Student Achievement Reports ******", locations.size()); } private InputStream getStudentAchievementReport(UUID studentGuid) { String accessTokenNext = tokenUtils.getAccessToken(); - InputStreamResource result = webClient.get().uri(String.format(educGraduationApiConstants.getStudentCredentialByType(), studentGuid, "ACHV")).headers(h -> h.setBearerAuth(accessTokenNext)).retrieve().bodyToMono(InputStreamResource.class).block(); - if (result != null) { - try { + try { + InputStreamResource result = webClient.get().uri(String.format(educGraduationApiConstants.getStudentCredentialByType(), studentGuid, "ACHV")).headers(h -> h.setBearerAuth(accessTokenNext)).retrieve().bodyToMono(InputStreamResource.class).block(); + if (result != null) { logger.debug("******** Fetched Achievement Report for {} ******", studentGuid); return result.getInputStream(); - } catch (IOException e) { - logger.debug("Error extracting report binary from stream: {}", e.getLocalizedMessage()); } + } catch (Exception e) { + logger.debug("Error extracting report binary from stream: {}", e.getLocalizedMessage()); } return null; } diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java index d5edf3e..60fbb7a 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java @@ -5,21 +5,52 @@ import com.itextpdf.text.pdf.PdfCopy; import com.itextpdf.text.pdf.PdfReader; import com.itextpdf.text.pdf.PdfSmartCopy; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; +import org.apache.pdfbox.io.MemoryUsageSetting; +import org.apache.pdfbox.multipdf.PDFMergerUtility; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; import java.util.ArrayList; import java.util.List; import java.util.Optional; public class EducGradBusinessUtil { + private static final Logger logger = LoggerFactory.getLogger(EducGradBusinessUtil.class); + + public static final String TMP_DIR = "/tmp"; + private EducGradBusinessUtil() {} private static final int BUFFER_SIZE = 250000; + public static byte[] mergeDocumentsPDFs(List locations) throws IOException { + File bufferDirectory = null; + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + ByteArrayInputStream result; + try { + bufferDirectory = IOUtils.createTempDirectory(TMP_DIR, "buffer"); + PDFMergerUtility pdfMergerUtility = new PDFMergerUtility(); + pdfMergerUtility.setDestinationStream(outputStream); + pdfMergerUtility.addSources(locations); + MemoryUsageSetting memoryUsageSetting = MemoryUsageSetting.setupMixed(50000000) + .setTempDir(bufferDirectory); + pdfMergerUtility.mergeDocuments(memoryUsageSetting); + result = new ByteArrayInputStream(outputStream.toByteArray()); + return result.readAllBytes(); + } catch (Exception e) { + logger.error("Error {}", e.getLocalizedMessage()); + } finally { + if (bufferDirectory != null) { + IOUtils.removeFileOrDirectory(bufferDirectory); + } + outputStream.close(); + } + return new byte[0]; + } + public static byte[] mergeDocuments(List locations) throws IOException { final byte[] result; diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/IOUtils.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/IOUtils.java new file mode 100644 index 0000000..8d54886 --- /dev/null +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/IOUtils.java @@ -0,0 +1,65 @@ +package ca.bc.gov.educ.api.gradbusiness.util; + +import org.apache.commons.lang3.SystemUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.FileSystemUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Set; + +public class IOUtils { + + private static final Logger logger = LoggerFactory.getLogger(IOUtils.class); + + private IOUtils(){} + + /** + * Creates a secured temp dir for processing files, it is up to + * calling method to also remove directory (see removeFileOrDirectory + * method in this class) + * + * @param location + * @param prefix + * @return + * @throws IOException + */ + public static File createTempDirectory(String location, String prefix) throws IOException { + File temp; + Path loc = Paths.get(location); + if (SystemUtils.IS_OS_UNIX) { + FileAttribute> attr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------")); + temp = Files.createTempDirectory(loc, prefix, attr).toFile(); // Compliant + } else { + temp = Files.createTempDirectory(loc, prefix).toFile(); // Compliant + temp.setReadable(true, true); + temp.setWritable(true, true); + temp.setExecutable(true, true); + } + return temp; + } + + /** + * Removes a directory or file recursively + * @param file + */ + public static void removeFileOrDirectory(File file) { + try { + if(file.isDirectory() && file.exists()){ + FileSystemUtils.deleteRecursively(file); + } else { + Files.deleteIfExists(Path.of(file.getAbsolutePath())); + } + } catch (IOException e) { + logger.error("Unable to delete file or folder {}", file.getAbsolutePath()); + } + } + +} diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/TokenUtils.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/TokenUtils.java index 87e8088..18ea101 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/TokenUtils.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/TokenUtils.java @@ -49,6 +49,7 @@ private ResponseObj getResponseObj() { constants.getUserName(), constants.getPassword()); MultiValueMap map= new LinkedMultiValueMap<>(); map.add("grant_type", "client_credentials"); + logger.debug("******** Fetch Access Token ********"); return this.webClient.post().uri(constants.getTokenUrl()) .headers(h -> h.addAll(httpHeaders)) .contentType(MediaType.APPLICATION_FORM_URLENCODED) diff --git a/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java b/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java index 733c306..993d1f8 100644 --- a/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java +++ b/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java @@ -324,8 +324,7 @@ void testgetAmalgamatedSchoolReportPDFByMincode() throws Exception { byteData = gradBusinessService.getAmalgamatedSchoolReportPDFByMincode(mincode, type, "accessToken"); assertNotNull(byteData); - assertNotNull(byteData.getBody()); - assertTrue(byteData.getStatusCode().is5xxServerError()); + assertNull(byteData.getBody()); }