diff --git a/api/pom.xml b/api/pom.xml
index ecb4c05..3436e57 100644
--- a/api/pom.xml
+++ b/api/pom.xml
@@ -175,6 +175,17 @@
json
20220320
+
+
+ org.apache.commons
+ commons-collections4
+ 4.4
+
+
+ org.apache.pdfbox
+ pdfbox
+ 2.0.27
+
diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java
index b2ea285..bac2109 100644
--- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java
+++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java
@@ -43,7 +43,7 @@ public WebClient webClient() {
.exchangeStrategies(ExchangeStrategies.builder()
.codecs(configurer -> configurer
.defaultCodecs()
- .maxInMemorySize(100 * 1024 * 1024))
+ .maxInMemorySize(300 * 1024 * 1024))
.build()).build();
}
diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java
index 6e34e2d..b6f7d12 100644
--- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java
+++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java
@@ -7,6 +7,7 @@
import ca.bc.gov.educ.api.gradbusiness.util.TokenUtils;
import io.github.resilience4j.retry.annotation.Retry;
import jakarta.transaction.Transactional;
+import org.apache.commons.collections4.ListUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -21,7 +22,6 @@
import org.springframework.web.reactive.function.BodyInserters;
import org.springframework.web.reactive.function.client.WebClient;
-import java.io.IOException;
import java.io.InputStream;
import java.util.*;
import java.util.concurrent.CompletableFuture;
@@ -201,14 +201,15 @@ public ResponseEntity getAmalgamatedSchoolReportPDFByMincode(String minc
List locations = new ArrayList<>();
if (studentList != null && !studentList.isEmpty()) {
logger.debug("******** Fetched {} students ******", studentList.size());
- getStudentAchievementReports(studentList, locations);
+ List> partitions = ListUtils.partition(studentList, 200);
+ getStudentAchievementReports(partitions, locations);
Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("PST"), Locale.CANADA);
int year = cal.get(Calendar.YEAR);
String month = "00";
String fileName = EducGradBusinessUtil.getFileNameSchoolReports(mincode, year, month, type);
try {
logger.debug("******** Merging Documents Started ******");
- byte[] res = EducGradBusinessUtil.mergeDocuments(locations);
+ byte[] res = EducGradBusinessUtil.mergeDocumentsPDFs(locations);
logger.debug("******** Merged {} Documents ******", locations.size());
HttpHeaders headers = new HttpHeaders();
headers.put(HttpHeaders.AUTHORIZATION, Collections.singletonList(BEARER + accessToken));
@@ -274,29 +275,32 @@ public ResponseEntity getStudentTranscriptPDFByType(String pen, String t
}
}
- private void getStudentAchievementReports(List studentList, List locations) {
+ private void getStudentAchievementReports(List> partitions, List locations) {
logger.debug("******** Getting Student Achievement Reports ******");
- List> futures = studentList.stream()
- .map(studentGuid -> CompletableFuture.supplyAsync(() -> getStudentAchievementReport(studentGuid)))
- .toList();
- CompletableFuture allFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[futures.size()]));
- CompletableFuture> result = allFutures.thenApply(v -> futures.stream()
- .map(CompletableFuture::join)
- .toList());
- locations.addAll(result.join());
- logger.debug("******** Fetched All Student Achievement Reports ******");
+ for(List studentList: partitions) {
+ logger.debug("******** Run partition with {} students ******", studentList.size());
+ List> futures = studentList.stream()
+ .map(studentGuid -> CompletableFuture.supplyAsync(() -> getStudentAchievementReport(studentGuid)))
+ .toList();
+ CompletableFuture allFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[futures.size()]));
+ CompletableFuture> result = allFutures.thenApply(v -> futures.stream()
+ .map(CompletableFuture::join)
+ .toList());
+ locations.addAll(result.join());
+ }
+ logger.debug("******** Fetched All {} Student Achievement Reports ******", locations.size());
}
private InputStream getStudentAchievementReport(UUID studentGuid) {
String accessTokenNext = tokenUtils.getAccessToken();
- InputStreamResource result = webClient.get().uri(String.format(educGraduationApiConstants.getStudentCredentialByType(), studentGuid, "ACHV")).headers(h -> h.setBearerAuth(accessTokenNext)).retrieve().bodyToMono(InputStreamResource.class).block();
- if (result != null) {
- try {
+ try {
+ InputStreamResource result = webClient.get().uri(String.format(educGraduationApiConstants.getStudentCredentialByType(), studentGuid, "ACHV")).headers(h -> h.setBearerAuth(accessTokenNext)).retrieve().bodyToMono(InputStreamResource.class).block();
+ if (result != null) {
logger.debug("******** Fetched Achievement Report for {} ******", studentGuid);
return result.getInputStream();
- } catch (IOException e) {
- logger.debug("Error extracting report binary from stream: {}", e.getLocalizedMessage());
}
+ } catch (Exception e) {
+ logger.debug("Error extracting report binary from stream: {}", e.getLocalizedMessage());
}
return null;
}
diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java
index d5edf3e..60fbb7a 100644
--- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java
+++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java
@@ -5,21 +5,52 @@
import com.itextpdf.text.pdf.PdfCopy;
import com.itextpdf.text.pdf.PdfReader;
import com.itextpdf.text.pdf.PdfSmartCopy;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
+import org.apache.pdfbox.io.MemoryUsageSetting;
+import org.apache.pdfbox.multipdf.PDFMergerUtility;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
public class EducGradBusinessUtil {
+ private static final Logger logger = LoggerFactory.getLogger(EducGradBusinessUtil.class);
+
+ public static final String TMP_DIR = "/tmp";
+
private EducGradBusinessUtil() {}
private static final int BUFFER_SIZE = 250000;
+ public static byte[] mergeDocumentsPDFs(List locations) throws IOException {
+ File bufferDirectory = null;
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ ByteArrayInputStream result;
+ try {
+ bufferDirectory = IOUtils.createTempDirectory(TMP_DIR, "buffer");
+ PDFMergerUtility pdfMergerUtility = new PDFMergerUtility();
+ pdfMergerUtility.setDestinationStream(outputStream);
+ pdfMergerUtility.addSources(locations);
+ MemoryUsageSetting memoryUsageSetting = MemoryUsageSetting.setupMixed(50000000)
+ .setTempDir(bufferDirectory);
+ pdfMergerUtility.mergeDocuments(memoryUsageSetting);
+ result = new ByteArrayInputStream(outputStream.toByteArray());
+ return result.readAllBytes();
+ } catch (Exception e) {
+ logger.error("Error {}", e.getLocalizedMessage());
+ } finally {
+ if (bufferDirectory != null) {
+ IOUtils.removeFileOrDirectory(bufferDirectory);
+ }
+ outputStream.close();
+ }
+ return new byte[0];
+ }
+
public static byte[] mergeDocuments(List locations) throws IOException {
final byte[] result;
diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/IOUtils.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/IOUtils.java
new file mode 100644
index 0000000..8d54886
--- /dev/null
+++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/IOUtils.java
@@ -0,0 +1,65 @@
+package ca.bc.gov.educ.api.gradbusiness.util;
+
+import org.apache.commons.lang3.SystemUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.FileSystemUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.FileAttribute;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.Set;
+
+public class IOUtils {
+
+ private static final Logger logger = LoggerFactory.getLogger(IOUtils.class);
+
+ private IOUtils(){}
+
+ /**
+ * Creates a secured temp dir for processing files, it is up to
+ * calling method to also remove directory (see removeFileOrDirectory
+ * method in this class)
+ *
+ * @param location
+ * @param prefix
+ * @return
+ * @throws IOException
+ */
+ public static File createTempDirectory(String location, String prefix) throws IOException {
+ File temp;
+ Path loc = Paths.get(location);
+ if (SystemUtils.IS_OS_UNIX) {
+ FileAttribute> attr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------"));
+ temp = Files.createTempDirectory(loc, prefix, attr).toFile(); // Compliant
+ } else {
+ temp = Files.createTempDirectory(loc, prefix).toFile(); // Compliant
+ temp.setReadable(true, true);
+ temp.setWritable(true, true);
+ temp.setExecutable(true, true);
+ }
+ return temp;
+ }
+
+ /**
+ * Removes a directory or file recursively
+ * @param file
+ */
+ public static void removeFileOrDirectory(File file) {
+ try {
+ if(file.isDirectory() && file.exists()){
+ FileSystemUtils.deleteRecursively(file);
+ } else {
+ Files.deleteIfExists(Path.of(file.getAbsolutePath()));
+ }
+ } catch (IOException e) {
+ logger.error("Unable to delete file or folder {}", file.getAbsolutePath());
+ }
+ }
+
+}
diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/TokenUtils.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/TokenUtils.java
index 87e8088..18ea101 100644
--- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/TokenUtils.java
+++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/TokenUtils.java
@@ -49,6 +49,7 @@ private ResponseObj getResponseObj() {
constants.getUserName(), constants.getPassword());
MultiValueMap map= new LinkedMultiValueMap<>();
map.add("grant_type", "client_credentials");
+ logger.debug("******** Fetch Access Token ********");
return this.webClient.post().uri(constants.getTokenUrl())
.headers(h -> h.addAll(httpHeaders))
.contentType(MediaType.APPLICATION_FORM_URLENCODED)
diff --git a/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java b/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java
index 733c306..993d1f8 100644
--- a/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java
+++ b/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java
@@ -324,8 +324,7 @@ void testgetAmalgamatedSchoolReportPDFByMincode() throws Exception {
byteData = gradBusinessService.getAmalgamatedSchoolReportPDFByMincode(mincode, type, "accessToken");
assertNotNull(byteData);
- assertNotNull(byteData.getBody());
- assertTrue(byteData.getStatusCode().is5xxServerError());
+ assertNull(byteData.getBody());
}