Merge pull request #124 from bcgov/develop/alex-GRAD2-2410-2

Develop/alex grad2 2410 2

api/pom.xml

@@ -175,6 +175,17 @@
             <artifactId>json</artifactId>
             <version>20220320</version>
         </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-collections4 -->
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-collections4</artifactId>
+            <version>4.4</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.pdfbox</groupId>
+            <artifactId>pdfbox</artifactId>
+            <version>2.0.27</version>
+        </dependency>
     </dependencies>
 
     <build>

api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java

@@ -43,7 +43,7 @@ public WebClient webClient() {
                 .exchangeStrategies(ExchangeStrategies.builder()
                 .codecs(configurer -> configurer
                         .defaultCodecs()
-                        .maxInMemorySize(100 * 1024 * 1024))
+                        .maxInMemorySize(300 * 1024 * 1024))
                 .build()).build();
     }
 

api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java

@@ -7,6 +7,7 @@
 import ca.bc.gov.educ.api.gradbusiness.util.TokenUtils;
 import io.github.resilience4j.retry.annotation.Retry;
 import jakarta.transaction.Transactional;
+import org.apache.commons.collections4.ListUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -21,7 +22,6 @@
 import org.springframework.web.reactive.function.BodyInserters;
 import org.springframework.web.reactive.function.client.WebClient;
 
-import java.io.IOException;
 import java.io.InputStream;
 import java.util.*;
 import java.util.concurrent.CompletableFuture;
@@ -201,14 +201,15 @@ public ResponseEntity<byte[]> getAmalgamatedSchoolReportPDFByMincode(String minc
         List<InputStream> locations = new ArrayList<>();
         if (studentList != null && !studentList.isEmpty()) {
             logger.debug("******** Fetched {} students ******", studentList.size());
-            getStudentAchievementReports(studentList, locations);
+            List<List<UUID>> partitions = ListUtils.partition(studentList, 200);
+            getStudentAchievementReports(partitions, locations);
             Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("PST"), Locale.CANADA);
             int year = cal.get(Calendar.YEAR);
             String month = "00";
             String fileName = EducGradBusinessUtil.getFileNameSchoolReports(mincode, year, month, type);
             try {
                 logger.debug("******** Merging Documents Started ******");
-                byte[] res = EducGradBusinessUtil.mergeDocuments(locations);
+                byte[] res = EducGradBusinessUtil.mergeDocumentsPDFs(locations);
                 logger.debug("******** Merged {} Documents ******", locations.size());
                 HttpHeaders headers = new HttpHeaders();
                 headers.put(HttpHeaders.AUTHORIZATION, Collections.singletonList(BEARER + accessToken));
@@ -274,29 +275,32 @@ public ResponseEntity<byte[]> getStudentTranscriptPDFByType(String pen, String t
         }
     }
 
-    private void getStudentAchievementReports(List<UUID> studentList, List<InputStream> locations) {
+    private void getStudentAchievementReports(List<List<UUID>> partitions, List<InputStream> locations) {
         logger.debug("******** Getting Student Achievement Reports ******");
-        List<CompletableFuture<InputStream>> futures = studentList.stream()
-                .map(studentGuid -> CompletableFuture.supplyAsync(() -> getStudentAchievementReport(studentGuid)))
-                .toList();
-        CompletableFuture<Void> allFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[futures.size()]));
-        CompletableFuture<List<InputStream>> result = allFutures.thenApply(v -> futures.stream()
-                .map(CompletableFuture::join)
-                .toList());
-        locations.addAll(result.join());
-        logger.debug("******** Fetched All Student Achievement Reports ******");
+        for(List<UUID> studentList: partitions) {
+            logger.debug("******** Run partition with {} students ******", studentList.size());
+            List<CompletableFuture<InputStream>> futures = studentList.stream()
+                    .map(studentGuid -> CompletableFuture.supplyAsync(() -> getStudentAchievementReport(studentGuid)))
+                    .toList();
+            CompletableFuture<Void> allFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[futures.size()]));
+            CompletableFuture<List<InputStream>> result = allFutures.thenApply(v -> futures.stream()
+                    .map(CompletableFuture::join)
+                    .toList());
+            locations.addAll(result.join());
+        }
+        logger.debug("******** Fetched All {} Student Achievement Reports ******", locations.size());
     }
 
     private InputStream getStudentAchievementReport(UUID studentGuid) {
         String accessTokenNext = tokenUtils.getAccessToken();
-        InputStreamResource result = webClient.get().uri(String.format(educGraduationApiConstants.getStudentCredentialByType(), studentGuid, "ACHV")).headers(h -> h.setBearerAuth(accessTokenNext)).retrieve().bodyToMono(InputStreamResource.class).block();
-        if (result != null) {
-            try {
+        try {
+            InputStreamResource result = webClient.get().uri(String.format(educGraduationApiConstants.getStudentCredentialByType(), studentGuid, "ACHV")).headers(h -> h.setBearerAuth(accessTokenNext)).retrieve().bodyToMono(InputStreamResource.class).block();
+            if (result != null) {
                 logger.debug("******** Fetched Achievement Report for {} ******", studentGuid);
                 return result.getInputStream();
-            } catch (IOException e) {
-                logger.debug("Error extracting report binary from stream: {}", e.getLocalizedMessage());
             }
+        } catch (Exception e) {
+            logger.debug("Error extracting report binary from stream: {}", e.getLocalizedMessage());
         }
         return null;
     }

api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java

@@ -5,21 +5,52 @@
 import com.itextpdf.text.pdf.PdfCopy;
 import com.itextpdf.text.pdf.PdfReader;
 import com.itextpdf.text.pdf.PdfSmartCopy;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
+import org.apache.pdfbox.io.MemoryUsageSetting;
+import org.apache.pdfbox.multipdf.PDFMergerUtility;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
 
 public class EducGradBusinessUtil {
 
+    private static final Logger logger = LoggerFactory.getLogger(EducGradBusinessUtil.class);
+
+    public static final String TMP_DIR = "/tmp";
+
     private EducGradBusinessUtil() {}
 
 
     private static final int BUFFER_SIZE = 250000;
 
+    public static byte[] mergeDocumentsPDFs(List<InputStream> locations) throws IOException {
+        File bufferDirectory = null;
+        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+        ByteArrayInputStream result;
+        try {
+            bufferDirectory = IOUtils.createTempDirectory(TMP_DIR, "buffer");
+            PDFMergerUtility pdfMergerUtility = new PDFMergerUtility();
+            pdfMergerUtility.setDestinationStream(outputStream);
+            pdfMergerUtility.addSources(locations);
+            MemoryUsageSetting memoryUsageSetting = MemoryUsageSetting.setupMixed(50000000)
+                    .setTempDir(bufferDirectory);
+            pdfMergerUtility.mergeDocuments(memoryUsageSetting);
+            result = new ByteArrayInputStream(outputStream.toByteArray());
+            return result.readAllBytes();
+        } catch (Exception e) {
+            logger.error("Error {}", e.getLocalizedMessage());
+        } finally {
+            if (bufferDirectory != null) {
+                IOUtils.removeFileOrDirectory(bufferDirectory);
+            }
+            outputStream.close();
+        }
+        return new byte[0];
+    }
+
     public static byte[] mergeDocuments(List<InputStream> locations) throws IOException {
         final byte[] result;
 

api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/IOUtils.java

@@ -0,0 +1,65 @@
+package ca.bc.gov.educ.api.gradbusiness.util;
+
+import org.apache.commons.lang3.SystemUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.FileSystemUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.FileAttribute;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.Set;
+
+public class IOUtils {
+
+    private static final Logger logger = LoggerFactory.getLogger(IOUtils.class);
+
+    private IOUtils(){}
+
+    /**
+     * Creates a secured temp dir for processing files, it is up to
+     * calling method to also remove directory (see removeFileOrDirectory
+     * method in this class)
+     *
+     * @param location
+     * @param prefix
+     * @return
+     * @throws IOException
+     */
+    public static File createTempDirectory(String location, String prefix) throws IOException {
+        File temp;
+        Path loc = Paths.get(location);
+        if (SystemUtils.IS_OS_UNIX) {
+            FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------"));
+            temp = Files.createTempDirectory(loc, prefix, attr).toFile(); // Compliant
+        } else {
+            temp = Files.createTempDirectory(loc, prefix).toFile();  // Compliant
+            temp.setReadable(true, true);
+            temp.setWritable(true, true);
+            temp.setExecutable(true, true);
+        }
+        return temp;
+    }
+
+    /**
+     * Removes a directory or file recursively
+     * @param file
+     */
+    public static void removeFileOrDirectory(File file) {
+        try {
+            if(file.isDirectory() && file.exists()){
+                FileSystemUtils.deleteRecursively(file);
+            } else {
+                Files.deleteIfExists(Path.of(file.getAbsolutePath()));
+            }
+        } catch (IOException e) {
+            logger.error("Unable to delete file or folder {}", file.getAbsolutePath());
+        }
+    }
+
+}

api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/TokenUtils.java

@@ -49,6 +49,7 @@ private ResponseObj getResponseObj() {
                 constants.getUserName(), constants.getPassword());
         MultiValueMap<String, String> map= new LinkedMultiValueMap<>();
         map.add("grant_type", "client_credentials");
+        logger.debug("******** Fetch Access Token ********");
         return this.webClient.post().uri(constants.getTokenUrl())
                 .headers(h -> h.addAll(httpHeaders))
                 .contentType(MediaType.APPLICATION_FORM_URLENCODED)

api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java

@@ -324,8 +324,7 @@ void testgetAmalgamatedSchoolReportPDFByMincode() throws Exception {
 
 		byteData = gradBusinessService.getAmalgamatedSchoolReportPDFByMincode(mincode, type, "accessToken");
 		assertNotNull(byteData);
-		assertNotNull(byteData.getBody());
-		assertTrue(byteData.getStatusCode().is5xxServerError());
+		assertNull(byteData.getBody());
 
 	}