feat(NoticeJdbcRepository): Batch insert를 위한 NoticeJdbcRepository 구현

ku-ring · Jan 14, 2024 · ed75bad · ed75bad
1 parent a46c2e6
commit ed75bad
Show file tree

Hide file tree

Showing 6 changed files with 82 additions and 11 deletions.
diff --git a/src/main/java/com/kustacks/kuring/notice/domain/NoticeJdbcRepository.java b/src/main/java/com/kustacks/kuring/notice/domain/NoticeJdbcRepository.java
@@ -0,0 +1,64 @@
+package com.kustacks.kuring.notice.domain;
+
+import lombok.RequiredArgsConstructor;
+import org.springframework.jdbc.core.BatchPreparedStatementSetter;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Repository;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.List;
+
+@Repository
+@RequiredArgsConstructor
+public class NoticeJdbcRepository {
+
+    private final JdbcTemplate jdbcTemplate;
+
+    @Transactional
+    public void saveAllCategoryNotices(List<Notice> notices) {
+        jdbcTemplate.batchUpdate("INSERT INTO notice (article_id, category_name, important, posted_dt, subject, updated_dt, url, dtype) values (?, ?, ?, ?, ?, ?, ?, 'Notice')",
+                new BatchPreparedStatementSetter() {
+                    @Override
+                    public void setValues(PreparedStatement ps, int i) throws SQLException {
+                        Notice notice = notices.get(i);
+                        ps.setString(1, notice.getArticleId());
+                        ps.setString(2, notice.getCategoryName().toUpperCase());
+                        ps.setInt(3, notice.isImportant() ? 1 : 0);
+                        ps.setString(4, notice.getPostedDate());
+                        ps.setString(5, notice.getSubject());
+                        ps.setString(6, notice.getUpdatedDate());
+                        ps.setString(7, notice.getUrl());
+                    }
+
+                    @Override
+                    public int getBatchSize() {
+                        return notices.size();
+                    }
+                });
+    }
+
+    @Transactional
+    public void saveAllDepartmentNotices(List<DepartmentNotice> departmentNotices) {
+        jdbcTemplate.batchUpdate("INSERT INTO notice (article_id, category_name, important, posted_dt, subject, updated_dt, url, dtype) values (?, ?, ?, ?, ?, ?, ?, 'DepartmentNotice')",
+                new BatchPreparedStatementSetter() {
+                    @Override
+                    public void setValues(PreparedStatement ps, int i) throws SQLException {
+                        DepartmentNotice departmentNotice = departmentNotices.get(i);
+                        ps.setString(1, departmentNotice.getArticleId());
+                        ps.setString(2, departmentNotice.getCategoryName().toUpperCase());
+                        ps.setInt(3, departmentNotice.isImportant() ? 1 : 0);
+                        ps.setString(4, departmentNotice.getPostedDate());
+                        ps.setString(5, departmentNotice.getSubject());
+                        ps.setString(6, departmentNotice.getUpdatedDate());
+                        ps.setString(7, departmentNotice.getUrl());
+                    }
+
+                    @Override
+                    public int getBatchSize() {
+                        return departmentNotices.size();
+                    }
+                });
+    }
+}
diff --git a/src/main/java/com/kustacks/kuring/worker/update/notice/CategoryNoticeUpdater.java b/src/main/java/com/kustacks/kuring/worker/update/notice/CategoryNoticeUpdater.java
@@ -3,6 +3,7 @@
 import com.kustacks.kuring.notice.domain.CategoryName;
 import com.kustacks.kuring.message.firebase.FirebaseService;
 import com.kustacks.kuring.notice.domain.Notice;
+import com.kustacks.kuring.notice.domain.NoticeJdbcRepository;
 import com.kustacks.kuring.notice.domain.NoticeRepository;
 import com.kustacks.kuring.worker.scrap.KuisNoticeScraperTemplate;
 import com.kustacks.kuring.worker.scrap.client.notice.LibraryNoticeApiClient;
@@ -30,6 +31,7 @@ public class CategoryNoticeUpdater {
     private final List<KuisNoticeInfo> kuisNoticeInfoList;
     private final KuisNoticeScraperTemplate scrapperTemplate;
     private final NoticeRepository noticeRepository;
+    private final NoticeJdbcRepository noticeJdbcRepository;
     private final FirebaseService firebaseService;
     private final LibraryNoticeApiClient libraryNoticeApiClient;
     private final ThreadPoolTaskExecutor noticeUpdaterThreadTaskExecutor;
@@ -61,15 +63,11 @@ private void updateLibrary() {
     }
 
     private List<CommonNoticeFormatDto> updateLibraryNotice(CategoryName categoryName) {
-        List<CommonNoticeFormatDto> scrapResults = libraryNoticeApiClient.request(categoryName);
-        Collections.reverse(scrapResults);
-        return scrapResults;
+        return libraryNoticeApiClient.request(categoryName);
     }
 
     private List<CommonNoticeFormatDto> updateKuisNoticeAsync(KuisNoticeInfo deptInfo, Function<KuisNoticeInfo, List<CommonNoticeFormatDto>> decisionMaker) {
-        List<CommonNoticeFormatDto> scrapResults = scrapperTemplate.scrap(deptInfo, decisionMaker);
-        Collections.reverse(scrapResults);
-        return scrapResults;
+        return scrapperTemplate.scrap(deptInfo, decisionMaker);
     }
 
     private List<Notice> compareLatestAndUpdateDB(List<CommonNoticeFormatDto> scrapResults, CategoryName categoryName) {
@@ -92,7 +90,7 @@ private List<Notice> synchronizationWithDb(List<CommonNoticeFormatDto> scrapResu
 
         List<String> deletedNoticesArticleIds = filteringSoonDeleteIds(savedArticleIds, scrapNoticeIds);
 
-        noticeRepository.saveAllAndFlush(newNotices);
+        noticeJdbcRepository.saveAllCategoryNotices(newNotices);
 
         if (!deletedNoticesArticleIds.isEmpty()) {
             noticeRepository.deleteAllByIdsAndCategory(categoryName, deletedNoticesArticleIds);

diff --git a/src/main/java/com/kustacks/kuring/worker/update/notice/DepartmentNoticeUpdater.java b/src/main/java/com/kustacks/kuring/worker/update/notice/DepartmentNoticeUpdater.java
@@ -1,10 +1,12 @@
 package com.kustacks.kuring.worker.update.notice;
 
-import com.kustacks.kuring.notice.domain.CategoryName;
+
 import com.kustacks.kuring.message.firebase.FirebaseService;
+import com.kustacks.kuring.notice.domain.CategoryName;
 import com.kustacks.kuring.notice.domain.DepartmentName;
 import com.kustacks.kuring.notice.domain.DepartmentNotice;
 import com.kustacks.kuring.notice.domain.DepartmentNoticeRepository;
+import com.kustacks.kuring.notice.domain.NoticeJdbcRepository;
 import com.kustacks.kuring.worker.scrap.DepartmentNoticeScraperTemplate;
 import com.kustacks.kuring.worker.scrap.deptinfo.DeptInfo;
 import com.kustacks.kuring.worker.scrap.dto.ComplexNoticeFormatDto;
@@ -34,6 +36,7 @@ public class DepartmentNoticeUpdater {
 
     private final List<DeptInfo> deptInfoList;
     private final DepartmentNoticeScraperTemplate scrapperTemplate;
+    private final NoticeJdbcRepository noticeJdbcRepository;
     private final DepartmentNoticeRepository departmentNoticeRepository;
     private final ThreadPoolTaskExecutor noticeUpdaterThreadTaskExecutor;
     private final FirebaseService firebaseService;
@@ -107,7 +110,7 @@ private List<DepartmentNotice> compareLatestAndUpdateDB(List<ComplexNoticeFormat
 
     private List<DepartmentNotice> saveNewNotices(List<CommonNoticeFormatDto> scrapResults, List<Integer> savedArticleIds, DepartmentName departmentNameEnum, boolean important) {
         List<DepartmentNotice> newNotices = filteringSoonSaveNotice(scrapResults, savedArticleIds, departmentNameEnum, important);
-        departmentNoticeRepository.saveAllAndFlush(newNotices);
+        noticeJdbcRepository.saveAllDepartmentNotices(newNotices);
         return newNotices;
     }
 
@@ -143,7 +146,7 @@ private void synchronizationWithDb(List<CommonNoticeFormatDto> scrapResults, Lis
 
         List<String> deletedNoticesArticleIds = filteringSoonDeleteIds(savedArticleIds, latestNoticeIds);
 
-        departmentNoticeRepository.saveAllAndFlush(newNotices);
+        noticeJdbcRepository.saveAllDepartmentNotices(newNotices);
 
         if (!deletedNoticesArticleIds.isEmpty()) {
             departmentNoticeRepository.deleteAllByIdsAndDepartment(departmentNameEnum, deletedNoticesArticleIds);

diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
@@ -11,6 +11,8 @@ spring:
       ddl-auto: none
     properties:
       hibernate:
+        jdbc:
+          batch_size: 100
         format_sql: true
         dialect: com.kustacks.kuring.config.CustomMariaDbDialect
     defer-datasource-initialization: false

diff --git a/src/test/java/com/kustacks/kuring/worker/update/DepartmentNoticeUpdaterTest.java b/src/test/java/com/kustacks/kuring/worker/update/DepartmentNoticeUpdaterTest.java
@@ -57,7 +57,7 @@ public void department_scrap_async_test() throws InterruptedException {
 
         // when
         departmentNoticeUpdater.update();
-        noticeUpdaterThreadTaskExecutor.getThreadPoolExecutor().awaitTermination(3, TimeUnit.SECONDS);
+        noticeUpdaterThreadTaskExecutor.getThreadPoolExecutor().awaitTermination(1, TimeUnit.SECONDS);
         List<DepartmentNotice> notices = departmentNoticeRepository.findAll();
 
 

diff --git a/src/test/resources/application-test.yml b/src/test/resources/application-test.yml
@@ -6,6 +6,8 @@ spring:
     driver-class-name: org.testcontainers.jdbc.ContainerDatabaseDriver
     p6spy:
       enable-logging: true
+    hikari:
+      jdbc-url: jdbc:tc:mariadb:10.2://localhost:3306/kuring?profileSQL=true&logger=Slf4JLogger&maxQuerySizeToLog=999999
   flyway:
     enabled: false
   jpa:
@@ -14,6 +16,8 @@ spring:
     show-sql: true
     properties:
       hibernate:
+        jdbc:
+          batch_size: 100
         format_sql: true
         dialect: com.kustacks.kuring.config.CustomMariaDbDialect
     defer-datasource-initialization: true