From ed75badc6b1a5ca101cfb87cba4897c8416e2d47 Mon Sep 17 00:00:00 2001 From: jiwoo Date: Sun, 14 Jan 2024 21:19:05 +0900 Subject: [PATCH] =?UTF-8?q?feat(NoticeJdbcRepository):=20Batch=20insert?= =?UTF-8?q?=EB=A5=BC=20=EC=9C=84=ED=95=9C=20NoticeJdbcRepository=20?= =?UTF-8?q?=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../notice/domain/NoticeJdbcRepository.java | 64 +++++++++++++++++++ .../update/notice/CategoryNoticeUpdater.java | 12 ++-- .../notice/DepartmentNoticeUpdater.java | 9 ++- src/main/resources/application.yml | 2 + .../update/DepartmentNoticeUpdaterTest.java | 2 +- src/test/resources/application-test.yml | 4 ++ 6 files changed, 82 insertions(+), 11 deletions(-) create mode 100644 src/main/java/com/kustacks/kuring/notice/domain/NoticeJdbcRepository.java diff --git a/src/main/java/com/kustacks/kuring/notice/domain/NoticeJdbcRepository.java b/src/main/java/com/kustacks/kuring/notice/domain/NoticeJdbcRepository.java new file mode 100644 index 00000000..4da21053 --- /dev/null +++ b/src/main/java/com/kustacks/kuring/notice/domain/NoticeJdbcRepository.java @@ -0,0 +1,64 @@ +package com.kustacks.kuring.notice.domain; + +import lombok.RequiredArgsConstructor; +import org.springframework.jdbc.core.BatchPreparedStatementSetter; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.List; + +@Repository +@RequiredArgsConstructor +public class NoticeJdbcRepository { + + private final JdbcTemplate jdbcTemplate; + + @Transactional + public void saveAllCategoryNotices(List notices) { + jdbcTemplate.batchUpdate("INSERT INTO notice (article_id, category_name, important, posted_dt, subject, updated_dt, url, dtype) values (?, ?, ?, ?, ?, ?, ?, 'Notice')", + new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + Notice notice = notices.get(i); + ps.setString(1, notice.getArticleId()); + ps.setString(2, notice.getCategoryName().toUpperCase()); + ps.setInt(3, notice.isImportant() ? 1 : 0); + ps.setString(4, notice.getPostedDate()); + ps.setString(5, notice.getSubject()); + ps.setString(6, notice.getUpdatedDate()); + ps.setString(7, notice.getUrl()); + } + + @Override + public int getBatchSize() { + return notices.size(); + } + }); + } + + @Transactional + public void saveAllDepartmentNotices(List departmentNotices) { + jdbcTemplate.batchUpdate("INSERT INTO notice (article_id, category_name, important, posted_dt, subject, updated_dt, url, dtype) values (?, ?, ?, ?, ?, ?, ?, 'DepartmentNotice')", + new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + DepartmentNotice departmentNotice = departmentNotices.get(i); + ps.setString(1, departmentNotice.getArticleId()); + ps.setString(2, departmentNotice.getCategoryName().toUpperCase()); + ps.setInt(3, departmentNotice.isImportant() ? 1 : 0); + ps.setString(4, departmentNotice.getPostedDate()); + ps.setString(5, departmentNotice.getSubject()); + ps.setString(6, departmentNotice.getUpdatedDate()); + ps.setString(7, departmentNotice.getUrl()); + } + + @Override + public int getBatchSize() { + return departmentNotices.size(); + } + }); + } +} diff --git a/src/main/java/com/kustacks/kuring/worker/update/notice/CategoryNoticeUpdater.java b/src/main/java/com/kustacks/kuring/worker/update/notice/CategoryNoticeUpdater.java index dfd6b143..bc5c48c0 100644 --- a/src/main/java/com/kustacks/kuring/worker/update/notice/CategoryNoticeUpdater.java +++ b/src/main/java/com/kustacks/kuring/worker/update/notice/CategoryNoticeUpdater.java @@ -3,6 +3,7 @@ import com.kustacks.kuring.notice.domain.CategoryName; import com.kustacks.kuring.message.firebase.FirebaseService; import com.kustacks.kuring.notice.domain.Notice; +import com.kustacks.kuring.notice.domain.NoticeJdbcRepository; import com.kustacks.kuring.notice.domain.NoticeRepository; import com.kustacks.kuring.worker.scrap.KuisNoticeScraperTemplate; import com.kustacks.kuring.worker.scrap.client.notice.LibraryNoticeApiClient; @@ -30,6 +31,7 @@ public class CategoryNoticeUpdater { private final List kuisNoticeInfoList; private final KuisNoticeScraperTemplate scrapperTemplate; private final NoticeRepository noticeRepository; + private final NoticeJdbcRepository noticeJdbcRepository; private final FirebaseService firebaseService; private final LibraryNoticeApiClient libraryNoticeApiClient; private final ThreadPoolTaskExecutor noticeUpdaterThreadTaskExecutor; @@ -61,15 +63,11 @@ private void updateLibrary() { } private List updateLibraryNotice(CategoryName categoryName) { - List scrapResults = libraryNoticeApiClient.request(categoryName); - Collections.reverse(scrapResults); - return scrapResults; + return libraryNoticeApiClient.request(categoryName); } private List updateKuisNoticeAsync(KuisNoticeInfo deptInfo, Function> decisionMaker) { - List scrapResults = scrapperTemplate.scrap(deptInfo, decisionMaker); - Collections.reverse(scrapResults); - return scrapResults; + return scrapperTemplate.scrap(deptInfo, decisionMaker); } private List compareLatestAndUpdateDB(List scrapResults, CategoryName categoryName) { @@ -92,7 +90,7 @@ private List synchronizationWithDb(List scrapResu List deletedNoticesArticleIds = filteringSoonDeleteIds(savedArticleIds, scrapNoticeIds); - noticeRepository.saveAllAndFlush(newNotices); + noticeJdbcRepository.saveAllCategoryNotices(newNotices); if (!deletedNoticesArticleIds.isEmpty()) { noticeRepository.deleteAllByIdsAndCategory(categoryName, deletedNoticesArticleIds); diff --git a/src/main/java/com/kustacks/kuring/worker/update/notice/DepartmentNoticeUpdater.java b/src/main/java/com/kustacks/kuring/worker/update/notice/DepartmentNoticeUpdater.java index e75a0455..dc54e034 100644 --- a/src/main/java/com/kustacks/kuring/worker/update/notice/DepartmentNoticeUpdater.java +++ b/src/main/java/com/kustacks/kuring/worker/update/notice/DepartmentNoticeUpdater.java @@ -1,10 +1,12 @@ package com.kustacks.kuring.worker.update.notice; -import com.kustacks.kuring.notice.domain.CategoryName; + import com.kustacks.kuring.message.firebase.FirebaseService; +import com.kustacks.kuring.notice.domain.CategoryName; import com.kustacks.kuring.notice.domain.DepartmentName; import com.kustacks.kuring.notice.domain.DepartmentNotice; import com.kustacks.kuring.notice.domain.DepartmentNoticeRepository; +import com.kustacks.kuring.notice.domain.NoticeJdbcRepository; import com.kustacks.kuring.worker.scrap.DepartmentNoticeScraperTemplate; import com.kustacks.kuring.worker.scrap.deptinfo.DeptInfo; import com.kustacks.kuring.worker.scrap.dto.ComplexNoticeFormatDto; @@ -34,6 +36,7 @@ public class DepartmentNoticeUpdater { private final List deptInfoList; private final DepartmentNoticeScraperTemplate scrapperTemplate; + private final NoticeJdbcRepository noticeJdbcRepository; private final DepartmentNoticeRepository departmentNoticeRepository; private final ThreadPoolTaskExecutor noticeUpdaterThreadTaskExecutor; private final FirebaseService firebaseService; @@ -107,7 +110,7 @@ private List compareLatestAndUpdateDB(List saveNewNotices(List scrapResults, List savedArticleIds, DepartmentName departmentNameEnum, boolean important) { List newNotices = filteringSoonSaveNotice(scrapResults, savedArticleIds, departmentNameEnum, important); - departmentNoticeRepository.saveAllAndFlush(newNotices); + noticeJdbcRepository.saveAllDepartmentNotices(newNotices); return newNotices; } @@ -143,7 +146,7 @@ private void synchronizationWithDb(List scrapResults, Lis List deletedNoticesArticleIds = filteringSoonDeleteIds(savedArticleIds, latestNoticeIds); - departmentNoticeRepository.saveAllAndFlush(newNotices); + noticeJdbcRepository.saveAllDepartmentNotices(newNotices); if (!deletedNoticesArticleIds.isEmpty()) { departmentNoticeRepository.deleteAllByIdsAndDepartment(departmentNameEnum, deletedNoticesArticleIds); diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 7c72eea6..69e4fbf5 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -11,6 +11,8 @@ spring: ddl-auto: none properties: hibernate: + jdbc: + batch_size: 100 format_sql: true dialect: com.kustacks.kuring.config.CustomMariaDbDialect defer-datasource-initialization: false diff --git a/src/test/java/com/kustacks/kuring/worker/update/DepartmentNoticeUpdaterTest.java b/src/test/java/com/kustacks/kuring/worker/update/DepartmentNoticeUpdaterTest.java index 5b11282a..1278935f 100644 --- a/src/test/java/com/kustacks/kuring/worker/update/DepartmentNoticeUpdaterTest.java +++ b/src/test/java/com/kustacks/kuring/worker/update/DepartmentNoticeUpdaterTest.java @@ -57,7 +57,7 @@ public void department_scrap_async_test() throws InterruptedException { // when departmentNoticeUpdater.update(); - noticeUpdaterThreadTaskExecutor.getThreadPoolExecutor().awaitTermination(3, TimeUnit.SECONDS); + noticeUpdaterThreadTaskExecutor.getThreadPoolExecutor().awaitTermination(1, TimeUnit.SECONDS); List notices = departmentNoticeRepository.findAll(); diff --git a/src/test/resources/application-test.yml b/src/test/resources/application-test.yml index f5f3a650..95666fbb 100644 --- a/src/test/resources/application-test.yml +++ b/src/test/resources/application-test.yml @@ -6,6 +6,8 @@ spring: driver-class-name: org.testcontainers.jdbc.ContainerDatabaseDriver p6spy: enable-logging: true + hikari: + jdbc-url: jdbc:tc:mariadb:10.2://localhost:3306/kuring?profileSQL=true&logger=Slf4JLogger&maxQuerySizeToLog=999999 flyway: enabled: false jpa: @@ -14,6 +16,8 @@ spring: show-sql: true properties: hibernate: + jdbc: + batch_size: 100 format_sql: true dialect: com.kustacks.kuring.config.CustomMariaDbDialect defer-datasource-initialization: true