feat(NoticeEmbeddingUpdater): 공지 embedding을 위한 Updater 구현

src/main/java/com/kustacks/kuring/worker/scrap/KuisHomepageNoticeScraperTemplate.java

@@ -2,15 +2,18 @@
 
 import com.kustacks.kuring.common.exception.InternalLogicException;
 import com.kustacks.kuring.common.exception.code.ErrorCode;
+import com.kustacks.kuring.notice.application.port.out.dto.NoticeDto;
 import com.kustacks.kuring.worker.dto.ComplexNoticeFormatDto;
 import com.kustacks.kuring.worker.dto.ScrapingResultDto;
-import com.kustacks.kuring.worker.scrap.noticeinfo.KuisHomepageNoticeInfo;
+import com.kustacks.kuring.worker.parser.notice.PageTextDto;
 import com.kustacks.kuring.worker.parser.notice.RowsDto;
+import com.kustacks.kuring.worker.scrap.noticeinfo.KuisHomepageNoticeInfo;
 import com.kustacks.kuring.worker.update.notice.dto.response.CommonNoticeFormatDto;
 import lombok.extern.slf4j.Slf4j;
 import org.jsoup.nodes.Document;
 import org.springframework.stereotype.Component;
 
+import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.function.Function;
@@ -34,6 +37,19 @@ public List<ComplexNoticeFormatDto> scrap(
         return noticeDtoList;
     }
 
+    public List<PageTextDto> scrapForEmbedding(
+            List<NoticeDto> scrapResults,
+            KuisHomepageNoticeInfo noticeInfo
+    ) throws InternalLogicException {
+        List<ScrapingResultDto> requestResults = requestWithDeptInfoForEmbedding(scrapResults, noticeInfo);
+
+        log.debug("[{}] Text extract begin", noticeInfo.getCategoryName());
+        List<PageTextDto> noticeDtoList = htmlTextParsingFromScrapingResult(noticeInfo, requestResults);
+        log.debug("[{}] Text extract end", noticeInfo.getCategoryName());
+
+        return noticeDtoList;
+    }
+
     private void validateScrapedNoticeCountIsNotZero(List<ComplexNoticeFormatDto> noticeDtoList) {
         for (ComplexNoticeFormatDto complexNoticeFormatDto : noticeDtoList) {
             if (complexNoticeFormatDto.getNormalNoticeSize() == 0) {
@@ -48,16 +64,52 @@ private List<ScrapingResultDto> requestWithDeptInfo(
     ) {
         long startTime = System.currentTimeMillis();
 
-        log.debug("[{}] HTML 요청", kuisNoticeInfo.getCategoryName());
+        log.debug("[{}] HTML SCRAP 요청", kuisNoticeInfo.getCategoryName());
         List<ScrapingResultDto> reqResults = decisionMaker.apply(kuisNoticeInfo);
-        log.debug("[{}] HTML 수신", kuisNoticeInfo.getCategoryName());
+        log.debug("[{}] HTML SCRAP 수신", kuisNoticeInfo.getCategoryName());
 
         long endTime = System.currentTimeMillis();
         log.debug("[{}] 파싱에 소요된 초 = {}", kuisNoticeInfo.getCategoryName(), (endTime - startTime) / 1000.0);
 
         return reqResults;
     }
 
+    private List<ScrapingResultDto> requestWithDeptInfoForEmbedding(
+            List<NoticeDto> scrapResults,
+            KuisHomepageNoticeInfo noticeInfo
+    ) {
+        long startTime = System.currentTimeMillis();
+
+        List<ScrapingResultDto> scrapResultDtos = new LinkedList<>();
+        for (NoticeDto scrapResult : scrapResults) {
+            log.debug("[{}] HTML SCRAP 요청", noticeInfo.getCategoryName());
+            scrapResultDtos.add(noticeInfo.scrapSinglePageHtml(scrapResult.getUrl()));
+            log.debug("[{}] HTML SCRAP 수신", noticeInfo.getCategoryName());
+        }
+
+        long endTime = System.currentTimeMillis();
+        log.debug("[{}] 파싱에 소요된 초 = {}", noticeInfo.getCategoryName(), (endTime - startTime) / 1000.0);
+
+        return scrapResultDtos;
+    }
+
+    private List<PageTextDto> htmlTextParsingFromScrapingResult(
+            KuisHomepageNoticeInfo noticeInfo,
+            List<ScrapingResultDto> results
+    ) {
+        List<PageTextDto> parsedTexts = new ArrayList<>();
+        for (ScrapingResultDto result : results) {
+            try {
+                PageTextDto parsedText = noticeInfo.parseText(result.getDocument());
+                parsedTexts.add(parsedText);
+            } catch (InternalLogicException e) {
+                log.error("Exception extracting url: {}", result.getViewUrl(), e);
+            }
+        }
+
+        return parsedTexts;
+    }
+
 
     private List<ComplexNoticeFormatDto> htmlParsingFromScrapingResult(
             KuisHomepageNoticeInfo kuisNoticeInfo,

src/main/java/com/kustacks/kuring/worker/scrap/client/notice/KuisHomepageNoticeApiClient.java

@@ -62,7 +62,7 @@ public List<ScrapingResultDto> requestAll(KuisHomepageNoticeInfo kuisHomepageNot
     @Override
     public ScrapingResultDto requestSinglePageWithUrl(KuisHomepageNoticeInfo noticeInfo, String url) {
         try {
-            Document document = jsoupClient.get(url, LATEST_SCRAP_ALL_TIMEOUT);
+            Document document = jsoupClient.get(url, LATEST_SCRAP_TIMEOUT);
             return new ScrapingResultDto(document, url);
         } catch (IOException e) {
             log.info("Notice Text Scrap IOException", e);
@@ -78,7 +78,7 @@ public int getTotalNoticeSize(String url) throws IOException, IndexOutOfBoundsEx
 
         Element totalNoticeSizeElement = document.selectFirst(".util-search strong");
 
-        if(totalNoticeSizeElement == null) { // 총 공지 개수가 없는 경우 650개로 가정
+        if (totalNoticeSizeElement == null) { // 총 공지 개수가 없는 경우 650개로 가정
             return TOTAL_KUIS_NOTICES_COUNT;
         }
 

src/main/java/com/kustacks/kuring/worker/update/notice/NoticeEmbeddingUpdater.java

@@ -0,0 +1,80 @@
+package com.kustacks.kuring.worker.update.notice;
+
+import com.kustacks.kuring.ai.application.port.out.CommandVectorStorePort;
+import com.kustacks.kuring.notice.application.port.out.NoticeCommandPort;
+import com.kustacks.kuring.notice.application.port.out.NoticeQueryPort;
+import com.kustacks.kuring.notice.application.port.out.dto.NoticeDto;
+import com.kustacks.kuring.notice.domain.CategoryName;
+import com.kustacks.kuring.worker.parser.notice.PageTextDto;
+import com.kustacks.kuring.worker.scrap.KuisHomepageNoticeScraperTemplate;
+import com.kustacks.kuring.worker.scrap.noticeinfo.KuisHomepageNoticeInfo;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+import org.springframework.stereotype.Component;
+
+import java.time.LocalDateTime;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class NoticeEmbeddingUpdater {
+
+    private final ThreadPoolTaskExecutor noticeUpdaterThreadTaskExecutor;
+    private final KuisHomepageNoticeScraperTemplate scrapperTemplate;
+    private final List<KuisHomepageNoticeInfo> kuisNoticeInfoList;
+    private final CommandVectorStorePort commandVectorStorePort;
+    private final NoticeCommandPort noticeCommandPort;
+    private final NoticeQueryPort noticeQueryPort;
+
+    /*
+    학사, 장학, 취창업, 국제, 학생, 산학, 일반, 공지 embedding
+    */
+    @Scheduled(cron = "0 0 21 * * *", zone = "Asia/Seoul") // 매일 오후 9시 embedding 작업 수행
+    public void update() {
+        log.info("========== KUIS Hompage Embedding 시작 ==========");
+
+        for (KuisHomepageNoticeInfo kuisNoticeInfo : kuisNoticeInfoList) {
+            CompletableFuture
+                    .supplyAsync(
+                            () -> lookupNotYetEmbeddingNotice(kuisNoticeInfo),
+                            noticeUpdaterThreadTaskExecutor
+                    ).thenApply(
+                            scrapResults -> scrapNoticeText(scrapResults, kuisNoticeInfo)
+                    ).thenAccept(
+                            scrapResults -> embeddingNotice(scrapResults, kuisNoticeInfo.getCategoryName())
+                    );
+        }
+    }
+
+    private List<NoticeDto> lookupNotYetEmbeddingNotice(KuisHomepageNoticeInfo noticeInfo) {
+        log.debug("lookupNotYetEmbeddingNotice {}", noticeInfo.getCategoryName());
+        LocalDateTime startDate = LocalDateTime.now().minusMonths(2);
+        return noticeQueryPort.findNotYetEmbeddingNotice(noticeInfo.getCategoryName(), startDate);
+    }
+
+    private List<PageTextDto> scrapNoticeText(
+            List<NoticeDto> scrapResults,
+            KuisHomepageNoticeInfo noticeInfo
+    ) {
+        return scrapperTemplate.scrapForEmbedding(scrapResults, noticeInfo);
+    }
+
+    private void embeddingNotice(List<PageTextDto> extractTextResults, CategoryName categoryName) {
+        if (extractTextResults.isEmpty()) {
+            log.debug("Embedding {} no more notice to embed", categoryName);
+            return;
+        }
+        log.info("Embedding {}, size = {}", categoryName, extractTextResults.size());
+
+        commandVectorStorePort.embedding(extractTextResults, categoryName);
+        List<String> articleIds = extractTextResults.stream()
+                .map(PageTextDto::articleId)
+                .toList();
+
+        noticeCommandPort.updateNoticeEmbeddingStatus(categoryName, articleIds);
+    }
+}