From a62379e95c8d29c3d1ec65a8581388c103917601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=80=EC=9B=90?= <110761377+andrewkimswe@users.noreply.github.com> Date: Tue, 11 Feb 2025 15:35:50 +0900 Subject: [PATCH 01/18] =?UTF-8?q?=08hotfix=20:=20headless=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../giftidea/service/CoupangApiService.java | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/team4/giftidea/service/CoupangApiService.java b/src/main/java/com/team4/giftidea/service/CoupangApiService.java index 6cf5afd..8b23b84 100644 --- a/src/main/java/com/team4/giftidea/service/CoupangApiService.java +++ b/src/main/java/com/team4/giftidea/service/CoupangApiService.java @@ -51,21 +51,14 @@ public List searchItems(String query) { System.setProperty("webdriver.chrome.driver", chromeDriverPath); ChromeOptions options = new ChromeOptions(); - options.setBinary("/opt/google/chrome/chrome"); // 크롬 바이너리 직접 지정 (AWS 환경) - options.addArguments("--disable-gpu"); + options.addArguments("--headless=new"); // Headless 모드 유지 + options.addArguments("--disable-gpu"); options.addArguments("--no-sandbox"); - options.addArguments("--disable-dev-shm-usage"); - options.addArguments("--remote-debugging-port=9222"); - options.addArguments("--window-size=1920,1080"); - options.addArguments("--disable-software-rasterizer"); - options.addArguments("--disable-crash-reporter"); - options.addArguments("--disable-extensions"); - options.addArguments("--disable-hang-monitor"); - - // 최신 User-Agent 추가 - options.addArguments("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.5672.63 Safari/537.36"); - - WebDriver driver = new ChromeDriver(options); + options.addArguments("--disable-dev-shm-usage"); + options.addArguments("--remote-debugging-port=9222"); + options.addArguments("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.90 Safari/537.36"); // 브라우저 속이기 + + WebDriver driver = new ChromeDriver(options); try { String encodedQuery = URLEncoder.encode(query, StandardCharsets.UTF_8); From a98776bb88b93f706fe981260428c61228730625 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=80=EC=9B=90?= <110761377+andrewkimswe@users.noreply.github.com> Date: Tue, 11 Feb 2025 15:36:16 +0900 Subject: [PATCH 02/18] =?UTF-8?q?hotfix=20:=20Kream=EC=97=90=20headless=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/team4/giftidea/service/KreamApiService.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/team4/giftidea/service/KreamApiService.java b/src/main/java/com/team4/giftidea/service/KreamApiService.java index 051318f..6059526 100644 --- a/src/main/java/com/team4/giftidea/service/KreamApiService.java +++ b/src/main/java/com/team4/giftidea/service/KreamApiService.java @@ -48,19 +48,12 @@ public List searchItems(String query) { System.setProperty("webdriver.chrome.driver", chromeDriverPath); ChromeOptions options = new ChromeOptions(); - options.setBinary("/opt/google/chrome/chrome"); // 크롬 바이너리 직접 지정 (AWS 환경) + options.addArguments("--headless=new"); // Headless 모드 유지 options.addArguments("--disable-gpu"); options.addArguments("--no-sandbox"); options.addArguments("--disable-dev-shm-usage"); options.addArguments("--remote-debugging-port=9222"); - options.addArguments("--window-size=1920,1080"); // 창 크기 설정 - options.addArguments("--disable-software-rasterizer"); - options.addArguments("--disable-crash-reporter"); - options.addArguments("--disable-extensions"); - options.addArguments("--disable-hang-monitor"); - - // 최신 User-Agent 추가 - options.addArguments("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.5672.63 Safari/537.36"); + options.addArguments("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.90 Safari/537.36"); // 브라우저 속이기 WebDriver driver = new ChromeDriver(options); From c23a859726b55a08c7d40d00298a0ada21780e66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=80=EC=9B=90?= <110761377+andrewkimswe@users.noreply.github.com> Date: Tue, 11 Feb 2025 15:54:39 +0900 Subject: [PATCH 03/18] =?UTF-8?q?=08hotfix=20:=20selenium=20=EA=B4=80?= =?UTF-8?q?=EB=A0=A8=20=EC=88=98=EC=A0=95=203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/com/team4/giftidea/service/KreamApiService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/team4/giftidea/service/KreamApiService.java b/src/main/java/com/team4/giftidea/service/KreamApiService.java index 6059526..c50e76a 100644 --- a/src/main/java/com/team4/giftidea/service/KreamApiService.java +++ b/src/main/java/com/team4/giftidea/service/KreamApiService.java @@ -48,6 +48,7 @@ public List searchItems(String query) { System.setProperty("webdriver.chrome.driver", chromeDriverPath); ChromeOptions options = new ChromeOptions(); + options.setBinary("/opt/google/chrome/chrome"); options.addArguments("--headless=new"); // Headless 모드 유지 options.addArguments("--disable-gpu"); options.addArguments("--no-sandbox"); From a52916c40c40602ec7aaf9ce5e64056a87059d32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=80=EC=9B=90?= <110761377+andrewkimswe@users.noreply.github.com> Date: Tue, 11 Feb 2025 15:55:02 +0900 Subject: [PATCH 04/18] =?UTF-8?q?=08hotfix=20:=20selenium=20=EA=B4=80?= =?UTF-8?q?=EB=A0=A8=20=EC=88=98=EC=A0=95=204?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/com/team4/giftidea/service/CoupangApiService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/team4/giftidea/service/CoupangApiService.java b/src/main/java/com/team4/giftidea/service/CoupangApiService.java index 8b23b84..4afbbc1 100644 --- a/src/main/java/com/team4/giftidea/service/CoupangApiService.java +++ b/src/main/java/com/team4/giftidea/service/CoupangApiService.java @@ -51,6 +51,7 @@ public List searchItems(String query) { System.setProperty("webdriver.chrome.driver", chromeDriverPath); ChromeOptions options = new ChromeOptions(); + options.setBinary("/opt/google/chrome/chrome"); options.addArguments("--headless=new"); // Headless 모드 유지 options.addArguments("--disable-gpu"); options.addArguments("--no-sandbox"); From 8ba1e5ba0c57682691a8fd3ff3b1f0f85ce74fe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=80=EC=9B=90?= <110761377+andrewkimswe@users.noreply.github.com> Date: Tue, 11 Feb 2025 16:31:55 +0900 Subject: [PATCH 05/18] Update CoupangApiService.java --- src/main/java/com/team4/giftidea/service/CoupangApiService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/team4/giftidea/service/CoupangApiService.java b/src/main/java/com/team4/giftidea/service/CoupangApiService.java index 4afbbc1..7126033 100644 --- a/src/main/java/com/team4/giftidea/service/CoupangApiService.java +++ b/src/main/java/com/team4/giftidea/service/CoupangApiService.java @@ -52,7 +52,7 @@ public List searchItems(String query) { ChromeOptions options = new ChromeOptions(); options.setBinary("/opt/google/chrome/chrome"); - options.addArguments("--headless=new"); // Headless 모드 유지 + options.addArguments("--headless"); // Headless 모드 유지 options.addArguments("--disable-gpu"); options.addArguments("--no-sandbox"); options.addArguments("--disable-dev-shm-usage"); From c4e36400b32c85459b60db17329b0c4f44d7686e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=80=EC=9B=90?= <110761377+andrewkimswe@users.noreply.github.com> Date: Tue, 11 Feb 2025 16:32:04 +0900 Subject: [PATCH 06/18] Update KreamApiService.java --- src/main/java/com/team4/giftidea/service/KreamApiService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/team4/giftidea/service/KreamApiService.java b/src/main/java/com/team4/giftidea/service/KreamApiService.java index c50e76a..5cc3519 100644 --- a/src/main/java/com/team4/giftidea/service/KreamApiService.java +++ b/src/main/java/com/team4/giftidea/service/KreamApiService.java @@ -49,7 +49,7 @@ public List searchItems(String query) { ChromeOptions options = new ChromeOptions(); options.setBinary("/opt/google/chrome/chrome"); - options.addArguments("--headless=new"); // Headless 모드 유지 + options.addArguments("--headless"); // Headless 모드 유지 options.addArguments("--disable-gpu"); options.addArguments("--no-sandbox"); options.addArguments("--disable-dev-shm-usage"); From a9fa7b35a29defdbd9a2c68f58609bef2e2b4f60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=80=EC=9B=90?= <110761377+andrewkimswe@users.noreply.github.com> Date: Tue, 11 Feb 2025 16:55:05 +0900 Subject: [PATCH 07/18] Update CoupangApiService.java --- .../giftidea/service/CoupangApiService.java | 125 +++++++++++------- 1 file changed, 74 insertions(+), 51 deletions(-) diff --git a/src/main/java/com/team4/giftidea/service/CoupangApiService.java b/src/main/java/com/team4/giftidea/service/CoupangApiService.java index 7126033..80749e8 100644 --- a/src/main/java/com/team4/giftidea/service/CoupangApiService.java +++ b/src/main/java/com/team4/giftidea/service/CoupangApiService.java @@ -41,57 +41,80 @@ public CoupangApiService(ProductService productService) { } /** - * 주어진 키워드에 대해 쿠팡에서 상품을 검색하고 리스트로 반환합니다. - * - * @param query 검색 키워드 - * @return 크롤링된 상품 리스트 - */ - public List searchItems(String query) { - List productList = new ArrayList<>(); - System.setProperty("webdriver.chrome.driver", chromeDriverPath); - - ChromeOptions options = new ChromeOptions(); - options.setBinary("/opt/google/chrome/chrome"); - options.addArguments("--headless"); // Headless 모드 유지 - options.addArguments("--disable-gpu"); - options.addArguments("--no-sandbox"); - options.addArguments("--disable-dev-shm-usage"); - options.addArguments("--remote-debugging-port=9222"); - options.addArguments("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.90 Safari/537.36"); // 브라우저 속이기 - - WebDriver driver = new ChromeDriver(options); - - try { - String encodedQuery = URLEncoder.encode(query, StandardCharsets.UTF_8); - String searchUrl = String.format(COUPANG_SEARCH_URL, encodedQuery); - - driver.get(searchUrl); - WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(90)); - wait.until(ExpectedConditions.presenceOfElementLocated(By.cssSelector(".search-product"))); - - List products = driver.findElements(By.cssSelector(".search-product")); - - for (WebElement productElement : products) { - try { - Product productEntity = extractProductInfo(productElement, query); - if (productEntity != null) { - productList.add(productEntity); - productService.saveItems(List.of(productEntity), query); - } - } catch (NoSuchElementException e) { - log.warn("요소를 찾을 수 없음: {}", e.getMessage()); - } - } - } catch (TimeoutException e) { - log.error("페이지 로딩 시간 초과: {}", e.getMessage()); - } catch (Exception e) { - log.error("크롤링 중 오류 발생: {}", e.getMessage()); - } finally { - driver.quit(); // 크롤링 후 브라우저 종료 - } - - return productList; - } + * 주어진 키워드에 대해 쿠팡에서 상품을 검색하고 리스트로 반환합니다. + * + * @param query 검색 키워드 + * @return 크롤링된 상품 리스트 + */ + public List searchItems(String query) { + List productList = new ArrayList<>(); + log.info("크롤링 시작: 키워드 = {}", query); + + System.setProperty("webdriver.chrome.driver", chromeDriverPath); + log.debug("ChromeDriver 경로: {}", chromeDriverPath); + + ChromeOptions options = new ChromeOptions(); + options.addArguments("--headless=new"); + options.addArguments("--disable-gpu"); + options.addArguments("--no-sandbox"); + options.addArguments("--disable-dev-shm-usage"); + options.addArguments("--remote-debugging-port=9222"); + options.addArguments("--disable-software-rasterizer"); + options.addArguments("--disable-extensions"); + options.addArguments("--disable-popup-blocking"); + options.addArguments("--window-size=1920,1080"); + options.addArguments(USER_AGENT); + + WebDriver driver = null; // 드라이버 선언 + + try { + driver = new ChromeDriver(options); // 드라이버 초기화 + log.info("ChromeDriver 초기화 성공"); + + String encodedQuery = URLEncoder.encode(query, StandardCharsets.UTF_8); + String searchUrl = String.format(COUPANG_SEARCH_URL, encodedQuery); + + log.info("쿠팡 검색 URL: {}", searchUrl); + driver.get(searchUrl); + + WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(90)); + wait.until(ExpectedConditions.presenceOfElementLocated(By.cssSelector(".search-product"))); + log.info("쿠팡 검색 페이지 로딩 완료"); + + + List products = driver.findElements(By.cssSelector(".search-product")); + log.info("검색된 상품 개수: {}", products.size()); + + for (WebElement productElement : products) { + try { + Product productEntity = extractProductInfo(productElement, query); + if (productEntity != null) { + productList.add(productEntity); + productService.saveItems(List.of(productEntity), query); + log.debug("상품 정보 추출 및 저장 성공: {}", productEntity); + } + } catch (NoSuchElementException e) { + log.warn("요소 찾기 실패: {}", e.getMessage(), e); // 예외 객체 e도 함께 로깅 + } + } + } catch (TimeoutException e) { + log.error("페이지 로딩 시간 초과: {}", e.getMessage(), e); // 예외 객체 e도 함께 로깅 + } catch (SessionNotCreatedException e) { + log.error("세션 생성 실패: {}", e.getMessage(), e); + } catch (WebDriverException e) { + log.error("WebDriver 오류: {}", e.getMessage(), e); + } catch (Exception e) { + log.error("크롤링 중 오류 발생: {}", e.getMessage(), e); // 예외 객체 e도 함께 로깅 + } finally { + if (driver != null) { // 드라이버가 null이 아닌 경우에만 종료 + driver.quit(); + log.info("ChromeDriver 종료"); + } + } + + log.info("크롤링 종료: 반환된 상품 개수 = {}", productList.size()); + return productList; + } /** * 개별 상품 정보를 추출하여 Product 객체를 생성합니다. From b4a366e3f7335ddaa054676d44a096e65e29e028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=80=EC=9B=90?= <110761377+andrewkimswe@users.noreply.github.com> Date: Tue, 11 Feb 2025 17:20:13 +0900 Subject: [PATCH 08/18] Update CoupangApiService.java --- src/main/java/com/team4/giftidea/service/CoupangApiService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/team4/giftidea/service/CoupangApiService.java b/src/main/java/com/team4/giftidea/service/CoupangApiService.java index 80749e8..cac2f8c 100644 --- a/src/main/java/com/team4/giftidea/service/CoupangApiService.java +++ b/src/main/java/com/team4/giftidea/service/CoupangApiService.java @@ -54,7 +54,6 @@ public List searchItems(String query) { log.debug("ChromeDriver 경로: {}", chromeDriverPath); ChromeOptions options = new ChromeOptions(); - options.addArguments("--headless=new"); options.addArguments("--disable-gpu"); options.addArguments("--no-sandbox"); options.addArguments("--disable-dev-shm-usage"); From 7ae0cfb4554910741dbecd6cf2679fb59dd0b9b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=80=EC=9B=90?= <110761377+andrewkimswe@users.noreply.github.com> Date: Tue, 11 Feb 2025 17:25:52 +0900 Subject: [PATCH 09/18] Update CoupangApiService.java --- src/main/java/com/team4/giftidea/service/CoupangApiService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/team4/giftidea/service/CoupangApiService.java b/src/main/java/com/team4/giftidea/service/CoupangApiService.java index cac2f8c..4ceb511 100644 --- a/src/main/java/com/team4/giftidea/service/CoupangApiService.java +++ b/src/main/java/com/team4/giftidea/service/CoupangApiService.java @@ -54,6 +54,7 @@ public List searchItems(String query) { log.debug("ChromeDriver 경로: {}", chromeDriverPath); ChromeOptions options = new ChromeOptions(); + options.addArguments("--headless"); options.addArguments("--disable-gpu"); options.addArguments("--no-sandbox"); options.addArguments("--disable-dev-shm-usage"); From f5eb7ae3fc8e5e3baa7fcec4762b63ac23ccb63a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=80=EC=9B=90?= <110761377+andrewkimswe@users.noreply.github.com> Date: Tue, 11 Feb 2025 17:32:29 +0900 Subject: [PATCH 10/18] Update CoupangApiService.java --- .../giftidea/service/CoupangApiService.java | 157 ++++++++++-------- 1 file changed, 89 insertions(+), 68 deletions(-) diff --git a/src/main/java/com/team4/giftidea/service/CoupangApiService.java b/src/main/java/com/team4/giftidea/service/CoupangApiService.java index 4ceb511..96c937c 100644 --- a/src/main/java/com/team4/giftidea/service/CoupangApiService.java +++ b/src/main/java/com/team4/giftidea/service/CoupangApiService.java @@ -47,74 +47,95 @@ public CoupangApiService(ProductService productService) { * @return 크롤링된 상품 리스트 */ public List searchItems(String query) { - List productList = new ArrayList<>(); - log.info("크롤링 시작: 키워드 = {}", query); - - System.setProperty("webdriver.chrome.driver", chromeDriverPath); - log.debug("ChromeDriver 경로: {}", chromeDriverPath); - - ChromeOptions options = new ChromeOptions(); - options.addArguments("--headless"); - options.addArguments("--disable-gpu"); - options.addArguments("--no-sandbox"); - options.addArguments("--disable-dev-shm-usage"); - options.addArguments("--remote-debugging-port=9222"); - options.addArguments("--disable-software-rasterizer"); - options.addArguments("--disable-extensions"); - options.addArguments("--disable-popup-blocking"); - options.addArguments("--window-size=1920,1080"); - options.addArguments(USER_AGENT); - - WebDriver driver = null; // 드라이버 선언 - - try { - driver = new ChromeDriver(options); // 드라이버 초기화 - log.info("ChromeDriver 초기화 성공"); - - String encodedQuery = URLEncoder.encode(query, StandardCharsets.UTF_8); - String searchUrl = String.format(COUPANG_SEARCH_URL, encodedQuery); - - log.info("쿠팡 검색 URL: {}", searchUrl); - driver.get(searchUrl); - - WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(90)); - wait.until(ExpectedConditions.presenceOfElementLocated(By.cssSelector(".search-product"))); - log.info("쿠팡 검색 페이지 로딩 완료"); - - - List products = driver.findElements(By.cssSelector(".search-product")); - log.info("검색된 상품 개수: {}", products.size()); - - for (WebElement productElement : products) { - try { - Product productEntity = extractProductInfo(productElement, query); - if (productEntity != null) { - productList.add(productEntity); - productService.saveItems(List.of(productEntity), query); - log.debug("상품 정보 추출 및 저장 성공: {}", productEntity); - } - } catch (NoSuchElementException e) { - log.warn("요소 찾기 실패: {}", e.getMessage(), e); // 예외 객체 e도 함께 로깅 - } - } - } catch (TimeoutException e) { - log.error("페이지 로딩 시간 초과: {}", e.getMessage(), e); // 예외 객체 e도 함께 로깅 - } catch (SessionNotCreatedException e) { - log.error("세션 생성 실패: {}", e.getMessage(), e); - } catch (WebDriverException e) { - log.error("WebDriver 오류: {}", e.getMessage(), e); - } catch (Exception e) { - log.error("크롤링 중 오류 발생: {}", e.getMessage(), e); // 예외 객체 e도 함께 로깅 - } finally { - if (driver != null) { // 드라이버가 null이 아닌 경우에만 종료 - driver.quit(); - log.info("ChromeDriver 종료"); - } - } - - log.info("크롤링 종료: 반환된 상품 개수 = {}", productList.size()); - return productList; - } + List productList = new ArrayList<>(); + System.setProperty("webdriver.chrome.driver", chromeDriverPath); + + ChromeOptions options = new ChromeOptions(); + options.setBinary("/opt/google/chrome/chrome"); // AWS 환경용 크롬 바이너리 경로 + options.addArguments("--no-sandbox"); + options.addArguments("--disable-gpu"); + options.addArguments("--disable-dev-shm-usage"); + options.addArguments("--disable-software-rasterizer"); + options.addArguments("--disable-crash-reporter"); + options.addArguments("--disable-extensions"); + options.addArguments("--disable-popup-blocking"); + options.addArguments("--disable-background-networking"); + options.addArguments("--disable-features=NetworkService,NetworkServiceInProcess"); + options.addArguments("--window-size=1920,1080"); + options.addArguments("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.5672.63 Safari/537.36"); + options.addArguments("--disable-blink-features=AutomationControlled"); // 탐지 우회 + + log.info("ChromeDriver 실행 준비 완료. 드라이버 경로: {}", chromeDriverPath); + + WebDriver driver = new ChromeDriver(options); + + try { + log.info("검색어: {}", query); + + String encodedQuery = URLEncoder.encode(query, StandardCharsets.UTF_8); + String searchUrl = String.format("https://www.coupang.com/np/search?q=%s&channel=user", encodedQuery); + + log.info("쿠팡 검색 URL: {}", searchUrl); + driver.get(searchUrl); + + log.info("쿠팡 페이지 접속 완료. 5초 대기 중..."); + Thread.sleep(5000); // 페이지 로딩 대기 + + // navigator.webdriver 감추기 + JavascriptExecutor js = (JavascriptExecutor) driver; + js.executeScript("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"); + + log.info("navigator.webdriver 속성 제거 완료."); + + // 현재 페이지의 HTML 확인 + String pageSource = driver.getPageSource(); + log.debug("현재 페이지 HTML (앞부분): {}", pageSource.substring(0, Math.min(1000, pageSource.length()))); + + // 페이지가 제대로 로드되었는지 확인 + if (pageSource.contains("captcha") || pageSource.contains("robot check")) { + log.error("쿠팡에서 봇 탐지를 수행함. 크롤링 차단됨."); + return Collections.emptyList(); + } + + log.info("상품 리스트 로딩 시작..."); + WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(90)); + wait.until(ExpectedConditions.presenceOfElementLocated(By.cssSelector(".search-product"))); + + List products = driver.findElements(By.cssSelector(".search-product")); + + if (products.isEmpty()) { + log.warn("검색 결과가 없음. 크롤링할 제품 없음."); + return Collections.emptyList(); + } + + log.info("총 {}개의 상품 발견", products.size()); + + for (WebElement productElement : products) { + try { + Product productEntity = extractProductInfo(productElement, query); + if (productEntity != null) { + productList.add(productEntity); + productService.saveItems(List.of(productEntity), query); + log.info("상품 저장 완료: {}", productEntity.getTitle()); + } + } catch (NoSuchElementException e) { + log.warn("요소를 찾을 수 없음: {}", e.getMessage()); + } + } + + } catch (TimeoutException e) { + log.error("페이지 로딩 시간 초과: {}", e.getMessage()); + } catch (WebDriverException e) { + log.error("ChromeDriver 관련 오류 발생: {}", e.getMessage()); + } catch (Exception e) { + log.error("크롤링 중 알 수 없는 오류 발생: {}", e.getMessage()); + } finally { + log.info("ChromeDriver 종료..."); + driver.quit(); + } + + return productList; + } /** * 개별 상품 정보를 추출하여 Product 객체를 생성합니다. From d0d2fe01c984158d419a5487927648c9ebc42fae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=80=EC=9B=90?= <110761377+andrewkimswe@users.noreply.github.com> Date: Tue, 11 Feb 2025 17:33:24 +0900 Subject: [PATCH 11/18] Update CoupangApiService.java --- src/main/java/com/team4/giftidea/service/CoupangApiService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/team4/giftidea/service/CoupangApiService.java b/src/main/java/com/team4/giftidea/service/CoupangApiService.java index 96c937c..3425e42 100644 --- a/src/main/java/com/team4/giftidea/service/CoupangApiService.java +++ b/src/main/java/com/team4/giftidea/service/CoupangApiService.java @@ -52,6 +52,7 @@ public List searchItems(String query) { ChromeOptions options = new ChromeOptions(); options.setBinary("/opt/google/chrome/chrome"); // AWS 환경용 크롬 바이너리 경로 + options.addArguments("--headless"); options.addArguments("--no-sandbox"); options.addArguments("--disable-gpu"); options.addArguments("--disable-dev-shm-usage"); From 31556ddf34b5e91d518b8f7613f622b5fe22276d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=80=EC=9B=90?= <110761377+andrewkimswe@users.noreply.github.com> Date: Tue, 11 Feb 2025 17:34:54 +0900 Subject: [PATCH 12/18] Update CoupangApiService.java --- src/main/java/com/team4/giftidea/service/CoupangApiService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/team4/giftidea/service/CoupangApiService.java b/src/main/java/com/team4/giftidea/service/CoupangApiService.java index 3425e42..7e73fea 100644 --- a/src/main/java/com/team4/giftidea/service/CoupangApiService.java +++ b/src/main/java/com/team4/giftidea/service/CoupangApiService.java @@ -10,6 +10,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import java.util.Collections; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.time.Duration; From 6e079f74aef3d890017382e9a9f37b3ecb993657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=80=EC=9B=90?= <110761377+andrewkimswe@users.noreply.github.com> Date: Tue, 11 Feb 2025 17:53:28 +0900 Subject: [PATCH 13/18] Update CoupangApiService.java --- .../giftidea/service/CoupangApiService.java | 47 +++++++++++-------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/team4/giftidea/service/CoupangApiService.java b/src/main/java/com/team4/giftidea/service/CoupangApiService.java index 7e73fea..f1e8a21 100644 --- a/src/main/java/com/team4/giftidea/service/CoupangApiService.java +++ b/src/main/java/com/team4/giftidea/service/CoupangApiService.java @@ -50,26 +50,33 @@ public CoupangApiService(ProductService productService) { public List searchItems(String query) { List productList = new ArrayList<>(); System.setProperty("webdriver.chrome.driver", chromeDriverPath); - - ChromeOptions options = new ChromeOptions(); - options.setBinary("/opt/google/chrome/chrome"); // AWS 환경용 크롬 바이너리 경로 - options.addArguments("--headless"); - options.addArguments("--no-sandbox"); - options.addArguments("--disable-gpu"); - options.addArguments("--disable-dev-shm-usage"); - options.addArguments("--disable-software-rasterizer"); - options.addArguments("--disable-crash-reporter"); - options.addArguments("--disable-extensions"); - options.addArguments("--disable-popup-blocking"); - options.addArguments("--disable-background-networking"); - options.addArguments("--disable-features=NetworkService,NetworkServiceInProcess"); - options.addArguments("--window-size=1920,1080"); - options.addArguments("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.5672.63 Safari/537.36"); - options.addArguments("--disable-blink-features=AutomationControlled"); // 탐지 우회 - - log.info("ChromeDriver 실행 준비 완료. 드라이버 경로: {}", chromeDriverPath); - - WebDriver driver = new ChromeDriver(options); + ChromeOptions options = new ChromeOptions(); + options.setBinary("/opt/google/chrome/chrome"); + options.addArguments("--headless=new"); // ✅ 기본적으로 유지 (테스트 후 필요 시 제거) + options.addArguments("--disable-gpu"); + options.addArguments("--no-sandbox"); + options.addArguments("--disable-dev-shm-usage"); + options.addArguments("--remote-debugging-port=9222"); + options.addArguments("--window-size=1920,1080"); + options.addArguments("--disable-software-rasterizer"); + options.addArguments("--disable-extensions"); + options.addArguments("--disable-popup-blocking"); + + // ✅ 최신 User-Agent 적용 (봇 탐지 우회) + options.addArguments("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.5672.63 Safari/537.36"); + + WebDriver driver = new ChromeDriver(options); + + // ✅ navigator.webdriver 속성 제거 (봇 탐지 우회) + JavascriptExecutor js = (JavascriptExecutor) driver; + js.executeScript("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"); + + // ✅ JavaScript 실행 대기 + Thread.sleep(5000); + + // ✅ HTML 확인 (디버깅용) + String pageSource = driver.getPageSource(); + System.out.println("Current Page Source: " + pageSource.substring(0, 500)); // 앞부분 500자만 출력 try { log.info("검색어: {}", query); From 235b4bf70df84e5a29efb72d935944541ce1c912 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=80=EC=9B=90?= <110761377+andrewkimswe@users.noreply.github.com> Date: Tue, 11 Feb 2025 17:55:53 +0900 Subject: [PATCH 14/18] Update CoupangApiService.java --- .../giftidea/service/CoupangApiService.java | 43 +++++++------------ 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/src/main/java/com/team4/giftidea/service/CoupangApiService.java b/src/main/java/com/team4/giftidea/service/CoupangApiService.java index f1e8a21..fdc38b0 100644 --- a/src/main/java/com/team4/giftidea/service/CoupangApiService.java +++ b/src/main/java/com/team4/giftidea/service/CoupangApiService.java @@ -50,33 +50,22 @@ public CoupangApiService(ProductService productService) { public List searchItems(String query) { List productList = new ArrayList<>(); System.setProperty("webdriver.chrome.driver", chromeDriverPath); - ChromeOptions options = new ChromeOptions(); - options.setBinary("/opt/google/chrome/chrome"); - options.addArguments("--headless=new"); // ✅ 기본적으로 유지 (테스트 후 필요 시 제거) - options.addArguments("--disable-gpu"); - options.addArguments("--no-sandbox"); - options.addArguments("--disable-dev-shm-usage"); - options.addArguments("--remote-debugging-port=9222"); - options.addArguments("--window-size=1920,1080"); - options.addArguments("--disable-software-rasterizer"); - options.addArguments("--disable-extensions"); - options.addArguments("--disable-popup-blocking"); - - // ✅ 최신 User-Agent 적용 (봇 탐지 우회) - options.addArguments("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.5672.63 Safari/537.36"); - - WebDriver driver = new ChromeDriver(options); - - // ✅ navigator.webdriver 속성 제거 (봇 탐지 우회) - JavascriptExecutor js = (JavascriptExecutor) driver; - js.executeScript("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"); - - // ✅ JavaScript 실행 대기 - Thread.sleep(5000); - - // ✅ HTML 확인 (디버깅용) - String pageSource = driver.getPageSource(); - System.out.println("Current Page Source: " + pageSource.substring(0, 500)); // 앞부분 500자만 출력 + ChromeOptions options = new ChromeOptions(); + options.setBinary("/opt/google/chrome/chrome"); // 크롬 바이너리 직접 지정 (AWS 환경) + options.addArguments("--headless"); // 기본적으로 headless 모드 유지 + options.addArguments("--disable-gpu"); + options.addArguments("--no-sandbox"); + options.addArguments("--disable-dev-shm-usage"); + options.addArguments("--remote-debugging-port=9222"); + options.addArguments("--window-size=1920,1080"); + options.addArguments("--disable-software-rasterizer"); + options.addArguments("--disable-extensions"); + options.addArguments("--disable-popup-blocking"); + + // 최신 User-Agent 추가 (봇 탐지 우회) + options.addArguments("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.5672.63 Safari/537.36"); + + WebDriver driver = new ChromeDriver(options); try { log.info("검색어: {}", query); From 8705792d7232e74f564968936f6553d0f55fd806 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=80=EC=9B=90?= <110761377+andrewkimswe@users.noreply.github.com> Date: Tue, 11 Feb 2025 18:02:59 +0900 Subject: [PATCH 15/18] Update CoupangApiService.java --- src/main/java/com/team4/giftidea/service/CoupangApiService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/team4/giftidea/service/CoupangApiService.java b/src/main/java/com/team4/giftidea/service/CoupangApiService.java index fdc38b0..baba671 100644 --- a/src/main/java/com/team4/giftidea/service/CoupangApiService.java +++ b/src/main/java/com/team4/giftidea/service/CoupangApiService.java @@ -64,7 +64,7 @@ public List searchItems(String query) { // 최신 User-Agent 추가 (봇 탐지 우회) options.addArguments("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.5672.63 Safari/537.36"); - + WebDriver driver = new ChromeDriver(options); try { From 759868c47bbd6cc41ad5b00e5e42973bf88f7f0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=80=EC=9B=90?= <110761377+andrewkimswe@users.noreply.github.com> Date: Wed, 12 Feb 2025 14:04:22 +0900 Subject: [PATCH 16/18] =?UTF-8?q?=08fix=20:=20=ED=85=8C=EC=8A=A4=ED=8A=B8?= =?UTF-8?q?=EC=9A=A9=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/team4/giftidea/configuration/SecurityConfig.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/com/team4/giftidea/configuration/SecurityConfig.java b/src/main/java/com/team4/giftidea/configuration/SecurityConfig.java index 0662e75..cd9c61b 100644 --- a/src/main/java/com/team4/giftidea/configuration/SecurityConfig.java +++ b/src/main/java/com/team4/giftidea/configuration/SecurityConfig.java @@ -60,11 +60,7 @@ public CorsConfigurationSource corsConfigurationSource() { CorsConfiguration configuration = new CorsConfiguration(); // 허용할 출처(Origin) 설정 - configuration.setAllowedOrigins(List.of( - "http://localhost:3000", // 로컬 개발 환경 - "https://presentalk.store", // 프론트엔드 배포 주소 - "https://app.presentalk.store" // 백엔드 API 주소 - )); + configuration.setAllowedOrigins(List.of("*")); // 허용할 HTTP 메서드 설정 configuration.setAllowedMethods(List.of("GET", "POST", "PUT", "DELETE", "OPTIONS")); From e159d612091a71a4849995ecb70df8593f66d74f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=80=EC=9B=90?= <110761377+andrewkimswe@users.noreply.github.com> Date: Wed, 12 Feb 2025 14:59:13 +0900 Subject: [PATCH 17/18] Update SecurityConfig.java --- .../configuration/SecurityConfig.java | 69 +++++++++---------- 1 file changed, 31 insertions(+), 38 deletions(-) diff --git a/src/main/java/com/team4/giftidea/configuration/SecurityConfig.java b/src/main/java/com/team4/giftidea/configuration/SecurityConfig.java index cd9c61b..72021c6 100644 --- a/src/main/java/com/team4/giftidea/configuration/SecurityConfig.java +++ b/src/main/java/com/team4/giftidea/configuration/SecurityConfig.java @@ -7,74 +7,67 @@ import org.springframework.web.cors.CorsConfiguration; import org.springframework.web.cors.CorsConfigurationSource; import org.springframework.web.cors.UrlBasedCorsConfigurationSource; +import org.springframework.web.filter.CorsFilter; +import org.springframework.boot.web.servlet.FilterRegistrationBean; import java.util.List; -/** - * Spring Security 및 CORS 설정을 담당하는 설정 클래스입니다. - */ @Configuration public class SecurityConfig { - /** - * HTTP 보안 설정을 구성하는 Bean입니다. - * - * - CORS 설정 적용 - * - CSRF 보호 비활성화 (JWT 사용 시 필요) - * - 특정 경로 보호 및 기본 요청 허용 설정 - * - * @param http Spring Security의 HTTP 보안 설정 객체 - * @return SecurityFilterChain 보안 필터 체인 - * @throws Exception 설정 과정에서 발생할 수 있는 예외 - */ @Bean public SecurityFilterChain securityFilterChain(HttpSecurity http) throws Exception { http - // CORS 설정 적용 .cors(cors -> cors.configurationSource(corsConfigurationSource())) - - // CSRF 보호 비활성화 (JWT 인증을 사용하는 경우 필요) .csrf(csrf -> csrf.disable()) - - // 접근 제어 설정 .authorizeHttpRequests(auth -> auth - .requestMatchers("/admin/**").authenticated() // "/admin/**" 경로는 인증 필요 - .anyRequest().permitAll() // 나머지 요청은 인증 없이 허용 + .requestMatchers("/admin/**").authenticated() + .anyRequest().permitAll() ); return http.build(); } - /** - * CORS 설정을 구성하는 Bean입니다. - * - * - 허용할 도메인(origin) 설정 - * - 허용할 HTTP 메서드(GET, POST 등) 지정 - * - 허용할 헤더 설정 - * - 쿠키 포함 요청 허용 - * - * @return CorsConfigurationSource CORS 설정 객체 - */ @Bean public CorsConfigurationSource corsConfigurationSource() { CorsConfiguration configuration = new CorsConfiguration(); + + // 특정 도메인만 허용 + configuration.setAllowedOrigins(List.of( + "http://localhost:5174", + "http://localhost:3000", + "https://presentalk.store", + "https://app.presentalk.store" + )); - // 허용할 출처(Origin) 설정 - configuration.setAllowedOrigins(List.of("*")); - - // 허용할 HTTP 메서드 설정 + // HTTP 메서드 허용 configuration.setAllowedMethods(List.of("GET", "POST", "PUT", "DELETE", "OPTIONS")); - // 허용할 요청 헤더 설정 - configuration.setAllowedHeaders(List.of("*")); // 모든 헤더 허용 + // 특정 헤더 허용 + configuration.setAllowedHeaders(List.of( + "Authorization", + "Cache-Control", + "Content-Type", + "X-Requested-With" + )); // 쿠키 포함 요청 허용 configuration.setAllowCredentials(true); // CORS 설정을 특정 경로에 적용 UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource(); - source.registerCorsConfiguration("/**", configuration); // 모든 경로에 적용 + source.registerCorsConfiguration("/**", configuration); return source; } + + /** + * CORS 필터가 Spring Security보다 먼저 실행되도록 설정 + */ + @Bean + public FilterRegistrationBean corsFilter() { + FilterRegistrationBean filterBean = new FilterRegistrationBean<>(new CorsFilter(corsConfigurationSource())); + filterBean.setOrder(0); // 가장 먼저 실행되도록 설정 + return filterBean; + } } From 22799ea47f7c29f5cbb9b48f9e3f1dae85fd55de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=80=EC=9B=90?= <110761377+andrewkimswe@users.noreply.github.com> Date: Wed, 12 Feb 2025 15:20:18 +0900 Subject: [PATCH 18/18] Update SecurityConfig.java --- .../configuration/SecurityConfig.java | 52 ++++++++++++------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/team4/giftidea/configuration/SecurityConfig.java b/src/main/java/com/team4/giftidea/configuration/SecurityConfig.java index 72021c6..993d918 100644 --- a/src/main/java/com/team4/giftidea/configuration/SecurityConfig.java +++ b/src/main/java/com/team4/giftidea/configuration/SecurityConfig.java @@ -2,8 +2,10 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.http.ResponseEntity; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.web.SecurityFilterChain; +import org.springframework.web.bind.annotation.*; import org.springframework.web.cors.CorsConfiguration; import org.springframework.web.cors.CorsConfigurationSource; import org.springframework.web.cors.UrlBasedCorsConfigurationSource; @@ -12,44 +14,50 @@ import java.util.List; +/** + * Spring Security 및 CORS 설정을 담당하는 설정 클래스입니다. + */ @Configuration +@RestController +@RequestMapping("/api") // 모든 API 요청 처리 public class SecurityConfig { + /** + * HTTP 보안 설정을 구성하는 Bean입니다. + */ @Bean public SecurityFilterChain securityFilterChain(HttpSecurity http) throws Exception { http - .cors(cors -> cors.configurationSource(corsConfigurationSource())) - .csrf(csrf -> csrf.disable()) + .cors(cors -> cors.configurationSource(corsConfigurationSource())) // CORS 적용 + .csrf(csrf -> csrf.disable()) // CSRF 비활성화 .authorizeHttpRequests(auth -> auth - .requestMatchers("/admin/**").authenticated() - .anyRequest().permitAll() + .requestMatchers("/admin/**").authenticated() // "/admin/**" 경로는 인증 필요 + .anyRequest().permitAll() // 나머지 요청은 인증 없이 허용 ); return http.build(); } + /** + * CORS 설정을 구성하는 Bean입니다. + */ @Bean public CorsConfigurationSource corsConfigurationSource() { CorsConfiguration configuration = new CorsConfiguration(); - // 특정 도메인만 허용 + // 허용할 출처 설정 configuration.setAllowedOrigins(List.of( - "http://localhost:5174", - "http://localhost:3000", - "https://presentalk.store", - "https://app.presentalk.store" + "http://localhost:5174", + "http://localhost:3000", + "https://presentalk.store", + "https://app.presentalk.store" )); - // HTTP 메서드 허용 + // 허용할 HTTP 메서드 설정 configuration.setAllowedMethods(List.of("GET", "POST", "PUT", "DELETE", "OPTIONS")); - - // 특정 헤더 허용 - configuration.setAllowedHeaders(List.of( - "Authorization", - "Cache-Control", - "Content-Type", - "X-Requested-With" - )); + + // 허용할 요청 헤더 설정 + configuration.setAllowedHeaders(List.of("*")); // 모든 헤더 허용 // 쿠키 포함 요청 허용 configuration.setAllowCredentials(true); @@ -70,4 +78,12 @@ public FilterRegistrationBean corsFilter() { filterBean.setOrder(0); // 가장 먼저 실행되도록 설정 return filterBean; } + + /** + * OPTIONS 요청을 수동으로 처리 (CORS 문제 해결) + */ + @RequestMapping(value = "/**", method = RequestMethod.OPTIONS) + public ResponseEntity handleOptions() { + return ResponseEntity.ok().build(); + } }