diff --git a/downloader/src/main/java/gleaners/domain/Product.java b/downloader/src/main/java/gleaners/domain/Product.java index d19a6b5..c318097 100644 --- a/downloader/src/main/java/gleaners/domain/Product.java +++ b/downloader/src/main/java/gleaners/domain/Product.java @@ -1,16 +1,49 @@ package gleaners.domain; -public class Product { - private String response; +import lombok.AccessLevel; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import lombok.experimental.FieldDefaults; - public Product(String response) { - this.response = response; - } +@ToString +@Setter +@NoArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class Product { + String idHash; + String id; + String title; + int pricePc; + String link; + String imageLink; + String categoryName1; + String shipping; + String flag; + String hash; - @Override - public String toString() { - return "Product{" + - "response='" + response + '\'' + - '}'; + @Builder + public Product( + String idHash, + String id, + String title, + int pricePc, + String link, + String imageLink, + String categoryName1, + String shipping, + String flag, + String hash) { + this.idHash = idHash; + this.id = id; + this.title = title; + this.pricePc = pricePc; + this.link = link; + this.imageLink = imageLink; + this.categoryName1 = categoryName1; + this.shipping = shipping; + this.flag = flag; + this.hash = hash; } } diff --git a/downloader/src/main/java/gleaners/domain/enums/Field.java b/downloader/src/main/java/gleaners/domain/enums/Field.java new file mode 100644 index 0000000..0b84406 --- /dev/null +++ b/downloader/src/main/java/gleaners/domain/enums/Field.java @@ -0,0 +1,37 @@ +package gleaners.domain.enums; + +import lombok.Getter; + +import java.util.AbstractMap; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Getter +public enum Field { + ID(List.of("id", "pid")), + TITLE(List.of("title")), + PRICE_PC(List.of("price_pc")), + LINK(List.of("link")), + IMAGE_LINK(List.of("image_link")), + CATEGORY_NAME1(List.of("category_name1")), + SHIPPING(List.of("shipping")), + CLASS(List.of("class")); + + private final List fields; + + Field(List fields) { + this.fields = fields; + } + + private static final Map FIELD_KEY_MAP = + Arrays.stream(values()) + .flatMap(f -> f.fields.stream() + .map(k -> new AbstractMap.SimpleEntry<>(k, f))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + public static Map getFieldKeyMap() { + return FIELD_KEY_MAP; + } +} diff --git a/downloader/src/main/java/gleaners/port/ProductSender.java b/downloader/src/main/java/gleaners/port/ProductSender.java index c7eb390..f6bf429 100644 --- a/downloader/src/main/java/gleaners/port/ProductSender.java +++ b/downloader/src/main/java/gleaners/port/ProductSender.java @@ -13,9 +13,11 @@ public class ProductSender { private final ReactiveKafkaProducerTemplate downloadSender; public void send(Product product) { + System.out.println("Product : " + product); + /* downloadSender.send("test-after-download", product.toString()) .doOnSuccess(senderResult -> log.info("send : {} \n offset : {}", product, senderResult.recordMetadata().offset())) - .subscribe(); + .subscribe();*/ } } diff --git a/downloader/src/main/java/gleaners/usecase/DownloadTask.java b/downloader/src/main/java/gleaners/usecase/DownloadTask.java index d955ad3..a11f4d9 100644 --- a/downloader/src/main/java/gleaners/usecase/DownloadTask.java +++ b/downloader/src/main/java/gleaners/usecase/DownloadTask.java @@ -1,7 +1,6 @@ package gleaners.usecase; import gleaners.domain.DownloadTarget; -import gleaners.domain.Product; import gleaners.port.ProductSender; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; @@ -13,10 +12,18 @@ public class DownloadTask { private final ProductSender sender; private final Downloader downloader; + private final Parser parser; public void downloadAndSend(DownloadTarget targetUrl) { downloader.extractLineByDelimiter(targetUrl) - .map(Product::new) + .mapNotNull(line -> { + if(parser.isEmptyFieldList()) { + parser.setFieldKey(line); + return null; + } else { + return parser.produceProduct(line); + } + }) .subscribe(sender::send); } } diff --git a/downloader/src/main/java/gleaners/usecase/Downloader.java b/downloader/src/main/java/gleaners/usecase/Downloader.java index 992fc07..72e9494 100644 --- a/downloader/src/main/java/gleaners/usecase/Downloader.java +++ b/downloader/src/main/java/gleaners/usecase/Downloader.java @@ -39,7 +39,6 @@ private Mono validate(ClientResponse response) { if(response.statusCode().isError()) { return Mono.empty(); } - return response.bodyToMono(String.class); } diff --git a/downloader/src/main/java/gleaners/usecase/Parser.java b/downloader/src/main/java/gleaners/usecase/Parser.java new file mode 100644 index 0000000..92c16ca --- /dev/null +++ b/downloader/src/main/java/gleaners/usecase/Parser.java @@ -0,0 +1,57 @@ +package gleaners.usecase; + +import gleaners.domain.enums.Field; +import gleaners.domain.Product; +import lombok.extern.log4j.Log4j2; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +@Log4j2 +public class Parser +{ + private static final String DELIMITER = "\\t"; + private List fieldList; + + public void setFieldKey(String fields) { + fieldList = Arrays.stream(fields.trim().split(DELIMITER, -1)) + .map(field -> Field.getFieldKeyMap().get(field)) + .collect(Collectors.toList()); + } + + public Product produceProduct(String line) { + Product product = new Product(); + try { + List valueList = Arrays.asList(line.split("\\t", -1)); + if(fieldList.size() != valueList.size()) { + throw new Exception("key와 value의 필드 개수가 동일하지 않습니다. field key size : " + + fieldList.size() + ", value size : " + valueList.size()); + } + + for(int i = 0; i < valueList.size(); i++) { + productSetter(product, fieldList.get(i), valueList.get(i)); + } + } catch(Exception e) { + log.error("Product 생성 에러 : {}", e.getMessage()); + } + return product; + } + + public boolean isEmptyFieldList() { + return fieldList.isEmpty(); + } + + private void productSetter(Product product, Field field, String value) { + switch (field) { + case ID -> product.setId(value); + case TITLE -> product.setTitle(value); + case PRICE_PC -> product.setPricePc(Integer.parseInt(value)); + case LINK -> product.setLink(value); + case IMAGE_LINK -> product.setImageLink(value); + case CATEGORY_NAME1 -> product.setCategoryName1(value); + case SHIPPING -> product.setShipping(value); + case CLASS -> product.setFlag(value); + } + } +} diff --git a/downloader/src/test/java/gleaners/usecase/DownloadTaskTest.java b/downloader/src/test/java/gleaners/usecase/DownloadTaskTest.java index 1047253..19204ed 100644 --- a/downloader/src/test/java/gleaners/usecase/DownloadTaskTest.java +++ b/downloader/src/test/java/gleaners/usecase/DownloadTaskTest.java @@ -10,7 +10,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import reactor.blockhound.BlockHound; -import reactor.test.StepVerifier; import java.io.IOException; @@ -52,10 +51,11 @@ void urlDownloadTest() throws InterruptedException { ProductSender productSender = new ProductSender(reactiveKafkaProducerTemplate); Downloader downloader = new Downloader(); + Parser tsvConvertor = new Parser(); - DownloadTask downloadTask = new DownloadTask(productSender, downloader); + DownloadTask downloadTask = new DownloadTask(productSender, downloader, tsvConvertor); - DownloadTarget downloadTarget = new DownloadTarget("https://www.naver.com"); + DownloadTarget downloadTarget = new DownloadTarget("https://korea-ne-wmf-ep-production.s3.ap-northeast-2.amazonaws.com/89f4a4733fd21df33816e84e97d8eb40/ep_total.tsv"); downloadTask diff --git a/downloader/src/test/java/gleaners/usecase/DownloaderTest.java b/downloader/src/test/java/gleaners/usecase/DownloaderTest.java index fd806b7..5eca559 100644 --- a/downloader/src/test/java/gleaners/usecase/DownloaderTest.java +++ b/downloader/src/test/java/gleaners/usecase/DownloaderTest.java @@ -1,6 +1,7 @@ package gleaners.usecase; import gleaners.domain.DownloadTarget; +import gleaners.port.ProductSender; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import reactor.test.StepVerifier; @@ -19,4 +20,18 @@ void downloadServiceTest() { .verifyComplete(); } + @Test + void test() { + ProductSender productSender = new ProductSender(null); + + Downloader downloader = new Downloader(); + Parser tsvConvertor = new Parser(); + + DownloadTask downloadTask = new DownloadTask(productSender, downloader, tsvConvertor); + + DownloadTarget downloadTarget = new DownloadTarget("https://korea-ne-wmf-ep-production.s3.ap-northeast-2.amazonaws.com/89f4a4733fd21df33816e84e97d8eb40/ep_total.tsv"); + + downloadTask + .downloadAndSend(downloadTarget); + } } diff --git a/downloader/src/test/java/gleaners/usecase/ParserTest.java b/downloader/src/test/java/gleaners/usecase/ParserTest.java new file mode 100644 index 0000000..d5e5eea --- /dev/null +++ b/downloader/src/test/java/gleaners/usecase/ParserTest.java @@ -0,0 +1,13 @@ +package gleaners.usecase; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +public class ParserTest +{ + @Test + @DisplayName("파일 1라인을 받아 Product를 생성할 수 있다.") + public void test2() { + + } +}