From d83fda32534d2545c126608a3009d7b83fac6842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=ED=9D=AC=EC=B0=AC?= Date: Tue, 22 Oct 2024 23:48:24 +0900 Subject: [PATCH 01/19] =?UTF-8?q?feature:=20KOPIS=20API=20=ED=98=B8?= =?UTF-8?q?=EC=B6=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.gradle | 3 + .../clacobatchserver/batch/FirstBatch.java | 95 ------------------- .../clacobatchserver/batch/KopisBatch.java | 55 +++++++++++ .../clacobatchserver/entity/BeforeEntity.java | 52 +++++++++- .../schedule/FirstSchedule.java | 2 +- .../service/KopisApiReader.java | 92 ++++++++++++++++++ .../service/KopisEntityWriter.java | 26 +++++ .../service/RestTemplateConfig.java | 24 +++++ 8 files changed, 251 insertions(+), 98 deletions(-) delete mode 100644 src/main/java/com/curateme/clacobatchserver/batch/FirstBatch.java create mode 100644 src/main/java/com/curateme/clacobatchserver/batch/KopisBatch.java create mode 100644 src/main/java/com/curateme/clacobatchserver/service/KopisApiReader.java create mode 100644 src/main/java/com/curateme/clacobatchserver/service/KopisEntityWriter.java create mode 100644 src/main/java/com/curateme/clacobatchserver/service/RestTemplateConfig.java diff --git a/build.gradle b/build.gradle index e05fcd8..4ee9ff9 100644 --- a/build.gradle +++ b/build.gradle @@ -28,6 +28,9 @@ dependencies { implementation 'org.springframework.boot:spring-boot-starter-data-jpa' implementation 'org.springframework.boot:spring-boot-starter-jdbc' implementation 'org.springframework.boot:spring-boot-starter-web' + implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.15.0' + implementation 'javax.xml.bind:jaxb-api:2.3.1' + implementation 'org.glassfish.jaxb:jaxb-runtime:2.3.1' compileOnly 'org.projectlombok:lombok' runtimeOnly 'com.mysql:mysql-connector-j' annotationProcessor 'org.projectlombok:lombok' diff --git a/src/main/java/com/curateme/clacobatchserver/batch/FirstBatch.java b/src/main/java/com/curateme/clacobatchserver/batch/FirstBatch.java deleted file mode 100644 index 88593f5..0000000 --- a/src/main/java/com/curateme/clacobatchserver/batch/FirstBatch.java +++ /dev/null @@ -1,95 +0,0 @@ -package com.curateme.clacobatchserver.batch; - -import com.curateme.clacobatchserver.entity.AfterEntity; -import com.curateme.clacobatchserver.entity.BeforeEntity; -import com.curateme.clacobatchserver.repository.AfterRepository; -import com.curateme.clacobatchserver.repository.BeforeRepository; -import java.util.Map; -import org.springframework.batch.core.Job; -import org.springframework.batch.core.Step; -import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.repository.JobRepository; -import org.springframework.batch.core.step.builder.StepBuilder; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.batch.item.data.RepositoryItemReader; -import org.springframework.batch.item.data.RepositoryItemWriter; -import org.springframework.batch.item.data.builder.RepositoryItemReaderBuilder; -import org.springframework.batch.item.data.builder.RepositoryItemWriterBuilder; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.domain.Sort; -import org.springframework.transaction.PlatformTransactionManager; - -@Configuration -public class FirstBatch { - private final JobRepository jobRepository; - private final PlatformTransactionManager platformTransactionManager; - - private final BeforeRepository beforeRepository; - private final AfterRepository afterRepository; - - public FirstBatch(JobRepository jobRepository, - PlatformTransactionManager platformTransactionManager, BeforeRepository beforeRepository, - AfterRepository afterRepository) { - this.jobRepository = jobRepository; - this.platformTransactionManager = platformTransactionManager; - this.beforeRepository = beforeRepository; - this.afterRepository = afterRepository; - } - - @Bean - public Job firstJob() { - - return new JobBuilder("firstJob", jobRepository) - .start(firstStep()) - .build(); - } - - @Bean - public Step firstStep() { - - return new StepBuilder("firstStep", jobRepository) - . chunk(10, platformTransactionManager) - .reader(beforeReader()) - .processor(middleProcessor()) - .writer(afterWriter()) - .build(); - } - - @Bean - public RepositoryItemReader beforeReader() { - - return new RepositoryItemReaderBuilder() - .name("beforeReader") - .pageSize(10) - .methodName("findAll") - .repository(beforeRepository) - .sorts(Map.of("id", Sort.Direction.ASC)) - .build(); - } - - @Bean - public ItemProcessor middleProcessor() { - - return new ItemProcessor<>() { - - @Override - public AfterEntity process(BeforeEntity item) throws Exception { - - AfterEntity afterEntity = new AfterEntity(); - afterEntity.setUsername(item.getUsername()); - - return afterEntity; - } - }; - } - - @Bean - public RepositoryItemWriter afterWriter() { - - return new RepositoryItemWriterBuilder() - .repository(afterRepository) - .methodName("save") - .build(); - } -} diff --git a/src/main/java/com/curateme/clacobatchserver/batch/KopisBatch.java b/src/main/java/com/curateme/clacobatchserver/batch/KopisBatch.java new file mode 100644 index 0000000..857c070 --- /dev/null +++ b/src/main/java/com/curateme/clacobatchserver/batch/KopisBatch.java @@ -0,0 +1,55 @@ +package com.curateme.clacobatchserver.batch; + +import com.curateme.clacobatchserver.entity.BeforeEntity; +import com.curateme.clacobatchserver.repository.AfterRepository; +import com.curateme.clacobatchserver.repository.BeforeRepository; +import com.curateme.clacobatchserver.service.KopisApiReader; +import com.curateme.clacobatchserver.service.KopisEntityWriter; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.job.builder.JobBuilder; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; + +@Configuration +public class KopisBatch { + private final JobRepository jobRepository; + private final PlatformTransactionManager platformTransactionManager; + + private final KopisApiReader kopisApiReader; + private final BeforeRepository beforeRepository; + private final AfterRepository afterRepository; + + public KopisBatch(JobRepository jobRepository, + PlatformTransactionManager platformTransactionManager, BeforeRepository beforeRepository, + AfterRepository afterRepository, KopisApiReader kopisApiReader) { + this.jobRepository = jobRepository; + this.platformTransactionManager = platformTransactionManager; + this.beforeRepository = beforeRepository; + this.afterRepository = afterRepository; + this.kopisApiReader = kopisApiReader; + } + + @Bean + public Job kopisJob(KopisEntityWriter writer){ + return new JobBuilder("kopisJob", jobRepository) + .start(firstStep(writer)) + .build(); + } + + @Bean + public Step firstStep(KopisEntityWriter writer) { + return new StepBuilder("firstStep", jobRepository) + .chunk(10, platformTransactionManager) + .reader(kopisApiReader) + .writer(writer) + .build(); + } + + + + +} diff --git a/src/main/java/com/curateme/clacobatchserver/entity/BeforeEntity.java b/src/main/java/com/curateme/clacobatchserver/entity/BeforeEntity.java index d28aca6..f75b4c4 100644 --- a/src/main/java/com/curateme/clacobatchserver/entity/BeforeEntity.java +++ b/src/main/java/com/curateme/clacobatchserver/entity/BeforeEntity.java @@ -1,18 +1,66 @@ package com.curateme.clacobatchserver.entity; +import jakarta.persistence.Column; import jakarta.persistence.Entity; import jakarta.persistence.GeneratedValue; import jakarta.persistence.GenerationType; import jakarta.persistence.Id; +import lombok.AllArgsConstructor; import lombok.Getter; +import lombok.NoArgsConstructor; @Entity @Getter +@NoArgsConstructor +@AllArgsConstructor public class BeforeEntity { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; - private String username; -} \ No newline at end of file + @Column(name = "concertId") + private String mt20id; + + @Column(name = "concertName") + private String prfnm; + + @Column(name = "startDate") + private String prfpdfrom; + + @Column(name = "endDate") + private String prfpdto; + + @Column(name = "facilityName") + private String fcltynm; + + @Column(name = "poster") + private String poster; + + @Column(name = "area") + private String area; + + @Column(name = "genre") + private String genrenm; + + @Column(name = "openrun") + private String openrun; + + @Column(name = "status") + private String prfstate; + + public void setConcertDetails(String mt20id, String prfnm, String prfpdfrom, String prfpdto, + String fcltynm, String poster, String area, String genrenm, + String openrun, String prfstate) { + this.mt20id = mt20id; + this.prfnm = prfnm; + this.prfpdfrom = prfpdfrom; + this.prfpdto = prfpdto; + this.fcltynm = fcltynm; + this.poster = poster; + this.area = area; + this.genrenm = genrenm; + this.openrun = openrun; + this.prfstate = prfstate; + } +} diff --git a/src/main/java/com/curateme/clacobatchserver/schedule/FirstSchedule.java b/src/main/java/com/curateme/clacobatchserver/schedule/FirstSchedule.java index 4933cb0..54a62e3 100644 --- a/src/main/java/com/curateme/clacobatchserver/schedule/FirstSchedule.java +++ b/src/main/java/com/curateme/clacobatchserver/schedule/FirstSchedule.java @@ -30,6 +30,6 @@ public void runFirstJob() throws Exception { .addString("date", date) .toJobParameters(); - jobLauncher.run(jobRegistry.getJob("firstJob"), jobParameters); + jobLauncher.run(jobRegistry.getJob("kopisJob"), jobParameters); } } \ No newline at end of file diff --git a/src/main/java/com/curateme/clacobatchserver/service/KopisApiReader.java b/src/main/java/com/curateme/clacobatchserver/service/KopisApiReader.java new file mode 100644 index 0000000..3a03554 --- /dev/null +++ b/src/main/java/com/curateme/clacobatchserver/service/KopisApiReader.java @@ -0,0 +1,92 @@ +package com.curateme.clacobatchserver.service; + +import com.curateme.clacobatchserver.entity.BeforeEntity; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.springframework.batch.item.ItemReader; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.client.HttpClientErrorException; + +@Component +public class KopisApiReader implements ItemReader { + + private final RestTemplate restTemplate; + private final String apiUrl = "http://www.kopis.or.kr/openApi/restful/pblprfr?service=f222668534db409b8769f640387de9c3"; + private int currentPage = 1; + private List beforeEntities = new ArrayList<>(); + private int index = 0; + + public KopisApiReader(RestTemplate restTemplate) { + this.restTemplate = restTemplate; + } + + @Override + public BeforeEntity read() throws Exception { + + if (index >= beforeEntities.size()) { + loadNextPage(); + index = 0; + } + + if (beforeEntities.isEmpty()) { + return null; + } + + return beforeEntities.get(index++); + } + + private void loadNextPage() { + LocalDate startDate = LocalDate.now(); + LocalDate endDate = startDate.plusMonths(3); + + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); + String formattedStartDate = startDate.format(formatter); + String formattedEndDate = endDate.format(formatter); + + String requestUrl = String.format("%s&stdate=%s&eddate=%s&rows=10&cpage=%d", + apiUrl, formattedStartDate, formattedEndDate, currentPage++); + + HttpHeaders headers = new HttpHeaders(); + headers.setAccept(Collections.singletonList(MediaType.APPLICATION_XML)); + HttpEntity entity = new HttpEntity<>(headers); + + try { + ResponseEntity response = restTemplate.exchange(requestUrl, HttpMethod.GET, entity, BeforeEntity[].class); + + if (response.getBody() != null && response.getBody().length > 0) { + beforeEntities.clear(); + for (BeforeEntity Beforeentity : response.getBody()) { + BeforeEntity beforeEntity = new BeforeEntity(); + beforeEntity.setConcertDetails( + Beforeentity.getMt20id(), + Beforeentity.getPrfnm(), + Beforeentity.getPrfpdfrom(), + Beforeentity.getPrfpdto(), + Beforeentity.getFcltynm(), + Beforeentity.getPoster(), + Beforeentity.getArea(), + Beforeentity.getGenrenm(), + Beforeentity.getOpenrun(), + Beforeentity.getPrfstate() + ); + + beforeEntities.add(beforeEntity); + } + } else { + beforeEntities.clear(); + } + } catch (HttpClientErrorException e) { + System.err.println("Error: " + e.getStatusCode() + " - " + e.getResponseBodyAsString()); + beforeEntities.clear(); + } + } +} diff --git a/src/main/java/com/curateme/clacobatchserver/service/KopisEntityWriter.java b/src/main/java/com/curateme/clacobatchserver/service/KopisEntityWriter.java new file mode 100644 index 0000000..c639f86 --- /dev/null +++ b/src/main/java/com/curateme/clacobatchserver/service/KopisEntityWriter.java @@ -0,0 +1,26 @@ +package com.curateme.clacobatchserver.service; + +import com.curateme.clacobatchserver.entity.BeforeEntity; +import com.curateme.clacobatchserver.repository.BeforeRepository; +import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemWriter; +import org.springframework.stereotype.Component; + +@Component +public class KopisEntityWriter implements ItemWriter { + + private final BeforeRepository beforeRepository; + + public KopisEntityWriter(BeforeRepository beforeRepository) { + + this.beforeRepository = beforeRepository; + } + + @Override + public void write(Chunk items) throws Exception { + + beforeRepository.saveAll(items.getItems()); + } + + +} \ No newline at end of file diff --git a/src/main/java/com/curateme/clacobatchserver/service/RestTemplateConfig.java b/src/main/java/com/curateme/clacobatchserver/service/RestTemplateConfig.java new file mode 100644 index 0000000..d83f9ab --- /dev/null +++ b/src/main/java/com/curateme/clacobatchserver/service/RestTemplateConfig.java @@ -0,0 +1,24 @@ +package com.curateme.clacobatchserver.service; + +import java.util.ArrayList; +import java.util.List; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.converter.HttpMessageConverter; +import org.springframework.http.converter.xml.MappingJackson2XmlHttpMessageConverter; +import org.springframework.web.client.RestTemplate; + +@Configuration +public class RestTemplateConfig { + + @Bean + public RestTemplate restTemplate() { + RestTemplate restTemplate = new RestTemplate(); + List> messageConverters = new ArrayList<>(); + + messageConverters.add(new MappingJackson2XmlHttpMessageConverter()); + restTemplate.setMessageConverters(messageConverters); + + return restTemplate; + } +} \ No newline at end of file From 3fe2e87d06df08fb0d28d50e469122abcc72dec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=ED=9D=AC=EC=B0=AC?= Date: Tue, 22 Oct 2024 23:50:12 +0900 Subject: [PATCH 02/19] =?UTF-8?q?feature:=20cron=20=ED=95=9C=EB=8B=AC=20?= =?UTF-8?q?=EB=8B=A8=EC=9C=84=EB=A1=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../schedule/FirstSchedule.java | 3 +- .../schedule/SecondSchedule.java | 34 ------------------- 2 files changed, 2 insertions(+), 35 deletions(-) delete mode 100644 src/main/java/com/curateme/clacobatchserver/schedule/SecondSchedule.java diff --git a/src/main/java/com/curateme/clacobatchserver/schedule/FirstSchedule.java b/src/main/java/com/curateme/clacobatchserver/schedule/FirstSchedule.java index 54a62e3..15be091 100644 --- a/src/main/java/com/curateme/clacobatchserver/schedule/FirstSchedule.java +++ b/src/main/java/com/curateme/clacobatchserver/schedule/FirstSchedule.java @@ -20,7 +20,8 @@ public FirstSchedule(JobLauncher jobLauncher, JobRegistry jobRegistry) { this.jobRegistry = jobRegistry; } - @Scheduled(cron = "10 * * * * *", zone = "Asia/Seoul") // 한국시간에 맞춰서 스케줄 실행 + @Scheduled(cron = "10 * * * * *", zone = "Asia/Seoul") + //@Scheduled(cron = "0 0 0 1 * *", zone = "Asia/Seoul") -> 추후 한달 단위로 수정 public void runFirstJob() throws Exception { SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-hh-mm-ss"); diff --git a/src/main/java/com/curateme/clacobatchserver/schedule/SecondSchedule.java b/src/main/java/com/curateme/clacobatchserver/schedule/SecondSchedule.java deleted file mode 100644 index a832ac9..0000000 --- a/src/main/java/com/curateme/clacobatchserver/schedule/SecondSchedule.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.curateme.clacobatchserver.schedule; - -import java.text.SimpleDateFormat; -import java.util.Date; -import org.springframework.batch.core.JobParameters; -import org.springframework.batch.core.JobParametersBuilder; -import org.springframework.batch.core.configuration.JobRegistry; -import org.springframework.batch.core.launch.JobLauncher; -import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.annotation.Scheduled; - -@Configuration -public class SecondSchedule { - private final JobLauncher jobLauncher; - private final JobRegistry jobRegistry; - - public SecondSchedule(JobLauncher jobLauncher, JobRegistry jobRegistry) { - this.jobLauncher = jobLauncher; - this.jobRegistry = jobRegistry; - } - - @Scheduled(cron = "5 * * * * *", zone = "Asia/Seoul") // 한국시간에 맞춰서 스케줄 실행 - public void runFirstJob() throws Exception { - - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-hh-mm-ss"); - String date = dateFormat.format(new Date()); - - JobParameters jobParameters = new JobParametersBuilder() - .addString("date", date) - .toJobParameters(); - - jobLauncher.run(jobRegistry.getJob("secondJob"), jobParameters); - } -} From e7f3ee96d50274eea4160e078e1b13619e2f0ff6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=ED=9D=AC=EC=B0=AC?= Date: Wed, 23 Oct 2024 23:18:27 +0900 Subject: [PATCH 03/19] feature: Step2 to Batch --- .../{KopisBatch.java => ConcertBatch.java} | 37 +++-- .../clacobatchserver/batch/SecondBatch.java | 83 ---------- .../RestTemplateConfig.java | 5 +- .../clacobatchserver/entity/AfterEntity.java | 22 --- .../clacobatchserver/entity/BeforeEntity.java | 66 -------- .../entity/ConcertEntity.java | 145 ++++++++++++++++++ .../clacobatchserver/entity/WinEntity.java | 22 --- .../repository/AfterRepository.java | 8 - .../repository/BeforeRepository.java | 8 - .../repository/ConcertRepository.java | 8 + .../repository/WinRepository.java | 11 -- ...irstSchedule.java => ConcertSchedule.java} | 6 +- ...Reader.java => KopisConcertApiReader.java} | 44 +++--- .../service/KopisDetailApiReader.java | 131 ++++++++++++++++ .../service/KopisEntityWriter.java | 16 +- 15 files changed, 342 insertions(+), 270 deletions(-) rename src/main/java/com/curateme/clacobatchserver/batch/{KopisBatch.java => ConcertBatch.java} (54%) delete mode 100644 src/main/java/com/curateme/clacobatchserver/batch/SecondBatch.java rename src/main/java/com/curateme/clacobatchserver/{service => config}/RestTemplateConfig.java (79%) delete mode 100644 src/main/java/com/curateme/clacobatchserver/entity/AfterEntity.java delete mode 100644 src/main/java/com/curateme/clacobatchserver/entity/BeforeEntity.java create mode 100644 src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java delete mode 100644 src/main/java/com/curateme/clacobatchserver/entity/WinEntity.java delete mode 100644 src/main/java/com/curateme/clacobatchserver/repository/AfterRepository.java delete mode 100644 src/main/java/com/curateme/clacobatchserver/repository/BeforeRepository.java create mode 100644 src/main/java/com/curateme/clacobatchserver/repository/ConcertRepository.java delete mode 100644 src/main/java/com/curateme/clacobatchserver/repository/WinRepository.java rename src/main/java/com/curateme/clacobatchserver/schedule/{FirstSchedule.java => ConcertSchedule.java} (88%) rename src/main/java/com/curateme/clacobatchserver/service/{KopisApiReader.java => KopisConcertApiReader.java} (63%) create mode 100644 src/main/java/com/curateme/clacobatchserver/service/KopisDetailApiReader.java diff --git a/src/main/java/com/curateme/clacobatchserver/batch/KopisBatch.java b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java similarity index 54% rename from src/main/java/com/curateme/clacobatchserver/batch/KopisBatch.java rename to src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java index 857c070..45cb3ce 100644 --- a/src/main/java/com/curateme/clacobatchserver/batch/KopisBatch.java +++ b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java @@ -1,9 +1,8 @@ package com.curateme.clacobatchserver.batch; -import com.curateme.clacobatchserver.entity.BeforeEntity; -import com.curateme.clacobatchserver.repository.AfterRepository; -import com.curateme.clacobatchserver.repository.BeforeRepository; -import com.curateme.clacobatchserver.service.KopisApiReader; +import com.curateme.clacobatchserver.entity.ConcertEntity; +import com.curateme.clacobatchserver.service.KopisConcertApiReader; +import com.curateme.clacobatchserver.service.KopisDetailApiReader; import com.curateme.clacobatchserver.service.KopisEntityWriter; import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; @@ -15,41 +14,47 @@ import org.springframework.transaction.PlatformTransactionManager; @Configuration -public class KopisBatch { +public class ConcertBatch { private final JobRepository jobRepository; private final PlatformTransactionManager platformTransactionManager; - private final KopisApiReader kopisApiReader; - private final BeforeRepository beforeRepository; - private final AfterRepository afterRepository; + private final KopisConcertApiReader kopisApiReader; + private final KopisDetailApiReader kopisDetailApiReader; - public KopisBatch(JobRepository jobRepository, - PlatformTransactionManager platformTransactionManager, BeforeRepository beforeRepository, - AfterRepository afterRepository, KopisApiReader kopisApiReader) { + public ConcertBatch(JobRepository jobRepository, + PlatformTransactionManager platformTransactionManager, + KopisConcertApiReader kopisApiReader, + KopisDetailApiReader kopisDetailApiReader) { this.jobRepository = jobRepository; this.platformTransactionManager = platformTransactionManager; - this.beforeRepository = beforeRepository; - this.afterRepository = afterRepository; this.kopisApiReader = kopisApiReader; + this.kopisDetailApiReader = kopisDetailApiReader; } @Bean public Job kopisJob(KopisEntityWriter writer){ return new JobBuilder("kopisJob", jobRepository) .start(firstStep(writer)) + .next(secondStep()) .build(); } + // 1. Kopis에서 해당 기간에 대한 공연 정보 가져 오기 @Bean public Step firstStep(KopisEntityWriter writer) { return new StepBuilder("firstStep", jobRepository) - .chunk(10, platformTransactionManager) + .chunk(10, platformTransactionManager) .reader(kopisApiReader) .writer(writer) .build(); } - - + // 2. Step1 에서 가져온 공연에 대해 상세 내역 가져 오기 + @Bean + public Step secondStep() { + return new StepBuilder("secondStep", jobRepository) + .tasklet(kopisDetailApiReader, platformTransactionManager) + .build(); + } } diff --git a/src/main/java/com/curateme/clacobatchserver/batch/SecondBatch.java b/src/main/java/com/curateme/clacobatchserver/batch/SecondBatch.java deleted file mode 100644 index 3dd2924..0000000 --- a/src/main/java/com/curateme/clacobatchserver/batch/SecondBatch.java +++ /dev/null @@ -1,83 +0,0 @@ -package com.curateme.clacobatchserver.batch; - -import com.curateme.clacobatchserver.entity.WinEntity; -import com.curateme.clacobatchserver.repository.WinRepository; -import java.util.Collections; -import java.util.Map; -import org.springframework.batch.core.Job; -import org.springframework.batch.core.Step; -import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.repository.JobRepository; -import org.springframework.batch.core.step.builder.StepBuilder; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.batch.item.data.RepositoryItemReader; -import org.springframework.batch.item.data.RepositoryItemWriter; -import org.springframework.batch.item.data.builder.RepositoryItemReaderBuilder; -import org.springframework.batch.item.data.builder.RepositoryItemWriterBuilder; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.domain.Sort; -import org.springframework.transaction.PlatformTransactionManager; - -@Configuration -public class SecondBatch { - private final JobRepository jobRepository; - private final PlatformTransactionManager platformTransactionManager; - private final WinRepository winRepository; - - public SecondBatch(JobRepository jobRepository, PlatformTransactionManager platformTransactionManager, WinRepository winRepository) { - this.jobRepository = jobRepository; - this.platformTransactionManager = platformTransactionManager; - this.winRepository = winRepository; - } - - @Bean - public Job secondJob() { - - return new JobBuilder("secondJob", jobRepository) - .start(secondStep()) - .build(); - } - - @Bean - public Step secondStep() { - - return new StepBuilder("secondStep", jobRepository) - . chunk(10, platformTransactionManager) - .reader(winReader()) - .processor(trueProcessor()) - .writer(winWriter()) - .build(); - } - - @Bean - public RepositoryItemReader winReader() { - - return new RepositoryItemReaderBuilder() - .name("winReader") - .pageSize(10) - .methodName("findByWinGreaterThanEqual") - .arguments(Collections.singletonList(10L)) - .repository(winRepository) - .sorts(Map.of("id", Sort.Direction.ASC)) - .build(); - } - - @Bean - public ItemProcessor trueProcessor() { - - return item -> { - item.setReward(true); - return item; - }; - } - - @Bean - public RepositoryItemWriter winWriter() { - - return new RepositoryItemWriterBuilder() - .repository(winRepository) - .methodName("save") - .build(); - } -} diff --git a/src/main/java/com/curateme/clacobatchserver/service/RestTemplateConfig.java b/src/main/java/com/curateme/clacobatchserver/config/RestTemplateConfig.java similarity index 79% rename from src/main/java/com/curateme/clacobatchserver/service/RestTemplateConfig.java rename to src/main/java/com/curateme/clacobatchserver/config/RestTemplateConfig.java index d83f9ab..dde3f36 100644 --- a/src/main/java/com/curateme/clacobatchserver/service/RestTemplateConfig.java +++ b/src/main/java/com/curateme/clacobatchserver/config/RestTemplateConfig.java @@ -1,10 +1,11 @@ -package com.curateme.clacobatchserver.service; +package com.curateme.clacobatchserver.config; import java.util.ArrayList; import java.util.List; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.http.converter.HttpMessageConverter; +import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; import org.springframework.http.converter.xml.MappingJackson2XmlHttpMessageConverter; import org.springframework.web.client.RestTemplate; @@ -17,6 +18,8 @@ public RestTemplate restTemplate() { List> messageConverters = new ArrayList<>(); messageConverters.add(new MappingJackson2XmlHttpMessageConverter()); + messageConverters.add(new MappingJackson2HttpMessageConverter()); + restTemplate.setMessageConverters(messageConverters); return restTemplate; diff --git a/src/main/java/com/curateme/clacobatchserver/entity/AfterEntity.java b/src/main/java/com/curateme/clacobatchserver/entity/AfterEntity.java deleted file mode 100644 index 0a659db..0000000 --- a/src/main/java/com/curateme/clacobatchserver/entity/AfterEntity.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.curateme.clacobatchserver.entity; - -import jakarta.persistence.Entity; -import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; -import jakarta.persistence.Id; -import lombok.Getter; - -@Entity -@Getter -public class AfterEntity { - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private Long id; - - private String username; - - public void setUsername(String username) { - this.username = username; - } -} \ No newline at end of file diff --git a/src/main/java/com/curateme/clacobatchserver/entity/BeforeEntity.java b/src/main/java/com/curateme/clacobatchserver/entity/BeforeEntity.java deleted file mode 100644 index f75b4c4..0000000 --- a/src/main/java/com/curateme/clacobatchserver/entity/BeforeEntity.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.curateme.clacobatchserver.entity; - -import jakarta.persistence.Column; -import jakarta.persistence.Entity; -import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; -import jakarta.persistence.Id; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; - -@Entity -@Getter -@NoArgsConstructor -@AllArgsConstructor -public class BeforeEntity { - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private Long id; - - @Column(name = "concertId") - private String mt20id; - - @Column(name = "concertName") - private String prfnm; - - @Column(name = "startDate") - private String prfpdfrom; - - @Column(name = "endDate") - private String prfpdto; - - @Column(name = "facilityName") - private String fcltynm; - - @Column(name = "poster") - private String poster; - - @Column(name = "area") - private String area; - - @Column(name = "genre") - private String genrenm; - - @Column(name = "openrun") - private String openrun; - - @Column(name = "status") - private String prfstate; - - public void setConcertDetails(String mt20id, String prfnm, String prfpdfrom, String prfpdto, - String fcltynm, String poster, String area, String genrenm, - String openrun, String prfstate) { - this.mt20id = mt20id; - this.prfnm = prfnm; - this.prfpdfrom = prfpdfrom; - this.prfpdto = prfpdto; - this.fcltynm = fcltynm; - this.poster = poster; - this.area = area; - this.genrenm = genrenm; - this.openrun = openrun; - this.prfstate = prfstate; - } -} diff --git a/src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java b/src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java new file mode 100644 index 0000000..6855bfa --- /dev/null +++ b/src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java @@ -0,0 +1,145 @@ +package com.curateme.clacobatchserver.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Entity +@Getter +@NoArgsConstructor +@AllArgsConstructor +public class ConcertEntity { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(name = "concertId") + private String mt20id; + + @Column(name = "concertName") + private String prfnm; + + @Column(name = "startDate") + private String prfpdfrom; + + @Column(name = "endDate") + private String prfpdto; + + @Column(name = "facilityName") + private String fcltynm; + + @Column(name = "poster") + private String poster; + + @Column(name = "area") + private String area; + + @Column(name = "genre") + private String genrenm; + + @Column(name = "openrun") + private String openrun; + + @Column(name = "status") + private String prfstate; + + @Column(name = "cast") + private String prfcast; + + @Column(name = "crew") + private String prfcrew; + + @Column(name = "runtime") + private String prfruntime; + + @Column(name = "age") + private String prfage; + + @Column(name = "companyName") + private String entrpsnm; + + @Column(name = "companyNameP") + private String entrpsnmP; + + @Column(name = "companyNameA") + private String entrpsnmA; + + @Column(name = "companyNameH") + private String entrpsnmH; + + @Column(name = "companyNameS") + private String entrpsnmS; + + @Column(name = "seatGuidance") + private String pcseguidance; + + @Column(name = "visit") + private String visit; + + @Column(name = "child") + private String child; + + @Column(name = "daehakro") + private String daehakro; + + @Column(name = "festival") + private String festival; + + @Column(name = "musicalLicense") + private String musicallicense; + + @Column(name = "musicalCreate") + private String musicalcreate; + + @Column(name = "updateDate") + private String updatedate; + + @Column(name = "scheduleGuidance", length = 1000) + private String dtguidance; + + public void setConcertDetails(String mt20id, String prfnm, String prfpdfrom, String prfpdto, + String fcltynm, String poster, String area, String genrenm, + String openrun, String prfstate) { + this.mt20id = mt20id; + this.prfnm = prfnm; + this.prfpdfrom = prfpdfrom; + this.prfpdto = prfpdto; + this.fcltynm = fcltynm; + this.poster = poster; + this.area = area; + this.genrenm = genrenm; + this.openrun = openrun; + this.prfstate = prfstate; + } + + public void setAdditionalConcertDetails(String prfcast, String prfcrew, String prfruntime, String prfage, + String entrpsnm, String entrpsnmP, String entrpsnmA, String entrpsnmH, + String entrpsnmS, String pcseguidance, String visit, String child, + String daehakro, String festival, String musicallicense, + String musicalcreate, String updatedate, String dtguidance) { + this.prfcast = prfcast; + this.prfcrew = prfcrew; + this.prfruntime = prfruntime; + this.prfage = prfage; + this.entrpsnm = entrpsnm; + this.entrpsnmP = entrpsnmP; + this.entrpsnmA = entrpsnmA; + this.entrpsnmH = entrpsnmH; + this.entrpsnmS = entrpsnmS; + this.pcseguidance = pcseguidance; + this.visit = visit; + this.child = child; + this.daehakro = daehakro; + this.festival = festival; + this.musicallicense = musicallicense; + this.musicalcreate = musicalcreate; + this.updatedate = updatedate; + this.dtguidance = dtguidance; + } +} diff --git a/src/main/java/com/curateme/clacobatchserver/entity/WinEntity.java b/src/main/java/com/curateme/clacobatchserver/entity/WinEntity.java deleted file mode 100644 index e06f4fa..0000000 --- a/src/main/java/com/curateme/clacobatchserver/entity/WinEntity.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.curateme.clacobatchserver.entity; - -import jakarta.persistence.Entity; -import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; -import jakarta.persistence.Id; -import lombok.Getter; -import lombok.Setter; - -@Entity -@Getter -@Setter -public class WinEntity { - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private Long id; - - private String username; - private Long win; - private Boolean reward; -} \ No newline at end of file diff --git a/src/main/java/com/curateme/clacobatchserver/repository/AfterRepository.java b/src/main/java/com/curateme/clacobatchserver/repository/AfterRepository.java deleted file mode 100644 index e9b23ed..0000000 --- a/src/main/java/com/curateme/clacobatchserver/repository/AfterRepository.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.curateme.clacobatchserver.repository; - -import com.curateme.clacobatchserver.entity.AfterEntity; -import org.springframework.data.jpa.repository.JpaRepository; - -public interface AfterRepository extends JpaRepository { - -} diff --git a/src/main/java/com/curateme/clacobatchserver/repository/BeforeRepository.java b/src/main/java/com/curateme/clacobatchserver/repository/BeforeRepository.java deleted file mode 100644 index 8e6b16b..0000000 --- a/src/main/java/com/curateme/clacobatchserver/repository/BeforeRepository.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.curateme.clacobatchserver.repository; - -import com.curateme.clacobatchserver.entity.BeforeEntity; -import org.springframework.data.jpa.repository.JpaRepository; - -public interface BeforeRepository extends JpaRepository { - -} diff --git a/src/main/java/com/curateme/clacobatchserver/repository/ConcertRepository.java b/src/main/java/com/curateme/clacobatchserver/repository/ConcertRepository.java new file mode 100644 index 0000000..71eda3c --- /dev/null +++ b/src/main/java/com/curateme/clacobatchserver/repository/ConcertRepository.java @@ -0,0 +1,8 @@ +package com.curateme.clacobatchserver.repository; + +import com.curateme.clacobatchserver.entity.ConcertEntity; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface ConcertRepository extends JpaRepository { + +} diff --git a/src/main/java/com/curateme/clacobatchserver/repository/WinRepository.java b/src/main/java/com/curateme/clacobatchserver/repository/WinRepository.java deleted file mode 100644 index d262af1..0000000 --- a/src/main/java/com/curateme/clacobatchserver/repository/WinRepository.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.curateme.clacobatchserver.repository; - -import com.curateme.clacobatchserver.entity.WinEntity; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.jpa.repository.JpaRepository; - -public interface WinRepository extends JpaRepository { - - Page findByWinGreaterThanEqual(Long win, Pageable pageable); -} diff --git a/src/main/java/com/curateme/clacobatchserver/schedule/FirstSchedule.java b/src/main/java/com/curateme/clacobatchserver/schedule/ConcertSchedule.java similarity index 88% rename from src/main/java/com/curateme/clacobatchserver/schedule/FirstSchedule.java rename to src/main/java/com/curateme/clacobatchserver/schedule/ConcertSchedule.java index 15be091..407a047 100644 --- a/src/main/java/com/curateme/clacobatchserver/schedule/FirstSchedule.java +++ b/src/main/java/com/curateme/clacobatchserver/schedule/ConcertSchedule.java @@ -10,18 +10,18 @@ import org.springframework.scheduling.annotation.Scheduled; @Configuration -public class FirstSchedule { +public class ConcertSchedule { private final JobLauncher jobLauncher; private final JobRegistry jobRegistry; - public FirstSchedule(JobLauncher jobLauncher, JobRegistry jobRegistry) { + public ConcertSchedule(JobLauncher jobLauncher, JobRegistry jobRegistry) { this.jobLauncher = jobLauncher; this.jobRegistry = jobRegistry; } @Scheduled(cron = "10 * * * * *", zone = "Asia/Seoul") - //@Scheduled(cron = "0 0 0 1 * *", zone = "Asia/Seoul") -> 추후 한달 단위로 수정 + //@Scheduled(cron = "0 0 0 1 * *", zone = "Asia/Seoul") -> 추후 한달 단위로 수정(매달 1일) public void runFirstJob() throws Exception { SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-hh-mm-ss"); diff --git a/src/main/java/com/curateme/clacobatchserver/service/KopisApiReader.java b/src/main/java/com/curateme/clacobatchserver/service/KopisConcertApiReader.java similarity index 63% rename from src/main/java/com/curateme/clacobatchserver/service/KopisApiReader.java rename to src/main/java/com/curateme/clacobatchserver/service/KopisConcertApiReader.java index 3a03554..af8a658 100644 --- a/src/main/java/com/curateme/clacobatchserver/service/KopisApiReader.java +++ b/src/main/java/com/curateme/clacobatchserver/service/KopisConcertApiReader.java @@ -1,6 +1,6 @@ package com.curateme.clacobatchserver.service; -import com.curateme.clacobatchserver.entity.BeforeEntity; +import com.curateme.clacobatchserver.entity.ConcertEntity; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -17,20 +17,20 @@ import org.springframework.web.client.HttpClientErrorException; @Component -public class KopisApiReader implements ItemReader { +public class KopisConcertApiReader implements ItemReader { private final RestTemplate restTemplate; private final String apiUrl = "http://www.kopis.or.kr/openApi/restful/pblprfr?service=f222668534db409b8769f640387de9c3"; private int currentPage = 1; - private List beforeEntities = new ArrayList<>(); + private List beforeEntities = new ArrayList<>(); private int index = 0; - public KopisApiReader(RestTemplate restTemplate) { + public KopisConcertApiReader(RestTemplate restTemplate) { this.restTemplate = restTemplate; } @Override - public BeforeEntity read() throws Exception { + public ConcertEntity read() throws Exception { if (index >= beforeEntities.size()) { loadNextPage(); @@ -46,7 +46,7 @@ public BeforeEntity read() throws Exception { private void loadNextPage() { LocalDate startDate = LocalDate.now(); - LocalDate endDate = startDate.plusMonths(3); + LocalDate endDate = startDate.plusDays(1); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); String formattedStartDate = startDate.format(formatter); @@ -60,26 +60,26 @@ private void loadNextPage() { HttpEntity entity = new HttpEntity<>(headers); try { - ResponseEntity response = restTemplate.exchange(requestUrl, HttpMethod.GET, entity, BeforeEntity[].class); - + ResponseEntity response = restTemplate.exchange(requestUrl, HttpMethod.GET, entity, ConcertEntity[].class); + System.out.println("response = " + response); if (response.getBody() != null && response.getBody().length > 0) { beforeEntities.clear(); - for (BeforeEntity Beforeentity : response.getBody()) { - BeforeEntity beforeEntity = new BeforeEntity(); - beforeEntity.setConcertDetails( - Beforeentity.getMt20id(), - Beforeentity.getPrfnm(), - Beforeentity.getPrfpdfrom(), - Beforeentity.getPrfpdto(), - Beforeentity.getFcltynm(), - Beforeentity.getPoster(), - Beforeentity.getArea(), - Beforeentity.getGenrenm(), - Beforeentity.getOpenrun(), - Beforeentity.getPrfstate() + for (ConcertEntity beforeentity : response.getBody()) { + ConcertEntity concertEntity = new ConcertEntity(); + concertEntity.setConcertDetails( + beforeentity.getMt20id(), + beforeentity.getPrfnm(), + beforeentity.getPrfpdfrom(), + beforeentity.getPrfpdto(), + beforeentity.getFcltynm(), + beforeentity.getPoster(), + beforeentity.getArea(), + beforeentity.getGenrenm(), + beforeentity.getOpenrun(), + beforeentity.getPrfstate() ); - beforeEntities.add(beforeEntity); + beforeEntities.add(concertEntity); } } else { beforeEntities.clear(); diff --git a/src/main/java/com/curateme/clacobatchserver/service/KopisDetailApiReader.java b/src/main/java/com/curateme/clacobatchserver/service/KopisDetailApiReader.java new file mode 100644 index 0000000..0dd112b --- /dev/null +++ b/src/main/java/com/curateme/clacobatchserver/service/KopisDetailApiReader.java @@ -0,0 +1,131 @@ +package com.curateme.clacobatchserver.service; + +import com.curateme.clacobatchserver.entity.ConcertEntity; +import com.curateme.clacobatchserver.repository.ConcertRepository; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.repeat.RepeatStatus; +import org.springframework.stereotype.Component; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; + +@Component +public class KopisDetailApiReader implements Tasklet { + + private final ConcertRepository concertRepository; + + public KopisDetailApiReader(ConcertRepository concertRepository) { + this.concertRepository = concertRepository; + } + + @Override + public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception { + List beforeEntities = concertRepository.findAll(); + + for (ConcertEntity concertEntity : beforeEntities) { + fetchAndSave(concertEntity.getMt20id(), concertEntity); + } + + return RepeatStatus.FINISHED; + } + + private void fetchAndSave(String mt20id, ConcertEntity concertEntity) { + try { + String urlString = String.format( + "http://www.kopis.or.kr/openApi/restful/pblprfr/%s?service=f222668534db409b8769f640387de9c3", + mt20id + ); + System.out.println("url = " + urlString); + + URL url = new URL(urlString); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("GET"); + connection.setRequestProperty("Accept", "application/xml"); + connection.setRequestProperty("Accept-Charset", "UTF-8"); + + int responseCode = connection.getResponseCode(); + System.out.println("Response Code: " + responseCode); + + if (responseCode == HttpURLConnection.HTTP_OK) { + BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8)); + String inputLine; + StringBuilder response = new StringBuilder(); + + while ((inputLine = in.readLine()) != null) { + response.append(inputLine); + } + in.close(); + + String xmlResponse = response.toString(); + System.out.println("Response: " + xmlResponse); + + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document document = builder.parse(new ByteArrayInputStream(xmlResponse.getBytes(StandardCharsets.UTF_8))); + + document.getDocumentElement().normalize(); + NodeList nodeList = document.getElementsByTagName("db"); + + for (int i = 0; i < nodeList.getLength(); i++) { + Node node = nodeList.item(i); + + if (node.getNodeType() == Node.ELEMENT_NODE) { + Element element = (Element) node; + + String prfcast = getTagValue("prfcast", element); + String prfcrew = getTagValue("prfcrew", element); + String prfruntime = getTagValue("prfruntime", element); + String prfage = getTagValue("prfage", element); + String entrpsnm = getTagValue("entrpsnm", element); + String entrpsnmP = getTagValue("entrpsnmP", element); + String entrpsnmA = getTagValue("entrpsnmA", element); + String entrpsnmH = getTagValue("entrpsnmH", element); + String entrpsnmS = getTagValue("entrpsnmS", element); + String pcseguidance = getTagValue("pcseguidance", element); + String visit = getTagValue("visit", element); + String child = getTagValue("child", element); + String daehakro = getTagValue("daehakro", element); + String festival = getTagValue("festival", element); + String musicallicense = getTagValue("musicallicense", element); + String musicalcreate = getTagValue("musicalcreate", element); + String updatedate = getTagValue("updatedate", element); + String dtguidance = getTagValue("dtguidance", element); + + concertEntity.setAdditionalConcertDetails( + prfcast, prfcrew, prfruntime, prfage, entrpsnm, entrpsnmP, entrpsnmA, entrpsnmH, + entrpsnmS, pcseguidance, visit, child, daehakro, festival, musicallicense, + musicalcreate, updatedate, dtguidance + ); + + concertRepository.save(concertEntity); + } + } + } else { + System.out.println("Error: Received HTTP " + responseCode); + } + + connection.disconnect(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private String getTagValue(String tag, Element element) { + NodeList nodeList = element.getElementsByTagName(tag).item(0).getChildNodes(); + Node node = nodeList.item(0); + return node != null ? node.getNodeValue() : ""; + } +} diff --git a/src/main/java/com/curateme/clacobatchserver/service/KopisEntityWriter.java b/src/main/java/com/curateme/clacobatchserver/service/KopisEntityWriter.java index c639f86..631f28d 100644 --- a/src/main/java/com/curateme/clacobatchserver/service/KopisEntityWriter.java +++ b/src/main/java/com/curateme/clacobatchserver/service/KopisEntityWriter.java @@ -1,25 +1,25 @@ package com.curateme.clacobatchserver.service; -import com.curateme.clacobatchserver.entity.BeforeEntity; -import com.curateme.clacobatchserver.repository.BeforeRepository; +import com.curateme.clacobatchserver.entity.ConcertEntity; +import com.curateme.clacobatchserver.repository.ConcertRepository; import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.stereotype.Component; @Component -public class KopisEntityWriter implements ItemWriter { +public class KopisEntityWriter implements ItemWriter { - private final BeforeRepository beforeRepository; + private final ConcertRepository concertRepository; - public KopisEntityWriter(BeforeRepository beforeRepository) { + public KopisEntityWriter(ConcertRepository concertRepository) { - this.beforeRepository = beforeRepository; + this.concertRepository = concertRepository; } @Override - public void write(Chunk items) throws Exception { + public void write(Chunk items) throws Exception { - beforeRepository.saveAll(items.getItems()); + concertRepository.saveAll(items.getItems()); } From 5bee67e8ee19e5bbf7c68464e490dd1e9f7b4adf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=ED=9D=AC=EC=B0=AC?= Date: Thu, 24 Oct 2024 01:11:30 +0900 Subject: [PATCH 04/19] =?UTF-8?q?feature:=20ConcertEntity=20Introduction?= =?UTF-8?q?=20=EC=86=8D=EC=84=B1=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../curateme/clacobatchserver/batch/ConcertBatch.java | 6 ++++++ .../clacobatchserver/entity/ConcertEntity.java | 7 +++++++ .../clacobatchserver/service/KopisDetailApiReader.java | 10 ++++++++++ 3 files changed, 23 insertions(+) diff --git a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java index 45cb3ce..59f45cc 100644 --- a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java +++ b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java @@ -57,4 +57,10 @@ public Step secondStep() { .build(); } + // 3. 공연 포스터 이미지를 통해서 Flask 서버로 부터 카테고리 추출 + @Bean + public Step thirdStep(){ + return null; + } + } diff --git a/src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java b/src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java index 6855bfa..535eecb 100644 --- a/src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java +++ b/src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java @@ -103,6 +103,9 @@ public class ConcertEntity { @Column(name = "scheduleGuidance", length = 1000) private String dtguidance; + @Column(name = "introduction") + private String styurl; + public void setConcertDetails(String mt20id, String prfnm, String prfpdfrom, String prfpdto, String fcltynm, String poster, String area, String genrenm, String openrun, String prfstate) { @@ -142,4 +145,8 @@ public void setAdditionalConcertDetails(String prfcast, String prfcrew, String p this.updatedate = updatedate; this.dtguidance = dtguidance; } + + public void setStyurl(String styurl){ + this.styurl =styurl; + } } diff --git a/src/main/java/com/curateme/clacobatchserver/service/KopisDetailApiReader.java b/src/main/java/com/curateme/clacobatchserver/service/KopisDetailApiReader.java index 0dd112b..fd1f8ec 100644 --- a/src/main/java/com/curateme/clacobatchserver/service/KopisDetailApiReader.java +++ b/src/main/java/com/curateme/clacobatchserver/service/KopisDetailApiReader.java @@ -104,6 +104,16 @@ private void fetchAndSave(String mt20id, ConcertEntity concertEntity) { String updatedate = getTagValue("updatedate", element); String dtguidance = getTagValue("dtguidance", element); + NodeList styurlsList = element.getElementsByTagName("styurls"); + if (styurlsList.getLength() > 0) { + Element styurlsElement = (Element) styurlsList.item(0); + NodeList styurlList = styurlsElement.getElementsByTagName("styurl"); + if (styurlList.getLength() > 0) { + String styurl = styurlList.item(0).getTextContent(); + concertEntity.setStyurl(styurl); + } + } + concertEntity.setAdditionalConcertDetails( prfcast, prfcrew, prfruntime, prfage, entrpsnm, entrpsnmP, entrpsnmA, entrpsnmH, entrpsnmS, pcseguidance, visit, child, daehakro, festival, musicallicense, From a85093e386556828dde8ad27e31d25aeab3fef45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=ED=9D=AC=EC=B0=AC?= Date: Thu, 24 Oct 2024 01:36:10 +0900 Subject: [PATCH 05/19] =?UTF-8?q?feature:=20Step3=20Concert=20=ED=8F=AC?= =?UTF-8?q?=EC=8A=A4=ED=84=B0=20=EB=B6=84=EC=84=9D=20=EB=B0=8F=20=EC=B9=B4?= =?UTF-8?q?=ED=85=8C=EA=B3=A0=EB=A6=AC=20=EC=A0=80=EC=9E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../clacobatchserver/batch/ConcertBatch.java | 26 +++++++- .../entity/ConcertEntity.java | 10 ++++ .../service/ConcertCategoryExtractor.java | 59 +++++++++++++++++++ 3 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 src/main/java/com/curateme/clacobatchserver/service/ConcertCategoryExtractor.java diff --git a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java index 59f45cc..f487977 100644 --- a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java +++ b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java @@ -1,14 +1,19 @@ package com.curateme.clacobatchserver.batch; import com.curateme.clacobatchserver.entity.ConcertEntity; +import com.curateme.clacobatchserver.service.ConcertCategoryExtractor; import com.curateme.clacobatchserver.service.KopisConcertApiReader; import com.curateme.clacobatchserver.service.KopisDetailApiReader; import com.curateme.clacobatchserver.service.KopisEntityWriter; import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; +import org.springframework.batch.core.StepContribution; import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.scope.context.ChunkContext; import org.springframework.batch.core.step.builder.StepBuilder; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.repeat.RepeatStatus; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.transaction.PlatformTransactionManager; @@ -20,15 +25,18 @@ public class ConcertBatch { private final KopisConcertApiReader kopisApiReader; private final KopisDetailApiReader kopisDetailApiReader; + private final ConcertCategoryExtractor concertCategoryExtractor; public ConcertBatch(JobRepository jobRepository, PlatformTransactionManager platformTransactionManager, KopisConcertApiReader kopisApiReader, - KopisDetailApiReader kopisDetailApiReader) { + KopisDetailApiReader kopisDetailApiReader, + ConcertCategoryExtractor concertCategoryExtractor) { this.jobRepository = jobRepository; this.platformTransactionManager = platformTransactionManager; this.kopisApiReader = kopisApiReader; this.kopisDetailApiReader = kopisDetailApiReader; + this.concertCategoryExtractor = concertCategoryExtractor; } @Bean @@ -36,6 +44,7 @@ public Job kopisJob(KopisEntityWriter writer){ return new JobBuilder("kopisJob", jobRepository) .start(firstStep(writer)) .next(secondStep()) + .next(thirdStep()) .build(); } @@ -59,8 +68,19 @@ public Step secondStep() { // 3. 공연 포스터 이미지를 통해서 Flask 서버로 부터 카테고리 추출 @Bean - public Step thirdStep(){ - return null; + public Step thirdStep() { + return new StepBuilder("thirdStep", jobRepository) + .tasklet(concertCategoryExtractorTasklet(), platformTransactionManager) + .build(); } + @Bean + public Tasklet concertCategoryExtractorTasklet() { + return new Tasklet() { + @Override + public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception { + return concertCategoryExtractor.execute(contribution, chunkContext); + } + }; + } } diff --git a/src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java b/src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java index 535eecb..3421d95 100644 --- a/src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java +++ b/src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java @@ -1,10 +1,12 @@ package com.curateme.clacobatchserver.entity; import jakarta.persistence.Column; +import jakarta.persistence.ElementCollection; import jakarta.persistence.Entity; import jakarta.persistence.GeneratedValue; import jakarta.persistence.GenerationType; import jakarta.persistence.Id; +import java.util.List; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; @@ -106,6 +108,10 @@ public class ConcertEntity { @Column(name = "introduction") private String styurl; + @ElementCollection + @Column(name = "categories") + private List categories; + public void setConcertDetails(String mt20id, String prfnm, String prfpdfrom, String prfpdto, String fcltynm, String poster, String area, String genrenm, String openrun, String prfstate) { @@ -149,4 +155,8 @@ public void setAdditionalConcertDetails(String prfcast, String prfcrew, String p public void setStyurl(String styurl){ this.styurl =styurl; } + + public void setCategories(List categories) { + this.categories = categories; + } } diff --git a/src/main/java/com/curateme/clacobatchserver/service/ConcertCategoryExtractor.java b/src/main/java/com/curateme/clacobatchserver/service/ConcertCategoryExtractor.java new file mode 100644 index 0000000..1420e3d --- /dev/null +++ b/src/main/java/com/curateme/clacobatchserver/service/ConcertCategoryExtractor.java @@ -0,0 +1,59 @@ +package com.curateme.clacobatchserver.service; + +import com.curateme.clacobatchserver.entity.ConcertEntity; +import com.curateme.clacobatchserver.repository.ConcertRepository; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.repeat.RepeatStatus; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; + +@Component +public class ConcertCategoryExtractor { + + private final RestTemplate restTemplate; + private final ConcertRepository concertRepository; + + @Autowired + public ConcertCategoryExtractor(RestTemplate restTemplate, ConcertRepository concertRepository) { + this.restTemplate = restTemplate; + this.concertRepository = concertRepository; + } + + public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception { + List concerts = concertRepository.findAll(); + + for (ConcertEntity concert : concerts) { + String introduction = concert.getStyurl(); + + Map requestBody = new HashMap<>(); + requestBody.put("image_url", introduction); + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + HttpEntity> requestEntity = new HttpEntity<>(requestBody, headers); + + ResponseEntity response = restTemplate.postForEntity("http://localhost:8080/categories", requestEntity, Map.class); + + if (response.getBody() != null) { + Map responseBody = (Map) response.getBody(); + Map clovaResponse = (Map) responseBody.get("clova_response"); + + List categories = (List) clovaResponse.get("categories"); + concert.setCategories(categories); + + concertRepository.save(concert); + } + } + return RepeatStatus.FINISHED; + } + +} From 42341dd9a023e8e00d801a67da33f757d74aaca2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=ED=9D=AC=EC=B0=AC?= Date: Thu, 24 Oct 2024 12:19:20 +0900 Subject: [PATCH 06/19] =?UTF-8?q?fix:=20Step3=20Concert=20=ED=8F=AC?= =?UTF-8?q?=EC=8A=A4=ED=84=B0=20=EB=B6=84=EC=84=9D=20=EB=B0=8F=20=EC=B9=B4?= =?UTF-8?q?=ED=85=8C=EA=B3=A0=EB=A6=AC=20HashMap=EC=9C=BC=EB=A1=9C=20?= =?UTF-8?q?=EC=A0=80=EC=9E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../clacobatchserver/entity/ConcertEntity.java | 6 +++--- .../service/ConcertCategoryExtractor.java | 14 ++++++++++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java b/src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java index 3421d95..9b30c02 100644 --- a/src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java +++ b/src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java @@ -7,6 +7,7 @@ import jakarta.persistence.GenerationType; import jakarta.persistence.Id; import java.util.List; +import java.util.Map; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; @@ -110,8 +111,7 @@ public class ConcertEntity { @ElementCollection @Column(name = "categories") - private List categories; - + private Map categories; public void setConcertDetails(String mt20id, String prfnm, String prfpdfrom, String prfpdto, String fcltynm, String poster, String area, String genrenm, String openrun, String prfstate) { @@ -156,7 +156,7 @@ public void setStyurl(String styurl){ this.styurl =styurl; } - public void setCategories(List categories) { + public void setCategories(Map categories) { this.categories = categories; } } diff --git a/src/main/java/com/curateme/clacobatchserver/service/ConcertCategoryExtractor.java b/src/main/java/com/curateme/clacobatchserver/service/ConcertCategoryExtractor.java index 1420e3d..e71a1eb 100644 --- a/src/main/java/com/curateme/clacobatchserver/service/ConcertCategoryExtractor.java +++ b/src/main/java/com/curateme/clacobatchserver/service/ConcertCategoryExtractor.java @@ -45,15 +45,25 @@ public RepeatStatus execute(StepContribution contribution, ChunkContext chunkCon if (response.getBody() != null) { Map responseBody = (Map) response.getBody(); - Map clovaResponse = (Map) responseBody.get("clova_response"); + List> clovaResponse = (List>) responseBody.get("clova_response"); + + Map categories = new HashMap<>(); + + for (Map categoryData : clovaResponse) { + String category = categoryData.get("name").toString(); + Double score = Double.parseDouble(categoryData.get("score").toString()); + categories.put(category, score); + } - List categories = (List) clovaResponse.get("categories"); concert.setCategories(categories); concertRepository.save(concert); } } + return RepeatStatus.FINISHED; } + + } From 405fec1bfd716f95ba7355559db57cce3a91d3a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=ED=9D=AC=EC=B0=AC?= Date: Sun, 27 Oct 2024 00:22:05 +0900 Subject: [PATCH 07/19] =?UTF-8?q?feature:=20S3=20Config=20=EC=84=A4?= =?UTF-8?q?=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.gradle | 3 + .../clacobatchserver/config/s3/S3Config.java | 33 ++++++++++ .../clacobatchserver/config/s3/S3Service.java | 61 +++++++++++++++++++ .../entity/ConcertEntity.java | 1 - 4 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/curateme/clacobatchserver/config/s3/S3Config.java create mode 100644 src/main/java/com/curateme/clacobatchserver/config/s3/S3Service.java diff --git a/build.gradle b/build.gradle index 4ee9ff9..76ff45c 100644 --- a/build.gradle +++ b/build.gradle @@ -37,6 +37,9 @@ dependencies { testImplementation 'org.springframework.boot:spring-boot-starter-test' testImplementation 'org.springframework.batch:spring-batch-test' testRuntimeOnly 'org.junit.platform:junit-platform-launcher' + + // AWS SDK for S3 + implementation 'com.amazonaws:aws-java-sdk-s3:1.12.500' } tasks.named('test') { diff --git a/src/main/java/com/curateme/clacobatchserver/config/s3/S3Config.java b/src/main/java/com/curateme/clacobatchserver/config/s3/S3Config.java new file mode 100644 index 0000000..1ffa93b --- /dev/null +++ b/src/main/java/com/curateme/clacobatchserver/config/s3/S3Config.java @@ -0,0 +1,33 @@ +package com.curateme.clacobatchserver.config.s3; + +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class S3Config { + @Value("${cloud.aws.credentials.access-key}") + private String accessKey; + + @Value("${cloud.aws.credentials.secret-key}") + private String secretKey; + + @Value("${cloud.aws.region.static}") + private String region; + + @Bean + public AmazonS3Client amazonS3Client() { + BasicAWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); + + return (AmazonS3Client) AmazonS3ClientBuilder + .standard() + .withRegion(region) + .withCredentials(new AWSStaticCredentialsProvider(credentials)) + .build(); + } + +} diff --git a/src/main/java/com/curateme/clacobatchserver/config/s3/S3Service.java b/src/main/java/com/curateme/clacobatchserver/config/s3/S3Service.java new file mode 100644 index 0000000..e91c7df --- /dev/null +++ b/src/main/java/com/curateme/clacobatchserver/config/s3/S3Service.java @@ -0,0 +1,61 @@ +package com.curateme.clacobatchserver.config.s3; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.ObjectMetadata; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Paths; + +@Service +@RequiredArgsConstructor +@Slf4j +public class S3Service { + + private final AmazonS3 amazonS3Client; + + @Value("${cloud.aws.s3.bucket}") + private String bucketName; + + public String downloadCsvFile(String folderPath, String fileName) { + String s3Key = folderPath + "/" + fileName; + String localFilePath = System.getProperty("java.io.tmpdir") + fileName; + + try { + log.info("Downloading file from S3: {}/{}", bucketName, s3Key); + amazonS3Client.getObject(new GetObjectRequest(bucketName, s3Key), new File(localFilePath)); + log.info("File downloaded successfully to: {}", localFilePath); + } catch (Exception e) { + log.error("Error downloading file from S3", e); + } + + return localFilePath; + } + + public void updateAndUploadCsvFile(String folderPath, String fileName, String filePath) { + try (BufferedWriter writer = new BufferedWriter(new FileWriter(filePath, true))) { + writer.write("\nUpdated Data"); + log.info("File updated successfully: {}", filePath); + } catch (IOException e) { + log.error("Error updating file", e); + return; + } + + // S3에 파일 업로드 + try (InputStream inputStream = Files.newInputStream(Paths.get(filePath))) { + String s3Key = folderPath + "/" + fileName; + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentLength(Files.size(Paths.get(filePath))); + + amazonS3Client.putObject(bucketName, s3Key, inputStream, metadata); + log.info("File uploaded successfully to S3: {}/{}", bucketName, s3Key); + } catch (Exception e) { + log.error("Error uploading file to S3", e); + } + } +} diff --git a/src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java b/src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java index 9b30c02..dd35545 100644 --- a/src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java +++ b/src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java @@ -6,7 +6,6 @@ import jakarta.persistence.GeneratedValue; import jakarta.persistence.GenerationType; import jakarta.persistence.Id; -import java.util.List; import java.util.Map; import lombok.AllArgsConstructor; import lombok.Getter; From ca9d997c6980b95a74cc1127eb8a4d99f46c4b51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=ED=9D=AC=EC=B0=AC?= Date: Sun, 27 Oct 2024 00:37:35 +0900 Subject: [PATCH 08/19] =?UTF-8?q?feature:=20S3=20csv=20=ED=8C=8C=EC=9D=BC?= =?UTF-8?q?=20=EB=8B=A4=EC=9A=B4=20=EB=B0=9B=EA=B8=B0=20=EB=A1=9C=EC=A7=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../clacobatchserver/batch/ConcertBatch.java | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java index f487977..0ebbab8 100644 --- a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java +++ b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java @@ -1,5 +1,6 @@ package com.curateme.clacobatchserver.batch; +import com.curateme.clacobatchserver.config.s3.S3Service; import com.curateme.clacobatchserver.entity.ConcertEntity; import com.curateme.clacobatchserver.service.ConcertCategoryExtractor; import com.curateme.clacobatchserver.service.KopisConcertApiReader; @@ -26,17 +27,20 @@ public class ConcertBatch { private final KopisConcertApiReader kopisApiReader; private final KopisDetailApiReader kopisDetailApiReader; private final ConcertCategoryExtractor concertCategoryExtractor; + private final S3Service s3Service; public ConcertBatch(JobRepository jobRepository, PlatformTransactionManager platformTransactionManager, KopisConcertApiReader kopisApiReader, KopisDetailApiReader kopisDetailApiReader, - ConcertCategoryExtractor concertCategoryExtractor) { + ConcertCategoryExtractor concertCategoryExtractor, + S3Service s3Service) { this.jobRepository = jobRepository; this.platformTransactionManager = platformTransactionManager; this.kopisApiReader = kopisApiReader; this.kopisDetailApiReader = kopisDetailApiReader; this.concertCategoryExtractor = concertCategoryExtractor; + this.s3Service = s3Service; } @Bean @@ -74,6 +78,28 @@ public Step thirdStep() { .build(); } + // 4. 추출한 카테고리 값을 CSV 파일에 저장 + @Bean + public Step fourthStep() { + return new StepBuilder("fourthStep", jobRepository) + .tasklet((StepContribution contribution, ChunkContext chunkContext) -> { + + String folderPath = "datasets"; + String fileName = "concerts.csv"; + String localFilePath = s3Service.downloadCsvFile(folderPath, fileName); + + + + if (localFilePath != null) { + s3Service.updateAndUploadCsvFile(folderPath, fileName, localFilePath); + } else { + System.out.println("concerts.csv 파일 다운로드 실패"); + } + + return RepeatStatus.FINISHED; + }, platformTransactionManager).build(); + } + @Bean public Tasklet concertCategoryExtractorTasklet() { return new Tasklet() { From f851865f5062c38fb4747d817b387f3bd80e4be0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=ED=9D=AC=EC=B0=AC?= Date: Mon, 28 Oct 2024 16:30:34 +0900 Subject: [PATCH 09/19] =?UTF-8?q?feature:=20csv=20=EB=8B=A4=EC=9A=B4=20?= =?UTF-8?q?=EC=9D=B4=ED=9B=84=20=EC=83=88=EB=A1=9C=20=EC=B6=94=EA=B0=80?= =?UTF-8?q?=EB=90=9C=20=ED=95=AD=EB=AA=A9=EB=93=A4=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../clacobatchserver/batch/ConcertBatch.java | 88 +++++++++++++++++-- .../repository/ConcertRepository.java | 5 +- .../service/ConcertCategoryExtractor.java | 20 ++++- 3 files changed, 100 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java index 0ebbab8..71cc3f7 100644 --- a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java +++ b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java @@ -2,10 +2,22 @@ import com.curateme.clacobatchserver.config.s3.S3Service; import com.curateme.clacobatchserver.entity.ConcertEntity; +import com.curateme.clacobatchserver.repository.ConcertRepository; import com.curateme.clacobatchserver.service.ConcertCategoryExtractor; import com.curateme.clacobatchserver.service.KopisConcertApiReader; import com.curateme.clacobatchserver.service.KopisDetailApiReader; import com.curateme.clacobatchserver.service.KopisEntityWriter; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.StringJoiner; import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.StepContribution; @@ -23,6 +35,7 @@ public class ConcertBatch { private final JobRepository jobRepository; private final PlatformTransactionManager platformTransactionManager; + private final ConcertRepository concertRepository; private final KopisConcertApiReader kopisApiReader; private final KopisDetailApiReader kopisDetailApiReader; @@ -30,13 +43,14 @@ public class ConcertBatch { private final S3Service s3Service; public ConcertBatch(JobRepository jobRepository, - PlatformTransactionManager platformTransactionManager, + PlatformTransactionManager platformTransactionManager, ConcertRepository concertRepository, KopisConcertApiReader kopisApiReader, KopisDetailApiReader kopisDetailApiReader, ConcertCategoryExtractor concertCategoryExtractor, S3Service s3Service) { this.jobRepository = jobRepository; this.platformTransactionManager = platformTransactionManager; + this.concertRepository = concertRepository; this.kopisApiReader = kopisApiReader; this.kopisDetailApiReader = kopisDetailApiReader; this.concertCategoryExtractor = concertCategoryExtractor; @@ -46,9 +60,7 @@ public ConcertBatch(JobRepository jobRepository, @Bean public Job kopisJob(KopisEntityWriter writer){ return new JobBuilder("kopisJob", jobRepository) - .start(firstStep(writer)) - .next(secondStep()) - .next(thirdStep()) + .start(fourthStep()) .build(); } @@ -88,18 +100,78 @@ public Step fourthStep() { String fileName = "concerts.csv"; String localFilePath = s3Service.downloadCsvFile(folderPath, fileName); + // 다운로드된 파일 경로 출력 및 파일 존재 여부 확인 + System.out.println("다운로드된 파일 경로: " + localFilePath); + File downloadedFile = new File(localFilePath); + if (!downloadedFile.exists()) { + System.err.println("파일이 존재하지 않습니다: " + localFilePath); + return RepeatStatus.FINISHED; + } + + List columns = Arrays.asList("concertId", "grand", "delicate", "classical", "modern", + "lyrical", "dynamic", "romantic", "tragic", "familiar", "novel"); + + // CSV 형식으로 변환할 데이터를 저장 + List concerts = concertRepository.getAllConcertsWithCategories(); + StringJoiner csvContent = new StringJoiner("\n"); + + csvContent.add(String.join(",", columns)); + + for (ConcertEntity concert : concerts) { + Map row = new HashMap<>(); + row.put("concertId", concert.getMt20id()); + // 모든 카테고리를 0으로 초기화한 뒤, 실제 데이터를 넣기 + for (String column : columns.subList(1, columns.size())) { + row.put(column, 0.0); + } - if (localFilePath != null) { - s3Service.updateAndUploadCsvFile(folderPath, fileName, localFilePath); - } else { - System.out.println("concerts.csv 파일 다운로드 실패"); + // Categories에서 값을 가져와 CSV에 맞게 추가 + if (concert.getCategories() != null) { + for (Entry entry : concert.getCategories().entrySet()) { + // 카테고리가 컬럼에 존재하는지 확인 + if (columns.contains(entry.getKey())) { + row.put(entry.getKey(), entry.getValue()); + } + } + } + + StringJoiner rowContent = new StringJoiner(","); + for (String column : columns) { + rowContent.add(String.valueOf(row.get(column))); + } + csvContent.add(rowContent.toString()); + } + + File tempFile = null; + try { + tempFile = File.createTempFile("concerts_", ".csv"); + + try (BufferedWriter writer = new BufferedWriter(new FileWriter(tempFile))) { + writer.write(csvContent.toString()); + } + + // S3에 파일 업로드 + s3Service.updateAndUploadCsvFile(folderPath, fileName, tempFile.getAbsolutePath()); + + } catch (IOException e) { + System.err.println("파일 쓰기 오류: " + e.getMessage()); + } finally { + if (tempFile != null && tempFile.exists()) { + tempFile.delete(); + } } return RepeatStatus.FINISHED; }, platformTransactionManager).build(); } + + + + + + @Bean public Tasklet concertCategoryExtractorTasklet() { return new Tasklet() { diff --git a/src/main/java/com/curateme/clacobatchserver/repository/ConcertRepository.java b/src/main/java/com/curateme/clacobatchserver/repository/ConcertRepository.java index 71eda3c..114f263 100644 --- a/src/main/java/com/curateme/clacobatchserver/repository/ConcertRepository.java +++ b/src/main/java/com/curateme/clacobatchserver/repository/ConcertRepository.java @@ -1,8 +1,11 @@ package com.curateme.clacobatchserver.repository; import com.curateme.clacobatchserver.entity.ConcertEntity; +import java.util.List; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; public interface ConcertRepository extends JpaRepository { - + @Query("SELECT c FROM ConcertEntity c LEFT JOIN FETCH c.categories") + List getAllConcertsWithCategories(); } diff --git a/src/main/java/com/curateme/clacobatchserver/service/ConcertCategoryExtractor.java b/src/main/java/com/curateme/clacobatchserver/service/ConcertCategoryExtractor.java index e71a1eb..4a74bd5 100644 --- a/src/main/java/com/curateme/clacobatchserver/service/ConcertCategoryExtractor.java +++ b/src/main/java/com/curateme/clacobatchserver/service/ConcertCategoryExtractor.java @@ -31,6 +31,18 @@ public ConcertCategoryExtractor(RestTemplate restTemplate, ConcertRepository con public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception { List concerts = concertRepository.findAll(); + Map translationMap = new HashMap<>(); + translationMap.put("웅장한", "grand"); + translationMap.put("섬세한", "delicate"); + translationMap.put("고전적인", "classical"); + translationMap.put("현대적인", "modern"); + translationMap.put("서정적인", "lyrical"); + translationMap.put("역동적인", "dynamic"); + translationMap.put("낭만적인", "romantic"); + translationMap.put("비극적인", "tragic"); + translationMap.put("친숙한", "familiar"); + translationMap.put("새로운", "novel"); + for (ConcertEntity concert : concerts) { String introduction = concert.getStyurl(); @@ -50,9 +62,11 @@ public RepeatStatus execute(StepContribution contribution, ChunkContext chunkCon Map categories = new HashMap<>(); for (Map categoryData : clovaResponse) { - String category = categoryData.get("name").toString(); + String koreanCategory = categoryData.get("name").toString(); Double score = Double.parseDouble(categoryData.get("score").toString()); - categories.put(category, score); + + String englishCategory = translationMap.getOrDefault(koreanCategory, koreanCategory); + categories.put(englishCategory, score); } concert.setCategories(categories); @@ -64,6 +78,4 @@ public RepeatStatus execute(StepContribution contribution, ChunkContext chunkCon return RepeatStatus.FINISHED; } - - } From 4516e49d77a3700c863d28738243b15b27500a15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=ED=9D=AC=EC=B0=AC?= Date: Mon, 28 Oct 2024 17:03:58 +0900 Subject: [PATCH 10/19] =?UTF-8?q?fix:=20csv=20=EB=8B=A4=EC=9A=B4=20?= =?UTF-8?q?=EC=9D=B4=ED=9B=84=20=EC=83=88=EB=A1=9C=20=EC=B6=94=EA=B0=80?= =?UTF-8?q?=EB=90=9C=20=ED=95=AD=EB=AA=A9=EB=93=A4=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../clacobatchserver/batch/ConcertBatch.java | 27 +++++++++++++++---- .../clacobatchserver/config/s3/S3Service.java | 1 - 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java index 71cc3f7..9815dfc 100644 --- a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java +++ b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java @@ -7,8 +7,10 @@ import com.curateme.clacobatchserver.service.KopisConcertApiReader; import com.curateme.clacobatchserver.service.KopisDetailApiReader; import com.curateme.clacobatchserver.service.KopisEntityWriter; +import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; +import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; @@ -105,18 +107,28 @@ public Step fourthStep() { File downloadedFile = new File(localFilePath); if (!downloadedFile.exists()) { System.err.println("파일이 존재하지 않습니다: " + localFilePath); - return RepeatStatus.FINISHED; + return RepeatStatus.FINISHED; // 파일이 없으면 작업 종료 } List columns = Arrays.asList("concertId", "grand", "delicate", "classical", "modern", "lyrical", "dynamic", "romantic", "tragic", "familiar", "novel"); - // CSV 형식으로 변환할 데이터를 저장 - List concerts = concertRepository.getAllConcertsWithCategories(); + // CSV 내용을 읽고 새 데이터를 추가하기 위한 StringJoiner StringJoiner csvContent = new StringJoiner("\n"); - csvContent.add(String.join(",", columns)); + // 기존 CSV 파일의 내용을 읽기 + try (BufferedReader reader = new BufferedReader(new FileReader(downloadedFile))) { + String line; + while ((line = reader.readLine()) != null) { + csvContent.add(line); // 기존 내용을 추가 + } + } catch (IOException e) { + System.err.println("기존 CSV 파일 읽기 오류: " + e.getMessage()); + return RepeatStatus.FINISHED; // 읽기 오류 발생 시 종료 + } + // 새 데이터 추가 + List concerts = concertRepository.getAllConcertsWithCategories(); for (ConcertEntity concert : concerts) { Map row = new HashMap<>(); row.put("concertId", concert.getMt20id()); @@ -128,7 +140,7 @@ public Step fourthStep() { // Categories에서 값을 가져와 CSV에 맞게 추가 if (concert.getCategories() != null) { - for (Entry entry : concert.getCategories().entrySet()) { + for (Map.Entry entry : concert.getCategories().entrySet()) { // 카테고리가 컬럼에 존재하는지 확인 if (columns.contains(entry.getKey())) { row.put(entry.getKey(), entry.getValue()); @@ -143,10 +155,13 @@ public Step fourthStep() { csvContent.add(rowContent.toString()); } + // 임시 파일 생성 및 최종 CSV 내용 쓰기 File tempFile = null; try { + // 임시 파일 생성 tempFile = File.createTempFile("concerts_", ".csv"); + // CSV 내용을 임시 파일에 쓰기 try (BufferedWriter writer = new BufferedWriter(new FileWriter(tempFile))) { writer.write(csvContent.toString()); } @@ -157,6 +172,7 @@ public Step fourthStep() { } catch (IOException e) { System.err.println("파일 쓰기 오류: " + e.getMessage()); } finally { + // 임시 파일 정리 if (tempFile != null && tempFile.exists()) { tempFile.delete(); } @@ -172,6 +188,7 @@ public Step fourthStep() { + @Bean public Tasklet concertCategoryExtractorTasklet() { return new Tasklet() { diff --git a/src/main/java/com/curateme/clacobatchserver/config/s3/S3Service.java b/src/main/java/com/curateme/clacobatchserver/config/s3/S3Service.java index e91c7df..23bdad3 100644 --- a/src/main/java/com/curateme/clacobatchserver/config/s3/S3Service.java +++ b/src/main/java/com/curateme/clacobatchserver/config/s3/S3Service.java @@ -39,7 +39,6 @@ public String downloadCsvFile(String folderPath, String fileName) { public void updateAndUploadCsvFile(String folderPath, String fileName, String filePath) { try (BufferedWriter writer = new BufferedWriter(new FileWriter(filePath, true))) { - writer.write("\nUpdated Data"); log.info("File updated successfully: {}", filePath); } catch (IOException e) { log.error("Error updating file", e); From 3e139a742385fc5c3915f6e71cb931a86d529aa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=ED=9D=AC=EC=B0=AC?= Date: Mon, 28 Oct 2024 17:10:56 +0900 Subject: [PATCH 11/19] =?UTF-8?q?feature:=20=ED=95=B4=EB=8B=B9=20Batch?= =?UTF-8?q?=EC=97=90=20=EC=A0=80=EC=9E=A5=EB=90=9C=20=EB=94=94=EB=B9=84=20?= =?UTF-8?q?=EA=B0=92=EB=93=A4=EC=9D=84=20=EC=A0=84=EB=B6=80=20=EC=82=AD?= =?UTF-8?q?=EC=A0=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../clacobatchserver/batch/ConcertBatch.java | 189 +++++++++++------- 1 file changed, 114 insertions(+), 75 deletions(-) diff --git a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java index 9815dfc..e0eac96 100644 --- a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java +++ b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java @@ -13,12 +13,10 @@ import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.StringJoiner; import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; @@ -62,7 +60,11 @@ public ConcertBatch(JobRepository jobRepository, @Bean public Job kopisJob(KopisEntityWriter writer){ return new JobBuilder("kopisJob", jobRepository) - .start(fourthStep()) + .start(firstStep(writer)) + .next(secondStep()) + .next(thirdStep()) + .next(fourthStep()) + .next(finalStep()) .build(); } @@ -100,88 +102,125 @@ public Step fourthStep() { String folderPath = "datasets"; String fileName = "concerts.csv"; - String localFilePath = s3Service.downloadCsvFile(folderPath, fileName); - // 다운로드된 파일 경로 출력 및 파일 존재 여부 확인 - System.out.println("다운로드된 파일 경로: " + localFilePath); - File downloadedFile = new File(localFilePath); - if (!downloadedFile.exists()) { - System.err.println("파일이 존재하지 않습니다: " + localFilePath); - return RepeatStatus.FINISHED; // 파일이 없으면 작업 종료 + String localFilePath = s3Service.downloadCsvFile(folderPath, fileName); + if (!isFileExists(localFilePath)) { + return RepeatStatus.FINISHED; } - List columns = Arrays.asList("concertId", "grand", "delicate", "classical", "modern", - "lyrical", "dynamic", "romantic", "tragic", "familiar", "novel"); - - // CSV 내용을 읽고 새 데이터를 추가하기 위한 StringJoiner - StringJoiner csvContent = new StringJoiner("\n"); - - // 기존 CSV 파일의 내용을 읽기 - try (BufferedReader reader = new BufferedReader(new FileReader(downloadedFile))) { - String line; - while ((line = reader.readLine()) != null) { - csvContent.add(line); // 기존 내용을 추가 - } - } catch (IOException e) { - System.err.println("기존 CSV 파일 읽기 오류: " + e.getMessage()); - return RepeatStatus.FINISHED; // 읽기 오류 발생 시 종료 - } + StringJoiner csvContent = readExistingCsvContent(localFilePath); + appendNewConcertData(csvContent); - // 새 데이터 추가 - List concerts = concertRepository.getAllConcertsWithCategories(); - for (ConcertEntity concert : concerts) { - Map row = new HashMap<>(); - row.put("concertId", concert.getMt20id()); - - // 모든 카테고리를 0으로 초기화한 뒤, 실제 데이터를 넣기 - for (String column : columns.subList(1, columns.size())) { - row.put(column, 0.0); - } - - // Categories에서 값을 가져와 CSV에 맞게 추가 - if (concert.getCategories() != null) { - for (Map.Entry entry : concert.getCategories().entrySet()) { - // 카테고리가 컬럼에 존재하는지 확인 - if (columns.contains(entry.getKey())) { - row.put(entry.getKey(), entry.getValue()); - } - } - } - - StringJoiner rowContent = new StringJoiner(","); - for (String column : columns) { - rowContent.add(String.valueOf(row.get(column))); - } - csvContent.add(rowContent.toString()); - } + saveAndUploadCsvFile(csvContent, folderPath, fileName); - // 임시 파일 생성 및 최종 CSV 내용 쓰기 - File tempFile = null; - try { - // 임시 파일 생성 - tempFile = File.createTempFile("concerts_", ".csv"); - - // CSV 내용을 임시 파일에 쓰기 - try (BufferedWriter writer = new BufferedWriter(new FileWriter(tempFile))) { - writer.write(csvContent.toString()); - } - - // S3에 파일 업로드 - s3Service.updateAndUploadCsvFile(folderPath, fileName, tempFile.getAbsolutePath()); - - } catch (IOException e) { - System.err.println("파일 쓰기 오류: " + e.getMessage()); - } finally { - // 임시 파일 정리 - if (tempFile != null && tempFile.exists()) { - tempFile.delete(); - } - } + return RepeatStatus.FINISHED; + }, platformTransactionManager).build(); + } + // 5. 해당 Batch에서 가져온 값들을 전부 삭제하는 로직 + @Bean + public Step finalStep() { + return new StepBuilder("finalStep", jobRepository) + .tasklet((StepContribution contribution, ChunkContext chunkContext) -> { + // 데이터베이스에서 ConcertEntity와 관련된 모든 데이터를 삭제 + deleteAllConcertData(); return RepeatStatus.FINISHED; }, platformTransactionManager).build(); } + // ConcertEntity와 관련된 모든 데이터를 삭제하는 메서드 + private void deleteAllConcertData() { + try { + concertRepository.deleteAll(); + } catch (Exception e) { + System.err.println("ConcertEntity 데이터 삭제 오류: " + e.getMessage()); + } + } + + + // 파일 존재 확인 메서드 + private boolean isFileExists(String localFilePath) { + File downloadedFile = new File(localFilePath); + if (!downloadedFile.exists()) { + System.err.println("파일이 존재하지 않습니다: " + localFilePath); + return false; + } + System.out.println("다운로드된 파일 경로: " + localFilePath); + return true; + } + + // 기존 CSV 파일 내용을 읽는 메서드 + private StringJoiner readExistingCsvContent(String localFilePath) { + StringJoiner csvContent = new StringJoiner("\n"); + try (BufferedReader reader = new BufferedReader(new FileReader(localFilePath))) { + String line; + while ((line = reader.readLine()) != null) { + csvContent.add(line); + } + } catch (IOException e) { + System.err.println("기존 CSV 파일 읽기 오류: " + e.getMessage()); + } + return csvContent; + } + + // 새로운 Concert 데이터를 CSV에 추가하는 메서드 + private void appendNewConcertData(StringJoiner csvContent) { + List columns = Arrays.asList("concertId", "grand", "delicate", "classical", "modern", + "lyrical", "dynamic", "romantic", "tragic", "familiar", "novel"); + + List concerts = concertRepository.getAllConcertsWithCategories(); + for (ConcertEntity concert : concerts) { + Map row = initializeRowData(columns, concert); + StringJoiner rowContent = new StringJoiner(","); + for (String column : columns) { + rowContent.add(String.valueOf(row.get(column))); + } + csvContent.add(rowContent.toString()); + } + } + + // 새로운 Concert 데이터 행을 초기화하는 메서드 + private Map initializeRowData(List columns, ConcertEntity concert) { + Map row = new HashMap<>(); + row.put("concertId", concert.getMt20id()); + + for (String column : columns.subList(1, columns.size())) { + row.put(column, 0.0); + } + + if (concert.getCategories() != null) { + for (Map.Entry entry : concert.getCategories().entrySet()) { + if (columns.contains(entry.getKey())) { + row.put(entry.getKey(), entry.getValue()); + } + } + } + + return row; + } + + // 임시 파일에 저장하고 S3에 업로드하는 메서드 + private void saveAndUploadCsvFile(StringJoiner csvContent, String folderPath, String fileName) { + File tempFile = null; + try { + tempFile = File.createTempFile("concerts_", ".csv"); + try (BufferedWriter writer = new BufferedWriter(new FileWriter(tempFile))) { + writer.write(csvContent.toString()); + } + + s3Service.updateAndUploadCsvFile(folderPath, fileName, tempFile.getAbsolutePath()); + + } catch (IOException e) { + System.err.println("파일 쓰기 오류: " + e.getMessage()); + } finally { + // 임시 파일 정리 + if (tempFile != null && tempFile.exists()) { + tempFile.delete(); + } + } + } + + From 9c6adf92566009d052ded0f784bb3a5fa8cbbd81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=ED=9D=AC=EC=B0=AC?= Date: Mon, 28 Oct 2024 17:11:25 +0900 Subject: [PATCH 12/19] =?UTF-8?q?feature:=20=ED=95=B4=EB=8B=B9=20Batch?= =?UTF-8?q?=EC=97=90=20=EC=A0=80=EC=9E=A5=EB=90=9C=20=EB=94=94=EB=B9=84=20?= =?UTF-8?q?=EA=B0=92=EB=93=A4=EC=9D=84=20=EC=A0=84=EB=B6=80=20=EC=82=AD?= =?UTF-8?q?=EC=A0=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/curateme/clacobatchserver/batch/ConcertBatch.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java index e0eac96..7cd9624 100644 --- a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java +++ b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java @@ -220,14 +220,6 @@ private void saveAndUploadCsvFile(StringJoiner csvContent, String folderPath, St } } - - - - - - - - @Bean public Tasklet concertCategoryExtractorTasklet() { return new Tasklet() { From 5ded32cd0e319a36a2f01093fdf7df8064327086 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=ED=9D=AC=EC=B0=AC?= Date: Tue, 29 Oct 2024 21:51:46 +0900 Subject: [PATCH 13/19] =?UTF-8?q?feature:=20Table=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../clacobatchserver/batch/ConcertBatch.java | 11 ++-- .../{ConcertEntity.java => Concert.java} | 12 +++- .../entity/ConcertCategory.java | 33 +++++++++++ .../clacobatchserver/global/ActiveStatus.java | 27 +++++++++ .../clacobatchserver/global/BaseEntity.java | 58 +++++++++++++++++++ .../repository/ConcertCategoryRepository.java | 8 +++ .../repository/ConcertRepository.java | 8 +-- .../service/ConcertCategoryExtractor.java | 55 ++++++++++-------- .../service/KopisConcertApiReader.java | 18 +++--- .../service/KopisDetailApiReader.java | 16 ++--- .../service/KopisEntityWriter.java | 6 +- 11 files changed, 196 insertions(+), 56 deletions(-) rename src/main/java/com/curateme/clacobatchserver/entity/{ConcertEntity.java => Concert.java} (91%) create mode 100644 src/main/java/com/curateme/clacobatchserver/entity/ConcertCategory.java create mode 100644 src/main/java/com/curateme/clacobatchserver/global/ActiveStatus.java create mode 100644 src/main/java/com/curateme/clacobatchserver/global/BaseEntity.java create mode 100644 src/main/java/com/curateme/clacobatchserver/repository/ConcertCategoryRepository.java diff --git a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java index 7cd9624..7efaffc 100644 --- a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java +++ b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java @@ -1,7 +1,7 @@ package com.curateme.clacobatchserver.batch; import com.curateme.clacobatchserver.config.s3.S3Service; -import com.curateme.clacobatchserver.entity.ConcertEntity; +import com.curateme.clacobatchserver.entity.Concert; import com.curateme.clacobatchserver.repository.ConcertRepository; import com.curateme.clacobatchserver.service.ConcertCategoryExtractor; import com.curateme.clacobatchserver.service.KopisConcertApiReader; @@ -64,7 +64,6 @@ public Job kopisJob(KopisEntityWriter writer){ .next(secondStep()) .next(thirdStep()) .next(fourthStep()) - .next(finalStep()) .build(); } @@ -72,7 +71,7 @@ public Job kopisJob(KopisEntityWriter writer){ @Bean public Step firstStep(KopisEntityWriter writer) { return new StepBuilder("firstStep", jobRepository) - .chunk(10, platformTransactionManager) + .chunk(10, platformTransactionManager) .reader(kopisApiReader) .writer(writer) .build(); @@ -168,8 +167,8 @@ private void appendNewConcertData(StringJoiner csvContent) { List columns = Arrays.asList("concertId", "grand", "delicate", "classical", "modern", "lyrical", "dynamic", "romantic", "tragic", "familiar", "novel"); - List concerts = concertRepository.getAllConcertsWithCategories(); - for (ConcertEntity concert : concerts) { + List concerts = concertRepository.getAllConcertsWithCategories(); + for (Concert concert : concerts) { Map row = initializeRowData(columns, concert); StringJoiner rowContent = new StringJoiner(","); for (String column : columns) { @@ -180,7 +179,7 @@ private void appendNewConcertData(StringJoiner csvContent) { } // 새로운 Concert 데이터 행을 초기화하는 메서드 - private Map initializeRowData(List columns, ConcertEntity concert) { + private Map initializeRowData(List columns, Concert concert) { Map row = new HashMap<>(); row.put("concertId", concert.getMt20id()); diff --git a/src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java b/src/main/java/com/curateme/clacobatchserver/entity/Concert.java similarity index 91% rename from src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java rename to src/main/java/com/curateme/clacobatchserver/entity/Concert.java index dd35545..236c669 100644 --- a/src/main/java/com/curateme/clacobatchserver/entity/ConcertEntity.java +++ b/src/main/java/com/curateme/clacobatchserver/entity/Concert.java @@ -1,11 +1,15 @@ package com.curateme.clacobatchserver.entity; +import com.curateme.clacobatchserver.global.BaseEntity; +import jakarta.persistence.CollectionTable; import jakarta.persistence.Column; import jakarta.persistence.ElementCollection; import jakarta.persistence.Entity; import jakarta.persistence.GeneratedValue; import jakarta.persistence.GenerationType; import jakarta.persistence.Id; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.MapKeyColumn; import java.util.Map; import lombok.AllArgsConstructor; import lombok.Getter; @@ -15,14 +19,14 @@ @Getter @NoArgsConstructor @AllArgsConstructor -public class ConcertEntity { +public class Concert extends BaseEntity { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; @Column(name = "concertId") - private String mt20id; + private String mt20id; @Column(name = "concertName") private String prfnm; @@ -109,7 +113,9 @@ public class ConcertEntity { private String styurl; @ElementCollection - @Column(name = "categories") + @CollectionTable(name = "ConcertCategory", joinColumns = @JoinColumn(name = "concertId")) + @MapKeyColumn(name = "category") + @Column(name = "score") private Map categories; public void setConcertDetails(String mt20id, String prfnm, String prfpdfrom, String prfpdto, String fcltynm, String poster, String area, String genrenm, diff --git a/src/main/java/com/curateme/clacobatchserver/entity/ConcertCategory.java b/src/main/java/com/curateme/clacobatchserver/entity/ConcertCategory.java new file mode 100644 index 0000000..b6c1222 --- /dev/null +++ b/src/main/java/com/curateme/clacobatchserver/entity/ConcertCategory.java @@ -0,0 +1,33 @@ +package com.curateme.clacobatchserver.entity; + +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Entity +@Getter +@NoArgsConstructor +@AllArgsConstructor +public class ConcertCategory { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(name = "category") + private String category; + + @Column(name = "score") + private Double score; + + @ManyToOne + @JoinColumn(name = "concertId", nullable = false) + private Concert concert; + + public ConcertCategory(String category, Double score, Concert concert) { + this.category = category; + this.score = score; + this.concert = concert; + } +} diff --git a/src/main/java/com/curateme/clacobatchserver/global/ActiveStatus.java b/src/main/java/com/curateme/clacobatchserver/global/ActiveStatus.java new file mode 100644 index 0000000..c6e68f0 --- /dev/null +++ b/src/main/java/com/curateme/clacobatchserver/global/ActiveStatus.java @@ -0,0 +1,27 @@ +package com.curateme.clacobatchserver.global; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * @packageName : com.curateme.claco.global.entity + * @fileName : ActiveStatus.java + * @author : 이 건 + * @date : 2024.10.15 + * @author devkeon(devkeon123@gmail.com) + * =========================================================== + * DATE AUTHOR NOTE + * ----------------------------------------------------------- + * 2024.10.15 이 건 최초 생성 + */ +@Getter +@AllArgsConstructor(access = AccessLevel.PRIVATE) +public enum ActiveStatus { + + // ACTIVE: 활성, DELETED: 비활성 + ACTIVE("ACTIVE"), DELETED("DELETED"); + + private final String activeStatus; + +} diff --git a/src/main/java/com/curateme/clacobatchserver/global/BaseEntity.java b/src/main/java/com/curateme/clacobatchserver/global/BaseEntity.java new file mode 100644 index 0000000..80ec3dc --- /dev/null +++ b/src/main/java/com/curateme/clacobatchserver/global/BaseEntity.java @@ -0,0 +1,58 @@ +package com.curateme.clacobatchserver.global; + +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.MappedSuperclass; +import jakarta.persistence.PrePersist; +import jakarta.persistence.PreUpdate; +import java.time.LocalDateTime; +import lombok.Getter; + +/** + * @packageName : com.curateme.claco.global.entity + * @fileName : BaseEntity.java + * @author : 이 건 + * @date : 2024.10.15 + * @author devkeon(devkeon123@gmail.com) + * =========================================================== + * DATE AUTHOR NOTE + * ----------------------------------------------------------- + * 2024.10.15 이 건 최초 생성 + */ +@Getter +@MappedSuperclass +public abstract class BaseEntity { + + // 활성 여부 + @Enumerated(value = EnumType.STRING) + private ActiveStatus activeStatus = ActiveStatus.ACTIVE; + // 생성일 + private LocalDateTime createdAt; + // 수정일 + private LocalDateTime updatedAt; + + @PrePersist + public void createdAt() { + LocalDateTime currentTime = LocalDateTime.now(); + this.createdAt = currentTime; + this.updatedAt = currentTime; + } + + @PreUpdate + public void updatedAt() { + this.updatedAt = LocalDateTime.now(); + } + + public void deleteEntity() { + this.activeStatus = ActiveStatus.DELETED; + } + + public void restoreEntity() { + this.activeStatus = ActiveStatus.ACTIVE; + } + + public void updateActiveStatus(ActiveStatus activeStatus) { + this.activeStatus = activeStatus; + } + +} diff --git a/src/main/java/com/curateme/clacobatchserver/repository/ConcertCategoryRepository.java b/src/main/java/com/curateme/clacobatchserver/repository/ConcertCategoryRepository.java new file mode 100644 index 0000000..ecee0f6 --- /dev/null +++ b/src/main/java/com/curateme/clacobatchserver/repository/ConcertCategoryRepository.java @@ -0,0 +1,8 @@ +package com.curateme.clacobatchserver.repository; + +import com.curateme.clacobatchserver.entity.ConcertCategory; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface ConcertCategoryRepository extends JpaRepository { + +} diff --git a/src/main/java/com/curateme/clacobatchserver/repository/ConcertRepository.java b/src/main/java/com/curateme/clacobatchserver/repository/ConcertRepository.java index 114f263..5215b9d 100644 --- a/src/main/java/com/curateme/clacobatchserver/repository/ConcertRepository.java +++ b/src/main/java/com/curateme/clacobatchserver/repository/ConcertRepository.java @@ -1,11 +1,11 @@ package com.curateme.clacobatchserver.repository; -import com.curateme.clacobatchserver.entity.ConcertEntity; +import com.curateme.clacobatchserver.entity.Concert; import java.util.List; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; -public interface ConcertRepository extends JpaRepository { - @Query("SELECT c FROM ConcertEntity c LEFT JOIN FETCH c.categories") - List getAllConcertsWithCategories(); +public interface ConcertRepository extends JpaRepository { + @Query("SELECT c FROM Concert c LEFT JOIN FETCH c.categories") + List getAllConcertsWithCategories(); } diff --git a/src/main/java/com/curateme/clacobatchserver/service/ConcertCategoryExtractor.java b/src/main/java/com/curateme/clacobatchserver/service/ConcertCategoryExtractor.java index 4a74bd5..ddc167f 100644 --- a/src/main/java/com/curateme/clacobatchserver/service/ConcertCategoryExtractor.java +++ b/src/main/java/com/curateme/clacobatchserver/service/ConcertCategoryExtractor.java @@ -1,6 +1,6 @@ package com.curateme.clacobatchserver.service; -import com.curateme.clacobatchserver.entity.ConcertEntity; +import com.curateme.clacobatchserver.entity.Concert; import com.curateme.clacobatchserver.repository.ConcertRepository; import java.util.HashMap; import java.util.List; @@ -28,8 +28,8 @@ public ConcertCategoryExtractor(RestTemplate restTemplate, ConcertRepository con this.concertRepository = concertRepository; } - public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception { - List concerts = concertRepository.findAll(); + public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) { + List concerts = concertRepository.findAll(); Map translationMap = new HashMap<>(); translationMap.put("웅장한", "grand"); @@ -43,39 +43,48 @@ public RepeatStatus execute(StepContribution contribution, ChunkContext chunkCon translationMap.put("친숙한", "familiar"); translationMap.put("새로운", "novel"); - for (ConcertEntity concert : concerts) { - String introduction = concert.getStyurl(); + for (Concert concert : concerts) { + try { + String introduction = concert.getStyurl(); - Map requestBody = new HashMap<>(); - requestBody.put("image_url", introduction); + Map requestBody = new HashMap<>(); + requestBody.put("image_url", introduction); - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_JSON); - HttpEntity> requestEntity = new HttpEntity<>(requestBody, headers); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + HttpEntity> requestEntity = new HttpEntity<>(requestBody, headers); - ResponseEntity response = restTemplate.postForEntity("http://localhost:8080/categories", requestEntity, Map.class); + // Flask 서버에 요청 보내기 + ResponseEntity response = restTemplate.postForEntity("http://localhost:8081/categories", requestEntity, Map.class); - if (response.getBody() != null) { - Map responseBody = (Map) response.getBody(); - List> clovaResponse = (List>) responseBody.get("clova_response"); + if (response.getBody() != null) { + Map responseBody = (Map) response.getBody(); + List> clovaResponse = (List>) responseBody.get("clova_response"); - Map categories = new HashMap<>(); + Map categories = new HashMap<>(); - for (Map categoryData : clovaResponse) { - String koreanCategory = categoryData.get("name").toString(); - Double score = Double.parseDouble(categoryData.get("score").toString()); + for (Map categoryData : clovaResponse) { + String koreanCategory = categoryData.get("name").toString(); + Double score = Double.parseDouble(categoryData.get("score").toString()); - String englishCategory = translationMap.getOrDefault(koreanCategory, koreanCategory); - categories.put(englishCategory, score); - } + String englishCategory = translationMap.getOrDefault(koreanCategory, koreanCategory); + categories.put(englishCategory, score); + } - concert.setCategories(categories); + concert.setCategories(categories); - concertRepository.save(concert); + // 업데이트된 concert 엔티티를 저장 + concertRepository.save(concert); + } + } catch (Exception e) { + // 특정 concert에서 예외 발생 시 로그를 남기고, 다음 concert로 계속 진행 + System.err.println("concert ID " + concert.getMt20id() + " 처리 중 오류 발생: " + e.getMessage()); + e.printStackTrace(); } } return RepeatStatus.FINISHED; } + } diff --git a/src/main/java/com/curateme/clacobatchserver/service/KopisConcertApiReader.java b/src/main/java/com/curateme/clacobatchserver/service/KopisConcertApiReader.java index af8a658..bb2e0e6 100644 --- a/src/main/java/com/curateme/clacobatchserver/service/KopisConcertApiReader.java +++ b/src/main/java/com/curateme/clacobatchserver/service/KopisConcertApiReader.java @@ -1,6 +1,6 @@ package com.curateme.clacobatchserver.service; -import com.curateme.clacobatchserver.entity.ConcertEntity; +import com.curateme.clacobatchserver.entity.Concert; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -17,12 +17,12 @@ import org.springframework.web.client.HttpClientErrorException; @Component -public class KopisConcertApiReader implements ItemReader { +public class KopisConcertApiReader implements ItemReader { private final RestTemplate restTemplate; private final String apiUrl = "http://www.kopis.or.kr/openApi/restful/pblprfr?service=f222668534db409b8769f640387de9c3"; private int currentPage = 1; - private List beforeEntities = new ArrayList<>(); + private List beforeEntities = new ArrayList<>(); private int index = 0; public KopisConcertApiReader(RestTemplate restTemplate) { @@ -30,7 +30,7 @@ public KopisConcertApiReader(RestTemplate restTemplate) { } @Override - public ConcertEntity read() throws Exception { + public Concert read() throws Exception { if (index >= beforeEntities.size()) { loadNextPage(); @@ -60,13 +60,13 @@ private void loadNextPage() { HttpEntity entity = new HttpEntity<>(headers); try { - ResponseEntity response = restTemplate.exchange(requestUrl, HttpMethod.GET, entity, ConcertEntity[].class); + ResponseEntity response = restTemplate.exchange(requestUrl, HttpMethod.GET, entity, Concert[].class); System.out.println("response = " + response); if (response.getBody() != null && response.getBody().length > 0) { beforeEntities.clear(); - for (ConcertEntity beforeentity : response.getBody()) { - ConcertEntity concertEntity = new ConcertEntity(); - concertEntity.setConcertDetails( + for (Concert beforeentity : response.getBody()) { + Concert concert = new Concert(); + concert.setConcertDetails( beforeentity.getMt20id(), beforeentity.getPrfnm(), beforeentity.getPrfpdfrom(), @@ -79,7 +79,7 @@ private void loadNextPage() { beforeentity.getPrfstate() ); - beforeEntities.add(concertEntity); + beforeEntities.add(concert); } } else { beforeEntities.clear(); diff --git a/src/main/java/com/curateme/clacobatchserver/service/KopisDetailApiReader.java b/src/main/java/com/curateme/clacobatchserver/service/KopisDetailApiReader.java index fd1f8ec..6dad892 100644 --- a/src/main/java/com/curateme/clacobatchserver/service/KopisDetailApiReader.java +++ b/src/main/java/com/curateme/clacobatchserver/service/KopisDetailApiReader.java @@ -1,6 +1,6 @@ package com.curateme.clacobatchserver.service; -import com.curateme.clacobatchserver.entity.ConcertEntity; +import com.curateme.clacobatchserver.entity.Concert; import com.curateme.clacobatchserver.repository.ConcertRepository; import java.io.BufferedReader; import java.io.InputStreamReader; @@ -33,16 +33,16 @@ public KopisDetailApiReader(ConcertRepository concertRepository) { @Override public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception { - List beforeEntities = concertRepository.findAll(); + List beforeEntities = concertRepository.findAll(); - for (ConcertEntity concertEntity : beforeEntities) { - fetchAndSave(concertEntity.getMt20id(), concertEntity); + for (Concert concert : beforeEntities) { + fetchAndSave(concert.getMt20id(), concert); } return RepeatStatus.FINISHED; } - private void fetchAndSave(String mt20id, ConcertEntity concertEntity) { + private void fetchAndSave(String mt20id, Concert concert) { try { String urlString = String.format( "http://www.kopis.or.kr/openApi/restful/pblprfr/%s?service=f222668534db409b8769f640387de9c3", @@ -110,17 +110,17 @@ private void fetchAndSave(String mt20id, ConcertEntity concertEntity) { NodeList styurlList = styurlsElement.getElementsByTagName("styurl"); if (styurlList.getLength() > 0) { String styurl = styurlList.item(0).getTextContent(); - concertEntity.setStyurl(styurl); + concert.setStyurl(styurl); } } - concertEntity.setAdditionalConcertDetails( + concert.setAdditionalConcertDetails( prfcast, prfcrew, prfruntime, prfage, entrpsnm, entrpsnmP, entrpsnmA, entrpsnmH, entrpsnmS, pcseguidance, visit, child, daehakro, festival, musicallicense, musicalcreate, updatedate, dtguidance ); - concertRepository.save(concertEntity); + concertRepository.save(concert); } } } else { diff --git a/src/main/java/com/curateme/clacobatchserver/service/KopisEntityWriter.java b/src/main/java/com/curateme/clacobatchserver/service/KopisEntityWriter.java index 631f28d..fb736ce 100644 --- a/src/main/java/com/curateme/clacobatchserver/service/KopisEntityWriter.java +++ b/src/main/java/com/curateme/clacobatchserver/service/KopisEntityWriter.java @@ -1,13 +1,13 @@ package com.curateme.clacobatchserver.service; -import com.curateme.clacobatchserver.entity.ConcertEntity; +import com.curateme.clacobatchserver.entity.Concert; import com.curateme.clacobatchserver.repository.ConcertRepository; import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; import org.springframework.stereotype.Component; @Component -public class KopisEntityWriter implements ItemWriter { +public class KopisEntityWriter implements ItemWriter { private final ConcertRepository concertRepository; @@ -17,7 +17,7 @@ public KopisEntityWriter(ConcertRepository concertRepository) { } @Override - public void write(Chunk items) throws Exception { + public void write(Chunk items) throws Exception { concertRepository.saveAll(items.getItems()); } From f40ab70c2d615e4c460326d9db09f33293352e2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=ED=9D=AC=EC=B0=AC?= Date: Wed, 30 Oct 2024 18:19:07 +0900 Subject: [PATCH 14/19] =?UTF-8?q?fix:=20Table=20Column=20Snake=20case?= =?UTF-8?q?=EB=A1=9C=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../clacobatchserver/entity/Concert.java | 34 ++++++++++--------- .../entity/ConcertCategory.java | 3 +- .../clacobatchserver/global/BaseEntity.java | 4 +++ 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/curateme/clacobatchserver/entity/Concert.java b/src/main/java/com/curateme/clacobatchserver/entity/Concert.java index 236c669..1c5398a 100644 --- a/src/main/java/com/curateme/clacobatchserver/entity/Concert.java +++ b/src/main/java/com/curateme/clacobatchserver/entity/Concert.java @@ -10,6 +10,7 @@ import jakarta.persistence.Id; import jakarta.persistence.JoinColumn; import jakarta.persistence.MapKeyColumn; +import jakarta.persistence.Table; import java.util.Map; import lombok.AllArgsConstructor; import lombok.Getter; @@ -19,25 +20,26 @@ @Getter @NoArgsConstructor @AllArgsConstructor +@Table(name = "concert") public class Concert extends BaseEntity { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; - @Column(name = "concertId") + @Column(name = "concert_id") private String mt20id; - @Column(name = "concertName") + @Column(name = "concert_name") private String prfnm; - @Column(name = "startDate") + @Column(name = "start_date") private String prfpdfrom; - @Column(name = "endDate") + @Column(name = "end_date") private String prfpdto; - @Column(name = "facilityName") + @Column(name = "facility_name") private String fcltynm; @Column(name = "poster") @@ -67,22 +69,22 @@ public class Concert extends BaseEntity { @Column(name = "age") private String prfage; - @Column(name = "companyName") + @Column(name = "company_name") private String entrpsnm; - @Column(name = "companyNameP") + @Column(name = "company_namep") private String entrpsnmP; - @Column(name = "companyNameA") + @Column(name = "company_namea") private String entrpsnmA; - @Column(name = "companyNameH") + @Column(name = "company_nameh") private String entrpsnmH; - @Column(name = "companyNameS") + @Column(name = "company_names") private String entrpsnmS; - @Column(name = "seatGuidance") + @Column(name = "seat_guidance") private String pcseguidance; @Column(name = "visit") @@ -97,23 +99,23 @@ public class Concert extends BaseEntity { @Column(name = "festival") private String festival; - @Column(name = "musicalLicense") + @Column(name = "musical_license") private String musicallicense; - @Column(name = "musicalCreate") + @Column(name = "musical_create") private String musicalcreate; - @Column(name = "updateDate") + @Column(name = "update_date") private String updatedate; - @Column(name = "scheduleGuidance", length = 1000) + @Column(name = "schedule_guidance", length = 1000) private String dtguidance; @Column(name = "introduction") private String styurl; @ElementCollection - @CollectionTable(name = "ConcertCategory", joinColumns = @JoinColumn(name = "concertId")) + @CollectionTable(name = "concert_category", joinColumns = @JoinColumn(name = "concert_id")) @MapKeyColumn(name = "category") @Column(name = "score") private Map categories; diff --git a/src/main/java/com/curateme/clacobatchserver/entity/ConcertCategory.java b/src/main/java/com/curateme/clacobatchserver/entity/ConcertCategory.java index b6c1222..0d269ce 100644 --- a/src/main/java/com/curateme/clacobatchserver/entity/ConcertCategory.java +++ b/src/main/java/com/curateme/clacobatchserver/entity/ConcertCategory.java @@ -9,6 +9,7 @@ @Getter @NoArgsConstructor @AllArgsConstructor +@Table(name = "concert_category") public class ConcertCategory { @Id @@ -22,7 +23,7 @@ public class ConcertCategory { private Double score; @ManyToOne - @JoinColumn(name = "concertId", nullable = false) + @JoinColumn(name = "concert_id", nullable = false) private Concert concert; public ConcertCategory(String category, Double score, Concert concert) { diff --git a/src/main/java/com/curateme/clacobatchserver/global/BaseEntity.java b/src/main/java/com/curateme/clacobatchserver/global/BaseEntity.java index 80ec3dc..f5749c8 100644 --- a/src/main/java/com/curateme/clacobatchserver/global/BaseEntity.java +++ b/src/main/java/com/curateme/clacobatchserver/global/BaseEntity.java @@ -1,5 +1,6 @@ package com.curateme.clacobatchserver.global; +import jakarta.persistence.Column; import jakarta.persistence.EnumType; import jakarta.persistence.Enumerated; import jakarta.persistence.MappedSuperclass; @@ -25,10 +26,13 @@ public abstract class BaseEntity { // 활성 여부 @Enumerated(value = EnumType.STRING) + @Column(name = "active_status") private ActiveStatus activeStatus = ActiveStatus.ACTIVE; // 생성일 + @Column(name = "created_at") private LocalDateTime createdAt; // 수정일 + @Column(name = "updated_at") private LocalDateTime updatedAt; @PrePersist From 7905fa64c472f10c30175072fe6cd8d5f9277344 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=ED=9D=AC=EC=B0=AC?= Date: Thu, 7 Nov 2024 00:06:10 +0900 Subject: [PATCH 15/19] =?UTF-8?q?fix:=20=EA=B3=B5=EC=97=B0=20=EC=84=B1?= =?UTF-8?q?=EA=B2=A9=20=EC=9E=85=EB=A0=A5=20=EB=B0=A9=EC=8B=9D=20=EB=B3=80?= =?UTF-8?q?=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../clacobatchserver/batch/ConcertBatch.java | 47 +++++++++++++++---- .../dto/CategoryScoreDto.java | 19 ++++++++ .../clacobatchserver/entity/Category.java | 26 ++++++++++ .../entity/ConcertCategory.java | 8 ++-- .../repository/ConcertCategoryRepository.java | 9 ++++ .../repository/ConcertRepository.java | 3 ++ .../schedule/ConcertSchedule.java | 2 +- 7 files changed, 99 insertions(+), 15 deletions(-) create mode 100644 src/main/java/com/curateme/clacobatchserver/dto/CategoryScoreDto.java create mode 100644 src/main/java/com/curateme/clacobatchserver/entity/Category.java diff --git a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java index 7efaffc..fd6bd1d 100644 --- a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java +++ b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java @@ -1,7 +1,9 @@ package com.curateme.clacobatchserver.batch; import com.curateme.clacobatchserver.config.s3.S3Service; +import com.curateme.clacobatchserver.dto.CategoryScoreDto; import com.curateme.clacobatchserver.entity.Concert; +import com.curateme.clacobatchserver.repository.ConcertCategoryRepository; import com.curateme.clacobatchserver.repository.ConcertRepository; import com.curateme.clacobatchserver.service.ConcertCategoryExtractor; import com.curateme.clacobatchserver.service.KopisConcertApiReader; @@ -13,6 +15,7 @@ import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -36,6 +39,7 @@ public class ConcertBatch { private final JobRepository jobRepository; private final PlatformTransactionManager platformTransactionManager; private final ConcertRepository concertRepository; + private final ConcertCategoryRepository concertCategoryRepository; private final KopisConcertApiReader kopisApiReader; private final KopisDetailApiReader kopisDetailApiReader; @@ -43,7 +47,7 @@ public class ConcertBatch { private final S3Service s3Service; public ConcertBatch(JobRepository jobRepository, - PlatformTransactionManager platformTransactionManager, ConcertRepository concertRepository, + PlatformTransactionManager platformTransactionManager, ConcertRepository concertRepository, ConcertCategoryRepository concertCategoryRepository, KopisConcertApiReader kopisApiReader, KopisDetailApiReader kopisDetailApiReader, ConcertCategoryExtractor concertCategoryExtractor, @@ -51,6 +55,7 @@ public ConcertBatch(JobRepository jobRepository, this.jobRepository = jobRepository; this.platformTransactionManager = platformTransactionManager; this.concertRepository = concertRepository; + this.concertCategoryRepository = concertCategoryRepository; this.kopisApiReader = kopisApiReader; this.kopisDetailApiReader = kopisDetailApiReader; this.concertCategoryExtractor = concertCategoryExtractor; @@ -60,10 +65,7 @@ public ConcertBatch(JobRepository jobRepository, @Bean public Job kopisJob(KopisEntityWriter writer){ return new JobBuilder("kopisJob", jobRepository) - .start(firstStep(writer)) - .next(secondStep()) - .next(thirdStep()) - .next(fourthStep()) + .start(fourthStep()) .build(); } @@ -167,17 +169,42 @@ private void appendNewConcertData(StringJoiner csvContent) { List columns = Arrays.asList("concertId", "grand", "delicate", "classical", "modern", "lyrical", "dynamic", "romantic", "tragic", "familiar", "novel"); - List concerts = concertRepository.getAllConcertsWithCategories(); - for (Concert concert : concerts) { - Map row = initializeRowData(columns, concert); + List concertIds = concertRepository.findAllConcertIds(); + + for (Long concertId : concertIds) { + // 각 Concert ID에 대해 category와 score 조회 + List categoryScores = concertCategoryRepository.findByConcertId(concertId); + System.out.println("categoryScores = " + categoryScores); + // 각 카테고리의 기본 값을 0.0으로 초기화한 Map 생성 + Map categoryScoreMap = new HashMap<>(); + for (String column : columns.subList(1, columns.size())) { + categoryScoreMap.put(column, 0.0); + } + + // categoryScores에서 각 카테고리의 점수를 Map에 업데이트 + for (CategoryScoreDto categoryScore : categoryScores) { + String category = categoryScore.getCategory().toLowerCase(); + if (categoryScoreMap.containsKey(category)) { + categoryScoreMap.put(category, categoryScore.getScore()); + } + } + + // CSV의 한 행을 구성하는 StringJoiner 생성 StringJoiner rowContent = new StringJoiner(","); - for (String column : columns) { - rowContent.add(String.valueOf(row.get(column))); + rowContent.add(String.valueOf(concertId)); // concertId 추가 + + // 각 카테고리의 점수를 순서대로 추가 + for (String column : columns.subList(1, columns.size())) { + rowContent.add(String.valueOf(categoryScoreMap.get(column))); } + + // 완성된 행을 csvContent에 추가 csvContent.add(rowContent.toString()); } } + + // 새로운 Concert 데이터 행을 초기화하는 메서드 private Map initializeRowData(List columns, Concert concert) { Map row = new HashMap<>(); diff --git a/src/main/java/com/curateme/clacobatchserver/dto/CategoryScoreDto.java b/src/main/java/com/curateme/clacobatchserver/dto/CategoryScoreDto.java new file mode 100644 index 0000000..6619b2e --- /dev/null +++ b/src/main/java/com/curateme/clacobatchserver/dto/CategoryScoreDto.java @@ -0,0 +1,19 @@ +package com.curateme.clacobatchserver.dto; + +public class CategoryScoreDto { + private String category; + private Double score; + + public CategoryScoreDto(String category, Double score) { + this.category = category; + this.score = score; + } + + public String getCategory() { + return category; + } + + public Double getScore() { + return score; + } +} \ No newline at end of file diff --git a/src/main/java/com/curateme/clacobatchserver/entity/Category.java b/src/main/java/com/curateme/clacobatchserver/entity/Category.java new file mode 100644 index 0000000..8d1e656 --- /dev/null +++ b/src/main/java/com/curateme/clacobatchserver/entity/Category.java @@ -0,0 +1,26 @@ +package com.curateme.clacobatchserver.entity; + +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Entity +@Getter +@NoArgsConstructor(access = AccessLevel.PROTECTED) +@AllArgsConstructor +@Table(name = "category") +public class Category { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + private String category; + + private String imageUrl; +} diff --git a/src/main/java/com/curateme/clacobatchserver/entity/ConcertCategory.java b/src/main/java/com/curateme/clacobatchserver/entity/ConcertCategory.java index 0d269ce..f4f2cc6 100644 --- a/src/main/java/com/curateme/clacobatchserver/entity/ConcertCategory.java +++ b/src/main/java/com/curateme/clacobatchserver/entity/ConcertCategory.java @@ -16,9 +16,6 @@ public class ConcertCategory { @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; - @Column(name = "category") - private String category; - @Column(name = "score") private Double score; @@ -26,7 +23,10 @@ public class ConcertCategory { @JoinColumn(name = "concert_id", nullable = false) private Concert concert; - public ConcertCategory(String category, Double score, Concert concert) { + @ManyToOne + @JoinColumn(name = "category_id", nullable = false) + private Category category; + public ConcertCategory(Category category, Double score, Concert concert) { this.category = category; this.score = score; this.concert = concert; diff --git a/src/main/java/com/curateme/clacobatchserver/repository/ConcertCategoryRepository.java b/src/main/java/com/curateme/clacobatchserver/repository/ConcertCategoryRepository.java index ecee0f6..a6ee3cd 100644 --- a/src/main/java/com/curateme/clacobatchserver/repository/ConcertCategoryRepository.java +++ b/src/main/java/com/curateme/clacobatchserver/repository/ConcertCategoryRepository.java @@ -1,8 +1,17 @@ package com.curateme.clacobatchserver.repository; +import com.curateme.clacobatchserver.dto.CategoryScoreDto; import com.curateme.clacobatchserver.entity.ConcertCategory; +import java.util.List; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; public interface ConcertCategoryRepository extends JpaRepository { + @Query("SELECT new com.curateme.clacobatchserver.dto.CategoryScoreDto(cc.category.category, cc.score) " + + "FROM ConcertCategory cc WHERE cc.concert.id = :concertId") + List findByConcertId(@Param("concertId") Long concertId); } + + diff --git a/src/main/java/com/curateme/clacobatchserver/repository/ConcertRepository.java b/src/main/java/com/curateme/clacobatchserver/repository/ConcertRepository.java index 5215b9d..d55c40d 100644 --- a/src/main/java/com/curateme/clacobatchserver/repository/ConcertRepository.java +++ b/src/main/java/com/curateme/clacobatchserver/repository/ConcertRepository.java @@ -8,4 +8,7 @@ public interface ConcertRepository extends JpaRepository { @Query("SELECT c FROM Concert c LEFT JOIN FETCH c.categories") List getAllConcertsWithCategories(); + + @Query("SELECT c.id FROM Concert c") + List findAllConcertIds(); } diff --git a/src/main/java/com/curateme/clacobatchserver/schedule/ConcertSchedule.java b/src/main/java/com/curateme/clacobatchserver/schedule/ConcertSchedule.java index 407a047..bc645b9 100644 --- a/src/main/java/com/curateme/clacobatchserver/schedule/ConcertSchedule.java +++ b/src/main/java/com/curateme/clacobatchserver/schedule/ConcertSchedule.java @@ -21,7 +21,7 @@ public ConcertSchedule(JobLauncher jobLauncher, JobRegistry jobRegistry) { } @Scheduled(cron = "10 * * * * *", zone = "Asia/Seoul") - //@Scheduled(cron = "0 0 0 1 * *", zone = "Asia/Seoul") -> 추후 한달 단위로 수정(매달 1일) + //@Scheduled(cron = "0 0 0 1 * *", zone = "Asia/Seoul") public void runFirstJob() throws Exception { SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-hh-mm-ss"); From d7e5cf9a9141a3614395a5f60e08f4fa4ff7fa69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=ED=9D=AC=EC=B0=AC?= Date: Sun, 17 Nov 2024 02:09:44 +0900 Subject: [PATCH 16/19] =?UTF-8?q?feature:=20concert=20summary=20Batch=20St?= =?UTF-8?q?ep=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../clacobatchserver/batch/ConcertBatch.java | 42 ++++++++++-- .../clacobatchserver/entity/Concert.java | 5 ++ .../schedule/ConcertSchedule.java | 4 +- .../service/ConcertSummaryExtractor.java | 64 +++++++++++++++++++ 4 files changed, 106 insertions(+), 9 deletions(-) create mode 100644 src/main/java/com/curateme/clacobatchserver/service/ConcertSummaryExtractor.java diff --git a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java index fd6bd1d..ec9faab 100644 --- a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java +++ b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java @@ -6,6 +6,7 @@ import com.curateme.clacobatchserver.repository.ConcertCategoryRepository; import com.curateme.clacobatchserver.repository.ConcertRepository; import com.curateme.clacobatchserver.service.ConcertCategoryExtractor; +import com.curateme.clacobatchserver.service.ConcertSummaryExtractor; import com.curateme.clacobatchserver.service.KopisConcertApiReader; import com.curateme.clacobatchserver.service.KopisDetailApiReader; import com.curateme.clacobatchserver.service.KopisEntityWriter; @@ -32,10 +33,12 @@ import org.springframework.batch.repeat.RepeatStatus; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.config.Task; import org.springframework.transaction.PlatformTransactionManager; @Configuration public class ConcertBatch { + private final JobRepository jobRepository; private final PlatformTransactionManager platformTransactionManager; private final ConcertRepository concertRepository; @@ -45,13 +48,16 @@ public class ConcertBatch { private final KopisDetailApiReader kopisDetailApiReader; private final ConcertCategoryExtractor concertCategoryExtractor; private final S3Service s3Service; + private final ConcertSummaryExtractor concertSummaryExtractor; public ConcertBatch(JobRepository jobRepository, - PlatformTransactionManager platformTransactionManager, ConcertRepository concertRepository, ConcertCategoryRepository concertCategoryRepository, + PlatformTransactionManager platformTransactionManager, ConcertRepository concertRepository, + ConcertCategoryRepository concertCategoryRepository, KopisConcertApiReader kopisApiReader, KopisDetailApiReader kopisDetailApiReader, ConcertCategoryExtractor concertCategoryExtractor, - S3Service s3Service) { + S3Service s3Service, + ConcertSummaryExtractor concertSummaryExtractor) { this.jobRepository = jobRepository; this.platformTransactionManager = platformTransactionManager; this.concertRepository = concertRepository; @@ -60,10 +66,11 @@ public ConcertBatch(JobRepository jobRepository, this.kopisDetailApiReader = kopisDetailApiReader; this.concertCategoryExtractor = concertCategoryExtractor; this.s3Service = s3Service; + this.concertSummaryExtractor = concertSummaryExtractor; } @Bean - public Job kopisJob(KopisEntityWriter writer){ + public Job kopisJob(KopisEntityWriter writer) { return new JobBuilder("kopisJob", jobRepository) .start(fourthStep()) .build(); @@ -118,6 +125,14 @@ public Step fourthStep() { }, platformTransactionManager).build(); } + // 5. 클라코 큐레이션: 공연 정보 요약 + @Bean + public Step fifthStep() { + return new StepBuilder("fifthStep", jobRepository) + .tasklet(concertSummaryExtractorTasklet(), platformTransactionManager) + .build(); + } + // 5. 해당 Batch에서 가져온 값들을 전부 삭제하는 로직 @Bean public Step finalStep() { @@ -166,14 +181,16 @@ private StringJoiner readExistingCsvContent(String localFilePath) { // 새로운 Concert 데이터를 CSV에 추가하는 메서드 private void appendNewConcertData(StringJoiner csvContent) { - List columns = Arrays.asList("concertId", "grand", "delicate", "classical", "modern", + List columns = Arrays.asList("concertId", "grand", "delicate", "classical", + "modern", "lyrical", "dynamic", "romantic", "tragic", "familiar", "novel"); List concertIds = concertRepository.findAllConcertIds(); for (Long concertId : concertIds) { // 각 Concert ID에 대해 category와 score 조회 - List categoryScores = concertCategoryRepository.findByConcertId(concertId); + List categoryScores = concertCategoryRepository.findByConcertId( + concertId); System.out.println("categoryScores = " + categoryScores); // 각 카테고리의 기본 값을 0.0으로 초기화한 Map 생성 Map categoryScoreMap = new HashMap<>(); @@ -204,7 +221,6 @@ private void appendNewConcertData(StringJoiner csvContent) { } - // 새로운 Concert 데이터 행을 초기화하는 메서드 private Map initializeRowData(List columns, Concert concert) { Map row = new HashMap<>(); @@ -250,9 +266,21 @@ private void saveAndUploadCsvFile(StringJoiner csvContent, String folderPath, St public Tasklet concertCategoryExtractorTasklet() { return new Tasklet() { @Override - public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception { + public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) + throws Exception { return concertCategoryExtractor.execute(contribution, chunkContext); } }; } + + @Bean + public Tasklet concertSummaryExtractorTasklet() { + return new Tasklet() { + @Override + public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) + throws Exception { + return concertSummaryExtractor.execute(contribution, chunkContext); + } + }; + } } diff --git a/src/main/java/com/curateme/clacobatchserver/entity/Concert.java b/src/main/java/com/curateme/clacobatchserver/entity/Concert.java index 1c5398a..40aa780 100644 --- a/src/main/java/com/curateme/clacobatchserver/entity/Concert.java +++ b/src/main/java/com/curateme/clacobatchserver/entity/Concert.java @@ -114,6 +114,9 @@ public class Concert extends BaseEntity { @Column(name = "introduction") private String styurl; + @Column(name = "summary", length = 1500) + private String summary; + @ElementCollection @CollectionTable(name = "concert_category", joinColumns = @JoinColumn(name = "concert_id")) @MapKeyColumn(name = "category") @@ -166,4 +169,6 @@ public void setStyurl(String styurl){ public void setCategories(Map categories) { this.categories = categories; } + + public void setSummary(String summary) {this.summary = summary; } } diff --git a/src/main/java/com/curateme/clacobatchserver/schedule/ConcertSchedule.java b/src/main/java/com/curateme/clacobatchserver/schedule/ConcertSchedule.java index bc645b9..745349d 100644 --- a/src/main/java/com/curateme/clacobatchserver/schedule/ConcertSchedule.java +++ b/src/main/java/com/curateme/clacobatchserver/schedule/ConcertSchedule.java @@ -20,8 +20,8 @@ public ConcertSchedule(JobLauncher jobLauncher, JobRegistry jobRegistry) { this.jobRegistry = jobRegistry; } - @Scheduled(cron = "10 * * * * *", zone = "Asia/Seoul") - //@Scheduled(cron = "0 0 0 1 * *", zone = "Asia/Seoul") + //@Scheduled(cron = "10 * * * * *", zone = "Asia/Seoul") + @Scheduled(cron = "0 0 0 1 * *", zone = "Asia/Seoul") public void runFirstJob() throws Exception { SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-hh-mm-ss"); diff --git a/src/main/java/com/curateme/clacobatchserver/service/ConcertSummaryExtractor.java b/src/main/java/com/curateme/clacobatchserver/service/ConcertSummaryExtractor.java new file mode 100644 index 0000000..9f62d02 --- /dev/null +++ b/src/main/java/com/curateme/clacobatchserver/service/ConcertSummaryExtractor.java @@ -0,0 +1,64 @@ +package com.curateme.clacobatchserver.service; + +import com.curateme.clacobatchserver.entity.Concert; +import com.curateme.clacobatchserver.repository.ConcertRepository; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.repeat.RepeatStatus; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; + +@Component +public class ConcertSummaryExtractor { + private final RestTemplate restTemplate; + private final ConcertRepository concertRepository; + + @Autowired + public ConcertSummaryExtractor(RestTemplate restTemplate, ConcertRepository concertRepository) { + this.restTemplate = restTemplate; + this.concertRepository = concertRepository; + } + + public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) { + List concerts = concertRepository.findAll(); + + for (Concert concert : concerts) { + try { + String introduction = concert.getStyurl(); + + Map requestBody = new HashMap<>(); + requestBody.put("image_url", introduction); + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + HttpEntity> requestEntity = new HttpEntity<>(requestBody, headers); + + // Flask 서버에 요청 보내기 + ResponseEntity response = restTemplate.postForEntity("http://localhost:8081/summaries", requestEntity, Map.class); + + // TODO: 점검 필요 + if (response.getBody() != null) { + String clovaResponse = (String) response.getBody().get("clova_response"); + + concert.setSummary(clovaResponse); + } + + } catch (Exception e) { + // 특정 concert에서 예외 발생 시 로그를 남기고, 다음 concert로 계속 진행 + System.err.println("concert ID " + concert.getMt20id() + " 처리 중 오류 발생: " + e.getMessage()); + e.printStackTrace(); + } + } + + return RepeatStatus.FINISHED; + } + +} From e696302fdd3218e24d82f1183f033a8c0acfd5fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=ED=9D=AC=EC=B0=AC?= Date: Sun, 17 Nov 2024 16:22:01 +0900 Subject: [PATCH 17/19] =?UTF-8?q?feature:=20concert=20summary=20Batch=20?= =?UTF-8?q?=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/curateme/clacobatchserver/batch/ConcertBatch.java | 2 +- .../curateme/clacobatchserver/schedule/ConcertSchedule.java | 4 ++-- .../clacobatchserver/service/ConcertSummaryExtractor.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java index ec9faab..a0d3009 100644 --- a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java +++ b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java @@ -72,7 +72,7 @@ public ConcertBatch(JobRepository jobRepository, @Bean public Job kopisJob(KopisEntityWriter writer) { return new JobBuilder("kopisJob", jobRepository) - .start(fourthStep()) + .start(fifthStep()) .build(); } diff --git a/src/main/java/com/curateme/clacobatchserver/schedule/ConcertSchedule.java b/src/main/java/com/curateme/clacobatchserver/schedule/ConcertSchedule.java index 745349d..bc645b9 100644 --- a/src/main/java/com/curateme/clacobatchserver/schedule/ConcertSchedule.java +++ b/src/main/java/com/curateme/clacobatchserver/schedule/ConcertSchedule.java @@ -20,8 +20,8 @@ public ConcertSchedule(JobLauncher jobLauncher, JobRegistry jobRegistry) { this.jobRegistry = jobRegistry; } - //@Scheduled(cron = "10 * * * * *", zone = "Asia/Seoul") - @Scheduled(cron = "0 0 0 1 * *", zone = "Asia/Seoul") + @Scheduled(cron = "10 * * * * *", zone = "Asia/Seoul") + //@Scheduled(cron = "0 0 0 1 * *", zone = "Asia/Seoul") public void runFirstJob() throws Exception { SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-hh-mm-ss"); diff --git a/src/main/java/com/curateme/clacobatchserver/service/ConcertSummaryExtractor.java b/src/main/java/com/curateme/clacobatchserver/service/ConcertSummaryExtractor.java index 9f62d02..a9049af 100644 --- a/src/main/java/com/curateme/clacobatchserver/service/ConcertSummaryExtractor.java +++ b/src/main/java/com/curateme/clacobatchserver/service/ConcertSummaryExtractor.java @@ -44,11 +44,11 @@ public RepeatStatus execute(StepContribution contribution, ChunkContext chunkCon // Flask 서버에 요청 보내기 ResponseEntity response = restTemplate.postForEntity("http://localhost:8081/summaries", requestEntity, Map.class); - // TODO: 점검 필요 if (response.getBody() != null) { String clovaResponse = (String) response.getBody().get("clova_response"); concert.setSummary(clovaResponse); + concertRepository.save(concert); } } catch (Exception e) { From 210f27f62e26f31e5be8f8dfb19c191cc5590aa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=ED=9D=AC=EC=B0=AC?= Date: Sun, 17 Nov 2024 16:25:30 +0900 Subject: [PATCH 18/19] fix: batch Step --- .../com/curateme/clacobatchserver/batch/ConcertBatch.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java index a0d3009..213fba8 100644 --- a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java +++ b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java @@ -72,7 +72,11 @@ public ConcertBatch(JobRepository jobRepository, @Bean public Job kopisJob(KopisEntityWriter writer) { return new JobBuilder("kopisJob", jobRepository) - .start(fifthStep()) + .start(firstStep(writer)) + .next(secondStep()) + .next(thirdStep()) + .next(fourthStep()) + .next(fifthStep()) .build(); } From 89935f7461074eb839963bacb3a017d8045311dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=ED=9D=AC=EC=B0=AC?= Date: Thu, 28 Nov 2024 16:30:13 +0900 Subject: [PATCH 19/19] Refactoring: Batch Step --- .../clacobatchserver/batch/ConcertBatch.java | 23 ++++++++++-- .../repository/ConcertCategoryRepository.java | 3 ++ .../service/ConcertCategoryExtractor.java | 32 ++++++++-------- .../service/KopisConcertApiReader.java | 37 +++++++++++-------- 4 files changed, 59 insertions(+), 36 deletions(-) diff --git a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java index 213fba8..7fa9a9c 100644 --- a/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java +++ b/src/main/java/com/curateme/clacobatchserver/batch/ConcertBatch.java @@ -189,6 +189,19 @@ private void appendNewConcertData(StringJoiner csvContent) { "modern", "lyrical", "dynamic", "romantic", "tragic", "familiar", "novel"); + Map translationMap = Map.of( + "웅장한", "grand", + "섬세한", "delicate", + "고전적인", "classical", + "현대적인", "modern", + "서정적인", "lyrical", + "역동적인", "dynamic", + "낭만적인", "romantic", + "비극적인", "tragic", + "친숙한", "familiar", + "새로운", "novel" + ); + List concertIds = concertRepository.findAllConcertIds(); for (Long concertId : concertIds) { @@ -204,9 +217,12 @@ private void appendNewConcertData(StringJoiner csvContent) { // categoryScores에서 각 카테고리의 점수를 Map에 업데이트 for (CategoryScoreDto categoryScore : categoryScores) { - String category = categoryScore.getCategory().toLowerCase(); - if (categoryScoreMap.containsKey(category)) { - categoryScoreMap.put(category, categoryScore.getScore()); + String koreanCategory = categoryScore.getCategory(); + System.out.println("koreanCategory = " + koreanCategory); + String englishCategory = translationMap.get(koreanCategory); + System.out.println("englishCategory = " + englishCategory); + if (categoryScoreMap.containsKey(englishCategory)) { + categoryScoreMap.put(englishCategory, categoryScore.getScore()); } } @@ -224,7 +240,6 @@ private void appendNewConcertData(StringJoiner csvContent) { } } - // 새로운 Concert 데이터 행을 초기화하는 메서드 private Map initializeRowData(List columns, Concert concert) { Map row = new HashMap<>(); diff --git a/src/main/java/com/curateme/clacobatchserver/repository/ConcertCategoryRepository.java b/src/main/java/com/curateme/clacobatchserver/repository/ConcertCategoryRepository.java index a6ee3cd..1fee3b1 100644 --- a/src/main/java/com/curateme/clacobatchserver/repository/ConcertCategoryRepository.java +++ b/src/main/java/com/curateme/clacobatchserver/repository/ConcertCategoryRepository.java @@ -12,6 +12,9 @@ public interface ConcertCategoryRepository extends JpaRepository findByConcertId(@Param("concertId") Long concertId); + @Query("SELECT cc FROM ConcertCategory cc WHERE cc.concert.id = :concertId") + List findAllByConcertId(@Param("concertId") Long concertId); + } diff --git a/src/main/java/com/curateme/clacobatchserver/service/ConcertCategoryExtractor.java b/src/main/java/com/curateme/clacobatchserver/service/ConcertCategoryExtractor.java index ddc167f..945841a 100644 --- a/src/main/java/com/curateme/clacobatchserver/service/ConcertCategoryExtractor.java +++ b/src/main/java/com/curateme/clacobatchserver/service/ConcertCategoryExtractor.java @@ -31,18 +31,6 @@ public ConcertCategoryExtractor(RestTemplate restTemplate, ConcertRepository con public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) { List concerts = concertRepository.findAll(); - Map translationMap = new HashMap<>(); - translationMap.put("웅장한", "grand"); - translationMap.put("섬세한", "delicate"); - translationMap.put("고전적인", "classical"); - translationMap.put("현대적인", "modern"); - translationMap.put("서정적인", "lyrical"); - translationMap.put("역동적인", "dynamic"); - translationMap.put("낭만적인", "romantic"); - translationMap.put("비극적인", "tragic"); - translationMap.put("친숙한", "familiar"); - translationMap.put("새로운", "novel"); - for (Concert concert : concerts) { try { String introduction = concert.getStyurl(); @@ -59,16 +47,28 @@ public RepeatStatus execute(StepContribution contribution, ChunkContext chunkCon if (response.getBody() != null) { Map responseBody = (Map) response.getBody(); - List> clovaResponse = (List>) responseBody.get("clova_response"); + Object clovaResponse = responseBody.get("clova_response"); Map categories = new HashMap<>(); - for (Map categoryData : clovaResponse) { + // clova_response 처리 + if (clovaResponse instanceof Map) { + // clova_response가 Map인 경우 + Map categoryData = (Map) clovaResponse; String koreanCategory = categoryData.get("name").toString(); Double score = Double.parseDouble(categoryData.get("score").toString()); - String englishCategory = translationMap.getOrDefault(koreanCategory, koreanCategory); - categories.put(englishCategory, score); + categories.put(koreanCategory, score); + } else if (clovaResponse instanceof List) { + // clova_response가 List인 경우 + List> categoryList = (List>) clovaResponse; + + for (Map categoryData : categoryList) { + String koreanCategory = categoryData.get("name").toString(); + Double score = Double.parseDouble(categoryData.get("score").toString()); + + categories.put(koreanCategory, score); + } } concert.setCategories(categories); diff --git a/src/main/java/com/curateme/clacobatchserver/service/KopisConcertApiReader.java b/src/main/java/com/curateme/clacobatchserver/service/KopisConcertApiReader.java index bb2e0e6..f756104 100644 --- a/src/main/java/com/curateme/clacobatchserver/service/KopisConcertApiReader.java +++ b/src/main/java/com/curateme/clacobatchserver/service/KopisConcertApiReader.java @@ -46,7 +46,7 @@ public Concert read() throws Exception { private void loadNextPage() { LocalDate startDate = LocalDate.now(); - LocalDate endDate = startDate.plusDays(1); + LocalDate endDate = startDate.plusDays(30); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); String formattedStartDate = startDate.format(formatter); @@ -62,24 +62,28 @@ private void loadNextPage() { try { ResponseEntity response = restTemplate.exchange(requestUrl, HttpMethod.GET, entity, Concert[].class); System.out.println("response = " + response); + if (response.getBody() != null && response.getBody().length > 0) { beforeEntities.clear(); + for (Concert beforeentity : response.getBody()) { - Concert concert = new Concert(); - concert.setConcertDetails( - beforeentity.getMt20id(), - beforeentity.getPrfnm(), - beforeentity.getPrfpdfrom(), - beforeentity.getPrfpdto(), - beforeentity.getFcltynm(), - beforeentity.getPoster(), - beforeentity.getArea(), - beforeentity.getGenrenm(), - beforeentity.getOpenrun(), - beforeentity.getPrfstate() - ); - - beforeEntities.add(concert); + if ("무용".equals(beforeentity.getGenrenm()) || "서양음악(클래식)".equals(beforeentity.getGenrenm())) { + Concert concert = new Concert(); + concert.setConcertDetails( + beforeentity.getMt20id(), + beforeentity.getPrfnm(), + beforeentity.getPrfpdfrom(), + beforeentity.getPrfpdto(), + beforeentity.getFcltynm(), + beforeentity.getPoster(), + beforeentity.getArea(), + beforeentity.getGenrenm(), + beforeentity.getOpenrun(), + beforeentity.getPrfstate() + ); + + beforeEntities.add(concert); + } } } else { beforeEntities.clear(); @@ -88,5 +92,6 @@ private void loadNextPage() { System.err.println("Error: " + e.getStatusCode() + " - " + e.getResponseBodyAsString()); beforeEntities.clear(); } + } }