Merge pull request #1 from CarToi/feat/peristalsis

[Feat/peristalsis] 데이터 수집 및 전처리 기초 세팅

Author: kimD0ngjun

build.gradle

@@ -19,8 +19,24 @@ repositories {
 
 dependencies {
     implementation 'org.springframework.boot:spring-boot-starter-web'
+    implementation 'org.springframework.boot:spring-boot-starter-webflux'
+
+    implementation 'org.projectlombok:lombok'
+    annotationProcessor 'org.projectlombok:lombok'
+    testCompileOnly 'org.projectlombok:lombok'
+    testAnnotationProcessor 'org.projectlombok:lombok'
+
     testImplementation 'org.springframework.boot:spring-boot-starter-test'
+    testImplementation 'io.projectreactor:reactor-test'
     testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
+
+    // selenium
+    // https://mvnrepository.com/artifact/org.seleniumhq.selenium/selenium-java
+    implementation("org.seleniumhq.selenium:selenium-java:4.33.0") // 셀레늄 의존성
+//    // https://mvnrepository.com/artifact/io.github.bonigarcia/webdrivermanager
+//    implementation("io.github.bonigarcia:webdrivermanager:6.1.0")
+    // https://mvnrepository.com/artifact/edu.stanford.nlp/stanford-corenlp
+    implementation("edu.stanford.nlp:stanford-corenlp:4.5.9") // 문장 분석을 통한 자연어 처리
 }
 
 tasks.named('test') {

src/main/java/org/jun/saemangeum/global/persistence/domain/Category.java

@@ -0,0 +1,5 @@
+package org.jun.saemangeum.global.persistence.domain;
+
+public enum Category {
+    FESTIVAL, EVENT, TOUR
+}

src/main/java/org/jun/saemangeum/global/persistence/domain/Content.java

@@ -0,0 +1,10 @@
+package org.jun.saemangeum.global.persistence.domain;
+
+// H2의 엔티티로 쓰이게 될 클래스
+public class Content {
+    private Long id;
+    private String title;
+    private Category category;
+    private String image;
+    private String introduction;
+}

src/main/java/org/jun/saemangeum/process/application/service/api/FestivalCollector.java

@@ -0,0 +1,35 @@
+package org.jun.saemangeum.process.application.service.api;
+
+import org.jun.saemangeum.process.application.service.base.OpenApiCollector;
+import org.jun.saemangeum.process.domain.dto.RefinedDataDTO;
+import org.jun.saemangeum.process.infrastructure.api.RestTemplateClient;
+import org.jun.saemangeum.process.presentation.dto.Festival;
+import org.jun.saemangeum.process.presentation.dto.FestivalResponse;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+
+/**
+ * 공공데이터 API 축제 CSV 받아오기
+ */
+@Component
+public class FestivalCollector extends OpenApiCollector {
+
+    private static final String LAST_PATH = "/15006172/v1/uddi:ede8925d-bfbd-49fc-9f3c-abf1ead5b402";
+
+    public FestivalCollector(RestTemplateClient restTemplateClient) {
+        super(restTemplateClient);
+    }
+
+    @Override
+    public List<RefinedDataDTO> collectData() {
+        FestivalResponse response = restTemplateClient.get(
+                LAST_PATH,
+                FestivalResponse.class,
+                q -> q.queryParam("page", 1).queryParam("perPage", 100)
+        );
+
+        return response.data().stream().map(Festival::convertToDTO).toList();
+    }
+
+}

src/main/java/org/jun/saemangeum/process/application/service/base/OpenApiCollector.java

@@ -0,0 +1,30 @@
+package org.jun.saemangeum.process.application.service.base;
+
+import org.jun.saemangeum.global.persistence.domain.Content;
+import org.jun.saemangeum.process.domain.dto.RefinedDataDTO;
+import org.jun.saemangeum.process.infrastructure.api.RestTemplateClient;
+
+import java.util.List;
+
+public abstract class OpenApiCollector implements Refiner {
+//    protected final WebClient webClient;
+    protected final RestTemplateClient restTemplateClient;
+
+//    public OpenApiCollector(WebClient webClient) {
+//        this.webClient = webClient;
+//    }
+
+    public OpenApiCollector(RestTemplateClient restTemplateClient) {
+        this.restTemplateClient = restTemplateClient;
+    }
+
+    @Override
+    public List<Content> refine() {
+        // 여기에 추상 메소드들 기반으로 로직 처리처리할 예정
+        // 생성한 것들을 바탕으로 프로세스까지 처리하는 걸 여기에 책임을 부여하자(팩토리 메서드 취지)
+        return List.of();
+    }
+
+    // 각 OpenAPI 호출과 관련된 추상메소드들
+    public abstract List<RefinedDataDTO> collectData();
+}

src/main/java/org/jun/saemangeum/process/application/service/base/Refiner.java

@@ -0,0 +1,9 @@
+package org.jun.saemangeum.process.application.service.base;
+
+import org.jun.saemangeum.global.persistence.domain.Content;
+
+import java.util.List;
+
+public interface Refiner {
+    List<Content> refine();
+}

src/main/java/org/jun/saemangeum/process/application/service/base/SeleniumCollector.java

@@ -0,0 +1,23 @@
+package org.jun.saemangeum.process.application.service.base;
+
+import org.jun.saemangeum.global.persistence.domain.Content;
+import org.openqa.selenium.WebDriver;
+
+import java.util.List;
+
+public abstract class SeleniumCollector implements Refiner {
+    protected final WebDriver webDriver;
+
+    public SeleniumCollector(WebDriver webDriver) {
+        this.webDriver = webDriver;
+    }
+
+    @Override
+    public List<Content> refine() {
+        // 여기에 추상 메소드들 기반으로 로직 처리처리할 예정
+        // 생성한 것들을 바탕으로 프로세스까지 처리하는 걸 여기에 책임을 부여하자(팩토리 메서드 취지)
+        return List.of();
+    }
+
+    // 각 페이지 크롤링과 관련된 추상메소드들
+}

src/main/java/org/jun/saemangeum/process/domain/dto/RefinedDataDTO.java

@@ -0,0 +1,12 @@
+package org.jun.saemangeum.process.domain.dto;
+
+import org.jun.saemangeum.global.persistence.domain.Category;
+
+public record RefinedDataDTO(
+        String title,
+        String position,
+        Category category,
+        String image,
+        String introduction
+) {
+}

src/main/java/org/jun/saemangeum/process/infrastructure/api/RestTemplateClient.java

@@ -0,0 +1,31 @@
+package org.jun.saemangeum.process.infrastructure.api;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import java.net.URI;
+import java.util.function.Consumer;
+
+@Component
+public class RestTemplateClient {
+
+    @Value("${dev.baseUrl}")
+    private String baseUrl;
+
+    private final RestTemplate restTemplate;
+
+    public RestTemplateClient(RestTemplate restTemplate) {
+        this.restTemplate = restTemplate;
+    }
+
+    public <T> T get(String path, Class<T> responseType, Consumer<UriComponentsBuilder> extraQuery) {
+        UriComponentsBuilder builder = UriComponentsBuilder.fromUriString(baseUrl).path(path);
+
+        if (extraQuery != null) extraQuery.accept(builder);
+
+        URI uri = builder.build(true).toUri();
+        return restTemplate.getForObject(uri, responseType);
+    }
+}

src/main/java/org/jun/saemangeum/process/infrastructure/config/RestTemplateConfig.java

@@ -0,0 +1,49 @@
+package org.jun.saemangeum.process.infrastructure.config;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.http.HttpRequest;
+import org.springframework.http.client.SimpleClientHttpRequestFactory;
+import org.springframework.http.client.support.HttpRequestWrapper;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import java.net.URI;
+import java.util.List;
+
+@Configuration
+public class RestTemplateConfig {
+
+    @Value("${dev.openApiKey}")
+    private String apiKey;
+
+    @Bean
+    public RestTemplate restTemplate() {
+        SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory();
+        requestFactory.setConnectTimeout(5000);
+        requestFactory.setReadTimeout(5000);
+
+        RestTemplate restTemplate = new RestTemplate(requestFactory);
+        restTemplate.setInterceptors(List.of((request, body, execution) -> {
+            URI uri = request.getURI();
+
+            URI updateUri = UriComponentsBuilder.fromUri(uri)
+                    .queryParam("serviceKey", apiKey)
+                    .build(true) // 인코딩 유지 설정을 통한 중복 인코딩 방지
+                    .toUri();
+
+            HttpRequest newRequest = new HttpRequestWrapper(request) {
+                @Override
+                public URI getURI() {
+                    return updateUri;
+                }
+            };
+
+            return execution.execute(newRequest, body);
+        }));
+
+        return restTemplate;
+    }
+
+}