From 2803e1501a3dc39e5d41fce88d537ed9705cbead Mon Sep 17 00:00:00 2001 From: GyuHwan Date: Tue, 18 Nov 2025 00:02:37 +0900 Subject: [PATCH 1/6] =?UTF-8?q?[#1]=20feat=20:=20=EC=9B=90=EB=B3=B8=20JSON?= =?UTF-8?q?=20=EB=8D=B0=EC=9D=B4=ED=84=B0=20=ED=8C=8C=EC=8B=B1=20-=20?= =?UTF-8?q?=EC=9B=90=EB=B3=B8=EC=9D=84=20=ED=95=84=EC=9A=94=ED=95=9C=20?= =?UTF-8?q?=EC=86=8D=EC=84=B1=EB=A7=8C=20dto=20=EB=B3=80=ED=99=98=20-=20dt?= =?UTF-8?q?o=EC=97=90=EC=84=9C=20=ED=95=84=EC=9A=94=EC=97=86=EB=8A=94=20?= =?UTF-8?q?=ED=95=AD=EB=AA=A9=20=ED=95=84=ED=84=B0=EB=A7=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 ++ build.gradle | 8 ++++ .../core/controller/MasterDataController.java | 24 ++++++++++ .../earseo/core/dto/etl/FilteredDataDto.java | 5 ++ .../core/service/MasterDataService.java | 47 +++++++++++++++++++ src/main/resources/application-local.yaml | 12 +++-- 6 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 src/main/java/com/earseo/core/controller/MasterDataController.java create mode 100644 src/main/java/com/earseo/core/dto/etl/FilteredDataDto.java create mode 100644 src/main/java/com/earseo/core/service/MasterDataService.java diff --git a/.gitignore b/.gitignore index c2065bc..4ea4adf 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,6 @@ out/ ### VS Code ### .vscode/ + +# macOS +.DS_Store \ No newline at end of file diff --git a/build.gradle b/build.gradle index 53d8b65..a09bb5f 100644 --- a/build.gradle +++ b/build.gradle @@ -22,6 +22,14 @@ dependencies { implementation 'org.springframework.boot:spring-boot-starter-web' implementation 'org.springframework.boot:spring-boot-starter-actuator' runtimeOnly 'io.micrometer:micrometer-registry-prometheus' + + implementation 'org.springframework.boot:spring-boot-starter-data-jpa' + runtimeOnly 'org.postgresql:postgresql' + + implementation 'org.hibernate.orm:hibernate-spatial' + implementation 'org.locationtech.jts:jts-core:1.19.0' + implementation 'com.fasterxml.jackson.core:jackson-databind' + implementation 'org.springframework.boot:spring-boot-starter-data-jpa' implementation 'org.springframework.boot:spring-boot-starter-validation' compileOnly 'org.projectlombok:lombok' diff --git a/src/main/java/com/earseo/core/controller/MasterDataController.java b/src/main/java/com/earseo/core/controller/MasterDataController.java new file mode 100644 index 0000000..7e8c6ef --- /dev/null +++ b/src/main/java/com/earseo/core/controller/MasterDataController.java @@ -0,0 +1,24 @@ +package com.earseo.core.controller; + +import com.earseo.core.common.BaseResponse; +import com.earseo.core.dto.etl.FilteredDataDto; +import com.earseo.core.service.MasterDataService; +import lombok.RequiredArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@RestController +@RequiredArgsConstructor +public class MasterDataController { + + private final MasterDataService masterDataService; + + @GetMapping("/admin/master") + public ResponseEntity> rawDataProcess(){ + List filteredData = masterDataService.getRawInfo(); + return ResponseEntity.ok(BaseResponse.ok(null)); + } +} diff --git a/src/main/java/com/earseo/core/dto/etl/FilteredDataDto.java b/src/main/java/com/earseo/core/dto/etl/FilteredDataDto.java new file mode 100644 index 0000000..613aee5 --- /dev/null +++ b/src/main/java/com/earseo/core/dto/etl/FilteredDataDto.java @@ -0,0 +1,5 @@ +package com.earseo.core.dto.etl; + +public record FilteredDataDto(String contentId, String contentTypeId, String cat1, + String cat2, String cat3, String outl) { +} diff --git a/src/main/java/com/earseo/core/service/MasterDataService.java b/src/main/java/com/earseo/core/service/MasterDataService.java new file mode 100644 index 0000000..4652378 --- /dev/null +++ b/src/main/java/com/earseo/core/service/MasterDataService.java @@ -0,0 +1,47 @@ +package com.earseo.core.service; + +import com.earseo.core.dto.etl.FilteredDataDto; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.ClassPathResource; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestClient; + +import java.io.InputStream; +import java.util.List; + +@Service +@RequiredArgsConstructor +public class MasterDataService { + + private final ObjectMapper objectMapper; + private final RestClient restClient; + + @Value("${API_KEY}") + private String ApiKeys; + + public List getRawInfo() { + try { + InputStream rawJson = new ClassPathResource("TourAPI_seoul.json").getInputStream(); + List rawJsonDtos = objectMapper.readValue( + rawJson, + new TypeReference>() { + } + ); + + List filtered = rawJsonDtos.stream() + .filter(dto -> !dto.contentTypeId().equals("25")) + .filter(dto -> !dto.contentTypeId().equals("32")) + .filter(dto -> !dto.cat3().equals("A04011000")) + .toList(); + + return filtered; + + } catch (Exception e) { + return List.of(); + } + } + +} diff --git a/src/main/resources/application-local.yaml b/src/main/resources/application-local.yaml index 32260f6..584bee0 100644 --- a/src/main/resources/application-local.yaml +++ b/src/main/resources/application-local.yaml @@ -1,9 +1,11 @@ spring: + application: + name: backend-core datasource: driver-class-name: org.postgresql.Driver - url: jdbc:postgresql://${DB_HOST:localhost}:${DB_PORT:15432}/${DB_NAME:core} - username: ${DB_USERNAME:postgres} - password: ${DB_PASSWORD:postgres} + url: jdbc:postgresql://localhost:${DB_PORT}/core + username: ${DB_USERNAME} + password: ${DB_PASSWORD} jpa: hibernate: @@ -12,7 +14,7 @@ spring: hibernate: show_sql: true format_sql: true - dialect: org.hibernate.dialect.PostgreSQLDialect + dialect: org.hibernate.spatial.dialect.postgis.PostgisPG95Dialect open-in-view: false kafka: bootstrap-servers: ${SPRING_KAFKA_BOOTSTRAP_SERVERS:${KAFKA_IP}:19092} @@ -61,6 +63,8 @@ management: metrics: tags: application: ${spring.application.name} + + tracing: enabled: false From d0f62677f8a8838567bb9dfec1db203610057da6 Mon Sep 17 00:00:00 2001 From: haribonyam Date: Thu, 20 Nov 2025 16:19:39 +0900 Subject: [PATCH 2/6] =?UTF-8?q?[#1]=20feat=20:=20=ED=8C=8C=EC=8B=B1?= =?UTF-8?q?=EB=90=9C=20=EB=8D=B0=EC=9D=B4=ED=84=B0=EB=A1=9C=20=EC=99=B8?= =?UTF-8?q?=EB=B6=80=20API=20=ED=98=B8=EC=B6=9C=20=ED=9B=84=20=EB=8D=B0?= =?UTF-8?q?=EC=9D=B4=ED=84=B0=20=EB=B3=91=ED=95=A9=ED=95=98=EC=97=AC=20?= =?UTF-8?q?=EC=A4=91=EA=B0=84=20=ED=85=8C=EC=9D=B4=EB=B8=94=20=EC=83=9D?= =?UTF-8?q?=EC=84=B1=20-=20=EC=B9=B4=ED=85=8C=EA=B3=A0=EB=A6=AC=20?= =?UTF-8?q?=EC=BD=94=EB=93=9C=20:=20=EB=84=A4=EC=9E=84=20=EB=A7=A4?= =?UTF-8?q?=EC=B9=AD=EC=9D=84=20=EC=9C=84=ED=95=9C=20=EC=B9=B4=ED=85=8C?= =?UTF-8?q?=EA=B3=A0=EB=A6=AC=20=ED=85=8C=EC=9D=B4=EB=B8=94=20=EC=B4=88?= =?UTF-8?q?=EA=B8=B0=ED=99=94=20=EC=9E=91=EC=97=85=20-=20=ED=8C=8C?= =?UTF-8?q?=EC=8B=B1=EB=90=9C=20=EB=8D=B0=EC=9D=B4=ED=84=B0=EB=A1=9C=20?= =?UTF-8?q?=EC=99=B8=EB=B6=80=20API=20=ED=98=B8=EC=B6=9C=20=ED=9B=84=20?= =?UTF-8?q?=EC=A4=91=EA=B0=84=20=ED=85=8C=EC=9D=B4=EB=B8=94=20=EC=83=9D?= =?UTF-8?q?=EC=84=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/common/config/RestClientConfig.java | 14 + .../core/controller/MasterDataController.java | 14 +- .../earseo/core/dto/etl/CategoryItemDto.java | 5 + .../earseo/core/dto/etl/CommonItemDto.java | 13 + .../earseo/core/dto/etl/DetailItemDto.java | 4 + .../com/earseo/core/dto/etl/ImageItemDto.java | 4 + .../earseo/core/dto/etl/MiddleDataDto.java | 8 + .../java/com/earseo/core/entity/Category.java | 25 ++ .../com/earseo/core/entity/MiddleData.java | 104 +++++++ .../core/repository/CategoryRepository.java | 10 + .../core/repository/MiddleRepository.java | 18 ++ .../core/service/MasterDataService.java | 261 +++++++++++++++++- 12 files changed, 476 insertions(+), 4 deletions(-) create mode 100644 src/main/java/com/earseo/core/common/config/RestClientConfig.java create mode 100644 src/main/java/com/earseo/core/dto/etl/CategoryItemDto.java create mode 100644 src/main/java/com/earseo/core/dto/etl/CommonItemDto.java create mode 100644 src/main/java/com/earseo/core/dto/etl/DetailItemDto.java create mode 100644 src/main/java/com/earseo/core/dto/etl/ImageItemDto.java create mode 100644 src/main/java/com/earseo/core/dto/etl/MiddleDataDto.java create mode 100644 src/main/java/com/earseo/core/entity/Category.java create mode 100644 src/main/java/com/earseo/core/entity/MiddleData.java create mode 100644 src/main/java/com/earseo/core/repository/CategoryRepository.java create mode 100644 src/main/java/com/earseo/core/repository/MiddleRepository.java diff --git a/src/main/java/com/earseo/core/common/config/RestClientConfig.java b/src/main/java/com/earseo/core/common/config/RestClientConfig.java new file mode 100644 index 0000000..fb0f6d4 --- /dev/null +++ b/src/main/java/com/earseo/core/common/config/RestClientConfig.java @@ -0,0 +1,14 @@ +package com.earseo.core.common.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.client.RestClient; + +@Configuration +public class RestClientConfig { + + @Bean + public RestClient.Builder restClientBuilder() { + return RestClient.builder(); + } +} \ No newline at end of file diff --git a/src/main/java/com/earseo/core/controller/MasterDataController.java b/src/main/java/com/earseo/core/controller/MasterDataController.java index 7e8c6ef..2a09730 100644 --- a/src/main/java/com/earseo/core/controller/MasterDataController.java +++ b/src/main/java/com/earseo/core/controller/MasterDataController.java @@ -2,10 +2,13 @@ import com.earseo.core.common.BaseResponse; import com.earseo.core.dto.etl.FilteredDataDto; +import com.earseo.core.dto.etl.MiddleDataDto; import com.earseo.core.service.MasterDataService; +import lombok.Getter; import lombok.RequiredArgsConstructor; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RestController; import java.util.List; @@ -16,9 +19,16 @@ public class MasterDataController { private final MasterDataService masterDataService; - @GetMapping("/admin/master") - public ResponseEntity> rawDataProcess(){ + @GetMapping("/admin/core/master/{start}") + public ResponseEntity> rawDataProcess(@PathVariable int start){ List filteredData = masterDataService.getRawInfo(); + List middleData = masterDataService.getMiddleData(filteredData,start); + return ResponseEntity.ok(BaseResponse.ok(null)); + } + + @GetMapping("/admin/core/init") + public ResponseEntity> init(){ + masterDataService.initData(); return ResponseEntity.ok(BaseResponse.ok(null)); } } diff --git a/src/main/java/com/earseo/core/dto/etl/CategoryItemDto.java b/src/main/java/com/earseo/core/dto/etl/CategoryItemDto.java new file mode 100644 index 0000000..c5a912f --- /dev/null +++ b/src/main/java/com/earseo/core/dto/etl/CategoryItemDto.java @@ -0,0 +1,5 @@ +package com.earseo.core.dto.etl; + +public record CategoryItemDto(String code, String name) { +} + diff --git a/src/main/java/com/earseo/core/dto/etl/CommonItemDto.java b/src/main/java/com/earseo/core/dto/etl/CommonItemDto.java new file mode 100644 index 0000000..f95c4a8 --- /dev/null +++ b/src/main/java/com/earseo/core/dto/etl/CommonItemDto.java @@ -0,0 +1,13 @@ +package com.earseo.core.dto.etl; + +public record CommonItemDto( + String title, + String addr1, + String addr2, + String mapX, + String mapY, + String modifiedTime, + String tel, + String mLevel, + String overview +) {} diff --git a/src/main/java/com/earseo/core/dto/etl/DetailItemDto.java b/src/main/java/com/earseo/core/dto/etl/DetailItemDto.java new file mode 100644 index 0000000..29585c2 --- /dev/null +++ b/src/main/java/com/earseo/core/dto/etl/DetailItemDto.java @@ -0,0 +1,4 @@ +package com.earseo.core.dto.etl; + +public record DetailItemDto(String usefee, String parking, String restdate, String usetime) { +} diff --git a/src/main/java/com/earseo/core/dto/etl/ImageItemDto.java b/src/main/java/com/earseo/core/dto/etl/ImageItemDto.java new file mode 100644 index 0000000..01a3385 --- /dev/null +++ b/src/main/java/com/earseo/core/dto/etl/ImageItemDto.java @@ -0,0 +1,4 @@ +package com.earseo.core.dto.etl; + +public record ImageItemDto(String imgrul, String smallimgurl) { +} diff --git a/src/main/java/com/earseo/core/dto/etl/MiddleDataDto.java b/src/main/java/com/earseo/core/dto/etl/MiddleDataDto.java new file mode 100644 index 0000000..968d661 --- /dev/null +++ b/src/main/java/com/earseo/core/dto/etl/MiddleDataDto.java @@ -0,0 +1,8 @@ +package com.earseo.core.dto.etl; + +public record MiddleDataDto(String contentId, String contentTypeId, String cat1, + String cat2, String cat3, String outl, String title, String addr1, String addr2, + String mapX, String mapY, String modifiedtime,String tel, String mLevel,String overview, + String originImgUrl, String smallImgUrl, String usetime, String restdate, String parking, String usefee + ) { +} diff --git a/src/main/java/com/earseo/core/entity/Category.java b/src/main/java/com/earseo/core/entity/Category.java new file mode 100644 index 0000000..944641b --- /dev/null +++ b/src/main/java/com/earseo/core/entity/Category.java @@ -0,0 +1,25 @@ +package com.earseo.core.entity; + +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Entity +@Getter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class Category { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + private String code; + private String name; +} diff --git a/src/main/java/com/earseo/core/entity/MiddleData.java b/src/main/java/com/earseo/core/entity/MiddleData.java new file mode 100644 index 0000000..880ec7f --- /dev/null +++ b/src/main/java/com/earseo/core/entity/MiddleData.java @@ -0,0 +1,104 @@ +package com.earseo.core.entity; + +import com.earseo.core.dto.etl.MiddleDataDto; +import jakarta.persistence.*; +import lombok.*; + +@Entity +@Table(name = "middle_data") +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class MiddleData { + + @Id + @Column(name = "content_id", nullable = false, unique = true) + private String contentId; + + @Column(name = "content_type_id") + private String contentTypeId; + + @Column(name = "cat1") + private String cat1; + + @Column(name = "cat2") + private String cat2; + + @Column(name = "cat3") + private String cat3; + + @Column(name = "outl", columnDefinition = "TEXT") + private String outl; + + @Column(name = "title", columnDefinition = "TEXT") + private String title; + + @Column(name = "addr1", columnDefinition = "TEXT") + private String addr1; + + @Column(name = "addr2", columnDefinition = "TEXT") + private String addr2; + + @Column(name = "map_x", columnDefinition = "TEXT") + private String mapX; + + @Column(name = "map_y", columnDefinition = "TEXT") + private String mapY; + + @Column(name = "modified_time", columnDefinition = "TEXT") + private String modifiedtime; + + @Column(name = "tel", columnDefinition = "TEXT") + private String tel; + + @Column(name = "m_level", columnDefinition = "TEXT") + private String mLevel; + + @Column(name = "overview", columnDefinition = "TEXT") + private String overview; + + @Column(name = "origin_img_url", columnDefinition = "TEXT") + private String originImgUrl; + + @Column(name = "small_img_url", columnDefinition = "TEXT") + private String smallImgUrl; + + @Column(name = "use_time", columnDefinition = "TEXT") + private String usetime; + + @Column(name = "rest_date", columnDefinition = "TEXT") + private String restdate; + + @Column(name = "parking", columnDefinition = "TEXT") + private String parking; + + @Column(name = "use_fee", columnDefinition = "TEXT") + private String usefee; + + + public MiddleData(MiddleDataDto dto) { + this.contentId = dto.contentId(); + this.contentTypeId = dto.contentTypeId(); + this.cat1 = dto.cat1(); + this.cat2 = dto.cat2(); + this.cat3 = dto.cat3(); + this.outl = dto.outl(); + this.title = dto.title(); + this.addr1 = dto.addr1(); + this.addr2 = dto.addr2(); + this.mapX = dto.mapX(); + this.mapY = dto.mapY(); + this.modifiedtime = dto.modifiedtime(); + this.tel = dto.tel(); + this.mLevel = dto.mLevel(); + this.overview = dto.overview(); + this.originImgUrl = dto.originImgUrl(); + this.smallImgUrl = dto.smallImgUrl(); + this.usetime = dto.usetime(); + this.restdate = dto.restdate(); + this.parking = dto.parking(); + this.usefee = dto.usefee(); + } +} diff --git a/src/main/java/com/earseo/core/repository/CategoryRepository.java b/src/main/java/com/earseo/core/repository/CategoryRepository.java new file mode 100644 index 0000000..7d7b8b8 --- /dev/null +++ b/src/main/java/com/earseo/core/repository/CategoryRepository.java @@ -0,0 +1,10 @@ +package com.earseo.core.repository; + +import com.earseo.core.entity.Category; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface CategoryRepository extends JpaRepository { + boolean existsByCode(String code); +} diff --git a/src/main/java/com/earseo/core/repository/MiddleRepository.java b/src/main/java/com/earseo/core/repository/MiddleRepository.java new file mode 100644 index 0000000..19cf5af --- /dev/null +++ b/src/main/java/com/earseo/core/repository/MiddleRepository.java @@ -0,0 +1,18 @@ +package com.earseo.core.repository; + +import com.earseo.core.entity.MiddleData; +import jakarta.transaction.Transactional; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +@Repository +public interface MiddleRepository extends JpaRepository { + + @Modifying + @Transactional + @Query("DELETE FROM MiddleData m WHERE m.contentTypeId = :contentTypeId") + int deleteByContentType(@Param("contentTypeId") String contentTypeId); +} diff --git a/src/main/java/com/earseo/core/service/MasterDataService.java b/src/main/java/com/earseo/core/service/MasterDataService.java index 4652378..b21f6bb 100644 --- a/src/main/java/com/earseo/core/service/MasterDataService.java +++ b/src/main/java/com/earseo/core/service/MasterDataService.java @@ -1,27 +1,45 @@ package com.earseo.core.service; -import com.earseo.core.dto.etl.FilteredDataDto; +import com.earseo.core.common.BaseResponse; +import com.earseo.core.dto.etl.*; +import com.earseo.core.entity.Category; +import com.earseo.core.entity.MiddleData; +import com.earseo.core.repository.CategoryRepository; +import com.earseo.core.repository.MiddleRepository; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import io.swagger.v3.core.util.Json; +import jakarta.transaction.Transactional; import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.ClassPathResource; +import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.client.RestClient; +import org.springframework.web.util.UriComponentsBuilder; import java.io.InputStream; +import java.net.URI; +import java.util.ArrayList; import java.util.List; +import java.util.Optional; @Service @RequiredArgsConstructor public class MasterDataService { private final ObjectMapper objectMapper; - private final RestClient restClient; + private final CategoryRepository categoryRepository; + private final MiddleRepository middleRepository; @Value("${API_KEY}") private String ApiKeys; + private String key; + private int index; + public List getRawInfo() { try { InputStream rawJson = new ClassPathResource("TourAPI_seoul.json").getInputStream(); @@ -44,4 +62,243 @@ public List getRawInfo() { } } + @Transactional + public void initData() { + List cat1s = fetchCategoryApi(null, null); + List allCategories = new ArrayList<>(); + allCategories.addAll(cat1s); + + for (CategoryItemDto cat1 : cat1s) { + List cat2s = fetchCategoryApi(cat1.code(), null); + allCategories.addAll(cat2s); + for (CategoryItemDto cat2 : cat2s) { + List cat3s = fetchCategoryApi(cat1.code(), cat2.code()); + allCategories.addAll(cat3s); + } + } + + List categories = allCategories.stream() + .map(c -> Category.builder() + .code(c.code()) + .name(c.name()) + .build()) + .toList(); + + categoryRepository.saveAll(categories); + } + + @Transactional + public List getMiddleData(List filteredData, int start) { + List middleData = new ArrayList<>(); + this.key = ApiKeys.split(",")[0]; + this.index = 0; + System.out.println(filteredData.size()); + for (FilteredDataDto filteredDataDto : filteredData) { + + String contentId = filteredDataDto.contentId(); + String contentTypeId = filteredDataDto.contentTypeId(); + + JsonNode common = fetchTourApi("https://apis.data.go.kr/B551011/KorService2/detailCommon2", contentId, null); + DetailItemDto detail = fetchTourDetailApi("https://apis.data.go.kr/B551011/KorService2/detailIntro2", contentId, contentTypeId); + JsonNode image = fetchTourApi("https://apis.data.go.kr/B551011/KorService2/detailImage2", contentId, null); + + ImageItemDto imageItemDto = null; + CommonItemDto commonItemDto = null; + + if(common == null || image == null) continue; + + JsonNode commonItems = common + .path("response") + .path("body") + .path("items") + .path("item"); + + JsonNode imageItems = image + .path("response") + .path("body") + .path("items") + .path("item"); + + JsonNode commonItem = commonItems.get(0); + JsonNode imageItem = imageItems.get(0); + + if(imageItem == null) imageItemDto = new ImageItemDto(null, null); + else imageItemDto = new ImageItemDto(imageItem.get("originimgurl").asText(),imageItem.get("smallimageurl").asText()); + + if(commonItem == null) commonItemDto = new CommonItemDto(null,null,null,null,null,null,null,null,null); + else commonItemDto = parseCommonItem(commonItem); + + middleData.add(new MiddleDataDto(contentId, contentTypeId, filteredDataDto.cat1(), filteredDataDto.cat2(), filteredDataDto.cat3(), + filteredDataDto.outl(), commonItemDto.title(),commonItemDto.addr1(), commonItemDto.addr2(), commonItemDto.mapX(), commonItemDto.mapY(), + commonItemDto.modifiedTime(), commonItemDto.tel(), commonItemDto.mLevel(),commonItemDto.overview(), + imageItemDto.imgrul(), imageItemDto.smallimgurl(), detail.usetime(), detail.restdate(), detail.parking(), detail.usefee() + )); + + } + List middleDataList = middleData.stream().map(MiddleData::new).toList(); + middleRepository.saveAll(middleDataList); + return middleData; + + } + + public JsonNode fetchTourApi(String url, String contentId, String contentTypeId) { + RestClient client = RestClient.create(); + + URI uri = UriComponentsBuilder + .fromHttpUrl(url) + .queryParam("serviceKey", this.key) + .queryParam("MobileApp", "AppTest") + .queryParam("MobileOS", "ETC") + .queryParam("pageNo", 1) + .queryParam("numOfRows", 10) + .queryParam("_type", "json") + .queryParamIfPresent("contentId", Optional.ofNullable(contentId)) + .queryParamIfPresent("contentTypeId", Optional.ofNullable(contentTypeId)) + .build(true) + .toUri(); + + JsonNode jsonNode = null; + + try { + jsonNode = client.get() + .uri(uri) + .retrieve() + .body(JsonNode.class); + + } catch (Exception e) { + if(this.index+1 != ApiKeys.split(",").length){ + this.key = ApiKeys.split(",")[index+1]; + this.index++; + return fetchTourApi(url, contentId, contentTypeId); + } + } + + return jsonNode; + } + + public DetailItemDto fetchTourDetailApi(String url, String contentId, String contentTypeId) { + RestClient client = RestClient.create(); + + URI uri = UriComponentsBuilder + .fromHttpUrl(url) + .queryParam("serviceKey", this.key) + .queryParam("MobileApp", "AppTest") + .queryParam("MobileOS", "ETC") + .queryParam("pageNo", 1) + .queryParam("numOfRows", 10) + .queryParam("_type", "json") + .queryParamIfPresent("contentId", Optional.ofNullable(contentId)) + .queryParamIfPresent("contentTypeId", Optional.ofNullable(contentTypeId)) + .build(true) + .toUri(); + JsonNode jsonNode = null; + try { + jsonNode = client.get() + .uri(uri) + .retrieve() + .body(JsonNode.class); + + } catch (Exception e) { + if(this.index+1 != ApiKeys.split(",").length){ + this.key = ApiKeys.split(",")[index+1]; + this.index++; + return fetchTourDetailApi(url, contentId, contentTypeId); + } + } + DetailItemDto detailItemDto; + + if(jsonNode == null) return detailItemDto = new DetailItemDto(null,null,null,null); + + JsonNode detailItem = jsonNode + .path("response") + .path("body") + .path("items") + .path("item") + .get(0); + + if(detailItem == null) return detailItemDto = new DetailItemDto(null,null,null,null); + switch (contentTypeId) { + case "12": + detailItemDto = new DetailItemDto(null, + detailItem.get("parking").asText(), detailItem.get("restdate").asText(), detailItem.get("usetime").asText()); + break; + case "14": + detailItemDto = new DetailItemDto(detailItem.get("usefee").asText(), + detailItem.get("parkingculture").asText(), detailItem.get("restdateculture").asText(), detailItem.get("usetimeculture").asText()); + break; + case "15": + detailItemDto = new DetailItemDto(detailItem.get("usetimefestival").asText(), + null, null, null); + break; + case "28": + detailItemDto = new DetailItemDto(detailItem.get("usefeeleports").asText(), + detailItem.get("parkingleports").asText(), detailItem.get("restdateleports").asText(), detailItem.get("usetimeleports").asText()); + break; + case "38": + detailItemDto = new DetailItemDto(null, + detailItem.get("parkingshopping").asText(), null, null); + break; + + default: + detailItemDto = new DetailItemDto(null, null, null, null); + } + + return detailItemDto; + } + + private CommonItemDto parseCommonItem(JsonNode commonItem) { + return new CommonItemDto( + commonItem.path("title").asText(), + commonItem.path("addr1").asText(), + commonItem.path("addr2").asText(), + commonItem.path("mapx").asText(), + commonItem.path("mapy").asText(), + commonItem.path("modifiedtime").asText(), + commonItem.path("tel").asText(), + commonItem.path("mlevel").asText(), + commonItem.path("overview").asText() + ); + } + + + public List fetchCategoryApi(String cat1, String cat2) { + String key = ApiKeys.split(",")[0]; + + RestClient client = RestClient.create(); + + URI uri = UriComponentsBuilder + .fromHttpUrl("https://apis.data.go.kr/B551011/KorService2/categoryCode2") + .queryParam("serviceKey", key) + .queryParam("MobileApp", "AppTest") + .queryParam("MobileOS", "ETC") + .queryParam("pageNo", 1) + .queryParam("numOfRows", 10) + .queryParam("_type", "json") + .queryParamIfPresent("cat1", Optional.ofNullable(cat1)) + .queryParamIfPresent("cat2", Optional.ofNullable(cat2)) + .build(true) + .toUri(); + try { + JsonNode jsonNode = client.get() + .uri(uri) + .retrieve() + .body(JsonNode.class); + JsonNode items = jsonNode + .path("response") + .path("body") + .path("items") + .path("item"); + + List list = new ArrayList<>(); + + for (JsonNode item : items) { + list.add(new CategoryItemDto(item.get("code").asText(), item.get("name").asText())); + } + return list; + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + } From 2432b6926861efcf866d9fd9b0b8c25f527b4171 Mon Sep 17 00:00:00 2001 From: haribonyam Date: Thu, 20 Nov 2025 17:14:02 +0900 Subject: [PATCH 3/6] =?UTF-8?q?[#1]=20feat=20:=20=EB=A7=88=EC=8A=A4?= =?UTF-8?q?=ED=84=B0=20=ED=85=8C=EC=9D=B4=EB=B8=94=20=EC=83=9D=EC=84=B1=20?= =?UTF-8?q?-=20=EC=B9=B4=ED=85=8C=EA=B3=A0=EB=A6=AC=20=EC=BD=94=EB=93=9C?= =?UTF-8?q?=EB=B2=88=ED=98=B8=20=EC=9D=B4=EB=A6=84=EA=B3=BC=20=EB=A7=A4?= =?UTF-8?q?=EC=B9=AD=ED=95=98=EC=97=AC=20=EC=86=8D=EC=84=B1=20=EC=83=9D?= =?UTF-8?q?=EC=84=B1=20-=20=EC=83=81=EC=84=B8=20=EC=A3=BC=EC=86=8C?= =?UTF-8?q?=EB=A1=9C=20=EC=84=9C=EC=9A=B8=EC=8B=9C=20+=2000=EA=B5=AC=20?= =?UTF-8?q?=EB=8B=A8=EC=9C=84=EC=9D=98=20=EC=A3=BC=EC=86=8C=20=EC=83=9D?= =?UTF-8?q?=EC=84=B1=20-=20=EA=B2=BD=EB=8F=84,=20=EC=9C=84=EB=8F=84?= =?UTF-8?q?=EB=A1=9C=20Geometry=20=EC=A0=95=EB=B3=B4(Points[x,y,ws4032])?= =?UTF-8?q?=20=EC=83=9D=EC=84=B1=20-=20=EB=A7=88=EC=8A=A4=ED=84=B0=20?= =?UTF-8?q?=ED=85=8C=EC=9D=B4=EB=B8=94=20=EC=A0=80=EC=9E=A5=20=EB=B0=8F=20?= =?UTF-8?q?json=20=EC=B6=94=EC=B6=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/controller/MasterDataController.java | 5 +- .../earseo/core/dto/etl/MasterItemDto.java | 10 ++ .../java/com/earseo/core/entity/Master.java | 99 +++++++++++ .../core/repository/MasterRepository.java | 9 + .../core/service/MasterDataService.java | 163 ++++++++++++++---- 5 files changed, 248 insertions(+), 38 deletions(-) create mode 100644 src/main/java/com/earseo/core/dto/etl/MasterItemDto.java create mode 100644 src/main/java/com/earseo/core/entity/Master.java create mode 100644 src/main/java/com/earseo/core/repository/MasterRepository.java diff --git a/src/main/java/com/earseo/core/controller/MasterDataController.java b/src/main/java/com/earseo/core/controller/MasterDataController.java index 2a09730..40e6300 100644 --- a/src/main/java/com/earseo/core/controller/MasterDataController.java +++ b/src/main/java/com/earseo/core/controller/MasterDataController.java @@ -11,6 +11,7 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RestController; +import java.io.IOException; import java.util.List; @RestController @@ -20,9 +21,10 @@ public class MasterDataController { private final MasterDataService masterDataService; @GetMapping("/admin/core/master/{start}") - public ResponseEntity> rawDataProcess(@PathVariable int start){ + public ResponseEntity> rawDataProcess(@PathVariable int start) throws IOException { List filteredData = masterDataService.getRawInfo(); List middleData = masterDataService.getMiddleData(filteredData,start); + masterDataService.createMasterTable(); return ResponseEntity.ok(BaseResponse.ok(null)); } @@ -31,4 +33,5 @@ public ResponseEntity> init(){ masterDataService.initData(); return ResponseEntity.ok(BaseResponse.ok(null)); } + } diff --git a/src/main/java/com/earseo/core/dto/etl/MasterItemDto.java b/src/main/java/com/earseo/core/dto/etl/MasterItemDto.java new file mode 100644 index 0000000..50a7369 --- /dev/null +++ b/src/main/java/com/earseo/core/dto/etl/MasterItemDto.java @@ -0,0 +1,10 @@ +package com.earseo.core.dto.etl; + +import org.locationtech.jts.geom.Point; + +public record MasterItemDto(String contentId, String contentTypeId, String cat1, + String cat2, String cat3, String ocat1, String ocat2, String ocat3, + String outl, String title, String addr1, String addr2, String addr3, + Double mapX, Double mapY, String modifiedtime, String tel, Integer mLevel, String overview, + String originImgUrl, String smallImgUrl, String usetime, String restdate, String parking, String usefee) { +} diff --git a/src/main/java/com/earseo/core/entity/Master.java b/src/main/java/com/earseo/core/entity/Master.java new file mode 100644 index 0000000..0ac42bf --- /dev/null +++ b/src/main/java/com/earseo/core/entity/Master.java @@ -0,0 +1,99 @@ +package com.earseo.core.entity; + +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import org.locationtech.jts.geom.Point; + +import java.awt.*; + +@Entity +@AllArgsConstructor +@NoArgsConstructor +@Builder @Getter +public class Master { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(name = "content_id", nullable = false, unique = true) + private String contentId; + + @Column(name = "content_type_id") + private String contentTypeId; + + @Column(name = "cat1") + private String cat1; + + @Column(name = "cat2") + private String cat2; + + @Column(name = "cat3") + private String cat3; + + @Column(name = "ocat1") + private String ocat1; + + @Column(name = "ocat2") + private String ocat2; + + @Column(name = "ocat3") + private String ocat3; + + @Column(name = "outl", columnDefinition = "TEXT") + private String outl; + + @Column(name = "title") + private String title; + + @Column(name = "addr1") + private String addr1; + + @Column(name = "addr2") + private String addr2; + + @Column(name = "addr3") + private String addr3; + + @Column(name = "map_x") + private Double mapX; + + @Column(name = "map_y") + private Double mapY; + + @Column(name = "modifiedtime") + private String modifiedtime; + + @Column(name = "tel") + private String tel; + + @Column(name = "m_level") + private Integer mLevel; + + @Column(name = "overview", columnDefinition = "TEXT") + private String overview; + + @Column(name = "origin_img_url", columnDefinition = "TEXT") + private String originImgUrl; + + @Column(name = "small_img_url", columnDefinition = "TEXT") + private String smallImgUrl; + + @Column(name = "use_time", columnDefinition = "TEXT") + private String usetime; + + @Column(name = "rest_date", columnDefinition = "TEXT") + private String restdate; + + @Column(name = "parking", columnDefinition = "TEXT") + private String parking; + + @Column(name = "use_fee", columnDefinition = "TEXT") + private String usefee; + + @Column(name = "geom", columnDefinition = "geometry(Point,4326)") + private Point geom; +} diff --git a/src/main/java/com/earseo/core/repository/MasterRepository.java b/src/main/java/com/earseo/core/repository/MasterRepository.java new file mode 100644 index 0000000..2b36ec1 --- /dev/null +++ b/src/main/java/com/earseo/core/repository/MasterRepository.java @@ -0,0 +1,9 @@ +package com.earseo.core.repository; + +import com.earseo.core.entity.Master; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface MasterRepository extends JpaRepository { +} diff --git a/src/main/java/com/earseo/core/service/MasterDataService.java b/src/main/java/com/earseo/core/service/MasterDataService.java index b21f6bb..97561df 100644 --- a/src/main/java/com/earseo/core/service/MasterDataService.java +++ b/src/main/java/com/earseo/core/service/MasterDataService.java @@ -3,15 +3,21 @@ import com.earseo.core.common.BaseResponse; import com.earseo.core.dto.etl.*; import com.earseo.core.entity.Category; +import com.earseo.core.entity.Master; import com.earseo.core.entity.MiddleData; import com.earseo.core.repository.CategoryRepository; +import com.earseo.core.repository.MasterRepository; import com.earseo.core.repository.MiddleRepository; +import com.fasterxml.jackson.core.StreamWriteConstraints; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import io.swagger.v3.core.util.Json; import jakarta.transaction.Transactional; import lombok.RequiredArgsConstructor; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.Point; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.ClassPathResource; import org.springframework.http.ResponseEntity; @@ -20,6 +26,7 @@ import org.springframework.web.client.RestClient; import org.springframework.web.util.UriComponentsBuilder; +import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.util.ArrayList; @@ -33,6 +40,8 @@ public class MasterDataService { private final ObjectMapper objectMapper; private final CategoryRepository categoryRepository; private final MiddleRepository middleRepository; + private final MasterRepository masterRepository; + private final GeometryFactory geometryFactory = new GeometryFactory(); @Value("${API_KEY}") private String ApiKeys; @@ -105,7 +114,7 @@ public List getMiddleData(List filteredData, int ImageItemDto imageItemDto = null; CommonItemDto commonItemDto = null; - if(common == null || image == null) continue; + if (common == null || image == null) continue; JsonNode commonItems = common .path("response") @@ -122,15 +131,17 @@ public List getMiddleData(List filteredData, int JsonNode commonItem = commonItems.get(0); JsonNode imageItem = imageItems.get(0); - if(imageItem == null) imageItemDto = new ImageItemDto(null, null); - else imageItemDto = new ImageItemDto(imageItem.get("originimgurl").asText(),imageItem.get("smallimageurl").asText()); + if (imageItem == null) imageItemDto = new ImageItemDto(null, null); + else + imageItemDto = new ImageItemDto(imageItem.get("originimgurl").asText(), imageItem.get("smallimageurl").asText()); - if(commonItem == null) commonItemDto = new CommonItemDto(null,null,null,null,null,null,null,null,null); + if (commonItem == null) + commonItemDto = new CommonItemDto(null, null, null, null, null, null, null, null, null); else commonItemDto = parseCommonItem(commonItem); middleData.add(new MiddleDataDto(contentId, contentTypeId, filteredDataDto.cat1(), filteredDataDto.cat2(), filteredDataDto.cat3(), - filteredDataDto.outl(), commonItemDto.title(),commonItemDto.addr1(), commonItemDto.addr2(), commonItemDto.mapX(), commonItemDto.mapY(), - commonItemDto.modifiedTime(), commonItemDto.tel(), commonItemDto.mLevel(),commonItemDto.overview(), + filteredDataDto.outl(), commonItemDto.title(), commonItemDto.addr1(), commonItemDto.addr2(), commonItemDto.mapX(), commonItemDto.mapY(), + commonItemDto.modifiedTime(), commonItemDto.tel(), commonItemDto.mLevel(), commonItemDto.overview(), imageItemDto.imgrul(), imageItemDto.smallimgurl(), detail.usetime(), detail.restdate(), detail.parking(), detail.usefee() )); @@ -166,8 +177,8 @@ public JsonNode fetchTourApi(String url, String contentId, String contentTypeId) .body(JsonNode.class); } catch (Exception e) { - if(this.index+1 != ApiKeys.split(",").length){ - this.key = ApiKeys.split(",")[index+1]; + if (this.index + 1 != ApiKeys.split(",").length) { + this.key = ApiKeys.split(",")[index + 1]; this.index++; return fetchTourApi(url, contentId, contentTypeId); } @@ -199,15 +210,15 @@ public DetailItemDto fetchTourDetailApi(String url, String contentId, String con .body(JsonNode.class); } catch (Exception e) { - if(this.index+1 != ApiKeys.split(",").length){ - this.key = ApiKeys.split(",")[index+1]; + if (this.index + 1 != ApiKeys.split(",").length) { + this.key = ApiKeys.split(",")[index + 1]; this.index++; return fetchTourDetailApi(url, contentId, contentTypeId); } } DetailItemDto detailItemDto; - if(jsonNode == null) return detailItemDto = new DetailItemDto(null,null,null,null); + if (jsonNode == null) return detailItemDto = new DetailItemDto(null, null, null, null); JsonNode detailItem = jsonNode .path("response") @@ -216,32 +227,32 @@ public DetailItemDto fetchTourDetailApi(String url, String contentId, String con .path("item") .get(0); - if(detailItem == null) return detailItemDto = new DetailItemDto(null,null,null,null); - switch (contentTypeId) { - case "12": - detailItemDto = new DetailItemDto(null, - detailItem.get("parking").asText(), detailItem.get("restdate").asText(), detailItem.get("usetime").asText()); - break; - case "14": - detailItemDto = new DetailItemDto(detailItem.get("usefee").asText(), - detailItem.get("parkingculture").asText(), detailItem.get("restdateculture").asText(), detailItem.get("usetimeculture").asText()); - break; - case "15": - detailItemDto = new DetailItemDto(detailItem.get("usetimefestival").asText(), - null, null, null); - break; - case "28": - detailItemDto = new DetailItemDto(detailItem.get("usefeeleports").asText(), - detailItem.get("parkingleports").asText(), detailItem.get("restdateleports").asText(), detailItem.get("usetimeleports").asText()); - break; - case "38": - detailItemDto = new DetailItemDto(null, - detailItem.get("parkingshopping").asText(), null, null); - break; - - default: - detailItemDto = new DetailItemDto(null, null, null, null); - } + if (detailItem == null) return detailItemDto = new DetailItemDto(null, null, null, null); + switch (contentTypeId) { + case "12": + detailItemDto = new DetailItemDto(null, + detailItem.get("parking").asText(), detailItem.get("restdate").asText(), detailItem.get("usetime").asText()); + break; + case "14": + detailItemDto = new DetailItemDto(detailItem.get("usefee").asText(), + detailItem.get("parkingculture").asText(), detailItem.get("restdateculture").asText(), detailItem.get("usetimeculture").asText()); + break; + case "15": + detailItemDto = new DetailItemDto(detailItem.get("usetimefestival").asText(), + null, null, null); + break; + case "28": + detailItemDto = new DetailItemDto(detailItem.get("usefeeleports").asText(), + detailItem.get("parkingleports").asText(), detailItem.get("restdateleports").asText(), detailItem.get("usetimeleports").asText()); + break; + case "38": + detailItemDto = new DetailItemDto(null, + detailItem.get("parkingshopping").asText(), null, null); + break; + + default: + detailItemDto = new DetailItemDto(null, null, null, null); + } return detailItemDto; } @@ -301,4 +312,82 @@ public List fetchCategoryApi(String cat1, String cat2) { return null; } + @Transactional + public void createMasterTable() throws IOException { + List middleDataList = middleRepository.findAll(); + List masterItemDtos = new ArrayList<>(); + List masters = new ArrayList<>(); + + for (MiddleData middle : middleDataList) { + String addr3 = null; + if (middle.getAddr1() != null && middle.getAddr1().startsWith("서울특별시")) { + addr3 = "서울시 " + middle.getAddr1().split(" ")[1]; + } + + Point geom = null; + try { + if (middle.getMapX() != null && middle.getMapY() != null) { + double x = Double.parseDouble(middle.getMapX()); + double y = Double.parseDouble(middle.getMapY()); + geom = geometryFactory.createPoint(new org.locationtech.jts.geom.Coordinate(x, y)); + } + } catch (NumberFormatException e) { + geom = null; + } + + Master master = Master.builder() + .contentId(middle.getContentId()) + .contentTypeId(middle.getContentTypeId()) + .cat1(middle.getCat1()) + .cat2(middle.getCat2()) + .cat3(middle.getCat3()) + .ocat1(middle.getCat1()) + .ocat2(middle.getCat2()) + .ocat3(middle.getCat3()) + .outl(middle.getOutl()) + .title(middle.getTitle()) + .addr1(middle.getAddr1()) + .addr2(middle.getAddr2()) + .addr3(addr3) + .mapX(middle.getMapX() != null ? Double.valueOf(middle.getMapX()) : null) + .mapY(middle.getMapY() != null ? Double.valueOf(middle.getMapY()) : null) + .modifiedtime(middle.getModifiedtime()) + .tel(middle.getTel()) + .mLevel( + (middle.getMLevel() != null && !middle.getMLevel().isBlank()) ? Integer.parseInt(middle.getMLevel().trim()) : null + ) + .overview(middle.getOverview()) + .originImgUrl(middle.getOriginImgUrl()) + .smallImgUrl(middle.getSmallImgUrl()) + .usetime(middle.getUsetime()) + .restdate(middle.getRestdate()) + .parking(middle.getParking()) + .usefee(middle.getUsefee()) + .geom(geom) + .build(); + + masters.add(master); + + MasterItemDto dto = new MasterItemDto( + master.getContentId(), master.getContentTypeId(), master.getCat1(), master.getCat2(), master.getCat3(), + master.getOcat1(), master.getOcat2(), master.getOcat3(), master.getOutl(), master.getTitle(), + master.getAddr1(), master.getAddr2(), master.getAddr3(), master.getMapX(), master.getMapY(), + master.getModifiedtime(), master.getTel(), master.getMLevel(), master.getOverview(), master.getOriginImgUrl(), + master.getSmallImgUrl(), master.getUsetime(), master.getRestdate(), master.getParking(), master.getUsefee() + ); + + masterItemDtos.add(dto); + } + + masterRepository.saveAll(masters); + objectMapper + .getFactory() + .setStreamWriteConstraints( + StreamWriteConstraints.builder() + .maxNestingDepth(3000) + .build() + ); + objectMapper.enable(SerializationFeature.INDENT_OUTPUT); + objectMapper.writeValue(new java.io.File("master_data.json"), masterItemDtos); // S3 저장으로 리팩토링 예정 + } } From a3110f40365e35f95b63e5c6cc93f0451ae72ba4 Mon Sep 17 00:00:00 2001 From: haribonyam Date: Wed, 26 Nov 2025 14:46:11 +0900 Subject: [PATCH 4/6] =?UTF-8?q?[#1]=20feat:=20=EB=A7=88=EC=8A=A4=ED=84=B0?= =?UTF-8?q?=20=ED=85=8C=EC=9D=B4=EB=B8=94=20=EB=8D=B0=EC=9D=B4=ED=84=B0=20?= =?UTF-8?q?=EC=99=B8=EB=B6=80=20=EC=A0=80=EC=9E=A5=EC=86=8C=EB=A1=9C=20?= =?UTF-8?q?=EC=A0=80=EC=9E=A5=20-=20=EB=A7=88=EC=8A=A4=ED=84=B0=20?= =?UTF-8?q?=ED=85=8C=EC=9D=B4=EB=B8=94=20json=20=ED=8C=8C=EC=9D=BC=20S3?= =?UTF-8?q?=EC=97=90=20=EC=A0=80=EC=9E=A5=20-=20=EC=A4=91=EA=B0=84=20?= =?UTF-8?q?=ED=85=8C=EC=9D=B4=EB=B8=94=20=EC=83=9D=EC=84=B1=20=EC=BD=94?= =?UTF-8?q?=EB=93=9C=20=EB=A6=AC=ED=8C=A9=ED=86=A0=EB=A7=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.gradle | 2 + .../earseo/core/common/config/S3Config.java | 29 ++ .../core/controller/MasterDataController.java | 8 +- .../core/service/MasterDataService.java | 376 ++++++++++-------- src/main/resources/application-local.yaml | 12 + 5 files changed, 246 insertions(+), 181 deletions(-) create mode 100644 src/main/java/com/earseo/core/common/config/S3Config.java diff --git a/build.gradle b/build.gradle index a09bb5f..8ce312e 100644 --- a/build.gradle +++ b/build.gradle @@ -30,6 +30,8 @@ dependencies { implementation 'org.locationtech.jts:jts-core:1.19.0' implementation 'com.fasterxml.jackson.core:jackson-databind' + implementation 'com.amazonaws:aws-java-sdk-s3:1.12.767' //S3 + implementation 'org.springframework.boot:spring-boot-starter-data-jpa' implementation 'org.springframework.boot:spring-boot-starter-validation' compileOnly 'org.projectlombok:lombok' diff --git a/src/main/java/com/earseo/core/common/config/S3Config.java b/src/main/java/com/earseo/core/common/config/S3Config.java new file mode 100644 index 0000000..6c0b4f0 --- /dev/null +++ b/src/main/java/com/earseo/core/common/config/S3Config.java @@ -0,0 +1,29 @@ +package com.earseo.core.common.config; + +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class S3Config { + @Value("${cloud.aws.credentials.access-key}") + private String accessKey; + @Value("${cloud.aws.credentials.secret-key}") + private String secretKey; + @Value("${cloud.aws.region.static}") + private String region; + + @Bean + public AmazonS3 amazonS3() { + BasicAWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); + return AmazonS3ClientBuilder + .standard() + .withRegion(region) + .withCredentials(new AWSStaticCredentialsProvider(credentials)) + .build(); + } +} \ No newline at end of file diff --git a/src/main/java/com/earseo/core/controller/MasterDataController.java b/src/main/java/com/earseo/core/controller/MasterDataController.java index 40e6300..2e599fe 100644 --- a/src/main/java/com/earseo/core/controller/MasterDataController.java +++ b/src/main/java/com/earseo/core/controller/MasterDataController.java @@ -4,11 +4,9 @@ import com.earseo.core.dto.etl.FilteredDataDto; import com.earseo.core.dto.etl.MiddleDataDto; import com.earseo.core.service.MasterDataService; -import lombok.Getter; import lombok.RequiredArgsConstructor; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RestController; import java.io.IOException; @@ -20,10 +18,10 @@ public class MasterDataController { private final MasterDataService masterDataService; - @GetMapping("/admin/core/master/{start}") - public ResponseEntity> rawDataProcess(@PathVariable int start) throws IOException { + @GetMapping("/admin/core/master") + public ResponseEntity> rawDataProcess() throws IOException { List filteredData = masterDataService.getRawInfo(); - List middleData = masterDataService.getMiddleData(filteredData,start); + List middleData = masterDataService.getMiddleData(filteredData); masterDataService.createMasterTable(); return ResponseEntity.ok(BaseResponse.ok(null)); } diff --git a/src/main/java/com/earseo/core/service/MasterDataService.java b/src/main/java/com/earseo/core/service/MasterDataService.java index 97561df..2998471 100644 --- a/src/main/java/com/earseo/core/service/MasterDataService.java +++ b/src/main/java/com/earseo/core/service/MasterDataService.java @@ -1,5 +1,7 @@ package com.earseo.core.service; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.PutObjectRequest; import com.earseo.core.common.BaseResponse; import com.earseo.core.dto.etl.*; import com.earseo.core.entity.Category; @@ -26,9 +28,12 @@ import org.springframework.web.client.RestClient; import org.springframework.web.util.UriComponentsBuilder; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.URI; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -42,9 +47,12 @@ public class MasterDataService { private final MiddleRepository middleRepository; private final MasterRepository masterRepository; private final GeometryFactory geometryFactory = new GeometryFactory(); + private final AmazonS3 amazonS3; @Value("${API_KEY}") private String ApiKeys; + @Value(("${cloud.aws.s3.bucket}")) + private String bucketName; private String key; private int index; @@ -61,6 +69,7 @@ public List getRawInfo() { List filtered = rawJsonDtos.stream() .filter(dto -> !dto.contentTypeId().equals("25")) .filter(dto -> !dto.contentTypeId().equals("32")) + .filter(dto -> !dto.contentTypeId().equals("39")) .filter(dto -> !dto.cat3().equals("A04011000")) .toList(); @@ -96,181 +105,6 @@ public void initData() { categoryRepository.saveAll(categories); } - @Transactional - public List getMiddleData(List filteredData, int start) { - List middleData = new ArrayList<>(); - this.key = ApiKeys.split(",")[0]; - this.index = 0; - System.out.println(filteredData.size()); - for (FilteredDataDto filteredDataDto : filteredData) { - - String contentId = filteredDataDto.contentId(); - String contentTypeId = filteredDataDto.contentTypeId(); - - JsonNode common = fetchTourApi("https://apis.data.go.kr/B551011/KorService2/detailCommon2", contentId, null); - DetailItemDto detail = fetchTourDetailApi("https://apis.data.go.kr/B551011/KorService2/detailIntro2", contentId, contentTypeId); - JsonNode image = fetchTourApi("https://apis.data.go.kr/B551011/KorService2/detailImage2", contentId, null); - - ImageItemDto imageItemDto = null; - CommonItemDto commonItemDto = null; - - if (common == null || image == null) continue; - - JsonNode commonItems = common - .path("response") - .path("body") - .path("items") - .path("item"); - - JsonNode imageItems = image - .path("response") - .path("body") - .path("items") - .path("item"); - - JsonNode commonItem = commonItems.get(0); - JsonNode imageItem = imageItems.get(0); - - if (imageItem == null) imageItemDto = new ImageItemDto(null, null); - else - imageItemDto = new ImageItemDto(imageItem.get("originimgurl").asText(), imageItem.get("smallimageurl").asText()); - - if (commonItem == null) - commonItemDto = new CommonItemDto(null, null, null, null, null, null, null, null, null); - else commonItemDto = parseCommonItem(commonItem); - - middleData.add(new MiddleDataDto(contentId, contentTypeId, filteredDataDto.cat1(), filteredDataDto.cat2(), filteredDataDto.cat3(), - filteredDataDto.outl(), commonItemDto.title(), commonItemDto.addr1(), commonItemDto.addr2(), commonItemDto.mapX(), commonItemDto.mapY(), - commonItemDto.modifiedTime(), commonItemDto.tel(), commonItemDto.mLevel(), commonItemDto.overview(), - imageItemDto.imgrul(), imageItemDto.smallimgurl(), detail.usetime(), detail.restdate(), detail.parking(), detail.usefee() - )); - - } - List middleDataList = middleData.stream().map(MiddleData::new).toList(); - middleRepository.saveAll(middleDataList); - return middleData; - - } - - public JsonNode fetchTourApi(String url, String contentId, String contentTypeId) { - RestClient client = RestClient.create(); - - URI uri = UriComponentsBuilder - .fromHttpUrl(url) - .queryParam("serviceKey", this.key) - .queryParam("MobileApp", "AppTest") - .queryParam("MobileOS", "ETC") - .queryParam("pageNo", 1) - .queryParam("numOfRows", 10) - .queryParam("_type", "json") - .queryParamIfPresent("contentId", Optional.ofNullable(contentId)) - .queryParamIfPresent("contentTypeId", Optional.ofNullable(contentTypeId)) - .build(true) - .toUri(); - - JsonNode jsonNode = null; - - try { - jsonNode = client.get() - .uri(uri) - .retrieve() - .body(JsonNode.class); - - } catch (Exception e) { - if (this.index + 1 != ApiKeys.split(",").length) { - this.key = ApiKeys.split(",")[index + 1]; - this.index++; - return fetchTourApi(url, contentId, contentTypeId); - } - } - - return jsonNode; - } - - public DetailItemDto fetchTourDetailApi(String url, String contentId, String contentTypeId) { - RestClient client = RestClient.create(); - - URI uri = UriComponentsBuilder - .fromHttpUrl(url) - .queryParam("serviceKey", this.key) - .queryParam("MobileApp", "AppTest") - .queryParam("MobileOS", "ETC") - .queryParam("pageNo", 1) - .queryParam("numOfRows", 10) - .queryParam("_type", "json") - .queryParamIfPresent("contentId", Optional.ofNullable(contentId)) - .queryParamIfPresent("contentTypeId", Optional.ofNullable(contentTypeId)) - .build(true) - .toUri(); - JsonNode jsonNode = null; - try { - jsonNode = client.get() - .uri(uri) - .retrieve() - .body(JsonNode.class); - - } catch (Exception e) { - if (this.index + 1 != ApiKeys.split(",").length) { - this.key = ApiKeys.split(",")[index + 1]; - this.index++; - return fetchTourDetailApi(url, contentId, contentTypeId); - } - } - DetailItemDto detailItemDto; - - if (jsonNode == null) return detailItemDto = new DetailItemDto(null, null, null, null); - - JsonNode detailItem = jsonNode - .path("response") - .path("body") - .path("items") - .path("item") - .get(0); - - if (detailItem == null) return detailItemDto = new DetailItemDto(null, null, null, null); - switch (contentTypeId) { - case "12": - detailItemDto = new DetailItemDto(null, - detailItem.get("parking").asText(), detailItem.get("restdate").asText(), detailItem.get("usetime").asText()); - break; - case "14": - detailItemDto = new DetailItemDto(detailItem.get("usefee").asText(), - detailItem.get("parkingculture").asText(), detailItem.get("restdateculture").asText(), detailItem.get("usetimeculture").asText()); - break; - case "15": - detailItemDto = new DetailItemDto(detailItem.get("usetimefestival").asText(), - null, null, null); - break; - case "28": - detailItemDto = new DetailItemDto(detailItem.get("usefeeleports").asText(), - detailItem.get("parkingleports").asText(), detailItem.get("restdateleports").asText(), detailItem.get("usetimeleports").asText()); - break; - case "38": - detailItemDto = new DetailItemDto(null, - detailItem.get("parkingshopping").asText(), null, null); - break; - - default: - detailItemDto = new DetailItemDto(null, null, null, null); - } - - return detailItemDto; - } - - private CommonItemDto parseCommonItem(JsonNode commonItem) { - return new CommonItemDto( - commonItem.path("title").asText(), - commonItem.path("addr1").asText(), - commonItem.path("addr2").asText(), - commonItem.path("mapx").asText(), - commonItem.path("mapy").asText(), - commonItem.path("modifiedtime").asText(), - commonItem.path("tel").asText(), - commonItem.path("mlevel").asText(), - commonItem.path("overview").asText() - ); - } - public List fetchCategoryApi(String cat1, String cat2) { String key = ApiKeys.split(",")[0]; @@ -379,6 +213,8 @@ public void createMasterTable() throws IOException { masterItemDtos.add(dto); } + File jsonFile = new File("master_data.json"); + masterRepository.saveAll(masters); objectMapper .getFactory() @@ -387,7 +223,195 @@ public void createMasterTable() throws IOException { .maxNestingDepth(3000) .build() ); + objectMapper.enable(SerializationFeature.INDENT_OUTPUT); - objectMapper.writeValue(new java.io.File("master_data.json"), masterItemDtos); // S3 저장으로 리팩토링 예정 + objectMapper.writeValue(jsonFile, masterItemDtos); // S3 저장으로 리팩토링 예정 + + String date = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyyMMdd")); + String s3Key = "master/master_data_" + date + ".json"; + + amazonS3.putObject( + new PutObjectRequest( + bucketName, + s3Key, + jsonFile + ) + ); + } + + @Transactional + public List getMiddleData(List filteredData) { + + this.key = ApiKeys.split(",")[0]; + this.index = 0; + + List middleDataList = new ArrayList<>(); + + for (FilteredDataDto filtered : filteredData) { + + String contentId = filtered.contentId(); + String contentTypeId = filtered.contentTypeId(); + + JsonNode commonNode = fetchTourApi( + "https://apis.data.go.kr/B551011/KorService2/detailCommon2", + contentId, + null + ); + + JsonNode detailNode = fetchTourApi( + "https://apis.data.go.kr/B551011/KorService2/detailIntro2", + contentId, + contentTypeId + ); + + JsonNode imageNode = fetchTourApi( + "https://apis.data.go.kr/B551011/KorService2/detailImage2", + contentId, + null + ); + + if (commonNode == null || detailNode == null || imageNode == null) continue; + + JsonNode commonItem = getItem(commonNode); + JsonNode detailItem = getItem(detailNode); + JsonNode imageItem = getItem(imageNode); + + CommonItemDto commonDto = parseCommon(commonItem); + DetailItemDto detailDto = parseDetail(detailItem, contentTypeId); + ImageItemDto imageDto = parseImage(imageItem); + + MiddleDataDto dto = new MiddleDataDto( + contentId, + contentTypeId, + filtered.cat1(), + filtered.cat2(), + filtered.cat3(), + filtered.outl(), + commonDto.title(), + commonDto.addr1(), + commonDto.addr2(), + commonDto.mapX(), + commonDto.mapY(), + commonDto.modifiedTime(), + commonDto.tel(), + commonDto.mLevel(), + commonDto.overview(), + imageDto.imgrul(), + imageDto.smallimgurl(), + detailDto.usetime(), + detailDto.restdate(), + detailDto.parking(), + detailDto.usefee() + ); + + middleDataList.add(dto); + } + + middleRepository.saveAll( + middleDataList.stream() + .map(MiddleData::new) + .toList() + ); + + return middleDataList; + } + + public JsonNode fetchTourApi(String url, String contentId, String contentTypeId) { + + RestClient client = RestClient.create(); + + URI uri = UriComponentsBuilder + .fromHttpUrl(url) + .queryParam("serviceKey", this.key) + .queryParam("MobileApp", "AppTest") + .queryParam("MobileOS", "ETC") + .queryParam("pageNo", 1) + .queryParam("numOfRows", 10) + .queryParam("_type", "json") + .queryParamIfPresent("contentId", Optional.ofNullable(contentId)) + .queryParamIfPresent("contentTypeId", Optional.ofNullable(contentTypeId)) + .build(true) + .toUri(); + + try { + return client.get().uri(uri).retrieve().body(JsonNode.class); + + } catch (Exception e) { + // key 변경 후 retry + if (this.index + 1 < ApiKeys.split(",").length) { + this.index++; + this.key = ApiKeys.split(",")[this.index]; + return fetchTourApi(url, contentId, contentTypeId); + } + return null; + } + } + + private JsonNode getItem(JsonNode root) { + return root.path("response").path("body") + .path("items").path("item").get(0); + } + + private CommonItemDto parseCommon(JsonNode item) { + if (item == null) { + return new CommonItemDto(null, null, null, null, null, null, null, null, null); + } + + return new CommonItemDto( + item.path("title").asText(), + item.path("addr1").asText(), + item.path("addr2").asText(), + item.path("mapx").asText(), + item.path("mapy").asText(), + item.path("modifiedtime").asText(), + item.path("tel").asText(), + item.path("mlevel").asText(), + item.path("overview").asText() + ); + } + + private ImageItemDto parseImage(JsonNode item) { + if (item == null) return new ImageItemDto(null, null); + + return new ImageItemDto( + item.path("originimgurl").asText(null), + item.path("smallimageurl").asText(null) + ); + } + + private DetailItemDto parseDetail(JsonNode item, String typeId) { + if (item == null) return new DetailItemDto(null, null, null, null); + + return switch (typeId) { + case "12" -> new DetailItemDto( + null, + item.path("parking").asText(), + item.path("restdate").asText(), + item.path("usetime").asText() + ); + case "14" -> new DetailItemDto( + item.path("usefee").asText(), + item.path("parkingculture").asText(), + item.path("restdateculture").asText(), + item.path("usetimeculture").asText() + ); + case "15" -> new DetailItemDto( + item.path("usetimefestival").asText(), + null, null, null + ); + case "28" -> new DetailItemDto( + item.path("usefeeleports").asText(), + item.path("parkingleports").asText(), + item.path("restdateleports").asText(), + item.path("usetimeleports").asText() + ); + case "38" -> new DetailItemDto( + null, + item.path("parkingshopping").asText(), + null, + null + ); + default -> new DetailItemDto(null, null, null, null); + }; } } diff --git a/src/main/resources/application-local.yaml b/src/main/resources/application-local.yaml index 584bee0..153a1d2 100644 --- a/src/main/resources/application-local.yaml +++ b/src/main/resources/application-local.yaml @@ -64,6 +64,18 @@ management: tags: application: ${spring.application.name} +cloud: + aws: + credentials: + access-key: ${AWS_CREDENTIAL_ACCESS_KEY} + secret-key: ${AWS_CREDENTIAL_SECRET_KEY} + region: + static: ap-northeast-2 + s3: + bucket: ${AWS_S3_BUCKET} + cloudfront: + domain: ${AWS_CLOUDFRONT_DOMAIN} + tracing: enabled: false From 7f303c10a0bf08660d03b05ce624e0cdc39715ea Mon Sep 17 00:00:00 2001 From: haribonyam Date: Wed, 26 Nov 2025 15:18:04 +0900 Subject: [PATCH 5/6] =?UTF-8?q?[#1]=20feat:=20=ED=85=8C=EC=8A=A4=ED=8A=B8?= =?UTF-8?q?=EC=9A=A9=20CI/CD=20=ED=99=98=EA=B2=BD=EB=B3=80=EC=88=98=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/resources/application-test.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main/resources/application-test.yaml b/src/main/resources/application-test.yaml index 33cac77..09c0014 100644 --- a/src/main/resources/application-test.yaml +++ b/src/main/resources/application-test.yaml @@ -61,3 +61,15 @@ management: logging: level: org.hibernate.SQL: ${LOGGING_LEVEL_ORG_HIBERNATE_SQL:debug} + +cloud: + aws: + credentials: + access-key: accesskey + secret-key: secretkey + region: + static: ap-northeast-2 + s3: + bucket: bucket + cloudfront: + domain: domain \ No newline at end of file From aa57765249f9ac124378a11081db377390389e73 Mon Sep 17 00:00:00 2001 From: haribonyam Date: Wed, 26 Nov 2025 15:23:45 +0900 Subject: [PATCH 6/6] =?UTF-8?q?[#1]=20feat:=20=ED=85=8C=EC=8A=A4=ED=8A=B8?= =?UTF-8?q?=EC=9A=A9=20CI/CD=20=ED=99=98=EA=B2=BD=EB=B3=80=EC=88=98=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/com/earseo/core/service/MasterDataService.java | 2 +- src/main/resources/application-local.yaml | 3 +++ src/main/resources/application-test.yaml | 5 ++++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/earseo/core/service/MasterDataService.java b/src/main/java/com/earseo/core/service/MasterDataService.java index 2998471..adb1bc6 100644 --- a/src/main/java/com/earseo/core/service/MasterDataService.java +++ b/src/main/java/com/earseo/core/service/MasterDataService.java @@ -49,7 +49,7 @@ public class MasterDataService { private final GeometryFactory geometryFactory = new GeometryFactory(); private final AmazonS3 amazonS3; - @Value("${API_KEY}") + @Value("${api.key}") private String ApiKeys; @Value(("${cloud.aws.s3.bucket}")) private String bucketName; diff --git a/src/main/resources/application-local.yaml b/src/main/resources/application-local.yaml index 153a1d2..7af9a03 100644 --- a/src/main/resources/application-local.yaml +++ b/src/main/resources/application-local.yaml @@ -83,3 +83,6 @@ cloud: logging: level: org.hibernate.SQL: ${LOGGING_LEVEL_ORG_HIBERNATE_SQL:debug} + +api: + key: ${API_KEY} diff --git a/src/main/resources/application-test.yaml b/src/main/resources/application-test.yaml index 09c0014..baa0d53 100644 --- a/src/main/resources/application-test.yaml +++ b/src/main/resources/application-test.yaml @@ -72,4 +72,7 @@ cloud: s3: bucket: bucket cloudfront: - domain: domain \ No newline at end of file + domain: domain + +api: + key: key \ No newline at end of file