diff --git a/.gitignore b/.gitignore index c2065bc..4ea4adf 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,6 @@ out/ ### VS Code ### .vscode/ + +# macOS +.DS_Store \ No newline at end of file diff --git a/build.gradle b/build.gradle index 53d8b65..8ce312e 100644 --- a/build.gradle +++ b/build.gradle @@ -22,6 +22,16 @@ dependencies { implementation 'org.springframework.boot:spring-boot-starter-web' implementation 'org.springframework.boot:spring-boot-starter-actuator' runtimeOnly 'io.micrometer:micrometer-registry-prometheus' + + implementation 'org.springframework.boot:spring-boot-starter-data-jpa' + runtimeOnly 'org.postgresql:postgresql' + + implementation 'org.hibernate.orm:hibernate-spatial' + implementation 'org.locationtech.jts:jts-core:1.19.0' + implementation 'com.fasterxml.jackson.core:jackson-databind' + + implementation 'com.amazonaws:aws-java-sdk-s3:1.12.767' //S3 + implementation 'org.springframework.boot:spring-boot-starter-data-jpa' implementation 'org.springframework.boot:spring-boot-starter-validation' compileOnly 'org.projectlombok:lombok' diff --git a/src/main/java/com/earseo/core/common/config/RestClientConfig.java b/src/main/java/com/earseo/core/common/config/RestClientConfig.java new file mode 100644 index 0000000..fb0f6d4 --- /dev/null +++ b/src/main/java/com/earseo/core/common/config/RestClientConfig.java @@ -0,0 +1,14 @@ +package com.earseo.core.common.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.client.RestClient; + +@Configuration +public class RestClientConfig { + + @Bean + public RestClient.Builder restClientBuilder() { + return RestClient.builder(); + } +} \ No newline at end of file diff --git a/src/main/java/com/earseo/core/common/config/S3Config.java b/src/main/java/com/earseo/core/common/config/S3Config.java new file mode 100644 index 0000000..6c0b4f0 --- /dev/null +++ b/src/main/java/com/earseo/core/common/config/S3Config.java @@ -0,0 +1,29 @@ +package com.earseo.core.common.config; + +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class S3Config { + @Value("${cloud.aws.credentials.access-key}") + private String accessKey; + @Value("${cloud.aws.credentials.secret-key}") + private String secretKey; + @Value("${cloud.aws.region.static}") + private String region; + + @Bean + public AmazonS3 amazonS3() { + BasicAWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); + return AmazonS3ClientBuilder + .standard() + .withRegion(region) + .withCredentials(new AWSStaticCredentialsProvider(credentials)) + .build(); + } +} \ No newline at end of file diff --git a/src/main/java/com/earseo/core/controller/MasterDataController.java b/src/main/java/com/earseo/core/controller/MasterDataController.java new file mode 100644 index 0000000..2e599fe --- /dev/null +++ b/src/main/java/com/earseo/core/controller/MasterDataController.java @@ -0,0 +1,35 @@ +package com.earseo.core.controller; + +import com.earseo.core.common.BaseResponse; +import com.earseo.core.dto.etl.FilteredDataDto; +import com.earseo.core.dto.etl.MiddleDataDto; +import com.earseo.core.service.MasterDataService; +import lombok.RequiredArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.io.IOException; +import java.util.List; + +@RestController +@RequiredArgsConstructor +public class MasterDataController { + + private final MasterDataService masterDataService; + + @GetMapping("/admin/core/master") + public ResponseEntity> rawDataProcess() throws IOException { + List filteredData = masterDataService.getRawInfo(); + List middleData = masterDataService.getMiddleData(filteredData); + masterDataService.createMasterTable(); + return ResponseEntity.ok(BaseResponse.ok(null)); + } + + @GetMapping("/admin/core/init") + public ResponseEntity> init(){ + masterDataService.initData(); + return ResponseEntity.ok(BaseResponse.ok(null)); + } + +} diff --git a/src/main/java/com/earseo/core/dto/etl/CategoryItemDto.java b/src/main/java/com/earseo/core/dto/etl/CategoryItemDto.java new file mode 100644 index 0000000..c5a912f --- /dev/null +++ b/src/main/java/com/earseo/core/dto/etl/CategoryItemDto.java @@ -0,0 +1,5 @@ +package com.earseo.core.dto.etl; + +public record CategoryItemDto(String code, String name) { +} + diff --git a/src/main/java/com/earseo/core/dto/etl/CommonItemDto.java b/src/main/java/com/earseo/core/dto/etl/CommonItemDto.java new file mode 100644 index 0000000..f95c4a8 --- /dev/null +++ b/src/main/java/com/earseo/core/dto/etl/CommonItemDto.java @@ -0,0 +1,13 @@ +package com.earseo.core.dto.etl; + +public record CommonItemDto( + String title, + String addr1, + String addr2, + String mapX, + String mapY, + String modifiedTime, + String tel, + String mLevel, + String overview +) {} diff --git a/src/main/java/com/earseo/core/dto/etl/DetailItemDto.java b/src/main/java/com/earseo/core/dto/etl/DetailItemDto.java new file mode 100644 index 0000000..29585c2 --- /dev/null +++ b/src/main/java/com/earseo/core/dto/etl/DetailItemDto.java @@ -0,0 +1,4 @@ +package com.earseo.core.dto.etl; + +public record DetailItemDto(String usefee, String parking, String restdate, String usetime) { +} diff --git a/src/main/java/com/earseo/core/dto/etl/FilteredDataDto.java b/src/main/java/com/earseo/core/dto/etl/FilteredDataDto.java new file mode 100644 index 0000000..613aee5 --- /dev/null +++ b/src/main/java/com/earseo/core/dto/etl/FilteredDataDto.java @@ -0,0 +1,5 @@ +package com.earseo.core.dto.etl; + +public record FilteredDataDto(String contentId, String contentTypeId, String cat1, + String cat2, String cat3, String outl) { +} diff --git a/src/main/java/com/earseo/core/dto/etl/ImageItemDto.java b/src/main/java/com/earseo/core/dto/etl/ImageItemDto.java new file mode 100644 index 0000000..01a3385 --- /dev/null +++ b/src/main/java/com/earseo/core/dto/etl/ImageItemDto.java @@ -0,0 +1,4 @@ +package com.earseo.core.dto.etl; + +public record ImageItemDto(String imgrul, String smallimgurl) { +} diff --git a/src/main/java/com/earseo/core/dto/etl/MasterItemDto.java b/src/main/java/com/earseo/core/dto/etl/MasterItemDto.java new file mode 100644 index 0000000..50a7369 --- /dev/null +++ b/src/main/java/com/earseo/core/dto/etl/MasterItemDto.java @@ -0,0 +1,10 @@ +package com.earseo.core.dto.etl; + +import org.locationtech.jts.geom.Point; + +public record MasterItemDto(String contentId, String contentTypeId, String cat1, + String cat2, String cat3, String ocat1, String ocat2, String ocat3, + String outl, String title, String addr1, String addr2, String addr3, + Double mapX, Double mapY, String modifiedtime, String tel, Integer mLevel, String overview, + String originImgUrl, String smallImgUrl, String usetime, String restdate, String parking, String usefee) { +} diff --git a/src/main/java/com/earseo/core/dto/etl/MiddleDataDto.java b/src/main/java/com/earseo/core/dto/etl/MiddleDataDto.java new file mode 100644 index 0000000..968d661 --- /dev/null +++ b/src/main/java/com/earseo/core/dto/etl/MiddleDataDto.java @@ -0,0 +1,8 @@ +package com.earseo.core.dto.etl; + +public record MiddleDataDto(String contentId, String contentTypeId, String cat1, + String cat2, String cat3, String outl, String title, String addr1, String addr2, + String mapX, String mapY, String modifiedtime,String tel, String mLevel,String overview, + String originImgUrl, String smallImgUrl, String usetime, String restdate, String parking, String usefee + ) { +} diff --git a/src/main/java/com/earseo/core/entity/Category.java b/src/main/java/com/earseo/core/entity/Category.java new file mode 100644 index 0000000..944641b --- /dev/null +++ b/src/main/java/com/earseo/core/entity/Category.java @@ -0,0 +1,25 @@ +package com.earseo.core.entity; + +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Entity +@Getter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class Category { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + private String code; + private String name; +} diff --git a/src/main/java/com/earseo/core/entity/Master.java b/src/main/java/com/earseo/core/entity/Master.java new file mode 100644 index 0000000..0ac42bf --- /dev/null +++ b/src/main/java/com/earseo/core/entity/Master.java @@ -0,0 +1,99 @@ +package com.earseo.core.entity; + +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import org.locationtech.jts.geom.Point; + +import java.awt.*; + +@Entity +@AllArgsConstructor +@NoArgsConstructor +@Builder @Getter +public class Master { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(name = "content_id", nullable = false, unique = true) + private String contentId; + + @Column(name = "content_type_id") + private String contentTypeId; + + @Column(name = "cat1") + private String cat1; + + @Column(name = "cat2") + private String cat2; + + @Column(name = "cat3") + private String cat3; + + @Column(name = "ocat1") + private String ocat1; + + @Column(name = "ocat2") + private String ocat2; + + @Column(name = "ocat3") + private String ocat3; + + @Column(name = "outl", columnDefinition = "TEXT") + private String outl; + + @Column(name = "title") + private String title; + + @Column(name = "addr1") + private String addr1; + + @Column(name = "addr2") + private String addr2; + + @Column(name = "addr3") + private String addr3; + + @Column(name = "map_x") + private Double mapX; + + @Column(name = "map_y") + private Double mapY; + + @Column(name = "modifiedtime") + private String modifiedtime; + + @Column(name = "tel") + private String tel; + + @Column(name = "m_level") + private Integer mLevel; + + @Column(name = "overview", columnDefinition = "TEXT") + private String overview; + + @Column(name = "origin_img_url", columnDefinition = "TEXT") + private String originImgUrl; + + @Column(name = "small_img_url", columnDefinition = "TEXT") + private String smallImgUrl; + + @Column(name = "use_time", columnDefinition = "TEXT") + private String usetime; + + @Column(name = "rest_date", columnDefinition = "TEXT") + private String restdate; + + @Column(name = "parking", columnDefinition = "TEXT") + private String parking; + + @Column(name = "use_fee", columnDefinition = "TEXT") + private String usefee; + + @Column(name = "geom", columnDefinition = "geometry(Point,4326)") + private Point geom; +} diff --git a/src/main/java/com/earseo/core/entity/MiddleData.java b/src/main/java/com/earseo/core/entity/MiddleData.java new file mode 100644 index 0000000..880ec7f --- /dev/null +++ b/src/main/java/com/earseo/core/entity/MiddleData.java @@ -0,0 +1,104 @@ +package com.earseo.core.entity; + +import com.earseo.core.dto.etl.MiddleDataDto; +import jakarta.persistence.*; +import lombok.*; + +@Entity +@Table(name = "middle_data") +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class MiddleData { + + @Id + @Column(name = "content_id", nullable = false, unique = true) + private String contentId; + + @Column(name = "content_type_id") + private String contentTypeId; + + @Column(name = "cat1") + private String cat1; + + @Column(name = "cat2") + private String cat2; + + @Column(name = "cat3") + private String cat3; + + @Column(name = "outl", columnDefinition = "TEXT") + private String outl; + + @Column(name = "title", columnDefinition = "TEXT") + private String title; + + @Column(name = "addr1", columnDefinition = "TEXT") + private String addr1; + + @Column(name = "addr2", columnDefinition = "TEXT") + private String addr2; + + @Column(name = "map_x", columnDefinition = "TEXT") + private String mapX; + + @Column(name = "map_y", columnDefinition = "TEXT") + private String mapY; + + @Column(name = "modified_time", columnDefinition = "TEXT") + private String modifiedtime; + + @Column(name = "tel", columnDefinition = "TEXT") + private String tel; + + @Column(name = "m_level", columnDefinition = "TEXT") + private String mLevel; + + @Column(name = "overview", columnDefinition = "TEXT") + private String overview; + + @Column(name = "origin_img_url", columnDefinition = "TEXT") + private String originImgUrl; + + @Column(name = "small_img_url", columnDefinition = "TEXT") + private String smallImgUrl; + + @Column(name = "use_time", columnDefinition = "TEXT") + private String usetime; + + @Column(name = "rest_date", columnDefinition = "TEXT") + private String restdate; + + @Column(name = "parking", columnDefinition = "TEXT") + private String parking; + + @Column(name = "use_fee", columnDefinition = "TEXT") + private String usefee; + + + public MiddleData(MiddleDataDto dto) { + this.contentId = dto.contentId(); + this.contentTypeId = dto.contentTypeId(); + this.cat1 = dto.cat1(); + this.cat2 = dto.cat2(); + this.cat3 = dto.cat3(); + this.outl = dto.outl(); + this.title = dto.title(); + this.addr1 = dto.addr1(); + this.addr2 = dto.addr2(); + this.mapX = dto.mapX(); + this.mapY = dto.mapY(); + this.modifiedtime = dto.modifiedtime(); + this.tel = dto.tel(); + this.mLevel = dto.mLevel(); + this.overview = dto.overview(); + this.originImgUrl = dto.originImgUrl(); + this.smallImgUrl = dto.smallImgUrl(); + this.usetime = dto.usetime(); + this.restdate = dto.restdate(); + this.parking = dto.parking(); + this.usefee = dto.usefee(); + } +} diff --git a/src/main/java/com/earseo/core/repository/CategoryRepository.java b/src/main/java/com/earseo/core/repository/CategoryRepository.java new file mode 100644 index 0000000..7d7b8b8 --- /dev/null +++ b/src/main/java/com/earseo/core/repository/CategoryRepository.java @@ -0,0 +1,10 @@ +package com.earseo.core.repository; + +import com.earseo.core.entity.Category; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface CategoryRepository extends JpaRepository { + boolean existsByCode(String code); +} diff --git a/src/main/java/com/earseo/core/repository/MasterRepository.java b/src/main/java/com/earseo/core/repository/MasterRepository.java new file mode 100644 index 0000000..2b36ec1 --- /dev/null +++ b/src/main/java/com/earseo/core/repository/MasterRepository.java @@ -0,0 +1,9 @@ +package com.earseo.core.repository; + +import com.earseo.core.entity.Master; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface MasterRepository extends JpaRepository { +} diff --git a/src/main/java/com/earseo/core/repository/MiddleRepository.java b/src/main/java/com/earseo/core/repository/MiddleRepository.java new file mode 100644 index 0000000..19cf5af --- /dev/null +++ b/src/main/java/com/earseo/core/repository/MiddleRepository.java @@ -0,0 +1,18 @@ +package com.earseo.core.repository; + +import com.earseo.core.entity.MiddleData; +import jakarta.transaction.Transactional; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +@Repository +public interface MiddleRepository extends JpaRepository { + + @Modifying + @Transactional + @Query("DELETE FROM MiddleData m WHERE m.contentTypeId = :contentTypeId") + int deleteByContentType(@Param("contentTypeId") String contentTypeId); +} diff --git a/src/main/java/com/earseo/core/service/MasterDataService.java b/src/main/java/com/earseo/core/service/MasterDataService.java new file mode 100644 index 0000000..adb1bc6 --- /dev/null +++ b/src/main/java/com/earseo/core/service/MasterDataService.java @@ -0,0 +1,417 @@ +package com.earseo.core.service; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.earseo.core.common.BaseResponse; +import com.earseo.core.dto.etl.*; +import com.earseo.core.entity.Category; +import com.earseo.core.entity.Master; +import com.earseo.core.entity.MiddleData; +import com.earseo.core.repository.CategoryRepository; +import com.earseo.core.repository.MasterRepository; +import com.earseo.core.repository.MiddleRepository; +import com.fasterxml.jackson.core.StreamWriteConstraints; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import io.swagger.v3.core.util.Json; +import jakarta.transaction.Transactional; +import lombok.RequiredArgsConstructor; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.Point; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.ClassPathResource; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.client.RestClient; +import org.springframework.web.util.UriComponentsBuilder; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +@Service +@RequiredArgsConstructor +public class MasterDataService { + + private final ObjectMapper objectMapper; + private final CategoryRepository categoryRepository; + private final MiddleRepository middleRepository; + private final MasterRepository masterRepository; + private final GeometryFactory geometryFactory = new GeometryFactory(); + private final AmazonS3 amazonS3; + + @Value("${api.key}") + private String ApiKeys; + @Value(("${cloud.aws.s3.bucket}")) + private String bucketName; + + private String key; + private int index; + + public List getRawInfo() { + try { + InputStream rawJson = new ClassPathResource("TourAPI_seoul.json").getInputStream(); + List rawJsonDtos = objectMapper.readValue( + rawJson, + new TypeReference>() { + } + ); + + List filtered = rawJsonDtos.stream() + .filter(dto -> !dto.contentTypeId().equals("25")) + .filter(dto -> !dto.contentTypeId().equals("32")) + .filter(dto -> !dto.contentTypeId().equals("39")) + .filter(dto -> !dto.cat3().equals("A04011000")) + .toList(); + + return filtered; + + } catch (Exception e) { + return List.of(); + } + } + + @Transactional + public void initData() { + List cat1s = fetchCategoryApi(null, null); + List allCategories = new ArrayList<>(); + allCategories.addAll(cat1s); + + for (CategoryItemDto cat1 : cat1s) { + List cat2s = fetchCategoryApi(cat1.code(), null); + allCategories.addAll(cat2s); + for (CategoryItemDto cat2 : cat2s) { + List cat3s = fetchCategoryApi(cat1.code(), cat2.code()); + allCategories.addAll(cat3s); + } + } + + List categories = allCategories.stream() + .map(c -> Category.builder() + .code(c.code()) + .name(c.name()) + .build()) + .toList(); + + categoryRepository.saveAll(categories); + } + + + public List fetchCategoryApi(String cat1, String cat2) { + String key = ApiKeys.split(",")[0]; + + RestClient client = RestClient.create(); + + URI uri = UriComponentsBuilder + .fromHttpUrl("https://apis.data.go.kr/B551011/KorService2/categoryCode2") + .queryParam("serviceKey", key) + .queryParam("MobileApp", "AppTest") + .queryParam("MobileOS", "ETC") + .queryParam("pageNo", 1) + .queryParam("numOfRows", 10) + .queryParam("_type", "json") + .queryParamIfPresent("cat1", Optional.ofNullable(cat1)) + .queryParamIfPresent("cat2", Optional.ofNullable(cat2)) + .build(true) + .toUri(); + try { + JsonNode jsonNode = client.get() + .uri(uri) + .retrieve() + .body(JsonNode.class); + JsonNode items = jsonNode + .path("response") + .path("body") + .path("items") + .path("item"); + + List list = new ArrayList<>(); + + for (JsonNode item : items) { + list.add(new CategoryItemDto(item.get("code").asText(), item.get("name").asText())); + } + return list; + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + @Transactional + public void createMasterTable() throws IOException { + List middleDataList = middleRepository.findAll(); + List masterItemDtos = new ArrayList<>(); + List masters = new ArrayList<>(); + + for (MiddleData middle : middleDataList) { + String addr3 = null; + if (middle.getAddr1() != null && middle.getAddr1().startsWith("서울특별시")) { + addr3 = "서울시 " + middle.getAddr1().split(" ")[1]; + } + + Point geom = null; + try { + if (middle.getMapX() != null && middle.getMapY() != null) { + double x = Double.parseDouble(middle.getMapX()); + double y = Double.parseDouble(middle.getMapY()); + geom = geometryFactory.createPoint(new org.locationtech.jts.geom.Coordinate(x, y)); + } + } catch (NumberFormatException e) { + geom = null; + } + + Master master = Master.builder() + .contentId(middle.getContentId()) + .contentTypeId(middle.getContentTypeId()) + .cat1(middle.getCat1()) + .cat2(middle.getCat2()) + .cat3(middle.getCat3()) + .ocat1(middle.getCat1()) + .ocat2(middle.getCat2()) + .ocat3(middle.getCat3()) + .outl(middle.getOutl()) + .title(middle.getTitle()) + .addr1(middle.getAddr1()) + .addr2(middle.getAddr2()) + .addr3(addr3) + .mapX(middle.getMapX() != null ? Double.valueOf(middle.getMapX()) : null) + .mapY(middle.getMapY() != null ? Double.valueOf(middle.getMapY()) : null) + .modifiedtime(middle.getModifiedtime()) + .tel(middle.getTel()) + .mLevel( + (middle.getMLevel() != null && !middle.getMLevel().isBlank()) ? Integer.parseInt(middle.getMLevel().trim()) : null + ) + .overview(middle.getOverview()) + .originImgUrl(middle.getOriginImgUrl()) + .smallImgUrl(middle.getSmallImgUrl()) + .usetime(middle.getUsetime()) + .restdate(middle.getRestdate()) + .parking(middle.getParking()) + .usefee(middle.getUsefee()) + .geom(geom) + .build(); + + masters.add(master); + + MasterItemDto dto = new MasterItemDto( + master.getContentId(), master.getContentTypeId(), master.getCat1(), master.getCat2(), master.getCat3(), + master.getOcat1(), master.getOcat2(), master.getOcat3(), master.getOutl(), master.getTitle(), + master.getAddr1(), master.getAddr2(), master.getAddr3(), master.getMapX(), master.getMapY(), + master.getModifiedtime(), master.getTel(), master.getMLevel(), master.getOverview(), master.getOriginImgUrl(), + master.getSmallImgUrl(), master.getUsetime(), master.getRestdate(), master.getParking(), master.getUsefee() + ); + + masterItemDtos.add(dto); + } + + File jsonFile = new File("master_data.json"); + + masterRepository.saveAll(masters); + objectMapper + .getFactory() + .setStreamWriteConstraints( + StreamWriteConstraints.builder() + .maxNestingDepth(3000) + .build() + ); + + objectMapper.enable(SerializationFeature.INDENT_OUTPUT); + objectMapper.writeValue(jsonFile, masterItemDtos); // S3 저장으로 리팩토링 예정 + + String date = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyyMMdd")); + String s3Key = "master/master_data_" + date + ".json"; + + amazonS3.putObject( + new PutObjectRequest( + bucketName, + s3Key, + jsonFile + ) + ); + } + + @Transactional + public List getMiddleData(List filteredData) { + + this.key = ApiKeys.split(",")[0]; + this.index = 0; + + List middleDataList = new ArrayList<>(); + + for (FilteredDataDto filtered : filteredData) { + + String contentId = filtered.contentId(); + String contentTypeId = filtered.contentTypeId(); + + JsonNode commonNode = fetchTourApi( + "https://apis.data.go.kr/B551011/KorService2/detailCommon2", + contentId, + null + ); + + JsonNode detailNode = fetchTourApi( + "https://apis.data.go.kr/B551011/KorService2/detailIntro2", + contentId, + contentTypeId + ); + + JsonNode imageNode = fetchTourApi( + "https://apis.data.go.kr/B551011/KorService2/detailImage2", + contentId, + null + ); + + if (commonNode == null || detailNode == null || imageNode == null) continue; + + JsonNode commonItem = getItem(commonNode); + JsonNode detailItem = getItem(detailNode); + JsonNode imageItem = getItem(imageNode); + + CommonItemDto commonDto = parseCommon(commonItem); + DetailItemDto detailDto = parseDetail(detailItem, contentTypeId); + ImageItemDto imageDto = parseImage(imageItem); + + MiddleDataDto dto = new MiddleDataDto( + contentId, + contentTypeId, + filtered.cat1(), + filtered.cat2(), + filtered.cat3(), + filtered.outl(), + commonDto.title(), + commonDto.addr1(), + commonDto.addr2(), + commonDto.mapX(), + commonDto.mapY(), + commonDto.modifiedTime(), + commonDto.tel(), + commonDto.mLevel(), + commonDto.overview(), + imageDto.imgrul(), + imageDto.smallimgurl(), + detailDto.usetime(), + detailDto.restdate(), + detailDto.parking(), + detailDto.usefee() + ); + + middleDataList.add(dto); + } + + middleRepository.saveAll( + middleDataList.stream() + .map(MiddleData::new) + .toList() + ); + + return middleDataList; + } + + public JsonNode fetchTourApi(String url, String contentId, String contentTypeId) { + + RestClient client = RestClient.create(); + + URI uri = UriComponentsBuilder + .fromHttpUrl(url) + .queryParam("serviceKey", this.key) + .queryParam("MobileApp", "AppTest") + .queryParam("MobileOS", "ETC") + .queryParam("pageNo", 1) + .queryParam("numOfRows", 10) + .queryParam("_type", "json") + .queryParamIfPresent("contentId", Optional.ofNullable(contentId)) + .queryParamIfPresent("contentTypeId", Optional.ofNullable(contentTypeId)) + .build(true) + .toUri(); + + try { + return client.get().uri(uri).retrieve().body(JsonNode.class); + + } catch (Exception e) { + // key 변경 후 retry + if (this.index + 1 < ApiKeys.split(",").length) { + this.index++; + this.key = ApiKeys.split(",")[this.index]; + return fetchTourApi(url, contentId, contentTypeId); + } + return null; + } + } + + private JsonNode getItem(JsonNode root) { + return root.path("response").path("body") + .path("items").path("item").get(0); + } + + private CommonItemDto parseCommon(JsonNode item) { + if (item == null) { + return new CommonItemDto(null, null, null, null, null, null, null, null, null); + } + + return new CommonItemDto( + item.path("title").asText(), + item.path("addr1").asText(), + item.path("addr2").asText(), + item.path("mapx").asText(), + item.path("mapy").asText(), + item.path("modifiedtime").asText(), + item.path("tel").asText(), + item.path("mlevel").asText(), + item.path("overview").asText() + ); + } + + private ImageItemDto parseImage(JsonNode item) { + if (item == null) return new ImageItemDto(null, null); + + return new ImageItemDto( + item.path("originimgurl").asText(null), + item.path("smallimageurl").asText(null) + ); + } + + private DetailItemDto parseDetail(JsonNode item, String typeId) { + if (item == null) return new DetailItemDto(null, null, null, null); + + return switch (typeId) { + case "12" -> new DetailItemDto( + null, + item.path("parking").asText(), + item.path("restdate").asText(), + item.path("usetime").asText() + ); + case "14" -> new DetailItemDto( + item.path("usefee").asText(), + item.path("parkingculture").asText(), + item.path("restdateculture").asText(), + item.path("usetimeculture").asText() + ); + case "15" -> new DetailItemDto( + item.path("usetimefestival").asText(), + null, null, null + ); + case "28" -> new DetailItemDto( + item.path("usefeeleports").asText(), + item.path("parkingleports").asText(), + item.path("restdateleports").asText(), + item.path("usetimeleports").asText() + ); + case "38" -> new DetailItemDto( + null, + item.path("parkingshopping").asText(), + null, + null + ); + default -> new DetailItemDto(null, null, null, null); + }; + } +} diff --git a/src/main/resources/application-local.yaml b/src/main/resources/application-local.yaml index 32260f6..7af9a03 100644 --- a/src/main/resources/application-local.yaml +++ b/src/main/resources/application-local.yaml @@ -1,9 +1,11 @@ spring: + application: + name: backend-core datasource: driver-class-name: org.postgresql.Driver - url: jdbc:postgresql://${DB_HOST:localhost}:${DB_PORT:15432}/${DB_NAME:core} - username: ${DB_USERNAME:postgres} - password: ${DB_PASSWORD:postgres} + url: jdbc:postgresql://localhost:${DB_PORT}/core + username: ${DB_USERNAME} + password: ${DB_PASSWORD} jpa: hibernate: @@ -12,7 +14,7 @@ spring: hibernate: show_sql: true format_sql: true - dialect: org.hibernate.dialect.PostgreSQLDialect + dialect: org.hibernate.spatial.dialect.postgis.PostgisPG95Dialect open-in-view: false kafka: bootstrap-servers: ${SPRING_KAFKA_BOOTSTRAP_SERVERS:${KAFKA_IP}:19092} @@ -61,9 +63,26 @@ management: metrics: tags: application: ${spring.application.name} + +cloud: + aws: + credentials: + access-key: ${AWS_CREDENTIAL_ACCESS_KEY} + secret-key: ${AWS_CREDENTIAL_SECRET_KEY} + region: + static: ap-northeast-2 + s3: + bucket: ${AWS_S3_BUCKET} + cloudfront: + domain: ${AWS_CLOUDFRONT_DOMAIN} + + tracing: enabled: false logging: level: org.hibernate.SQL: ${LOGGING_LEVEL_ORG_HIBERNATE_SQL:debug} + +api: + key: ${API_KEY} diff --git a/src/main/resources/application-test.yaml b/src/main/resources/application-test.yaml index 33cac77..baa0d53 100644 --- a/src/main/resources/application-test.yaml +++ b/src/main/resources/application-test.yaml @@ -61,3 +61,18 @@ management: logging: level: org.hibernate.SQL: ${LOGGING_LEVEL_ORG_HIBERNATE_SQL:debug} + +cloud: + aws: + credentials: + access-key: accesskey + secret-key: secretkey + region: + static: ap-northeast-2 + s3: + bucket: bucket + cloudfront: + domain: domain + +api: + key: key \ No newline at end of file