[#1] 마스터 테이블 ETL

.gitignore

@@ -35,3 +35,6 @@ out/
 
 ### VS Code ###
 .vscode/
+
+# macOS
+.DS_Store

build.gradle

@@ -22,6 +22,16 @@ dependencies {
     implementation 'org.springframework.boot:spring-boot-starter-web'
     implementation 'org.springframework.boot:spring-boot-starter-actuator'
     runtimeOnly 'io.micrometer:micrometer-registry-prometheus'
+
+    implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
+    runtimeOnly 'org.postgresql:postgresql'
+
+    implementation 'org.hibernate.orm:hibernate-spatial'
+    implementation 'org.locationtech.jts:jts-core:1.19.0'
+    implementation 'com.fasterxml.jackson.core:jackson-databind'
+
+    implementation 'com.amazonaws:aws-java-sdk-s3:1.12.767' //S3
+
     implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
     implementation 'org.springframework.boot:spring-boot-starter-validation'
     compileOnly 'org.projectlombok:lombok'

src/main/java/com/earseo/core/common/config/RestClientConfig.java

@@ -0,0 +1,14 @@
+package com.earseo.core.common.config;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.web.client.RestClient;
+
+@Configuration
+public class RestClientConfig {
+
+    @Bean
+    public RestClient.Builder restClientBuilder() {
+        return RestClient.builder();
+    }
+}

src/main/java/com/earseo/core/common/config/S3Config.java

@@ -0,0 +1,29 @@
+package com.earseo.core.common.config;
+
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class S3Config {
+    @Value("${cloud.aws.credentials.access-key}")
+    private String accessKey;
+    @Value("${cloud.aws.credentials.secret-key}")
+    private String secretKey;
+    @Value("${cloud.aws.region.static}")
+    private String region;
+
+    @Bean
+    public AmazonS3 amazonS3() {
+        BasicAWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
+        return AmazonS3ClientBuilder
+            .standard()
+            .withRegion(region)
+            .withCredentials(new AWSStaticCredentialsProvider(credentials))
+            .build();
+    }
+}

src/main/java/com/earseo/core/controller/MasterDataController.java

@@ -0,0 +1,35 @@
+package com.earseo.core.controller;
+
+import com.earseo.core.common.BaseResponse;
+import com.earseo.core.dto.etl.FilteredDataDto;
+import com.earseo.core.dto.etl.MiddleDataDto;
+import com.earseo.core.service.MasterDataService;
+import lombok.RequiredArgsConstructor;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.io.IOException;
+import java.util.List;
+
+@RestController
+@RequiredArgsConstructor
+public class MasterDataController {
+
+    private final MasterDataService masterDataService;
+
+    @GetMapping("/admin/core/master")
+    public ResponseEntity<BaseResponse<String>> rawDataProcess() throws IOException {
+        List<FilteredDataDto> filteredData = masterDataService.getRawInfo();
+        List<MiddleDataDto> middleData = masterDataService.getMiddleData(filteredData);
+        masterDataService.createMasterTable();
+        return ResponseEntity.ok(BaseResponse.ok(null));
+    }
+
+    @GetMapping("/admin/core/init")
+    public ResponseEntity<BaseResponse<String>> init(){
+        masterDataService.initData();
+        return ResponseEntity.ok(BaseResponse.ok(null));
+    }
+
+}

src/main/java/com/earseo/core/dto/etl/CategoryItemDto.java

@@ -0,0 +1,5 @@
+package com.earseo.core.dto.etl;
+
+public record CategoryItemDto(String code, String name) {
+}
+

src/main/java/com/earseo/core/dto/etl/CommonItemDto.java

@@ -0,0 +1,13 @@
+package com.earseo.core.dto.etl;
+
+public record CommonItemDto(
+        String title,
+        String addr1,
+        String addr2,
+        String mapX,
+        String mapY,
+        String modifiedTime,
+        String tel,
+        String mLevel,
+        String overview
+) {}

src/main/java/com/earseo/core/dto/etl/DetailItemDto.java

@@ -0,0 +1,4 @@
+package com.earseo.core.dto.etl;
+
+public record DetailItemDto(String usefee, String parking, String restdate, String usetime) {
+}

src/main/java/com/earseo/core/dto/etl/FilteredDataDto.java

@@ -0,0 +1,5 @@
+package com.earseo.core.dto.etl;
+
+public record FilteredDataDto(String contentId, String contentTypeId, String cat1,
+                              String cat2, String cat3, String outl) {
+}

src/main/java/com/earseo/core/dto/etl/ImageItemDto.java

@@ -0,0 +1,4 @@
+package com.earseo.core.dto.etl;
+
+public record ImageItemDto(String imgrul, String smallimgurl) {
+}

src/main/java/com/earseo/core/dto/etl/MasterItemDto.java

@@ -0,0 +1,10 @@
+package com.earseo.core.dto.etl;
+
+import org.locationtech.jts.geom.Point;
+
+public record MasterItemDto(String contentId, String contentTypeId, String cat1,
+                            String cat2, String cat3, String ocat1, String ocat2, String ocat3,
+                            String outl, String title, String addr1, String addr2, String addr3,
+                            Double mapX, Double mapY, String modifiedtime, String tel, Integer mLevel, String overview,
+                            String originImgUrl, String smallImgUrl, String usetime, String restdate, String parking, String usefee) {
+}

src/main/java/com/earseo/core/dto/etl/MiddleDataDto.java

@@ -0,0 +1,8 @@
+package com.earseo.core.dto.etl;
+
+public record MiddleDataDto(String contentId, String contentTypeId, String cat1,
+                            String cat2, String cat3, String outl, String title, String addr1, String addr2,
+                            String mapX, String mapY, String modifiedtime,String tel, String mLevel,String overview,
+                            String originImgUrl, String smallImgUrl, String usetime, String restdate, String parking, String usefee
+                            ) {
+}

src/main/java/com/earseo/core/entity/Category.java

@@ -0,0 +1,25 @@
+package com.earseo.core.entity;
+
+import jakarta.persistence.Entity;
+import jakarta.persistence.GeneratedValue;
+import jakarta.persistence.GenerationType;
+import jakarta.persistence.Id;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+@Entity
+@Getter
+@NoArgsConstructor
+@AllArgsConstructor
+@Builder
+public class Category {
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+
+    private String code;
+    private String name;
+}

src/main/java/com/earseo/core/entity/Master.java

@@ -0,0 +1,99 @@
+package com.earseo.core.entity;
+
+import jakarta.persistence.*;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import org.locationtech.jts.geom.Point;
+
+import java.awt.*;
+
+@Entity
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder @Getter
+public class Master {
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+
+    @Column(name = "content_id", nullable = false, unique = true)
+    private String contentId;
+
+    @Column(name = "content_type_id")
+    private String contentTypeId;
+
+    @Column(name = "cat1")
+    private String cat1;
+
+    @Column(name = "cat2")
+    private String cat2;
+
+    @Column(name = "cat3")
+    private String cat3;
+
+    @Column(name = "ocat1")
+    private String ocat1;
+
+    @Column(name = "ocat2")
+    private String ocat2;
+
+    @Column(name = "ocat3")
+    private String ocat3;
+
+    @Column(name = "outl", columnDefinition = "TEXT")
+    private String outl;
+
+    @Column(name = "title")
+    private String title;
+
+    @Column(name = "addr1")
+    private String addr1;
+
+    @Column(name = "addr2")
+    private String addr2;
+
+    @Column(name = "addr3")
+    private String addr3;
+
+    @Column(name = "map_x")
+    private Double mapX;
+
+    @Column(name = "map_y")
+    private Double mapY;
+
+    @Column(name = "modifiedtime")
+    private String modifiedtime;
+
+    @Column(name = "tel")
+    private String tel;
+
+    @Column(name = "m_level")
+    private Integer mLevel;
+
+    @Column(name = "overview", columnDefinition = "TEXT")
+    private String overview;
+
+    @Column(name = "origin_img_url", columnDefinition = "TEXT")
+    private String originImgUrl;
+
+    @Column(name = "small_img_url", columnDefinition = "TEXT")
+    private String smallImgUrl;
+
+    @Column(name = "use_time", columnDefinition = "TEXT")
+    private String usetime;
+
+    @Column(name = "rest_date", columnDefinition = "TEXT")
+    private String restdate;
+
+    @Column(name = "parking", columnDefinition = "TEXT")
+    private String parking;
+
+    @Column(name = "use_fee", columnDefinition = "TEXT")
+    private String usefee;
+
+    @Column(name = "geom", columnDefinition = "geometry(Point,4326)")
+    private Point geom;
+}

src/main/java/com/earseo/core/entity/MiddleData.java

@@ -0,0 +1,104 @@
+package com.earseo.core.entity;
+
+import com.earseo.core.dto.etl.MiddleDataDto;
+import jakarta.persistence.*;
+import lombok.*;
+
+@Entity
+@Table(name = "middle_data")
+@Getter
+@Setter
+@NoArgsConstructor
+@AllArgsConstructor
+@Builder
+public class MiddleData {
+
+    @Id
+    @Column(name = "content_id", nullable = false, unique = true)
+    private String contentId;
+
+    @Column(name = "content_type_id")
+    private String contentTypeId;
+
+    @Column(name = "cat1")
+    private String cat1;
+
+    @Column(name = "cat2")
+    private String cat2;
+
+    @Column(name = "cat3")
+    private String cat3;
+
+    @Column(name = "outl", columnDefinition = "TEXT")
+    private String outl;
+
+    @Column(name = "title", columnDefinition = "TEXT")
+    private String title;
+
+    @Column(name = "addr1", columnDefinition = "TEXT")
+    private String addr1;
+
+    @Column(name = "addr2", columnDefinition = "TEXT")
+    private String addr2;
+
+    @Column(name = "map_x", columnDefinition = "TEXT")
+    private String mapX;
+
+    @Column(name = "map_y", columnDefinition = "TEXT")
+    private String mapY;
+
+    @Column(name = "modified_time", columnDefinition = "TEXT")
+    private String modifiedtime;
+
+    @Column(name = "tel", columnDefinition = "TEXT")
+    private String tel;
+
+    @Column(name = "m_level", columnDefinition = "TEXT")
+    private String mLevel;
+
+    @Column(name = "overview", columnDefinition = "TEXT")
+    private String overview;
+
+    @Column(name = "origin_img_url", columnDefinition = "TEXT")
+    private String originImgUrl;
+
+    @Column(name = "small_img_url", columnDefinition = "TEXT")
+    private String smallImgUrl;
+
+    @Column(name = "use_time", columnDefinition = "TEXT")
+    private String usetime;
+
+    @Column(name = "rest_date", columnDefinition = "TEXT")
+    private String restdate;
+
+    @Column(name = "parking", columnDefinition = "TEXT")
+    private String parking;
+
+    @Column(name = "use_fee", columnDefinition = "TEXT")
+    private String usefee;
+
+
+    public MiddleData(MiddleDataDto dto) {
+        this.contentId = dto.contentId();
+        this.contentTypeId = dto.contentTypeId();
+        this.cat1 = dto.cat1();
+        this.cat2 = dto.cat2();
+        this.cat3 = dto.cat3();
+        this.outl = dto.outl();
+        this.title = dto.title();
+        this.addr1 = dto.addr1();
+        this.addr2 = dto.addr2();
+        this.mapX = dto.mapX();
+        this.mapY = dto.mapY();
+        this.modifiedtime = dto.modifiedtime();
+        this.tel = dto.tel();
+        this.mLevel = dto.mLevel();
+        this.overview = dto.overview();
+        this.originImgUrl = dto.originImgUrl();
+        this.smallImgUrl = dto.smallImgUrl();
+        this.usetime = dto.usetime();
+        this.restdate = dto.restdate();
+        this.parking = dto.parking();
+        this.usefee = dto.usefee();
+    }
+}