From 6d910e3a779d199a72582eb618b16e6104102822 Mon Sep 17 00:00:00 2001 From: Venki Korukanti Date: Thu, 28 Oct 2021 19:11:46 -0700 Subject: [PATCH 1/5] disable presto-docs module --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f2439ec4214f..34ffe720ae7c 100644 --- a/pom.xml +++ b/pom.xml @@ -124,7 +124,7 @@ presto-benchmark-driver presto-server presto-server-rpm - presto-docs + presto-verifier presto-testing-server-launcher presto-plugin-toolkit From cccd80068a7e361b25403e664e6dba66925e673d Mon Sep 17 00:00:00 2001 From: George Chow Date: Mon, 27 Sep 2021 22:08:18 -0700 Subject: [PATCH 2/5] Shaded Delta standalone reader jar Shade the Delta standalone reader jar and its dependencies to avoid conflicts with Presto library and its dependencies. --- pom.xml | 7 ++ presto-delta-driver/pom.xml | 229 ++++++++++++++++++++++++++++++++++++ 2 files changed, 236 insertions(+) create mode 100644 presto-delta-driver/pom.xml diff --git a/pom.xml b/pom.xml index 34ffe720ae7c..feb25862b581 100644 --- a/pom.xml +++ b/pom.xml @@ -147,6 +147,7 @@ presto-pinot presto-oracle presto-druid + presto-delta-driver @@ -1193,6 +1194,12 @@ ${dep.jackson.version} + + io.delta + delta-standalone_2.12 + 0.2.0 + + org.javassist diff --git a/presto-delta-driver/pom.xml b/presto-delta-driver/pom.xml new file mode 100644 index 000000000000..4e03a01b8ae8 --- /dev/null +++ b/presto-delta-driver/pom.xml @@ -0,0 +1,229 @@ + + + 4.0.0 + + io.prestosql + presto-root + 0.264-SNAPSHOT + + + + io.prestosql.delta + presto-delta-driver + presto-delta-driver + Presto - Delta Driver (Shaded Delta Standalone Reader) + jar + + + false + 2.6.7 + io.prestosql.delta.internal + + + + + io.delta + delta-standalone_2.12 + 0.2.0 + + + + org.scala-lang + scala-library + 2.12.12 + + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + + commons-io + commons-io + 2.8.0 + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.1.1 + + + + package + + shade + + + true + true + true + true + ${project.build.directory} + + + + io.delta:* + + + com.fasterxml.jackson.core:jackson-core + com.fasterxml.jackson.core:jackson-databind + com.fasterxml.jackson.core:jackson-annotations + com.fasterxml.jackson.module:jackson-module-scala_2.12 + com.fasterxml.jackson.module:jackson-module-paranamer + org.scala-lang:scala-library + commons-io:commons-io + + + com.thoughtworks.paranamer:paranamer + org.json4s:json4s-jackson_2.12 + org.json4s:json4s-core_2.12 + org.json4s:json4s-ast_2.12 + org.json4s:json4s-scalap_2.12 + org.scala-lang.modules:scala-xml_2.12 + org.apache.hadoop:hadoop-client + org.apache.hadoop:hadoop-common + com.github.mjakubowski84:parquet4s-core_2.12 + com.chuusai:shapeless_2.12 + org.typelevel:macro-compat_2.12 + org.scala-lang.modules:scala-collection-compat_2.12 + + + + io.prestosql.* + + + + + + + com.fasterxml.jackson + ${shadeBase}.com.fasterxml.jackson + + + + + com.fasterxml.jackson.module + ${shadeBase}.com.fasterxml.jackson.module + + + + + org.scala-lang + ${shadeBase}.org.scala-lang + + + + org.apache.commons.io + ${shadeBase}.org.apache.commons.io + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + META-INF/maven/** + + + + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + org.scala-lang:scala-library:jar + io.delta:delta-standalone_2.12:jar + commons-io:commons-io:jar + com.fasterxml.jackson.module:jackson-module-scala_2.12:jar + com.fasterxml.jackson.core:jackson-core:jar + com.fasterxml.jackson.core:jackson-annotations:jar + com.fasterxml.jackson.core:jackson-databind:jar + + + + + unpack + package + + unpack + + + + + io.prestosql.delta + presto-delta-driver + shaded + ${project.version} + jar + true + ${project.build.directory}/classes + **/** + + + + + + + + + + org.gaul + modernizer-maven-plugin + 1.7.1 + + true + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + + + org.alluxio:alluxio-shaded-client + org.codehaus.plexus:plexus-utils + com.fasterxml.jackson.core:jackson-core + com.google.guava:guava + com.fasterxml.jackson.core:jackson-annotations + com.fasterxml.jackson.core:jackson-databind + + + + + + + + + From 94606b42f98ac36b71b861071fdcb15a2335a409 Mon Sep 17 00:00:00 2001 From: Venki Korukanti Date: Tue, 28 Sep 2021 09:34:28 -0700 Subject: [PATCH 3/5] Add test Delta tables (copied from github.com/delta-io/connectors/golden-tables) Currently these golden tables are not part of any test jars that Delta provides. Make a PR in Delta to publish these golden tables as a maven artifact and in Presto add a maven dependency in tests. --- ...1f-b560-60792edbe807-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...ff-8e56-2431d0adf4bb-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...b2-8059-a4688c2d59a9-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...b2-9064-c459e9d26a87-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...d7-9bda-766b1255e6f6-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...2e-8991-5fc96e11200a-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...34-a932-a4e288cadf35-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...ac-9d75-dfd59ec37250-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...4d-95d3-42521e10e10d-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...ee-b58d-5cf32a3b8349-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...c8-84c9-85df909dba39-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...23-b0b6-08014cea263a-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...27-af5c-88e4536e5bd3-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...a8-8d69-bbf7e516b710-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...bc-8b56-e81b14aa1817-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...fd-aa12-1117de3f9071-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...43-9635-64f6e9ce9e3b-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...7a-b452-41539eadc475-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...3a-9b57-82d2123ba08e-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...28-a317-5b30608bb45c-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ....00000000000000000010.checkpoint.parquet.crc | Bin 0 -> 100 bytes .../00000000000000000010.checkpoint.parquet | Bin 0 -> 11345 bytes .../_delta_log/00000000000000000011.json | 2 ++ .../_delta_log/00000000000000000012.json | 2 ++ .../_delta_log/00000000000000000013.json | 2 ++ .../_delta_log/00000000000000000014.json | 2 ++ .../_delta_log/00000000000000000015.json | 2 ++ .../_delta_log/00000000000000000016.json | 2 ++ .../_delta_log/00000000000000000017.json | 2 ++ .../_delta_log/00000000000000000018.json | 2 ++ .../_delta_log/00000000000000000019.json | 2 ++ .../_delta_log/_last_checkpoint | 1 + ...5-491f-b560-60792edbe807-c000.snappy.parquet | Bin 0 -> 678 bytes ...f-4eff-8e56-2431d0adf4bb-c000.snappy.parquet | Bin 0 -> 687 bytes ...d-4eb2-8059-a4688c2d59a9-c000.snappy.parquet | Bin 0 -> 678 bytes ...f-48b2-9064-c459e9d26a87-c000.snappy.parquet | Bin 0 -> 687 bytes ...b-42d7-9bda-766b1255e6f6-c000.snappy.parquet | Bin 0 -> 687 bytes ...6-4e2e-8991-5fc96e11200a-c000.snappy.parquet | Bin 0 -> 687 bytes ...3-4134-a932-a4e288cadf35-c000.snappy.parquet | Bin 0 -> 678 bytes ...9-4dac-9d75-dfd59ec37250-c000.snappy.parquet | Bin 0 -> 687 bytes ...b-484d-95d3-42521e10e10d-c000.snappy.parquet | Bin 0 -> 678 bytes ...b-47ee-b58d-5cf32a3b8349-c000.snappy.parquet | Bin 0 -> 687 bytes ...f-42c8-84c9-85df909dba39-c000.snappy.parquet | Bin 0 -> 687 bytes ...c-4823-b0b6-08014cea263a-c000.snappy.parquet | Bin 0 -> 678 bytes ...a-4e27-af5c-88e4536e5bd3-c000.snappy.parquet | Bin 0 -> 678 bytes ...b-40a8-8d69-bbf7e516b710-c000.snappy.parquet | Bin 0 -> 678 bytes ...0-4bbc-8b56-e81b14aa1817-c000.snappy.parquet | Bin 0 -> 678 bytes ...2-47fd-aa12-1117de3f9071-c000.snappy.parquet | Bin 0 -> 678 bytes ...6-4c43-9635-64f6e9ce9e3b-c000.snappy.parquet | Bin 0 -> 687 bytes ...6-417a-b452-41539eadc475-c000.snappy.parquet | Bin 0 -> 678 bytes ...2-483a-9b57-82d2123ba08e-c000.snappy.parquet | Bin 0 -> 687 bytes ...9-4528-a317-5b30608bb45c-c000.snappy.parquet | Bin 0 -> 687 bytes ...b0-b208-438bbe179c93-c000.snappy.parquet.crc | Bin 0 -> 32 bytes ...77-9822-b0985158cd66-c000.snappy.parquet.crc | Bin 0 -> 32 bytes .../_delta_log/00000000000000000000.json | 5 +++++ ...3-4bb0-b208-438bbe179c93-c000.snappy.parquet | Bin 0 -> 2830 bytes ...1-4e77-9822-b0985158cd66-c000.snappy.parquet | Bin 0 -> 2832 bytes ...0d-a5cb-8d9d483ed390-c000.snappy.parquet.crc | Bin 0 -> 40 bytes ...73-8c72-423ee747abc0-c000.snappy.parquet.crc | Bin 0 -> 40 bytes .../_delta_log/00000000000000000000.json | 5 +++++ ...f-470d-a5cb-8d9d483ed390-c000.snappy.parquet | Bin 0 -> 3627 bytes ...5-4b73-8c72-423ee747abc0-c000.snappy.parquet | Bin 0 -> 3644 bytes ...28-9ee6-b3401faa46ba-c000.snappy.parquet.crc | Bin 0 -> 40 bytes ...1e-a396-12af34a70812-c000.snappy.parquet.crc | Bin 0 -> 40 bytes .../_delta_log/00000000000000000000.json | 5 +++++ ...b-4728-9ee6-b3401faa46ba-c000.snappy.parquet | Bin 0 -> 3638 bytes ...e-471e-a396-12af34a70812-c000.snappy.parquet | Bin 0 -> 3655 bytes ...28-8462-cc9c56edfebb-c000.snappy.parquet.crc | Bin 0 -> 20 bytes ...9a-8f93-92a40d9f065d-c000.snappy.parquet.crc | Bin 0 -> 20 bytes .../_delta_log/00000000000000000000.json | 5 +++++ ...9-4628-8462-cc9c56edfebb-c000.snappy.parquet | Bin 0 -> 1432 bytes ...b-449a-8f93-92a40d9f065d-c000.snappy.parquet | Bin 0 -> 1439 bytes .../_delta_log/00000000000000000000.crc | 1 + .../_delta_log/00000000000000000000.json | 6 ++++++ ...d-43f4-9d13-7c5354deb1f6.c000.snappy.parquet | Bin 0 -> 465 bytes ...c-4cba-b8c8-2297ef36e5ff.c000.snappy.parquet | Bin 0 -> 465 bytes ...7-462c-b88b-f8b49370d785.c000.snappy.parquet | Bin 0 -> 465 bytes ...7b-96a1-e2bb0f246b06-c000.snappy.parquet.crc | Bin 0 -> 28 bytes ...75-a810-fbd8e7994971-c000.snappy.parquet.crc | Bin 0 -> 32 bytes .../_delta_log/00000000000000000000.json | 5 +++++ ...3-4e7b-96a1-e2bb0f246b06-c000.snappy.parquet | Bin 0 -> 2482 bytes ...2-4475-a810-fbd8e7994971-c000.snappy.parquet | Bin 0 -> 2568 bytes .../_delta_log/00000000000000000000.json | 8 ++++++++ ...11-b8b3-71d02178554f.c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...c-4511-b8b3-71d02178554f.c000.snappy.parquet | Bin 0 -> 628 bytes ...0a-8ba3-1499f599b8f1.c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...3-400a-8ba3-1499f599b8f1.c000.snappy.parquet | Bin 0 -> 650 bytes ...88-ba9b-31fb8aeb8093.c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...e-4b88-ba9b-31fb8aeb8093.c000.snappy.parquet | Bin 0 -> 628 bytes ...fe-bba8-3dbf43b4eb86.c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...1-43fe-bba8-3dbf43b4eb86.c000.snappy.parquet | Bin 0 -> 639 bytes ...05-a6cc-04759d5f692b.c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...6-4405-a6cc-04759d5f692b.c000.snappy.parquet | Bin 0 -> 650 bytes .../_delta_log/00000000000000000000.json | 7 +++++++ ...01-bd22-d361a60627e3.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...70-bf02-5d1872324d6f.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...1-4401-bd22-d361a60627e3.c000.snappy.parquet | Bin 0 -> 436 bytes ...d-4d70-bf02-5d1872324d6f.c000.snappy.parquet | Bin 0 -> 431 bytes ...4f-9a4c-de0b22b62bbd.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...5a-9c6a-25f7559cd150.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...7-454f-9a4c-de0b22b62bbd.c000.snappy.parquet | Bin 0 -> 431 bytes ...5-465a-9c6a-25f7559cd150.c000.snappy.parquet | Bin 0 -> 436 bytes ...0e-83a1-212aa6c84c73-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...a1-9ca9-f0cd8a9cfd11-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...b5-a3d6-5b9ae1745045-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...ed-9cf8-74a843b32c8c-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...a7-b94e-07357ec28a7d-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...1b-bf10-28dc03aa9d2f-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...61-8a8a-fa958802b523-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...e6-998b-c61d1cfb0b1c-c000.snappy.parquet.crc | Bin 0 -> 16 bytes .../_delta_log/00000000000000000000.json | 5 +++++ .../_delta_log/00000000000000000001.json | 3 +++ .../_delta_log/00000000000000000002.json | 7 +++++++ .../_delta_log/00000000000000000003.json | 3 +++ ...1-400e-83a1-212aa6c84c73-c000.snappy.parquet | Bin 0 -> 650 bytes ...8-4fa1-9ca9-f0cd8a9cfd11-c000.snappy.parquet | Bin 0 -> 649 bytes ...2-44b5-a3d6-5b9ae1745045-c000.snappy.parquet | Bin 0 -> 649 bytes ...b-46ed-9cf8-74a843b32c8c-c000.snappy.parquet | Bin 0 -> 687 bytes ...4-4fa7-b94e-07357ec28a7d-c000.snappy.parquet | Bin 0 -> 650 bytes ...5-411b-bf10-28dc03aa9d2f-c000.snappy.parquet | Bin 0 -> 705 bytes ...a-4461-8a8a-fa958802b523-c000.snappy.parquet | Bin 0 -> 649 bytes ...c-4ee6-998b-c61d1cfb0b1c-c000.snappy.parquet | Bin 0 -> 649 bytes .../_delta_log/00000000000000000000.json | 13 +++++++++++++ .../_delta_log/00000000000000000001.json | 16 ++++++++++++++++ ...97-91da-55c4d3266fbe.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...09-9b2c-3bfd9debc2f0.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...8-4a97-91da-55c4d3266fbe.c000.snappy.parquet | Bin 0 -> 442 bytes ...3-4d09-9b2c-3bfd9debc2f0.c000.snappy.parquet | Bin 0 -> 437 bytes ...7a-be01-16dc85260296.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...aa-8d62-9e6ab59afa8b.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...b-487a-be01-16dc85260296.c000.snappy.parquet | Bin 0 -> 437 bytes ...3-4eaa-8d62-9e6ab59afa8b.c000.snappy.parquet | Bin 0 -> 442 bytes ...f5-9850-b5e8a72f4917.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...c6-bc0e-c19d164ac82d.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...f-4df5-9850-b5e8a72f4917.c000.snappy.parquet | Bin 0 -> 429 bytes ...c-46c6-bc0e-c19d164ac82d.c000.snappy.parquet | Bin 0 -> 429 bytes ...31-a17f-388265ab04b7.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...29-a6e5-3d578b0964f4.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...8-4131-a17f-388265ab04b7.c000.snappy.parquet | Bin 0 -> 429 bytes ...1-4829-a6e5-3d578b0964f4.c000.snappy.parquet | Bin 0 -> 429 bytes ...39-9076-37e523cd62f1.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...5c-aeca-5cfa6b38615c.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...f-4d39-9076-37e523cd62f1.c000.snappy.parquet | Bin 0 -> 429 bytes ...7-4c5c-aeca-5cfa6b38615c.c000.snappy.parquet | Bin 0 -> 429 bytes ...f8-b957-08583e2a1d1b.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...8e-b882-31c4aac05038.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...a-4df8-b957-08583e2a1d1b.c000.snappy.parquet | Bin 0 -> 429 bytes ...4-4d8e-b882-31c4aac05038.c000.snappy.parquet | Bin 0 -> 429 bytes ...d3-a6e7-5980a370461a.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...7f-b8c4-f0ada8cf9663.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...f-48d3-a6e7-5980a370461a.c000.snappy.parquet | Bin 0 -> 429 bytes ...7-4d7f-b8c4-f0ada8cf9663.c000.snappy.parquet | Bin 0 -> 429 bytes ...bc-8ca9-5c10997e307b-c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...af-80a5-7ff8a959bd53-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...44-a79c-21a6f3d53138-c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...77-9b5d-4e831fa723e1-c000.snappy.parquet.crc | Bin 0 -> 16 bytes .../_delta_log/00000000000000000000.json | 5 +++++ .../_delta_log/00000000000000000001.json | 4 ++++ ...c-40bc-8ca9-5c10997e307b-c000.snappy.parquet | Bin 0 -> 449 bytes ...8-4caf-80a5-7ff8a959bd53-c000.snappy.parquet | Bin 0 -> 711 bytes ...6-4d44-a79c-21a6f3d53138-c000.snappy.parquet | Bin 0 -> 451 bytes ...8-4e77-9b5d-4e831fa723e1-c000.snappy.parquet | Bin 0 -> 711 bytes 162 files changed, 122 insertions(+) create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-106da899-cbd5-491f-b560-60792edbe807-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-12ef2b67-494f-4eff-8e56-2431d0adf4bb-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-197ae400-931d-4eb2-8059-a4688c2d59a9-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-27b57b04-e80f-48b2-9064-c459e9d26a87-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-2e12af63-21db-42d7-9bda-766b1255e6f6-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-3a090e55-ad26-4e2e-8991-5fc96e11200a-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-4a9efe8d-12b3-4134-a932-a4e288cadf35-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-4aa85334-6e99-4dac-9d75-dfd59ec37250-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-4fb7f274-cd1b-484d-95d3-42521e10e10d-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-5d1ee207-355b-47ee-b58d-5cf32a3b8349-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-7f8faf7f-2d0f-42c8-84c9-85df909dba39-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-999ae312-021c-4823-b0b6-08014cea263a-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-9c2ae306-eeca-4e27-af5c-88e4536e5bd3-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-a12a3c66-8abb-40a8-8d69-bbf7e516b710-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-a90c681e-2310-4bbc-8b56-e81b14aa1817-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-bf15b1cf-2d02-47fd-aa12-1117de3f9071-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-df985307-35c6-4c43-9635-64f6e9ce9e3b-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-e048b59b-38c6-417a-b452-41539eadc475-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-e22924ef-f192-483a-9b57-82d2123ba08e-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-e877c5f2-5fb9-4528-a317-5b30608bb45c-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/_delta_log/.00000000000000000010.checkpoint.parquet.crc create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/_delta_log/00000000000000000010.checkpoint.parquet create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/_delta_log/00000000000000000011.json create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/_delta_log/00000000000000000012.json create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/_delta_log/00000000000000000013.json create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/_delta_log/00000000000000000014.json create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/_delta_log/00000000000000000015.json create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/_delta_log/00000000000000000016.json create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/_delta_log/00000000000000000017.json create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/_delta_log/00000000000000000018.json create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/_delta_log/00000000000000000019.json create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/_delta_log/_last_checkpoint create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/part-00000-106da899-cbd5-491f-b560-60792edbe807-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/part-00000-12ef2b67-494f-4eff-8e56-2431d0adf4bb-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/part-00000-197ae400-931d-4eb2-8059-a4688c2d59a9-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/part-00000-27b57b04-e80f-48b2-9064-c459e9d26a87-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/part-00000-2e12af63-21db-42d7-9bda-766b1255e6f6-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/part-00000-3a090e55-ad26-4e2e-8991-5fc96e11200a-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/part-00000-4a9efe8d-12b3-4134-a932-a4e288cadf35-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/part-00000-4aa85334-6e99-4dac-9d75-dfd59ec37250-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/part-00000-4fb7f274-cd1b-484d-95d3-42521e10e10d-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/part-00000-5d1ee207-355b-47ee-b58d-5cf32a3b8349-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/part-00000-7f8faf7f-2d0f-42c8-84c9-85df909dba39-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/part-00000-999ae312-021c-4823-b0b6-08014cea263a-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/part-00000-9c2ae306-eeca-4e27-af5c-88e4536e5bd3-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/part-00000-a12a3c66-8abb-40a8-8d69-bbf7e516b710-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/part-00000-a90c681e-2310-4bbc-8b56-e81b14aa1817-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/part-00000-bf15b1cf-2d02-47fd-aa12-1117de3f9071-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/part-00000-df985307-35c6-4c43-9635-64f6e9ce9e3b-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/part-00000-e048b59b-38c6-417a-b452-41539eadc475-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/part-00000-e22924ef-f192-483a-9b57-82d2123ba08e-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/checkpointed-delta-table/part-00000-e877c5f2-5fb9-4528-a317-5b30608bb45c-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/data-reader-array-complex-objects/.part-00000-a7d58b1a-7743-4bb0-b208-438bbe179c93-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/data-reader-array-complex-objects/.part-00001-7b211746-0a31-4e77-9822-b0985158cd66-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/data-reader-array-complex-objects/_delta_log/00000000000000000000.json create mode 100644 presto-delta/src/test/resources/data-reader-array-complex-objects/part-00000-a7d58b1a-7743-4bb0-b208-438bbe179c93-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/data-reader-array-complex-objects/part-00001-7b211746-0a31-4e77-9822-b0985158cd66-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/data-reader-array-primitives/.part-00000-182665f0-30df-470d-a5cb-8d9d483ed390-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/data-reader-array-primitives/.part-00001-2e274fe7-eb75-4b73-8c72-423ee747abc0-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/data-reader-array-primitives/_delta_log/00000000000000000000.json create mode 100644 presto-delta/src/test/resources/data-reader-array-primitives/part-00000-182665f0-30df-470d-a5cb-8d9d483ed390-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/data-reader-array-primitives/part-00001-2e274fe7-eb75-4b73-8c72-423ee747abc0-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/data-reader-map/.part-00000-d9004e55-077b-4728-9ee6-b3401faa46ba-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/data-reader-map/.part-00001-3d30d085-4cde-471e-a396-12af34a70812-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/data-reader-map/_delta_log/00000000000000000000.json create mode 100644 presto-delta/src/test/resources/data-reader-map/part-00000-d9004e55-077b-4728-9ee6-b3401faa46ba-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/data-reader-map/part-00001-3d30d085-4cde-471e-a396-12af34a70812-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/data-reader-nested-struct/.part-00000-f2547b28-9219-4628-8462-cc9c56edfebb-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/data-reader-nested-struct/.part-00001-0f755735-3b5b-449a-8f93-92a40d9f065d-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/data-reader-nested-struct/_delta_log/00000000000000000000.json create mode 100644 presto-delta/src/test/resources/data-reader-nested-struct/part-00000-f2547b28-9219-4628-8462-cc9c56edfebb-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/data-reader-nested-struct/part-00001-0f755735-3b5b-449a-8f93-92a40d9f065d-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/data-reader-partition-values/_delta_log/00000000000000000000.crc create mode 100644 presto-delta/src/test/resources/data-reader-partition-values/_delta_log/00000000000000000000.json create mode 100644 presto-delta/src/test/resources/data-reader-partition-values/as_int=0/as_long=0/as_byte=0/as_short=0/as_boolean=true/as_float=0.0/as_double=0.0/as_string=0/as_date=2021-09-08/as_timestamp=2021-09-08 11%3A11%3A11/as_big_decimal=0/part-00005-2d4e572d-5bdd-43f4-9d13-7c5354deb1f6.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/data-reader-partition-values/as_int=1/as_long=1/as_byte=1/as_short=1/as_boolean=false/as_float=1.0/as_double=1.0/as_string=1/as_date=2021-09-08/as_timestamp=2021-09-08 11%3A11%3A11/as_big_decimal=1/part-00007-00fd7022-5e9c-4cba-b8c8-2297ef36e5ff.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/data-reader-partition-values/as_int=__HIVE_DEFAULT_PARTITION__/as_long=__HIVE_DEFAULT_PARTITION__/as_byte=__HIVE_DEFAULT_PARTITION__/as_short=__HIVE_DEFAULT_PARTITION__/as_boolean=__HIVE_DEFAULT_PARTITION__/as_float=__HIVE_DEFAULT_PARTITION__/as_double=__HIVE_DEFAULT_PARTITION__/as_string=__HIVE_DEFAULT_PARTITION__/as_date=__HIVE_DEFAULT_PARTITION__/as_timestamp=__HIVE_DEFAULT_PARTITION__/as_big_decimal=__HIVE_DEFAULT_PARTITION__/part-00002-b31c437d-93a7-462c-b88b-f8b49370d785.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/data-reader-primitives/.part-00000-4f2f0b9f-50b3-4e7b-96a1-e2bb0f246b06-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/data-reader-primitives/.part-00001-09e47b80-36c2-4475-a810-fbd8e7994971-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/data-reader-primitives/_delta_log/00000000000000000000.json create mode 100644 presto-delta/src/test/resources/data-reader-primitives/part-00000-4f2f0b9f-50b3-4e7b-96a1-e2bb0f246b06-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/data-reader-primitives/part-00001-09e47b80-36c2-4475-a810-fbd8e7994971-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/deltatbl-partition-prune/_delta_log/00000000000000000000.json create mode 100644 presto-delta/src/test/resources/deltatbl-partition-prune/date=20180512/city=sh/.part-00001-c87aeb63-6d9c-4511-b8b3-71d02178554f.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/deltatbl-partition-prune/date=20180512/city=sh/part-00001-c87aeb63-6d9c-4511-b8b3-71d02178554f.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/deltatbl-partition-prune/date=20180520/city=bj/.part-00001-4c732f0f-a473-400a-8ba3-1499f599b8f1.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/deltatbl-partition-prune/date=20180520/city=bj/part-00001-4c732f0f-a473-400a-8ba3-1499f599b8f1.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/deltatbl-partition-prune/date=20180520/city=hz/.part-00000-de1d5bcd-ad7e-4b88-ba9b-31fb8aeb8093.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/deltatbl-partition-prune/date=20180520/city=hz/part-00000-de1d5bcd-ad7e-4b88-ba9b-31fb8aeb8093.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/deltatbl-partition-prune/date=20180718/city=hz/.part-00000-f888e95b-c831-43fe-bba8-3dbf43b4eb86.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/deltatbl-partition-prune/date=20180718/city=hz/part-00000-f888e95b-c831-43fe-bba8-3dbf43b4eb86.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/deltatbl-partition-prune/date=20181212/city=sz/.part-00001-529ff89b-55c6-4405-a6cc-04759d5f692b.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/deltatbl-partition-prune/date=20181212/city=sz/part-00001-529ff89b-55c6-4405-a6cc-04759d5f692b.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/deltatbl-partitioned/_delta_log/00000000000000000000.json create mode 100644 presto-delta/src/test/resources/deltatbl-partitioned/c2=foo0/.part-00000-2bcc9ff6-0551-4401-bd22-d361a60627e3.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/deltatbl-partitioned/c2=foo0/.part-00001-ca647ee7-f1ad-4d70-bf02-5d1872324d6f.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/deltatbl-partitioned/c2=foo0/part-00000-2bcc9ff6-0551-4401-bd22-d361a60627e3.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/deltatbl-partitioned/c2=foo0/part-00001-ca647ee7-f1ad-4d70-bf02-5d1872324d6f.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/deltatbl-partitioned/c2=foo1/.part-00000-786c7455-9587-454f-9a4c-de0b22b62bbd.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/deltatbl-partitioned/c2=foo1/.part-00001-1c702e73-89b5-465a-9c6a-25f7559cd150.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/deltatbl-partitioned/c2=foo1/part-00000-786c7455-9587-454f-9a4c-de0b22b62bbd.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/deltatbl-partitioned/c2=foo1/part-00001-1c702e73-89b5-465a-9c6a-25f7559cd150.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/snapshot-data3/.part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/snapshot-data3/.part-00000-64680d94-9e18-4fa1-9ca9-f0cd8a9cfd11-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/snapshot-data3/.part-00000-842017c2-3e02-44b5-a3d6-5b9ae1745045-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/snapshot-data3/.part-00000-cb078bc1-0aeb-46ed-9cf8-74a843b32c8c-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/snapshot-data3/.part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/snapshot-data3/.part-00001-9bf4b8f8-1b95-411b-bf10-28dc03aa9d2f-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/snapshot-data3/.part-00001-b8249b87-0b7a-4461-8a8a-fa958802b523-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/snapshot-data3/.part-00001-e62ca5a1-923c-4ee6-998b-c61d1cfb0b1c-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/snapshot-data3/_delta_log/00000000000000000000.json create mode 100644 presto-delta/src/test/resources/snapshot-data3/_delta_log/00000000000000000001.json create mode 100644 presto-delta/src/test/resources/snapshot-data3/_delta_log/00000000000000000002.json create mode 100644 presto-delta/src/test/resources/snapshot-data3/_delta_log/00000000000000000003.json create mode 100644 presto-delta/src/test/resources/snapshot-data3/part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/snapshot-data3/part-00000-64680d94-9e18-4fa1-9ca9-f0cd8a9cfd11-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/snapshot-data3/part-00000-842017c2-3e02-44b5-a3d6-5b9ae1745045-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/snapshot-data3/part-00000-cb078bc1-0aeb-46ed-9cf8-74a843b32c8c-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/snapshot-data3/part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/snapshot-data3/part-00001-9bf4b8f8-1b95-411b-bf10-28dc03aa9d2f-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/snapshot-data3/part-00001-b8249b87-0b7a-4461-8a8a-fa958802b523-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/snapshot-data3/part-00001-e62ca5a1-923c-4ee6-998b-c61d1cfb0b1c-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/_delta_log/00000000000000000000.json create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/_delta_log/00000000000000000001.json create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part2=0/.part-00000-7bce012e-f358-4a97-91da-55c4d3266fbe.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part2=0/.part-00001-2a830e69-78f3-4d09-9b2c-3bfd9debc2f0.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part2=0/part-00000-7bce012e-f358-4a97-91da-55c4d3266fbe.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part2=0/part-00001-2a830e69-78f3-4d09-9b2c-3bfd9debc2f0.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part2=1/.part-00000-82368d1d-588b-487a-be01-16dc85260296.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part2=1/.part-00001-0a72544a-fb83-4eaa-8d62-9e6ab59afa8b.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part2=1/part-00000-82368d1d-588b-487a-be01-16dc85260296.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part2=1/part-00001-0a72544a-fb83-4eaa-8d62-9e6ab59afa8b.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part5=0/.part-00000-67b6882e-f49f-4df5-9850-b5e8a72f4917.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part5=0/.part-00001-4f02a740-31dc-46c6-bc0e-c19d164ac82d.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part5=0/part-00000-67b6882e-f49f-4df5-9850-b5e8a72f4917.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part5=0/part-00001-4f02a740-31dc-46c6-bc0e-c19d164ac82d.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part5=1/.part-00000-8a40c3d2-f658-4131-a17f-388265ab04b7.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part5=1/.part-00001-3dcad520-b001-4829-a6e5-3d578b0964f4.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part5=1/part-00000-8a40c3d2-f658-4131-a17f-388265ab04b7.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part5=1/part-00001-3dcad520-b001-4829-a6e5-3d578b0964f4.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part5=2/.part-00000-ec6e3a2e-ecbf-4d39-9076-37e523cd62f1.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part5=2/.part-00001-e20bae81-3f27-4c5c-aeca-5cfa6b38615c.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part5=2/part-00000-ec6e3a2e-ecbf-4d39-9076-37e523cd62f1.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part5=2/part-00001-e20bae81-3f27-4c5c-aeca-5cfa6b38615c.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part5=3/.part-00000-eaf1edf4-b9da-4df8-b957-08583e2a1d1b.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part5=3/.part-00001-b9c6b926-a274-4d8e-b882-31c4aac05038.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part5=3/part-00000-eaf1edf4-b9da-4df8-b957-08583e2a1d1b.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part5=3/part-00001-b9c6b926-a274-4d8e-b882-31c4aac05038.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part5=4/.part-00000-ce66c2ca-8fdf-48d3-a6e7-5980a370461a.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part5=4/.part-00001-5705917d-d837-4d7f-b8c4-f0ada8cf9663.c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part5=4/part-00000-ce66c2ca-8fdf-48d3-a6e7-5980a370461a.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/time-travel-partition-changes-b/part5=4/part-00001-5705917d-d837-4d7f-b8c4-f0ada8cf9663.c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/time-travel-schema-changes-b/.part-00000-83680aa8-547c-40bc-8ca9-5c10997e307b-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/time-travel-schema-changes-b/.part-00000-a830a49c-6cc8-4caf-80a5-7ff8a959bd53-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/time-travel-schema-changes-b/.part-00001-3c1f89ce-a996-4d44-a79c-21a6f3d53138-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/time-travel-schema-changes-b/.part-00001-5fdfd303-d5e8-4e77-9b5d-4e831fa723e1-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/time-travel-schema-changes-b/_delta_log/00000000000000000000.json create mode 100644 presto-delta/src/test/resources/time-travel-schema-changes-b/_delta_log/00000000000000000001.json create mode 100644 presto-delta/src/test/resources/time-travel-schema-changes-b/part-00000-83680aa8-547c-40bc-8ca9-5c10997e307b-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/time-travel-schema-changes-b/part-00000-a830a49c-6cc8-4caf-80a5-7ff8a959bd53-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/time-travel-schema-changes-b/part-00001-3c1f89ce-a996-4d44-a79c-21a6f3d53138-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/time-travel-schema-changes-b/part-00001-5fdfd303-d5e8-4e77-9b5d-4e831fa723e1-c000.snappy.parquet diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-106da899-cbd5-491f-b560-60792edbe807-c000.snappy.parquet.crc b/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-106da899-cbd5-491f-b560-60792edbe807-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..9e1378be772249a9348beb1eecb73588f979d29c GIT binary patch literal 16 XcmYc;N@ieSU}ESEn6m#iW8nz^B~S%` literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-12ef2b67-494f-4eff-8e56-2431d0adf4bb-c000.snappy.parquet.crc b/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-12ef2b67-494f-4eff-8e56-2431d0adf4bb-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..34a63575cea7c2ec8dfffc2eb7c6a5005fae2cdd GIT binary patch literal 16 XcmYc;N@ieSU}E^T`s$mjZvqklFMJ0N literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-197ae400-931d-4eb2-8059-a4688c2d59a9-c000.snappy.parquet.crc b/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-197ae400-931d-4eb2-8059-a4688c2d59a9-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..4b6155c669388f4fa1359140fcc9723d3d1ac086 GIT binary patch literal 16 XcmYc;N@ieSU}AVYVfED8jD;rvDR>40 literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-27b57b04-e80f-48b2-9064-c459e9d26a87-c000.snappy.parquet.crc b/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-27b57b04-e80f-48b2-9064-c459e9d26a87-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..f8ab1bd81a179efe326a1378db9861d071959ea1 GIT binary patch literal 16 XcmYc;N@ieSU}D(*`mf5>Hvx$NEKCM) literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-2e12af63-21db-42d7-9bda-766b1255e6f6-c000.snappy.parquet.crc b/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-2e12af63-21db-42d7-9bda-766b1255e6f6-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..35c6c50fe42b6b2f1462d056896427e84ccb7ed0 GIT binary patch literal 16 XcmYc;N@ieSU}9hl7m2z0CLj?29x?=V literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-3a090e55-ad26-4e2e-8991-5fc96e11200a-c000.snappy.parquet.crc b/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-3a090e55-ad26-4e2e-8991-5fc96e11200a-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..1a79612892062f9e3351792d492235aaa3db9f2f GIT binary patch literal 16 XcmYc;N@ieSU}A7QU3LEIn}9?BC;CLj?2Bmo6U literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-999ae312-021c-4823-b0b6-08014cea263a-c000.snappy.parquet.crc b/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-999ae312-021c-4823-b0b6-08014cea263a-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..6976dff400e307c382f69accfa93da0433ba77c9 GIT binary patch literal 16 XcmYc;N@ieSU}E?da_Q7<#=;W-DqjXR literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-9c2ae306-eeca-4e27-af5c-88e4536e5bd3-c000.snappy.parquet.crc b/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-9c2ae306-eeca-4e27-af5c-88e4536e5bd3-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..9f40fa723d33e8effc1f1d68b48f12f49c1769f5 GIT binary patch literal 16 XcmYc;N@ieSU}C6_kO;laSa<>eAN~Z8 literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-a12a3c66-8abb-40a8-8d69-bbf7e516b710-c000.snappy.parquet.crc b/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-a12a3c66-8abb-40a8-8d69-bbf7e516b710-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..6fb286b647fb2c8192ff054b3f177bc57c2a83bf GIT binary patch literal 16 XcmYc;N@ieSU}7-0aol>FvG4={AMgaZ literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-a90c681e-2310-4bbc-8b56-e81b14aa1817-c000.snappy.parquet.crc b/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-a90c681e-2310-4bbc-8b56-e81b14aa1817-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..4052974e23ec5b901f97aa6419fbe2a05491ae55 GIT binary patch literal 16 XcmYc;N@ieSU}EsGU_Wr1vG4={A6EpM literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-bf15b1cf-2d02-47fd-aa12-1117de3f9071-c000.snappy.parquet.crc b/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-bf15b1cf-2d02-47fd-aa12-1117de3f9071-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..e27c2130040bc91a71bdeaa6bc730cebe7bd3c3b GIT binary patch literal 16 XcmYc;N@ieSU}D&1(5-cwvG4={BDn_Hvx$ND{lsV literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-e048b59b-38c6-417a-b452-41539eadc475-c000.snappy.parquet.crc b/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-e048b59b-38c6-417a-b452-41539eadc475-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..71f6b68fcf7f21dce8fafc0e340fced830dca28f GIT binary patch literal 16 XcmYc;N@ieSU}9j1iBq`ESa<>e9Nh#e literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-e22924ef-f192-483a-9b57-82d2123ba08e-c000.snappy.parquet.crc b/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-e22924ef-f192-483a-9b57-82d2123ba08e-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..c36454b7fe8eec037be19bd35b4d71dcff24a2cb GIT binary patch literal 16 XcmYc;N@ieSU}8v*apb-FCLj?2AqWJ% literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-e877c5f2-5fb9-4528-a317-5b30608bb45c-c000.snappy.parquet.crc b/presto-delta/src/test/resources/checkpointed-delta-table/.part-00000-e877c5f2-5fb9-4528-a317-5b30608bb45c-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..9dcd9876ad9dfb02213afd65a6a474373b614092 GIT binary patch literal 16 XcmYc;N@ieSU}E@QK6(4qHvx$ND}x4Q literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/_delta_log/.00000000000000000010.checkpoint.parquet.crc b/presto-delta/src/test/resources/checkpointed-delta-table/_delta_log/.00000000000000000010.checkpoint.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..b7d844826a413aff3ded58d45edfcd788c6bb4df GIT binary patch literal 100 zcmV-q0Gt0~a$^7h00IE?mD?a4ZH%dl1V25#HZ{K(MK(u8D?2kmqh;@A>{q0DhamB< z^R<{cXa?|qhIZjX56-a+0@+xVVV5AVe4HE+oYY3?_ChBnn5Sbwo_{>mb{;0ugb{6v G&BmJrYb(wG literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/_delta_log/00000000000000000010.checkpoint.parquet b/presto-delta/src/test/resources/checkpointed-delta-table/_delta_log/00000000000000000010.checkpoint.parquet new file mode 100644 index 0000000000000000000000000000000000000000..46a216cea9daa03137c5dd89cb3539ac0c3ecd5d GIT binary patch literal 11345 zcmeGiTZr7&RWqI)kF#TE*Ip^w%`_~7J*=J8MAB$9BeNu~U6OTii*e&L(3+sf)q2Q0 zHZxk+uD!(+V@keUQ;K6uF!@Npv=mZsOA78sAuUenSGyR01b>A>!Q`jLA13tNE9puj zjdwNc*j_@HSxIws-sjwN?mc%7fBT5UGHjk5n`NiqAJ-B;*@-k3~(p;9+hL6K^@U?^1?o~jiM>6WR~s)7lSeR6(qZl42rVRx7%I1=2OqRO=EFqY)!051W?g6WTUzzNU{O)N)<)W zwTcWnA{mqfYgDQPP%~waQ3PJd)WH~XP0;Ns@UJ7KT2WEe0DXy!lB`!uRTb(w*ijMn zx=^=NO)w0*hN_Zk)If9q{6&J`I`S_Z-7m03KSHx^=s#O`L!Sl*Ko!|i1ZsSr3$+=C z(Ak{_FYaX63_H#4JOsB4!;H;+hymd5citVyOtCLuG8r<@_?-WS&-t)iW@f-B5ZNkH zaZ)2`%<624wGXjlnAg6YmwrF~;7p2Pa-VIK0txo0cUiE z;xGy9e)#CoR3IZxOrCf8?7r{f&V$=Awe2hy@Fodt%QWHV(RoY>ZdI~8TkvJ_WY5pK zG2>oA>;xcBfJKSj;|mIo%Q~p%9JKs9$InDYz?nPT%$bA7Ed09|tGNM!bX7Nb?`PQA zkR+S^gzrCqEdC?!KLFv0L&Vj2c*L{O!&*CfcmO71tuebP+~3pey;MlR_fn^e&Z$LI z+*|B9-6hj0&KB(kYFfSG-XEPVw)GZ%Wp|@LP29^!nxPhhH$d-d}{V;WlL_3~)@4y;b?-S{gLZoK{MWhQfMdF>fB1_C9??pXLCsDB$HC=kq;CvTwiDGmoQ|&gPkFn&0G{Y_nO=aaWojf#v!^YZvI8fp42npl+|xX}jM> zn|jYVx(JC5Sp-kFEOwiVms!-EsYH}HSC6|V~VdgS}@FjXwTq$d*Ha>#?cps z=0y1R9Np=;l4rZ9)j5HpIuTQhTd0YA5df#P7?*%G&ZcBGu-vOn?Zs$b9LS6Um z<_-Z}mkaW4F#dqlN0;0fjiCdb=2EMT>1J^dWtxqi1Mx&+W;3{?h1!mbv(@ah?Z(1V zmqf~XqHfgeA{^W#%CNOSN2dde0IYNxOf=SNw2#347Iok7Cx$=#!*0U~h5>n#X_;)! z74)&pj&()>H#J$9d@xhW71;b_HOu5CXK(^=CqA3zrZ|<`TguJBQhc%inGZCF6j<6{ zU^5|bKQmB-{ZWK7{RrXYL6Ocxk*=pgQbBqlh9ls2w{d@argS|ur*5QHMA$-Kjbhn% zWtqwOO(1$S4F>5Cr#+(6Ja?${)o9cPH}eVUjJU}ZyX+8mxO6igrF40#>w^sTF}5bo zs)X}DQrt?abb0IC5Ifn)S-d^Qa}V!fNl<*bt)D?GR}*G`OmiQkOJ8mSX7N4gLw2eI zm|Yx)aP&hEv~ zY#K}>Vht|FA5gq7^Bsp*Cj%mlu#Dl~cy0~+h|o8li%M{g4}y)Bd?@wA{uff*uTrIR z{G8t>@b9n;j`j<}v*!|g@gtZLC5r|c#g`_dlH3}E6kc3V+*>i+OH-6#oifb2%MiN! zAo!P4-2180rKvgETc8i1l;3ZlV||6~`t!cE-;6Tx7c~PshZFAg=cC%5FAPB|P7svkXnP}`^+Z`v?zrY_bDe-#h+U_W+dqa?l%*lH(VxR5_$^q3fyl)^n zT}g3QQ>9P$z>3H3T?9`MZUaX9O!>u&1vc$17U(76lpp@cu51;RmR1SAiA>yvmz!2a@q)p&X^Nl9W!! zv$*Q6#3)^PDex>7slUo!c`3#j;QhrU24AJbuf&kAh^rw-WW6Frcys?$IM|)d@_eVe zP}Uc9Xy%kjU#ILfZ1|B%SuBeTzdHued#C9i$CkUDpOpRHHUHbGBn?T}D$+CB6DJ=C!pj>NI$#9e+^%v0J)EP^{_6k%wC(8h?{T%&|2 z=a#+*gRsSXogQzv7AL#KfrAG+ZAWjkd*6p+4fjYRx%PNVNrfm5=bdP0Z@qMaR$xHE z=yaM$$FZ95um;axivHBBiJOsBiiE*2M69|EPRf9)3DfdnXJC&OuCcBu$k6|(~?ARBgoGQXH-x?$U!hY0Uow`g_`J8;-N)wH@p-- ziN=0!o+1s+)NAKc0+j0$DKA4rMELB0N;|Z=bS8Vp4U`hqTkm%DQ!wluIYjFyoIR|& zg!Y|V#5q3^iw6-OIs>7yB+Au8IL$v-p$w>*S1Cl%vloh7Onf%(|E>c_^=){DCN+ev zus5LOlc@P<^_B+mH(+_VkxSCCWrqJQ7#shpRcLjmy?x5GRF=fDG(BT1HJa8m{Jhn& x>$V2JoHSHqt7@fUiK?h-s4AM8QPXOofecI94F_@shIuL4qH6x^g%Lh(>aH*6pWSu`lbB~gq({()Y4 z=r8T_Gl9|;mSysGX7>8DyrqapnO)5VLy>J6yR5I zEhkibp7Ms(u*ngD!4YRw5MNyHj1<5J8J4yFO(~i2`1vszK8Xp(1%&0QObWwkXd_(O z!)6sBRn~OPG!z|F?IKdN9SO^-tVU{VDE&WzXgV|x)^NG8)Nv{2(A+wFO6r$nEc;o{ z8Tutt&QpDdE|6vtGL6TK&p3o(+g$8L#7QuX$Z3mN%tEC8gOSWb1|yg+M9SjnE1QUv zZze=Oq!mryW`g!;lE#Z+N=-TnMCd29cfX<$pKOjclY)aiF^Ezz7BLi}MHq4~1PW+{ zNe{NPw_2~wyu6vo;hrEG|EGRfBXYZMxiN{$`;29q*%q_QHE$7wewlY&f7BhhZma1z lV&pijmTx(h6#5`x79BOD6{{6r_9lChy^b+b8(mqg+SZLJ{*A%@!5;P_ zcm3EbjBQZD$;~+@_uO=Nbz@UCr1bt$4#aV2IqslSrPEJ6f0VPP)_C|e!Ndc4{|DS9$}@Xko;&CS_qd9 z(5@k*Dw?jDMnMNv`wSJdJqatSqDE?L6#9Rp(R4@v)^NMA(s4QCVDB8>bLvOVSoJeq zXW*~FGQR37bb>UKkXbxoVlE&M+ht=kXEv3o z*p7+3<<>NPnM>NENg6LlDK+UhkfEQ@-tC%3V!A!rP74kW#2`xLM8;5vmSHHo5GbIP zCjGLdz4c~eX65!#4)+An* z5B^eLKNAoYmSysGX73gYwYt&t3PFT;}ByD2AA9zQ=OgC{ZOxQMV+kw|ec2`z-n zJ7`xBQY2MVjjF7Hs#QX=x+P&rk(5Y@t8(v;AgTtAc7@u+o!#26ti-?n85|QI3Gnt7<{RbnNg$zb8n~RjilUFtt zDPK>Bd=!>6eVYo}rAZpk2Prk^Fc6`i(C+<`Mtr${A)#VnIqdd-^$p|A6f>km6a*KOLK mBZiLCZ2G2SI<9D$1J`T2ZPOE;Z`D;pLkJc+ga6Zq7y1o-%9z3c literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/part-00000-27b57b04-e80f-48b2-9064-c459e9d26a87-c000.snappy.parquet b/presto-delta/src/test/resources/checkpointed-delta-table/part-00000-27b57b04-e80f-48b2-9064-c459e9d26a87-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a0bc805945e612120441c62497c6eb7482ab3da0 GIT binary patch literal 687 zcmah{O>4qH6kJm)T09im4I9WomKwCgB~gqKSR6LFIvAXdc4ig8-%>0o!%;O}@aXYAb|1uy<08URT_Qzy9+(JM z_pn_@NR|{;(Y3M)s%{CDl^qF7vLuIcq?Lz%q)}8z0M>B3wo-95=g{6dycgt;g0>pw zy3WC0gXMhXSLg&OIwA9D%J_mqAhylMoEL!vLq5xnvT4D=p6G{(n2HDrVHyP734j7x zq0=v0I#_QudS2c>$l)G8oc^c2+aPkiZ@D##YWs{elbHrHYYiv$1Fy#Wwm0cdY`fET mEHSaHPRBDW!?H!!aBZh&_Y6lkp4n7%6(RW0IlR#re$`J=BAU|x literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/part-00000-2e12af63-21db-42d7-9bda-766b1255e6f6-c000.snappy.parquet b/presto-delta/src/test/resources/checkpointed-delta-table/part-00000-2e12af63-21db-42d7-9bda-766b1255e6f6-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d79953afa7dbaa1f26cc69a4b1e61c784e2b0315 GIT binary patch literal 687 zcmah{!D_-l6kJm)TD+8Y!v=DYr3Ni=Ng_rgUV4+Bq}Ni)#;i6piJHWgD*i^Hf6zle zvQJHGptOas%)XtOee*V>s~Zy|e2$5ZB>*K{R!Q*`A+%p*)X43id~zYddQ{j_fUn`p zoUr2Yl(&qQ*(C=E7Kcr)4hCn#y;%kDw-igtU|3D&Jbt`S#t$OoxQMV+mq>9u3rvKo z8MNvM$&#Wfx>i;})hVH}vL|6lmgGo|wesMPG>Qrdz#4AXRw}OM9ITzgdqMsvXsbc4 z>m2+wSk6~|g-(#76EcgZjL$g)V!LeYNyLdii^y@C8O%VW`Gb)x0tRDPEJe!V*)t18 z%C}=8Z-q5YU*>}LX_CgvaY}VM@kQVzw12y%5f8UV+iAf;M)adpOhpWZXc+|D4S)h# zq0=v0+Fx%rdR}fF`r)Y)5pAvEz1~uHg#TGn+ literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/part-00000-3a090e55-ad26-4e2e-8991-5fc96e11200a-c000.snappy.parquet b/presto-delta/src/test/resources/checkpointed-delta-table/part-00000-3a090e55-ad26-4e2e-8991-5fc96e11200a-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..bf6353490ee854ebc59c251df75f491e64fe68a2 GIT binary patch literal 687 zcmah{!A`c+wdmoYK01fYOrjpR=eLc3K#wagC6C+8BZN4b;&d>vn8 zghh`hykR!1OLBlwp}BQ<&nX`{ebvu& zoq@jw%lIm<&3I z4EkkDd+W`{$jX}sIo#uilmFCr8$_=6EjOl7WuLKPG0S9DrRFaEz^m}CnklM~>TZI;JaJ&#J42h7f$H3~w}qU-c6Q+?uZd literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/part-00000-4a9efe8d-12b3-4134-a932-a4e288cadf35-c000.snappy.parquet b/presto-delta/src/test/resources/checkpointed-delta-table/part-00000-4a9efe8d-12b3-4134-a932-a4e288cadf35-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..90d1ce6d3f23be1c15afd30ac0fb29dda2cec95e GIT binary patch literal 678 zcmah{!D_-l6kJnVLh(>aH*6pWSyU9_l9(Eecq#NydXipCDI2re&=@s|Emi!3{y;B1 z^h^8HXac1zEX(BW%suDv{!F8kh)| zQrM~@q)4i!>NQydRr>_V>W+jZMN&c~s>%I7f~Xoa57uybZK>f>&Y`t+_$(-2h1#;8 z^_-z!GUYs#cjyACIw8|&%=nB$7`DyDUPKi8(~uln%wPs0jUSA79xxced?6ARO<&nW zBz!X=@=;jP@*fOb++>;rKuG!y1v>wB_a`DyJFCCNm9Ymh0}q54~-QbPQLxp4m`!4Ixc+wdS1~cL44{Zh8Yxr|Li<%gjoc2(XBRT8C(@Pzd>vor zgk_(nyk)j5n;aon95=Z-7@QCHW@W(NQY@>3VI^7c`0+j&KZqH}1%&0gObX+9XdzrV zK)a5RBCDEe=p_wQogyl!dlHrvS&5WbFAe@kqiT=!QMH%OUj3&uLilU zbMV(-IbY=!Izg&I$UL4hzTgmu?Xs~a5huYsBByO;G82*J4@R;K8H{0>iIm0jXEqZl z-;RmANo$(EEClV-B#pCiN)0**MCd29f4im;pKXt}(}IHoF^Ezz6)_Z|EDX690tK|f zpkKDMzus(&yxcy@;hrFx{-?g%AaZ?ZxiyPxhm19gSthe;4KE8qzs7s6Kj}?ew{3fl lm^e})A^1=g-e?5B>L4qH6kJm)p?D~z8#a)GEE*Kzk|;(aUJ5;wo}|}O$|hNDXpEY~mMZ>({()Y4 z=r8T_Gl9|;!ZQ1IX72C7a8$?A@TB}GyqCD!G^A3;a zOQxKs@(x`f)gWXVj~SnF2*b9y*o%mhU>cFrHZz%tNaF`1nTHHUFkgt2#nV?d5h>qH zhxC$A_)Bo?>YXcH*%bI%eBSG kw%cvrv`yO)9n*8%uG2ML;rdoXH8h0aL+9{5L-<3#0a=KcumAu6 literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/part-00000-5d1ee207-355b-47ee-b58d-5cf32a3b8349-c000.snappy.parquet b/presto-delta/src/test/resources/checkpointed-delta-table/part-00000-5d1ee207-355b-47ee-b58d-5cf32a3b8349-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..dd70fbff5ebb9a28e477be061b557e30f3042e05 GIT binary patch literal 687 zcmah{O;5r=6x>!MpxktuZPEkTNCT#9S_p*_F5ZYI;?)?lr3*He0{zek;cuAu4?OrI zeFc;zMopT{?Aw{yH*Y(-y0I|A=a?8+22jK$jTBB1Lc3K#_1q51Cl@lTM^Z)szK$<* z!m`H`-ZYz*MGg=w4x3yZ49W}lCWY}lun?~7 zVXKCaBCDEe=p_wQogyl!I}(-^SqYU$FAe@kqiT=p|y2*my{1lUk!3y z=islwa=ywdbb?faka;v^e8C|Q+h$`=B8vTaNRHdgWF{hwAB=bzFc`x!6$y*x&uk_V zo{fpTNo$(CEClV-IEm76LJd0cMc~D>f4in3pJhkcwBTS*^ut6>aE~8O|5M*>5V_vB+?++#ea5QAER$K)x|{leSLHp&oAf4*({8!8 lnAmo^?U}Y|JECKb9k=UrO;@;{)ldx$A^6ZaywM1L)lZq^nt1>K literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/part-00000-7f8faf7f-2d0f-42c8-84c9-85df909dba39-c000.snappy.parquet b/presto-delta/src/test/resources/checkpointed-delta-table/part-00000-7f8faf7f-2d0f-42c8-84c9-85df909dba39-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..4968c8854b3e2ecf73f4c2698b984eec22b4e236 GIT binary patch literal 687 zcmah{O;5r=6kIA2Fr3tFwn-0UBMq3cX-g=SaPdYw5wFIWEnTp&6zGRW2!F%Gf8fC% z=>w`XF>2CmX5Y@tzIofx)s2Y}KF37I5`YpetE6~}5ZbRYYUFlMKDm%!Jt}M|z}N6) zPFV4H!dphm?2!Wmi^C>Y2ZOWW-mC)nTZ$!RFs#N49zEX2;|DS0xQMV+mq>9u4@`us z8MNvM$&#Wfx>i;})hVH}vL|6lmgG>5wDRDOG>Qrdz#4AXRw}OM9ITzgdqMsvXsbc4 z>m2+wSk6~|g-(#76Ecscj4wC@V!LeYNkp+f56N+x8O%VW`GXNJ0|sMQrXpd{{F%)} z!nb20Z-q5YUKWD(X`DpqIH5Y7_#*IP+P_`Xkk7V9+iAf;M)bo(Ohp8RFbx9k20#I= z(CL>g?XNc*JukNoa=6D2r~j$%Hi%qjEw^S-Ez4LlnQ1Vy)^Jll@M^s0c$415aoUz^ mi-~Qw+n!+?wj(;m*m1i~*KmdFnN3Ak5rPk$!yAp@SN#N4qH6kJm)p?D~z8#a)GEGi0d*%*vQycBvUJxQ;nl#N+!XpEY~mMZ>({()Y4 z=r8T_J5bt&uuR_0%)WWq{?(m@5x&I4#0r1{Rt=Iz@N8EN)iPgDJ}WD*p5)dP;FobN zBUF8niUx0RazJ2kz{e_xFRr&nD&V~gD|+vym`p|d{Fn@$E4!hm#dYTl{b%aI46ik|EV8Vh}`a4Zj9s7E@R1J7H3we=FNl9FNu!p4?9EGwVR$J khmK?0K6kj|$`&8EUfXSRPkO#p*G&T4qH6x>8xLh(>aH*6pWSyU9_k{FCeycBvUJxQ;nl#SWg&=@s|Emiyn{R6%9 z&|li;cc8R|WtqI4nSJxJgR46WBYcU8fn@-vSkXzbfDqd5DyrwcpnO)5VJ}E)3h*nq zo)aoQ&v?^p+T?)1;DC=+5MNwxjTFFp8J4yFO(~u7`!#-|*@uuU%ZB9b(m#N@)I{P l5rN~_wr@J73&v?sdIpl!A;DC=+5MNwxjby-k8J5)kO*x(NWG!ZWE zpjAUikyK69>#_!_b_vPqmV_lmQX(a(%l$uss2Vg6)^K@!q2Y4Q!P+={7L>0-ebGPm zd_=!w%6Tg9&;?R;LgHl1_>@B!w#mg_M3M$^L{4pHFawe14@No*8H`{y7a2?9S2htD zUr&g96qYo5n+n>aX_m}~8P(}15TT#a-u;qBe6l`TPYMop#30JVSR_!0=3&Ua5GbG( zI^EmS-g33l^K$DbhkJr({Ga+^g~;u$<<=yr>@rqNW*W?_G`x8b`W4=F{ZV)1y0+yx lV&pis?Hi8axT0+gU9aPI3{QBz*;I86Az0`f{?7nj=r<7rn2Z1b literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/part-00000-a90c681e-2310-4bbc-8b56-e81b14aa1817-c000.snappy.parquet b/presto-delta/src/test/resources/checkpointed-delta-table/part-00000-a90c681e-2310-4bbc-8b56-e81b14aa1817-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..691fb0e58f855d19d7213ce2667f5eda1bfd2a44 GIT binary patch literal 678 zcmah{O-sW-6x_yIgY+Qch7IIU78_bfmn5ZWQ+g3Rh^OLJL^jQ~2Gg`nQpJ}31OI>* z5B^eLKNAoYmSysGX7suDv{z~5?TnC zchIgPq)4i!8oI23s#QX=x+P&rk(5Y@b-DLP5LJWb!5S{t7aA_-9PEw5XF>TY=!;&~ zbB2D&l=D>Hp$nuMgiPWQ<5Lb{*d`Zy5pfbsB68ehCNmMK|6nAukiiIMbCI%m^2){{ zzDt>q)`Eju=F#7>O7P(L4;f7Xk&e z!k~Lw+Fhw4+omTx->R#Ih7c@t2LGoIFZ3HVMwp%e literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/part-00000-bf15b1cf-2d02-47fd-aa12-1117de3f9071-c000.snappy.parquet b/presto-delta/src/test/resources/checkpointed-delta-table/part-00000-bf15b1cf-2d02-47fd-aa12-1117de3f9071-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..df73e724ab3759be33bfd1801710c3e12d3a9caa GIT binary patch literal 678 zcmah{!D_-l6kJm+p?D~z8#a)GEE*Kzk{FCeycBvUJxQ;nl#N+!XpEY~mMZ>1f1sBh z`lWqp)Iezq%QAU8GyCRc2Um9%M)(pF14{r-v8<6|2_dxG6;#iCLHVpI!G2QMP=H^> z^_)=ldCHq+vrP^N3=a5M1@Xo8&PWElmtjfm-;|RXkDnit;ggtfTtrx^Nu)TOh8Du* z47O?rDUzzGhAwNMYM&xm-I1`QNJ^x{y4?RGh^j&JU=5e+OAVKE4y~=jXF>TY=*#}G z=Og+hQ_fR)hc1w65HgL&jL$fPVcT5nMZ`%kjY!F6CNmLf{9q*WkiiJ%3z4#T`pPCE z<(mnSkHU(kZ!yNr4*R@-o mBSwy6+rH_Tjw{;c(DgcQ$Ml5fTMgCF5Q2ry;r|Tag?c+wd*Dx`#1mFypR8p)Wg!Ze98o3>m&o3lcPYPQK@O6Bd z6PA6R@|M}MTylh9aoptUU{D?I&B}nkrC3r1!%DK?@#B3meh@Q`iwH|~i4@24&_cL! zfOZ`rSyEKR&`T<)I%lY)>`7RXB{`B~y)^hEjiN#Vu!h_9m5M7l2Ycu6UXVWu`f8Bt zItPCZmh+Wgp%bJSgv{e9;|mUf*e)A;5^)mDBXZhiCNmLf{$M1_kii(1nMheYe`Yh0 z^6i+&TVYMpmxZ8xnxt_yPN_jBfe8JC_HWlT;#!B+czE4aYe@*yI$Arnx61{tEm_&Lhzv)ywM1L)la_*n+X5_ literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/part-00000-e048b59b-38c6-417a-b452-41539eadc475-c000.snappy.parquet b/presto-delta/src/test/resources/checkpointed-delta-table/part-00000-e048b59b-38c6-417a-b452-41539eadc475-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..b94d46f524483d6220b9be264ec08daf5ff076be GIT binary patch literal 678 zcmah{%WA?v6dhA5p|~ie6DE*_3@Qq7$OEGhH-#=rSJG`MWnxAf8lxt$rHX&hALyow zera!f21;89!^yqpoVn*FgR46eBYcU8jwJvkENi4FA%wQ4f*QFuD4$ic=1CztfnUXq zoKW_8%3DUuA_oKp2Yjr8_~LqNBm>^du%z~H%E^q!&yUIQNlZ8{A}rM;QXEc06XEg> zS~Y|eN!3)nE^DA_myoP(Nmx=OB~s$L-2Wqps=@GJ4VTxK8ZPG?tc}BGLHR1wm;K|+ zNAydkoTu^*Qy^6*WEzhdpK%DoHo4e~h?8I%kyD!)%s{01gOSWb1}m5^M9SjnE1QUv zXB#3Pg%wTTW`g!;lE#Z+N_9F4MCd29cfX<$pJc6UQ_$EEgD4ea5knzbgdz7rpnz8B zbZ<+0tMyvX%dMjv?g^stf9i)dBDcGdTa&o5%UCg)X)v?W@D@SnS9sU;N8OR@+Lq^t lk>l95Z#ahIincLyy^h;4JmL9fQ`I$u;6dl`eFpG{egl8)m_z^o literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/part-00000-e22924ef-f192-483a-9b57-82d2123ba08e-c000.snappy.parquet b/presto-delta/src/test/resources/checkpointed-delta-table/part-00000-e22924ef-f192-483a-9b57-82d2123ba08e-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c752e4280d5c6404fc42364d404719608725ecc9 GIT binary patch literal 687 zcmah{O;5r=6kIA2FnH2!wn-1oMj9|>)6!5V;o^;WB3_L#+jhamQlKpwA^Z&!|A7a8 zq>m4!iBXegGy8UC_RZT4uWl@i@Hr+1mH?D+StZ3&gwSr4Q9ZYV^2vn+>rr7t0ltne zbHYl%Gu||tmQ4;2EDoDo9SqI}JF^PlZz-0P{-B!9c=C9k`VV5laS>suCXu2)jVy$# zd)TTWBuk2_7_!%|mC=^kU(%$Wg#(c6l+Dr=$_QWvG#8@Oyh!;`By$C3v z6$bsXrM=a9ZRF*xgB804sSGsU-c8?Dw>@D literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/checkpointed-delta-table/part-00000-e877c5f2-5fb9-4528-a317-5b30608bb45c-c000.snappy.parquet b/presto-delta/src/test/resources/checkpointed-delta-table/part-00000-e877c5f2-5fb9-4528-a317-5b30608bb45c-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9ab06ab743e0672ab115830389cb45568fc1636e GIT binary patch literal 687 zcmah{O>4qH6kJm)T7(|j4I9WomKwCgB~gq^_JY$3=vtxW>Wt3w?X83-*Rgf)%F={CNm9Y)*4Rg2VRZ$ZEw<_*mkGw mSYl#XosMT%hGmPc;o45m?ir47JhQ3jDnjs~b9kdM{HmWUf11Gn literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/data-reader-array-complex-objects/.part-00000-a7d58b1a-7743-4bb0-b208-438bbe179c93-c000.snappy.parquet.crc b/presto-delta/src/test/resources/data-reader-array-complex-objects/.part-00000-a7d58b1a-7743-4bb0-b208-438bbe179c93-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..2950d858b40afce9c0cda925d690a7007f60f74b GIT binary patch literal 32 ocmYc;N@ieSU}Bj0+RR^ewfe&AEw@c-4`?y`)stEOcJ9J;0J&BTbN~PV literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/data-reader-array-complex-objects/.part-00001-7b211746-0a31-4e77-9822-b0985158cd66-c000.snappy.parquet.crc b/presto-delta/src/test/resources/data-reader-array-complex-objects/.part-00001-7b211746-0a31-4e77-9822-b0985158cd66-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..d52f585dc0de1dcfe9a79c0c48bf7e63c6e990ce GIT binary patch literal 32 ocmYc;N@ieSU}BiKet{gz?#I)z9$#I_<-~oGeYV8|fBVa80KFv*BLDyZ literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/data-reader-array-complex-objects/_delta_log/00000000000000000000.json b/presto-delta/src/test/resources/data-reader-array-complex-objects/_delta_log/00000000000000000000.json new file mode 100644 index 000000000000..f859e2df129f --- /dev/null +++ b/presto-delta/src/test/resources/data-reader-array-complex-objects/_delta_log/00000000000000000000.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1603724039052,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"38be1738-32ad-448f-9e29-912a7536d4ca","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"i\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"3d_int_list\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"containsNull\":true},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"4d_int_list\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"containsNull\":true},\"containsNull\":true},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"list_of_maps\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"long\",\"valueContainsNull\":true},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"list_of_records\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"val\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724038935}} +{"add":{"path":"part-00000-a7d58b1a-7743-4bb0-b208-438bbe179c93-c000.snappy.parquet","partitionValues":{},"size":2830,"modificationTime":1603724039000,"dataChange":true}} +{"add":{"path":"part-00001-7b211746-0a31-4e77-9822-b0985158cd66-c000.snappy.parquet","partitionValues":{},"size":2832,"modificationTime":1603724039000,"dataChange":true}} diff --git a/presto-delta/src/test/resources/data-reader-array-complex-objects/part-00000-a7d58b1a-7743-4bb0-b208-438bbe179c93-c000.snappy.parquet b/presto-delta/src/test/resources/data-reader-array-complex-objects/part-00000-a7d58b1a-7743-4bb0-b208-438bbe179c93-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..fa057de4dd61dee26727ab0f676ab36a4812a33e GIT binary patch literal 2830 zcmcguJ#5=X6uu)VSypVc34F}4ARK|H1f^gJ%93Tv(2ycRYXnAN6deK-g+PfXhcG4C zB<0#Lyksgm7>K8!QxFtI$4r?r2br>U>5`0rC(mt%zN3CD|FINmRFLj?@7;ah`@1Ku zfALL$AW|aBI!VJrlB85BNeCgz2PC8X*a`%n*aD2=9j?Ui3s*4n-9#dJqOSoEtmf80 za&ZeHYvi6zc42RlB%`@<>P>*S`Rd(YhKOn*)$Wu?5ye!XB9-Ft{fNn$PEK$f##+e$ z#ygp>eo__hmp4xir%_D=$qAkoO=MzIFh!H|E|Ai|Pvv7C1`{W$1^%$Whaw+J5KeMyu@in*-KI+LhE z@Zq_TqIBI95r$o0pW)8byRCZL+VZ146u9%#n{2PXW4S#Bw~NeV5fwp_2bz0d_(=jwO}Mgb{dnCdp^$PSeTL2z+z$3oO6Hxnrq1Y81*vC!Zx%BE<*q*F^|_ zB*>42+_PkLf<|$P0nft<;rU%XUutI@Wn z!`iZ-M;KwdF#K)TfWEQaV}apye=yoCuxu-^=rQ z@AkEh)#fS(q=Q}1VAuxpK+x&AuGMfs4+oj94KcNq{euI2bat#MjZj9MmgiY}ARe@) zZ$nG|i&i_eui5PcmecXS20QyU9_U(pVNCbNZq5JTUUX=#&-qKk6f+{jX3CrNq#=Pb za0G65zY=&&I-Ly=g}r(pbi1db`>~Teq4k5mf=L~SxJ2N>2yHA=k#>E5RmuL#DMTL( zuKVY0FSElu-|2QT1*2dZg-pKDb6h)PEm!Q;a;s7)E;P!lRW28cwpljI6}D(LD~+Yf YlG$JlyKq-=RfNuc(ILDrt;3t}Kkw9qVE_OC literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/data-reader-array-complex-objects/part-00001-7b211746-0a31-4e77-9822-b0985158cd66-c000.snappy.parquet b/presto-delta/src/test/resources/data-reader-array-complex-objects/part-00001-7b211746-0a31-4e77-9822-b0985158cd66-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..e99ac42ae864569f76355802f37c6a134bde4ae8 GIT binary patch literal 2832 zcmcguJ#5=X6uu)V>L)Ie6h7uy5RO2^f>JO9Wm~dkXmC*>H3}mzf(!wIK%m5vON5df zl1gkCUV|WmfqE)BWhjh}oicSUI(6%kr5%e*p4$$6NBvm-V<;pjA)dtd?tS0;yC-gb zalb?mSt2V2$-xhqWHc&e5kgc-!K$F*#1jZUa|Bq$n_MZworXIDH#Z>?DH3A?fFy)w zH!j?Q$Oc(6$PVn4i5$;W(5?gO^`HOv#}rW=apIk{ND;+Updyu$@x8=k!yv~v31fZ9 z1d^NiuU=`Y|HH4}9%NBn1j#YZiWai4C0L@xc^AmiFi!Ovh7Dz$s#W-pWxlNNr3&dZ z$$~*q2V6g#=anoR|igOOjKv$upRG)8X}-gGkZg`rEso z5<&e}Tb~KGZU0FlXnn&vHpw9(%nKB|s4UPci%!Gsh7HdRLLv|ul_e?-9=JNqJl0{| zX;cyT$S$H-`q&gPj-6wl;lVU|?MBDm4&oyec<|XLY`5{u_WBIY7nzwyR0PR9ZyHEi zD$?+%u8weh*6R6AJoF&rA{Zkjr)sKK$Y@ZSat6sN6oLp!PC3BJAueQUF@r(Xpcw%R zN?iCJbA)+GR(B?A7}ai+{mUPKaYy+JD}Q1DilsE5`y`+QUj7Ble+wh#@2*Ug_Hwez zHQ>fER(|9wz)tDMJ4qumQ3f8KD)K41lXCJh0^h9i6IR~f!aJ!p9u?~OAYaQGk&=TC z`XYiq6qFZ2;k8_wVo+S*z){p8yuRyaTkTYUys3b9PWc5Zzv04Bd;^^eR5{8iIizaz zcF*53?QI)ogc*$sGkEHmFgKp|S!nvb=Vph6wqu7j{XM;>hr8QMU)6)q@3%tT(AzHa zoIqcFw5NCN4p+G#9qxhw!?u_Yf^OgQ?4}2LFl2^4!qivy_VUBfg?FRS2&w-EohMruQFuaLV^FM?apPG*s{Do;sn32&i5ri;Co2XQ*Uym z`t|U*-|6`IXUq@WUN>JdOO{#67n^<8bMp2|)oHJ^tJTGYW`(sYmBmHJs#ujOD_gB< bbGf=~HCfXs-Bvvfp)((J0Pj|t@Gkrx5iN$| literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/data-reader-array-primitives/.part-00000-182665f0-30df-470d-a5cb-8d9d483ed390-c000.snappy.parquet.crc b/presto-delta/src/test/resources/data-reader-array-primitives/.part-00000-182665f0-30df-470d-a5cb-8d9d483ed390-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..003ddb680f8212ad5a24da28200bdb5bc66eba99 GIT binary patch literal 40 wcmYc;N@ieSU}A`5JLucx(qLNGI-Qqu=lS`WcberkGwf*mzJudlBg0=i02aa!B>(^b literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/data-reader-array-primitives/.part-00001-2e274fe7-eb75-4b73-8c72-423ee747abc0-c000.snappy.parquet.crc b/presto-delta/src/test/resources/data-reader-array-primitives/.part-00001-2e274fe7-eb75-4b73-8c72-423ee747abc0-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..0890a315fa1fa2596a7358949ecb559b9aeec718 GIT binary patch literal 40 wcmYc;N@ieSU}9Ks_*(4zVx`K&gY5IQMC+4MHcGsI@Kbf$nNFc)^JJyu05oY3HUIzs literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/data-reader-array-primitives/_delta_log/00000000000000000000.json b/presto-delta/src/test/resources/data-reader-array-primitives/_delta_log/00000000000000000000.json new file mode 100644 index 000000000000..4928f2bfa118 --- /dev/null +++ b/presto-delta/src/test/resources/data-reader-array-primitives/_delta_log/00000000000000000000.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1603724038064,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"caaa1362-3717-449b-ab9b-f7d8d536018d","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"as_array_int\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"as_array_long\",\"type\":{\"type\":\"array\",\"elementType\":\"long\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"as_array_byte\",\"type\":{\"type\":\"array\",\"elementType\":\"byte\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"as_array_short\",\"type\":{\"type\":\"array\",\"elementType\":\"short\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"as_array_boolean\",\"type\":{\"type\":\"array\",\"elementType\":\"boolean\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"as_array_float\",\"type\":{\"type\":\"array\",\"elementType\":\"float\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"as_array_double\",\"type\":{\"type\":\"array\",\"elementType\":\"double\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"as_array_string\",\"type\":{\"type\":\"array\",\"elementType\":\"string\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"as_array_binary\",\"type\":{\"type\":\"array\",\"elementType\":\"binary\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"as_array_big_decimal\",\"type\":{\"type\":\"array\",\"elementType\":\"decimal(1,0)\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724037970}} +{"add":{"path":"part-00000-182665f0-30df-470d-a5cb-8d9d483ed390-c000.snappy.parquet","partitionValues":{},"size":3627,"modificationTime":1603724038000,"dataChange":true}} +{"add":{"path":"part-00001-2e274fe7-eb75-4b73-8c72-423ee747abc0-c000.snappy.parquet","partitionValues":{},"size":3644,"modificationTime":1603724038000,"dataChange":true}} diff --git a/presto-delta/src/test/resources/data-reader-array-primitives/part-00000-182665f0-30df-470d-a5cb-8d9d483ed390-c000.snappy.parquet b/presto-delta/src/test/resources/data-reader-array-primitives/part-00000-182665f0-30df-470d-a5cb-8d9d483ed390-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d156253ed6823ff4ccf597e156e69b0d8c10bd40 GIT binary patch literal 3627 zcmb_fQES^)6uwuM6~}YB4tj+Q>gI{rOPFWuCUN6nsMn`aN-1UZWo+TfzD~8tQYy(C zAp}Y(r4M`957<-5V}8aS_CrSLLmvj~;~+a%vL)Hq2`We^H@a8neBU|ep8MH;_|2mZ zLF6vE+azUpP>7;yvWyVY_<&TjnVP}E=b8j#0Tr-d&K$XuajCTe~dNE>lprr9Ix%nDhSVo zivkgoLFm~aDSt8p2ci7%_71EM;D*0vgRuPfQuMQ;Umq-%AAI0SuflYx{5P@u&z+48 zG*~ZvQ5tZ2?g0$<$^C?(3#sjBxD78G$@Xa%1>PBLT9fp#UW zLj42kdGz?easna|1?|e3Y+F7|5RvMVw#IBWVNSrUH)3nsbq)n{`BHQ9IADT#8omB0 z$G;Zi`%hf&>Uiok?r}Z#FkRPX)Dc+FQcwxU*mkMNQChP9IA$2RQ_~jHC|~N}2cET} zOqG;F9Xe6c5)RXHsCO(Rk#^w@H{zg7>+oR2hSr4Ig4aTUx#USmEm!sGd84Egsai)$ zb#I{o!i}GAIFZz{`Ui%)QR-hIlvN#&IW`=MGprLi0$%JD5&3Vd{)NGzudUF>OH?yW z^-hX&NT_(9iBwS%@1(3*ypm(+q+7y{ybSx5oE7`52<3_@B=oL`{-bI(&We3)QCQPu zc&X^56yYVYhMg^xbb$KL(R20 z4fehqH^a(PCnCPU>I>X>U*?Euolc>u=M}XK>1$iAci5sw6ePJ7s&mVKYPUe9pH5lO z^4wor6Bf`B4XF0-`iT)7A2DO!fU-=7fzdR^7PCjbvH$&v;m`?x6$=CCIT%C^v12Ip zusqZB?;Q`w4foHNTcunzL|gi`D{Nr zeB368d_lf!kRsd&L{KO%A%rNOhh9e8u_qY#m~znPP!0=EZgLDFz*T@tgo_0N%Mawc z7@f#QS)(Guvj((lM&U&#qAxLe2_r!&@TgE0`%Hkf1zq;@&NvMUsm9WPF~w}*v;VM+ zdk^NBg!|l0j%9yEAEC`d$|LmqfNewc1Odz2z#s_y3D@6np+g2uZUel5rVi*lkz0&* zejgtNP;48}4Hh3X_SpVX;myZK}oTfSbmA? z7g8U*FtFr!PF@w)Pz5I|N%aQHuW|iVkx?zoIZpO=BP@osr!L9(3zjc%{p|*0j2m@~ zRlX|8MOZ%FaoppkcA|kLH)rnL^j=s^u;~|5!*9CI&&`S9Ya`9q=)cmL>Ytt%>VXPr znGSump^i<%8hPr$(=*l9ChS#c4B+Pg2m->unbpJa+|!=~bO1Gu4Hw3Tj_qrv?R_(~ zEcF0XjPr(S!-EFS7YsDi#f;R0v-9&tQki`=q?MjyGtp&-KdtT@n~Cns8PWueaCI4Sm$!saz=t-SLO+Zy@vtemDLP|2jex literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/data-reader-map/.part-00000-d9004e55-077b-4728-9ee6-b3401faa46ba-c000.snappy.parquet.crc b/presto-delta/src/test/resources/data-reader-map/.part-00000-d9004e55-077b-4728-9ee6-b3401faa46ba-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..cb0434e0b3fc61325a600b419bca3272f57b616f GIT binary patch literal 40 ycmV+@0N4Lxa$^7h00ICAqyNmGH=uEn5Efx}f~yx2HXY)i3>AZiLF1XDujp-|E=s~y>!thCN* zm12w`gq(WlA*T?64?TvSL-4t$(vt}}lpK34^baVcZ+2JOQM9(y*kqArci+7Cz3=^) zC;P^iH>(7ZHL_kJDR_`cGOJ1wLP+HcQczKN1Hq?84A%H1zKP-My}_MpgZ))p5t`u?9kp~ITGy-xhZfZ+3a?@TAXOK!^v{)r1=N~Pex7V`eQp6BHpyn_p$?&~ zskj20g|L9?^BC{j3Qjbu{9%PJt9)5|H(&r5Z+X4Ml zN{K8H3C`}6BlpR734llxfBL1!E4?Jg6X@}^KfnHn#4`V0yq50oQ0MOLetVmBDOp4$ zmXE2}L^W6`iIEtpN$UJvwtM?NwFV4Gfd*3JAn|n*p4`*dqxGc-{rr+=(T=tlddTNs zyZ|4;hY@X&9QjC%>$7P6bfBMy0mdWtJ>>Z`goq*bPPE;S8Q1Y`Ku_j5k}R|D`o?q2 zVqMnbZytCYQiPOLCYyOvNM<4LwIq@=kYy56Qp(R*d5jCm<_QxxlLcO!88r1%%n|Yc zu#Rb17+w$$?ip5|VbGct??o&UZx|d9^budgW<@-TXNeRr)FT-Fg_S>W;V9lzBL?p= z<535sdu|9Llrey`F=D1sM?V1cpB-dg1k4Qym}2Vw5kT zQ<4FOaV)2oyl=vp4A~RjkcjtcK1))QcmgM=%Wt^wY98tWjRu!Ek9l1trDGnRoXe7Q z;DJRjeu$NaxbWm$6Gp5*qeXsC(_sw+Yk}V*tUQ9wPIG&4M2jc~3n5A;f`eblgN0cu zdAR7|NpfZz3wS;&@5m1iJ3r)b0GE{Cv2uh9hl?k=D8cBK_kTHkdBZC5L49g|r` zU)%V8PwUYx-!efOXqi2iZ7~M~y@6#>-GY5EWF>8gscr1-@0UilbYwg7)TKKhglY4w z?>WTueh8waU)epEn!Wxv;MRA!KQjk?ItR<{ZGo0ZK;L!QP;?sa?djh8+qT0~Kjz-H zEk=7o>1h}k(;0Luo4(7yum`-p0j<*z;2?ej9oq!Ze&$384AwTg)GAg>mF2VG(V5O8 zvKdFLspHUH011Ac{Z<|Ao^XVqvLoMG=;OpgbPGDX9o6<;oBaQJ+}cO#^lpcEMA;(fm=_!PsgHNF+AA0N|hvGvIy7oUPxNr7DJF-`@QkxFm)$W`3e((1_ z=1IP@{Y9A|a*J#fNCF-dqNa5@ju6u2DBLxa7v4bdjwQi8@*dw*;EKVe!o^L%{v+~< z52VZndS7>dPy{P?Ur5&qcNUof_Z7J(uv0l_zI2uA^9A9CNMWNAuuXEdag3CBOh&N< zwa&xm%ZK~VX_(6UnzNcF86gcA>zD`bd_owDx4go@bL1?-yW*UGgqF5-8KIxIczXnI zU@%!#u7c2SaQy`r+UBYNcdiVu1IyUby+kf^SnpYsf|5`MbcU5B5G`+E1(fv#6{Q!{ z4>DUIgJo=iqaCsn1X_g9~a|aHm8f5~|30{1H2N_=MU+2Bbg(=@F2)x=}~&=$qmC za)^F*#j$9FTlgNb8JIV~NAO{VTc~3niAjAHuAlVuvoO(k#IA=dKL`*p#NG_I^OP&J5c62h0(&0I-2+P#9hi2ku9#{e(elUc6%|6weP12)c;J(!7YTBWWUg49yUR zBdq;_bFU+HJ!B5^vEUEcU}hsR1J9!YV*&pHSkpQ2z6qu?WKVdT$l55LCW(u90w=dFqd3$B8V!~> zn0Z|;N+&$LypbkJ&jSl#{1j_Xaqi`fIt*ENN{jIxCxaRY)&jq8u=Xv^9VfZHIHX0C zR|^43GlYX%$yW>WR`PVw!IPEL9v1LiR?d<44?92PZ~#}d-<|e8U7YEn0Ha&x^8q;@ z@T=t4B`pEz)K|N`-6GwmFp!JRY%ca6+eH}CkB4kf>~+5`w%LGMbU^jLV~57zV4oSA zMt{&7HU>t)XttPb^^MIh4~-6O^Q{(0J*`$}z;;;=1f8L6Q`3fhFk}V8$22w%kB$ms zTROHKdur2t5W=*%>v|3_oiBoL>Brs9fVMjQ&%v$h@@Q@j=H(phZf6&?L;~i)fcc`U zcyC@u&%o*qd3`-vS0TVb+y0amNJheJbx&kK$j|0_#icb#$078xkEC2ui literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/data-reader-nested-struct/.part-00001-0f755735-3b5b-449a-8f93-92a40d9f065d-c000.snappy.parquet.crc b/presto-delta/src/test/resources/data-reader-nested-struct/.part-00001-0f755735-3b5b-449a-8f93-92a40d9f065d-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..976c62f866fe6772db051e882b3dcadf0c32d40c GIT binary patch literal 20 bcmYc;N@ieSU}DH_y1eWAys0T|!ICclMq&sa literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/data-reader-nested-struct/_delta_log/00000000000000000000.json b/presto-delta/src/test/resources/data-reader-nested-struct/_delta_log/00000000000000000000.json new file mode 100644 index 000000000000..4046b145ff4f --- /dev/null +++ b/presto-delta/src/test/resources/data-reader-nested-struct/_delta_log/00000000000000000000.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1603724040818,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"975ef365-8dec-4bbf-ab88-264c10987001","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"a\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aa\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ab\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ac\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aca\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"acb\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"b\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724040747}} +{"add":{"path":"part-00000-f2547b28-9219-4628-8462-cc9c56edfebb-c000.snappy.parquet","partitionValues":{},"size":1432,"modificationTime":1603724040000,"dataChange":true}} +{"add":{"path":"part-00001-0f755735-3b5b-449a-8f93-92a40d9f065d-c000.snappy.parquet","partitionValues":{},"size":1439,"modificationTime":1603724040000,"dataChange":true}} diff --git a/presto-delta/src/test/resources/data-reader-nested-struct/part-00000-f2547b28-9219-4628-8462-cc9c56edfebb-c000.snappy.parquet b/presto-delta/src/test/resources/data-reader-nested-struct/part-00000-f2547b28-9219-4628-8462-cc9c56edfebb-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d1b86143ec4194806a8401636a50ef56648d8434 GIT binary patch literal 1432 zcmbtU&2G~`5Z<+&$cn-NiM4F7L}bw@m4dOG#EqjI3Ma&=;z|oz+nY4P`Kj#`P!y@> z3L(T}@C=;$44gUf2))6qO&tRjNkrsK#xwK%tS9o}7=A2V>f1co8^BDNdiih2c)1z;SLJ-?xaqB;PL_D$^v|O zg-?Ttm&F69a>@Z6Rfdp$fbfPYv>xTeC5&*X3<%+td|T76Ie@*rvCX?dYV1DO{L7}G zy?Z4mig*Bd!1;WD;F2 z;d4VN>LXS@;rbQs|6_+m?F&&S(jQEj{#UGg!SxHVzr_-Z%$HK`QNlP(a@wb3ac{Ht(oy@Xu|qV=PWU!=^Jy*5*pBis^7(hk-VERqi4iPH1`vu zX$(dz3}U1AVqr|^n71fYv#E`ySL2*?G#LVXG7m%Qg}??ftZ7_3FnWvSvbj?7a;fdH zevaM9reE+mnj~zzH`vHhC z#!ulV@dNC^vnM~o-t-N1OH~v$PMG2G-uwNTzJ|kx&l&`g2jo$Wlpx3xS+`LYJ1TA= zgp8U9r}#<0@YTY(p3Q?cKRWzy{u67H?pjxsia@DKIXD$m&HRGIt&^3L?;@|}VHaS_ zusH@?ydY0AOft{%rOLdYFJRn|g~EZ6G8Zw*7Yh<9+>m>mo3AS-(6~=J8I6p;KiuB* z1%QgTH<}f+<>PWRrQxfWk$1vIlt`#0sF;#GBIP|n6)4?c&Uh@g~|Gv!T1?xlm(bhr%w!DHnd{|azZz&YNG zxCDfdNOZV_#|^De@3Ha`S1xhyAEV4_FGZb5zcD5HU$F8SSC(Q=-7vJjj55o7A*CKw zoKU{Q$_HGzkajeue9b8(r0MGZBpg_DN?}yiYHC(=8dxyC(;17caPrz3v6%Wars_9w zZpLq?%duCNZ_tk8_d9*ZY2I^f*0=3u)34ig+hMJ`=eTXB XU3Z!5H}4qH5Qf(+N=m(M*pxklfTEyFVl*b=rMLD{q?BGtDVt=~U=lSS77=f~^w?k9 zTj?m4LWO0P;eBVG*`3MFgGCTg$u)WZ`uM(N1VdUQ3ZoShj*42x8zFSn!VoVi6^JIm zTIaT&uSB+Z&4am21i^^Z8<;2%{7B=moQYt2cmL=)6d_8_3>%r{(K4k~n_JvOw0-8t*D;65 z*QG3YwtVA>EJP>@L7#UGs+hWAI_0u$Bod^9P+ecz6H+ literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/data-reader-partition-values/as_int=1/as_long=1/as_byte=1/as_short=1/as_boolean=false/as_float=1.0/as_double=1.0/as_string=1/as_date=2021-09-08/as_timestamp=2021-09-08 11%3A11%3A11/as_big_decimal=1/part-00007-00fd7022-5e9c-4cba-b8c8-2297ef36e5ff.c000.snappy.parquet b/presto-delta/src/test/resources/data-reader-partition-values/as_int=1/as_long=1/as_byte=1/as_short=1/as_boolean=false/as_float=1.0/as_double=1.0/as_string=1/as_date=2021-09-08/as_timestamp=2021-09-08 11%3A11%3A11/as_big_decimal=1/part-00007-00fd7022-5e9c-4cba-b8c8-2297ef36e5ff.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..dee0f7863329179897f68baa5085fd6e7851abc3 GIT binary patch literal 465 zcmZXRO>4qH5Qf(++CaT-*pxklfTEy_YcwX}rMLD{q?BGtDVt=~VB$wUEF%7eLjOgN zz4ZTe6icDPGRyG3GtcbK_~tMgws-fBo{kQOpINXPeq^5={eQ^*%Z z97{h23)HHX_1o1N?)LkaJ)%!1W;;pE(~zd%qQgb|K=;!qm}j2Z@QWx8&0yjLZr7f; n{lNAeFYI|r^ar-;xvFbBqUQ#Kj;Gvi(_jof>;u*^hGqT$@yB== literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/data-reader-partition-values/as_int=__HIVE_DEFAULT_PARTITION__/as_long=__HIVE_DEFAULT_PARTITION__/as_byte=__HIVE_DEFAULT_PARTITION__/as_short=__HIVE_DEFAULT_PARTITION__/as_boolean=__HIVE_DEFAULT_PARTITION__/as_float=__HIVE_DEFAULT_PARTITION__/as_double=__HIVE_DEFAULT_PARTITION__/as_string=__HIVE_DEFAULT_PARTITION__/as_date=__HIVE_DEFAULT_PARTITION__/as_timestamp=__HIVE_DEFAULT_PARTITION__/as_big_decimal=__HIVE_DEFAULT_PARTITION__/part-00002-b31c437d-93a7-462c-b88b-f8b49370d785.c000.snappy.parquet b/presto-delta/src/test/resources/data-reader-partition-values/as_int=__HIVE_DEFAULT_PARTITION__/as_long=__HIVE_DEFAULT_PARTITION__/as_byte=__HIVE_DEFAULT_PARTITION__/as_short=__HIVE_DEFAULT_PARTITION__/as_boolean=__HIVE_DEFAULT_PARTITION__/as_float=__HIVE_DEFAULT_PARTITION__/as_double=__HIVE_DEFAULT_PARTITION__/as_string=__HIVE_DEFAULT_PARTITION__/as_date=__HIVE_DEFAULT_PARTITION__/as_timestamp=__HIVE_DEFAULT_PARTITION__/as_big_decimal=__HIVE_DEFAULT_PARTITION__/part-00002-b31c437d-93a7-462c-b88b-f8b49370d785.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..07cce42752a23b23e8bbe6eb1d0c4595d77f38aa GIT binary patch literal 465 zcmZXR&uYRz5XRRnN8-sKDW#WE$|hMgm_*GVi-^bGdg(Lt z$@(T8#ZsuS%rbo6%x`vQdh=isL{xH3zCPZ+>x^JXYeZqRV!=^S`*b6O>TL}1s#1Yy z5v+A@8~Ivf%hx#x&Ma7-?ujrG~V#M6in0P+hc?$v=k1 zfQ_IL6o)EC+Jv%IHs@k30Krc*9?Q81c6axWo?N6g9#S{C+2>~ONcXcSSmvJI_RA;^^%J=O0d(2;>6*jxY=; literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/data-reader-primitives/.part-00001-09e47b80-36c2-4475-a810-fbd8e7994971-c000.snappy.parquet.crc b/presto-delta/src/test/resources/data-reader-primitives/.part-00001-09e47b80-36c2-4475-a810-fbd8e7994971-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..852ffc4e2fc81dbc172846a8aaa354a3e6b4bbb0 GIT binary patch literal 32 qcmV+*0N?*(a$^7h00IEh-rd!l`nSTowzL;wmFMM{+YB#Qh9i&o0T1f{ literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/data-reader-primitives/_delta_log/00000000000000000000.json b/presto-delta/src/test/resources/data-reader-primitives/_delta_log/00000000000000000000.json new file mode 100644 index 000000000000..9c9a0d11558f --- /dev/null +++ b/presto-delta/src/test/resources/data-reader-primitives/_delta_log/00000000000000000000.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1607520163636,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputBytes":"5050","numOutputRows":"11"}}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"as_int\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_long\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_byte\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_short\",\"type\":\"short\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_boolean\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_float\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_string\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_binary\",\"type\":\"binary\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_big_decimal\",\"type\":\"decimal(1,0)\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1607520161353}} +{"add":{"path":"part-00000-4f2f0b9f-50b3-4e7b-96a1-e2bb0f246b06-c000.snappy.parquet","partitionValues":{},"size":2482,"modificationTime":1607520163000,"dataChange":true}} +{"add":{"path":"part-00001-09e47b80-36c2-4475-a810-fbd8e7994971-c000.snappy.parquet","partitionValues":{},"size":2568,"modificationTime":1607520163000,"dataChange":true}} diff --git a/presto-delta/src/test/resources/data-reader-primitives/part-00000-4f2f0b9f-50b3-4e7b-96a1-e2bb0f246b06-c000.snappy.parquet b/presto-delta/src/test/resources/data-reader-primitives/part-00000-4f2f0b9f-50b3-4e7b-96a1-e2bb0f246b06-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..b0442b0085a0f143109ce0c76f430d70b5130162 GIT binary patch literal 2482 zcmb`J&r2IY6vt;azt%-96ld9#MZ!XYLb_~>MiT<2w^DkL;;j_fxSN=Ev#XnpMMUVS zhaUTPc<@xH^wdM~zLW`sH&mQ$v{i_a-SUpB$ip%cQK zHn`WtnG`$IIfeOf@m7$NoUUM${ku8$tkj{i=se)Ak`2afKmc_G8!-67igto-twC=! zdIJPPb#QC*TMTYOCKE=C(BeYun1{N-`BjXm7KG56xtk7fO<7h#@z?9w(Z1<_+8OQc z*+Y{oAR?3nC2x*)25x{z7DNGrUV3!voErK5Hd45nXL7l>MiB9l5$2MI= zL@>}9J;yTvCxM&+X?f$WVrvnj+Dit_<0;=vaJTVIkoZh;p<+7(`97-*7nj&zktOS;KI;Bl{#Uxe~Aquj7hAZcJ$U5u}Tzvo0tlzgz zp>GQ+ndQ(?eTC?cvRk<=w4q6&w&Y{GhGOE~q9P344&}qjwMgBPw&WczXV~W4sn*cj zmHo=)4E5$alOxE#C0Rnns-k#)UpMznXh%KlO#RUIM+47Qa&_I%4WumJn=v|YbZFv( zu^s5X_gNp>foYk6sr<|xYr)aJt+h1Bz<4*%s#Cjv*469uS?1j4;vZ5mBKt#6+h@L`BaK6a7Dk$zjbn@#(!(s~XE-eU$d$1SxOg zl72b#OK}gBYI@DkYo(>`c;HwibE9eXHhRrweWly5dyPiDZW#@u(X`i$-DY>axo&jr Vu2owuo~j5v@OkR7#Hd2MRspoJ0SBkV6ll=Nxk?eX~2O87IbUAd69sA?gEsH#G5qjgEaz@1cq{?>gqsX^1B8PH5<%nTS2Ig(%|8)XkQ znGJgYdd($+fmM5Hr$0ptraRH^-SP@SI! z$xp}glSkn+AZv&y4Tb8#{CF};5m7-HAfk9S&Zn3foTXvoPOeSR^G9)#HtLY_u)M++ z#Bme`Gek@nU@JyZ900BY(gHb*7lSB7q{{U?O(wH(1B#{?8BAtDa+Ys#n-M4D<6(F* znFdj#0-h_#uGXB|tETA?rM7MAs8*bsD0bEU72ChzrrEopgli6P7zv@~AJ`!(#}xi% zoPV+XH-_mRy{1rE8WnXYiWtMsxm z)}ejWe5v*5)l!um`Uuu0w_d(k=^P)yS}Irwk)*DK=7r&q3MYxA$WRdVH@Nx2fT+XU zfu}_A%7=UhSy7eHd46T+XZEw^JF^GHk{7IO_SU*;mM39m_6xQzar3Rk42x2gP2X3< z>-i*K`L;jf=KBgWmy1+F*#CWFS5d7+X`GC`;52}W^s?ggo`lJK63?i%>9xH!q8p#B zm_Lp@sN=_rF!hr7sW%PNU>KwU{Zl)4)3eji-E$!bi<8uCxuZ!K4d?FOgL8KlOxY9^ z7?e9q@ueWbahSlsY!O8P)T+A&)?v#n!0z7p#YIa7uq#3kxGHbX?i69pReAI53KI5( zEDzvUmOzMs@{sJB6No~V2jmx{AS`uxM4sycvDD=edEN`eQhy0zd0z_-zrfc!t@Z|t z-bj0Rf#?rBrN5shbzTGYj@M~>o%;G6ht06)j% zNLQ`G1t}sZk9Kf;dR1z72$r*=-KS%`inkyYjmg-&7LY$~+b2Tj8{_6QCf%;KV zC8U{z%+e|2a}K6i;h(*VGz(`5Iq5KqS%|cM5ZNMP(CT6-a+c2CSS)hx^PH3S%7*6a zxu64@<>_*gQ+>@f;C2PN(Brj^((b bYfW6Q=k_d5c)s1%OdTN@=(B>*82;NA_kWHR literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/deltatbl-partition-prune/date=20180520/city=bj/.part-00001-4c732f0f-a473-400a-8ba3-1499f599b8f1.c000.snappy.parquet.crc b/presto-delta/src/test/resources/deltatbl-partition-prune/date=20180520/city=bj/.part-00001-4c732f0f-a473-400a-8ba3-1499f599b8f1.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..8c26ac11ed50250792f542f378645076718d0f78 GIT binary patch literal 16 XcmYc;N@ieSU}AVaEA(H5%olb5DDefJ literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/deltatbl-partition-prune/date=20180520/city=bj/part-00001-4c732f0f-a473-400a-8ba3-1499f599b8f1.c000.snappy.parquet b/presto-delta/src/test/resources/deltatbl-partition-prune/date=20180520/city=bj/part-00001-4c732f0f-a473-400a-8ba3-1499f599b8f1.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..4787f1ca94e6c869008f1894585bc272ade859b4 GIT binary patch literal 650 zcmah{O-sW-6x^gA6e)Od!v=zs#g-Q0l31EHr5A7Fsfafb*)-c4Owx9fC{hX@Jo*Ft zmHq}_lPVDfA-v7Yo0&J0WO{yOV}y?}HL(geb=)vW?OJBhYJX^``=dt)9gb9}6{Q9> zhXU~klafMJff30o@Jk&$J~%1ZI|x?w$wf0+artnU%J7cXU{zDK7e#wjI*rLc)^!7%f+TG2yeS53FNAjbxlXJ}7)%Q1 zfBG*;XdjBEBHbipDHn{dI0RsM7WOD)5-ej<>obd4i1dCTl6A-+_BsX!2(DZpF=$Iy{%w{Py={yjjpV0BmhVEVqjPk=Q0fTq}z&Hy-?uCE=sxax7DjjdO zTl1R@aRI#F62z%k2>BoTc8kd6-tlQ9Tl;LchgFu0J2mUAKScIb!ZO j{l0HGmg9(*u(dOtxv$VD+mgse%T_>l^@fErPLLt7Ak6!uKfK z;58N(tcu9osPK5_@S97CKDcZZnam&Wi{XPxBy9j~9TC<@WulG%D8ZsWOiR#+Zs=yG zY0#G5h3%%k#89FUE!MJ5bMS{<-9UdRguOd22DV(X7ort8)$-2EVCnW(&l>{Tdrc#t zn~aU~k&u~0)1vY(o>X2$$Il{Br3_>#*;{SK%hyct zJ}=6AIxM-#!$`$J!TYx}zIY0}sy1^27^NcwrqeiO7Qw47 zQCg~z%Ku~E&4FF59G@n6do^O)7Pckq_OU;W;-D>iUJ&*|&vVXvSB0+YIDzF_uBW=z a(DTo|bIVtLV4vuw0e}lWY5>>x-#!7KP>tvS literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/deltatbl-partition-prune/date=20180718/city=hz/.part-00000-f888e95b-c831-43fe-bba8-3dbf43b4eb86.c000.snappy.parquet.crc b/presto-delta/src/test/resources/deltatbl-partition-prune/date=20180718/city=hz/.part-00000-f888e95b-c831-43fe-bba8-3dbf43b4eb86.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..811ff4f5a12edcf0b328b386e0995dedc8155da1 GIT binary patch literal 16 XcmYc;N@ieSU}Ct#`qfk6!N%PHCD#SU literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/deltatbl-partition-prune/date=20180718/city=hz/part-00000-f888e95b-c831-43fe-bba8-3dbf43b4eb86.c000.snappy.parquet b/presto-delta/src/test/resources/deltatbl-partition-prune/date=20180718/city=hz/part-00000-f888e95b-c831-43fe-bba8-3dbf43b4eb86.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..019d84d3fc80a8c86cdc8e6d02b28bdbc9b23902 GIT binary patch literal 639 zcmah{O-sW-5Z$Cvixf|8*gy_pv89E$B*vyq>BXD$RJ@AFrrFkDKH4O$Xz9gY;Scn0 z2u|8+i-N*3yR-A&n>Wi2uWoFNu#OEA%WycvRh=kVhIDz zDJJ{mBd8Fij3s%O!Nu{}hFo4^S?gcd@+r?A@AJ`vh&fgeuGM9tBs>=N5&=nNloUvn zHC;0sRUOx~Hae_o8w{3JSzV|vjcWf7&zcS%K@55}UL_s+ZX`eNDNJz(v3S27vfe#Cds4*)3nV)ky3Es(hu+} z{RVH-DiMW3xRaZE&bj9#xjw(KFv7=}8d!##CRTLPxQdd1?+z99e)R~U!;uWNCN-dz zP#_**Qc+{0A}PxQ=kEBa?cRXSd+ z*TxSU;sSWLB}#HJ73n_q-5Qb0o#WG4+TKNMTg)<<)$Vyk6bEfSa)WRbx~_fZIU;l% i+YU^}bX+koeb*bhL(>ypVD&XaM+h!dgRgN7f8iJ8K$Nxs literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/deltatbl-partitioned/_delta_log/00000000000000000000.json b/presto-delta/src/test/resources/deltatbl-partitioned/_delta_log/00000000000000000000.json new file mode 100644 index 000000000000..df728b31bb46 --- /dev/null +++ b/presto-delta/src/test/resources/deltatbl-partitioned/_delta_log/00000000000000000000.json @@ -0,0 +1,7 @@ +{"commitInfo":{"timestamp":1629873032991,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"c2\"]"},"isBlindAppend":true,"operationMetrics":{"numFiles":"4","numOutputBytes":"1734","numOutputRows":"10"}}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"c1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["c2"],"configuration":{},"createdTime":1629873029858}} +{"add":{"path":"c2=foo0/part-00000-2bcc9ff6-0551-4401-bd22-d361a60627e3.c000.snappy.parquet","partitionValues":{"c2":"foo0"},"size":436,"modificationTime":1629873032000,"dataChange":true}} +{"add":{"path":"c2=foo1/part-00000-786c7455-9587-454f-9a4c-de0b22b62bbd.c000.snappy.parquet","partitionValues":{"c2":"foo1"},"size":431,"modificationTime":1629873032000,"dataChange":true}} +{"add":{"path":"c2=foo0/part-00001-ca647ee7-f1ad-4d70-bf02-5d1872324d6f.c000.snappy.parquet","partitionValues":{"c2":"foo0"},"size":431,"modificationTime":1629873032000,"dataChange":true}} +{"add":{"path":"c2=foo1/part-00001-1c702e73-89b5-465a-9c6a-25f7559cd150.c000.snappy.parquet","partitionValues":{"c2":"foo1"},"size":436,"modificationTime":1629873032000,"dataChange":true}} diff --git a/presto-delta/src/test/resources/deltatbl-partitioned/c2=foo0/.part-00000-2bcc9ff6-0551-4401-bd22-d361a60627e3.c000.snappy.parquet.crc b/presto-delta/src/test/resources/deltatbl-partitioned/c2=foo0/.part-00000-2bcc9ff6-0551-4401-bd22-d361a60627e3.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..36aeb8d3d89c8dd92b8c7136e2e8e7cdb70e573a GIT binary patch literal 12 TcmYc;N@ieSU}7-UyO{<65cUGy literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/deltatbl-partitioned/c2=foo0/.part-00001-ca647ee7-f1ad-4d70-bf02-5d1872324d6f.c000.snappy.parquet.crc b/presto-delta/src/test/resources/deltatbl-partitioned/c2=foo0/.part-00001-ca647ee7-f1ad-4d70-bf02-5d1872324d6f.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..dc43e92d33a435c0e5f1a18f4ffa4732b494ad45 GIT binary patch literal 12 TcmYc;N@ieSU}8vAp5Orh5Y+;g literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/deltatbl-partitioned/c2=foo0/part-00000-2bcc9ff6-0551-4401-bd22-d361a60627e3.c000.snappy.parquet b/presto-delta/src/test/resources/deltatbl-partitioned/c2=foo0/part-00000-2bcc9ff6-0551-4401-bd22-d361a60627e3.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c7bda6510defa907862eb01a0f8e3dbf95a80b08 GIT binary patch literal 436 zcmaKp!Arw16vneUYY(1;loWa>LB|F)B$jpS@FI8+Ps5vtNZO{e*mhf+DpKabzuZ@u zAP8O_FW>v+eILp4>P}#UPx08tCNw6tTrwOYgh*eIjmE!DAe>wqP!Ep!Vc+!>HXQ%v zxY@}1<*7-YRVlFn;+a8=L?GLAT*n<*E?`sBLQbz@!#3>?`*Sd|F8}d!T*w5gzyxR$ zIX)rldd1{Mf)D$)WUs1j@^wWH1I8IgWb%h-wgrQ9+g52-uisdyv`nRzsz@8Uc<5*)%YKvykoQ<#X|+-{2vu7YGA;mso64s@JL#g^?H+fC-0mIE z%i7yV^aK-}32z*?d69Z@9;Mkli=yBn4pkP0L6Gv0hmo4{B#LLzjK?ZY#l&%4c!AXb Ip(Xs+7klDiTmS$7 literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/deltatbl-partitioned/c2=foo0/part-00001-ca647ee7-f1ad-4d70-bf02-5d1872324d6f.c000.snappy.parquet b/presto-delta/src/test/resources/deltatbl-partitioned/c2=foo0/part-00001-ca647ee7-f1ad-4d70-bf02-5d1872324d6f.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..2297174763f7a134b1a7dc197a504f8c1abb3473 GIT binary patch literal 431 zcmaKp!Arw16vks$M-SpHC50SHz_CFMiFMsNym%8&!<&dm+NQJEc3av`bnJikRVE06 zmq+rwU*7levb=qe7-5S20NZdFVuupjK?o6h2-QX7UpElWuPvy9LEr2#YKAY^S; zTx}HC@NY-H)21obH8~Br-~y4!AEMco9Kvlo-Ey;j=ap_%rdmZF_pJTcXtrQYYr3>$ z0m}{% zC6|Irf1GqhnfYoSXZbvj0gqN35t@z0wARx+_bSlmNg>?em|Re6c?qo)3uJA000%;~gV< zfe54sym8VMW#*}QoaOU8j>D@Y(s>kxVJ0IP#d<2!IGM#Wndl@7CX_MQKo09(!dHC( D_~u~{ literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/deltatbl-partitioned/c2=foo1/part-00001-1c702e73-89b5-465a-9c6a-25f7559cd150.c000.snappy.parquet b/presto-delta/src/test/resources/deltatbl-partitioned/c2=foo1/part-00001-1c702e73-89b5-465a-9c6a-25f7559cd150.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..8143adb67f742316adca1a5e99c6bf523667d455 GIT binary patch literal 436 zcmaKp!Arw16vneUy9%C!loWC(LB|F)w3c=2@FI8+Ps5vtNZO{e*mhf*Dl+E5zuZ@u zAP64vNWS;W`#zGz)m?xQKE)#so3JslMM?h%Aw>FzP;E5&bpqk!+JJg6=$Zpq9f}Rd zyE$&wLcKgS>9ec^Hb6Wxh>-@!HXZ8Fp+y0km=_F>pF<%Ni~G!o{ zbJl3prkZ&yFJzfDZ2quiwWzvQ8i2gVidxI1R3OxCS&F0t0B%Z;{mf+Z?QZwDL*#bv zcwQ-YAJGkX;Pb#8C2dh=u9(GHKFi}cyhtLMM^PANe&k27ocL*+OyjAa$RrEK4yEt{ K%N{}t_^&U>J7JRm literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/snapshot-data3/.part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet.crc b/presto-delta/src/test/resources/snapshot-data3/.part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..f66b5b65e0e6a4879d49d2fe07f4c919e296a440 GIT binary patch literal 16 XcmYc;N@ieSU}CUS&0YVi->wn>A>su3 literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/snapshot-data3/.part-00000-64680d94-9e18-4fa1-9ca9-f0cd8a9cfd11-c000.snappy.parquet.crc b/presto-delta/src/test/resources/snapshot-data3/.part-00000-64680d94-9e18-4fa1-9ca9-f0cd8a9cfd11-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..02716266145c8b6d3b5b1453e07cd8fd6324d2b0 GIT binary patch literal 16 XcmYc;N@ieSU}A`PANl>^d+iMXDfI?@ literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/snapshot-data3/.part-00000-842017c2-3e02-44b5-a3d6-5b9ae1745045-c000.snappy.parquet.crc b/presto-delta/src/test/resources/snapshot-data3/.part-00000-842017c2-3e02-44b5-a3d6-5b9ae1745045-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..4f8c36bdd094fec7c3348bbf6160359add983090 GIT binary patch literal 16 XcmYc;N@ieSU}C6=+9dz*z4itGB;W;& literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/snapshot-data3/.part-00000-cb078bc1-0aeb-46ed-9cf8-74a843b32c8c-c000.snappy.parquet.crc b/presto-delta/src/test/resources/snapshot-data3/.part-00000-cb078bc1-0aeb-46ed-9cf8-74a843b32c8c-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..2b1ab4061b33b665bdd25cf67896e7e31e058791 GIT binary patch literal 16 XcmYc;N@ieSU}A_{Tj|7~B*+W^AHD;I literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/snapshot-data3/.part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet.crc b/presto-delta/src/test/resources/snapshot-data3/.part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..0ea3aa2df28c746c3ebc565b1a79aa689db0cd23 GIT binary patch literal 16 XcmYc;N@ieSU}AWEV$z0R{dScAETRT* literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/snapshot-data3/.part-00001-9bf4b8f8-1b95-411b-bf10-28dc03aa9d2f-c000.snappy.parquet.crc b/presto-delta/src/test/resources/snapshot-data3/.part-00001-9bf4b8f8-1b95-411b-bf10-28dc03aa9d2f-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..692cee1e438a672f2ed8b277c5acbefc2eb71810 GIT binary patch literal 16 XcmYc;N@ieSU}AVT`-;sn(=`GBDBJ~W literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/snapshot-data3/.part-00001-b8249b87-0b7a-4461-8a8a-fa958802b523-c000.snappy.parquet.crc b/presto-delta/src/test/resources/snapshot-data3/.part-00001-b8249b87-0b7a-4461-8a8a-fa958802b523-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..7db783ac002c31c5f48a4e0c21a62692fd8667be GIT binary patch literal 16 XcmYc;N@ieSU}BgrEnxfbz4itGAqfQS literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/snapshot-data3/.part-00001-e62ca5a1-923c-4ee6-998b-c61d1cfb0b1c-c000.snappy.parquet.crc b/presto-delta/src/test/resources/snapshot-data3/.part-00001-e62ca5a1-923c-4ee6-998b-c61d1cfb0b1c-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..5c04eeeb2ec1fa234255e0640201d990243b8fe0 GIT binary patch literal 16 YcmYc;N@ieSU}D(4?)TM)@3l7o04`_;9{>OV literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/snapshot-data3/_delta_log/00000000000000000000.json b/presto-delta/src/test/resources/snapshot-data3/_delta_log/00000000000000000000.json new file mode 100644 index 000000000000..491571a22c3f --- /dev/null +++ b/presto-delta/src/test/resources/snapshot-data3/_delta_log/00000000000000000000.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1603723967632,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isBlindAppend":true}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"93351cf1-c931-4326-88f0-d10e29e71b21","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"col1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"col2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603723967515}} +{"add":{"path":"part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet","partitionValues":{},"size":650,"modificationTime":1603723967000,"dataChange":true}} +{"add":{"path":"part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet","partitionValues":{},"size":650,"modificationTime":1603723967000,"dataChange":true}} diff --git a/presto-delta/src/test/resources/snapshot-data3/_delta_log/00000000000000000001.json b/presto-delta/src/test/resources/snapshot-data3/_delta_log/00000000000000000001.json new file mode 100644 index 000000000000..db30a2fad634 --- /dev/null +++ b/presto-delta/src/test/resources/snapshot-data3/_delta_log/00000000000000000001.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1603723969055,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isBlindAppend":true}} +{"add":{"path":"part-00000-64680d94-9e18-4fa1-9ca9-f0cd8a9cfd11-c000.snappy.parquet","partitionValues":{},"size":649,"modificationTime":1603723969000,"dataChange":true}} +{"add":{"path":"part-00001-b8249b87-0b7a-4461-8a8a-fa958802b523-c000.snappy.parquet","partitionValues":{},"size":649,"modificationTime":1603723969000,"dataChange":true}} diff --git a/presto-delta/src/test/resources/snapshot-data3/_delta_log/00000000000000000002.json b/presto-delta/src/test/resources/snapshot-data3/_delta_log/00000000000000000002.json new file mode 100644 index 000000000000..53eb414f095b --- /dev/null +++ b/presto-delta/src/test/resources/snapshot-data3/_delta_log/00000000000000000002.json @@ -0,0 +1,7 @@ +{"commitInfo":{"timestamp":1603723970832,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":1,"isBlindAppend":false}} +{"add":{"path":"part-00000-842017c2-3e02-44b5-a3d6-5b9ae1745045-c000.snappy.parquet","partitionValues":{},"size":649,"modificationTime":1603723970000,"dataChange":true}} +{"add":{"path":"part-00001-e62ca5a1-923c-4ee6-998b-c61d1cfb0b1c-c000.snappy.parquet","partitionValues":{},"size":649,"modificationTime":1603723970000,"dataChange":true}} +{"remove":{"path":"part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet","deletionTimestamp":1603723970832,"dataChange":true}} +{"remove":{"path":"part-00000-64680d94-9e18-4fa1-9ca9-f0cd8a9cfd11-c000.snappy.parquet","deletionTimestamp":1603723970832,"dataChange":true}} +{"remove":{"path":"part-00001-b8249b87-0b7a-4461-8a8a-fa958802b523-c000.snappy.parquet","deletionTimestamp":1603723970832,"dataChange":true}} +{"remove":{"path":"part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet","deletionTimestamp":1603723970832,"dataChange":true}} diff --git a/presto-delta/src/test/resources/snapshot-data3/_delta_log/00000000000000000003.json b/presto-delta/src/test/resources/snapshot-data3/_delta_log/00000000000000000003.json new file mode 100644 index 000000000000..6ba370acbe8c --- /dev/null +++ b/presto-delta/src/test/resources/snapshot-data3/_delta_log/00000000000000000003.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1603723972251,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":2,"isBlindAppend":true}} +{"add":{"path":"part-00000-cb078bc1-0aeb-46ed-9cf8-74a843b32c8c-c000.snappy.parquet","partitionValues":{},"size":687,"modificationTime":1603723972000,"dataChange":true}} +{"add":{"path":"part-00001-9bf4b8f8-1b95-411b-bf10-28dc03aa9d2f-c000.snappy.parquet","partitionValues":{},"size":705,"modificationTime":1603723972000,"dataChange":true}} diff --git a/presto-delta/src/test/resources/snapshot-data3/part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet b/presto-delta/src/test/resources/snapshot-data3/part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..7d1ff564e1e6720459d44a362fb5105d297d51b7 GIT binary patch literal 650 zcmah{PfNov6o0Kl%IGzvm5L*~nH~kQP z9#0K$H{}H67rn4SgWIdR}7g)$6qcA)*Av-3>BPR72I8wwM#&U$j0DU z&1O8kxypvuV!}xUh+LQT%5WMxK-5j?*0JHLVH#FLF;R7(z_#&4BC=N19wqHbYAAz0 z0vaZUKy&P?@gkW-Et$A$lQ&6wl@J^J60YQ+ec-8JnEL)S9ou}yv3Yx)!_MZhje+J1 zk=yC?&Yp;z7Z;&FOG}#H&jjt$EKe81oLY1gi8#z?|6)lKJ}ICgHv$`Cl;mP8Qrt)u zam<4lDX0||{kEn3=qH<&Ta&bQE7B~QCN;-)nC;Y>!6J&o w8t?hxs5kQcwi|e2lG`37lz*WB>pF literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/snapshot-data3/part-00000-64680d94-9e18-4fa1-9ca9-f0cd8a9cfd11-c000.snappy.parquet b/presto-delta/src/test/resources/snapshot-data3/part-00000-64680d94-9e18-4fa1-9ca9-f0cd8a9cfd11-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..198eea06e5f107d1a64bfdaa6dfd9561093f0261 GIT binary patch literal 649 zcmah{%SyvQ6rIE(p=3KDkrc~7OABepqfMLAjp9OF6<30YOqxjzrfHj`ij?kLyKtx9 z;0L*K>34Y32hB!>dEB}8oHO?fGd#I8F~V)!({LGrLoBFrse}-c^94z$Q8+>3$md}_ zIL?b4%=HQO91Z+u4qxq+9Mlken$ecoYRkLG0OaY}Mn!;l3ot4oGGSU3(Xb2oHs+PV zxtPvqa(|nS?%0Il62g3qmrJ8*WFjnXL#qb;mK9ae>VgWaj(`r84G8m6S$g55SFSD$ z{_v-$-~q&;t@=BsVzD4%twlaL>4SsX;1_QN1?d}ZMT#mnr^#5SGYZY?Yae<#k906J zU$9J1rjPoBWi)>W`HfqW>}keGpQKr`7-d8wLC7LMCH?CqiRmOS<-Vb?1%`3P#w>wE zyoe&|MF0UV)5uPi^p~rZR-{{dI@}Y+H|yCa2i)l-86Kw+VB=( x34Y32hB!>dEB}8oHO?fGd#I8F~V)!({LGrLoBFrse}-c^94z$Q8+>3$md}_ zIL?b4%=HQO91Z+u4qxq+9Mlken$eckYRkLG0OaY}Mn!;l3ot4oGGSU3(Xb2oHs+PV zxtPvqa(|nS?%0Il62g3qmrJ8*WFjnXL#qb;mK9ae>VgWaj(`r84G8m6S$g55SFSD$ z{_v-$-~q&;t@=BsVzD4%twlaL>4SsX;1_QN1?d}ZMT#mnr^#5SGYZY?Yae<#k906J zU$9J1rjPoBWi)>W`HfqW>}keGpQKr`7-d8wLC7LMCH?CqiRmOS<-Vb?1%`3P#w>wE zyoe&|MF0UV)5uPi^p~rZR-{{dI@}Y+H|yCa2i)l-86Kw+VB=( x6wL z3;BQMMd2OGyop}3ZU`;!)Hvc@zy&)9VO-^@A}=)c&{9wUVVP^ zt)v@*v#%l--~oKVCqNGn0Q&o?GpKPxBAKIOwOH`{_s?SdOUyWt5s_MwA&=*&i-@`@ zy%wa>Ov|*}iUq2kf*R%$h3^Zc|K))!C`pznu8T*DT6>~OHs0Xe!*s<Awjok&EnduZAgLhzwK@ZL`0slEV_+K+tz literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/snapshot-data3/part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet b/presto-delta/src/test/resources/snapshot-data3/part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..43b74e3d07f80abcf6c754b813100b331ea39e4d GIT binary patch literal 650 zcmah{KTpCy6o0LVm2|zPH8qk;AefY<1u2xU=wzITqsEvkJ+QG9Xp6=WHfIMn{Sba0 zCr3YruY#eAn)F@X-S5x6-=+QIbDID;A{~R2uqhHrR|~rUpcb}qRiKvJ;lZ9S;<|N^ zfkmtxED0-Wi?T>BE%ME=|&SK1iuShXD`$gmy0%G-Bfn%5o#HCI(T;M?A)Z zXdZ^l3z33aZqOfF+FdM{Mqb|7$nl;a8vUofT7r7E9=Se_%Xd6Zf=N`iP1`c`AB)le literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/snapshot-data3/part-00001-9bf4b8f8-1b95-411b-bf10-28dc03aa9d2f-c000.snappy.parquet b/presto-delta/src/test/resources/snapshot-data3/part-00001-9bf4b8f8-1b95-411b-bf10-28dc03aa9d2f-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..33d14ac582759815ed6d205de41c55f80007fc05 GIT binary patch literal 705 zcmah{%TC)s6rHgw6iVm=JeDna5sgF?BxmFYapEW&c8C?~wyoN6JV`AaC)iG_D$1TE zD89D4D}tb zj#`x!Y_KZ@Si`cp0jL8u0q+4H03QLL09ycQ1DA>(Q zK6rLthsDc`!@%E_LOBKmEI1d)J)5?n>7nmof=v*pNEK|Yx*_TZ*a3V_{Xc57Ni7oxODRtTclnw@t#cX z75xE2Z}8Xf6%YMV)iq>V#_lv9Gd|;RefE@u6(=c!NGEeqvV8iRO+?A%I~f;vLCZfg zLHo2Q^ZBr(HXS7*jSJfUvY;8C$d*hFjn!b1m0~P%*vRH-%A*uUKr3weVoUpr<JANY?^HirfHj`ij>|wd+?^; z;0Jl~=yy2LN^?fFSF}5%yjEsW(2Z5Zgl^m!`K1*4{Zn)}C)<)#v(YH#*Tmjl;861pGS+=^lx=Tc@ ze=H|c7T;VYgKIu!qyR*$ifUmn32h+qDs5JA-jb$kMorR@)s~>BeS?Uol$1w7c@k<; z{||qfjvi1P$Ev*uI+1fC%`cHRL3tIh*Z;*^PC@y=TS3#+y-7SW*_7e%=9dpMnS~}A zn$3A?#*;g9%u|-VgZeBiXnH^8v`3RPo)1!L&|$zsKcT&g1&!D^Yh}K%u>uBB%11oL zLNpIU=7k7BE;s0pEbT3pOCwKjuIYGB5RLw0UoJsCS&iHn$CXTBb3>^R$ybA!Q_z6D&Tw{yO literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/snapshot-data3/part-00001-e62ca5a1-923c-4ee6-998b-c61d1cfb0b1c-c000.snappy.parquet b/presto-delta/src/test/resources/snapshot-data3/part-00001-e62ca5a1-923c-4ee6-998b-c61d1cfb0b1c-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..5eeb30b719feb37facc2df7b2773dc9b9a7d39d0 GIT binary patch literal 649 zcmah{!AiqG5S_$YW65#FHQ*n`T>sX#avX_S!QS6dpmF0(}Odc0NE!UgH$jn6G>M~n*gAe)^S&$UN~W4D-dyC z+sT_{j13G4Bc~w$Q6T3(N-or;fMu-VG(7b?Ya#ORXsMDjSAf>G3@%1{Tedodx<^E< zcPyt9mfT#W{cApAqy$9V71dII9N9qRRqE{GycJE?jGCk)t0h5MTY`wFRFo${c@}C? z?+<^Pjvi1P$Ev*wI*|(^&O+o}P~HUW^?vbIP*6VcR?u{HZ=4KGHeootx$t48w~>j4 zZl^pmlkvSd;u*`|L46TsG<%qE+NEihO#2x%=pf`#kkanOjK*w~w{qXuSOLR0<3pZc zA)ZDN^CN^HmmBn3mUd_JxlyD$KXkk&jEDcRFXy11tVV8(lIk5#(r_GCZPT_)yIS|B yVH8wZ#|s9Xf#)?H-{k|>Z8ihTwOo(4tiI>By|(3ZKd|>SLkGYG9|FKBe!?$j35(JI literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/time-travel-partition-changes-b/_delta_log/00000000000000000000.json b/presto-delta/src/test/resources/time-travel-partition-changes-b/_delta_log/00000000000000000000.json new file mode 100644 index 000000000000..192815d32895 --- /dev/null +++ b/presto-delta/src/test/resources/time-travel-partition-changes-b/_delta_log/00000000000000000000.json @@ -0,0 +1,13 @@ +{"commitInfo":{"timestamp":1603724026157,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part5\"]"},"isBlindAppend":true}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"9ce7bb6f-507b-4925-a820-f33601e5d700","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"part5\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["part5"],"configuration":{},"createdTime":1603724025794}} +{"add":{"path":"part5=0/part-00000-67b6882e-f49f-4df5-9850-b5e8a72f4917.c000.snappy.parquet","partitionValues":{"part5":"0"},"size":429,"modificationTime":1603724025000,"dataChange":true}} +{"add":{"path":"part5=1/part-00000-8a40c3d2-f658-4131-a17f-388265ab04b7.c000.snappy.parquet","partitionValues":{"part5":"1"},"size":429,"modificationTime":1603724026000,"dataChange":true}} +{"add":{"path":"part5=2/part-00000-ec6e3a2e-ecbf-4d39-9076-37e523cd62f1.c000.snappy.parquet","partitionValues":{"part5":"2"},"size":429,"modificationTime":1603724026000,"dataChange":true}} +{"add":{"path":"part5=3/part-00000-eaf1edf4-b9da-4df8-b957-08583e2a1d1b.c000.snappy.parquet","partitionValues":{"part5":"3"},"size":429,"modificationTime":1603724026000,"dataChange":true}} +{"add":{"path":"part5=4/part-00000-ce66c2ca-8fdf-48d3-a6e7-5980a370461a.c000.snappy.parquet","partitionValues":{"part5":"4"},"size":429,"modificationTime":1603724026000,"dataChange":true}} +{"add":{"path":"part5=0/part-00001-4f02a740-31dc-46c6-bc0e-c19d164ac82d.c000.snappy.parquet","partitionValues":{"part5":"0"},"size":429,"modificationTime":1603724025000,"dataChange":true}} +{"add":{"path":"part5=1/part-00001-3dcad520-b001-4829-a6e5-3d578b0964f4.c000.snappy.parquet","partitionValues":{"part5":"1"},"size":429,"modificationTime":1603724026000,"dataChange":true}} +{"add":{"path":"part5=2/part-00001-e20bae81-3f27-4c5c-aeca-5cfa6b38615c.c000.snappy.parquet","partitionValues":{"part5":"2"},"size":429,"modificationTime":1603724026000,"dataChange":true}} +{"add":{"path":"part5=3/part-00001-b9c6b926-a274-4d8e-b882-31c4aac05038.c000.snappy.parquet","partitionValues":{"part5":"3"},"size":429,"modificationTime":1603724026000,"dataChange":true}} +{"add":{"path":"part5=4/part-00001-5705917d-d837-4d7f-b8c4-f0ada8cf9663.c000.snappy.parquet","partitionValues":{"part5":"4"},"size":429,"modificationTime":1603724026000,"dataChange":true}} diff --git a/presto-delta/src/test/resources/time-travel-partition-changes-b/_delta_log/00000000000000000001.json b/presto-delta/src/test/resources/time-travel-partition-changes-b/_delta_log/00000000000000000001.json new file mode 100644 index 000000000000..7cf1e9b774f4 --- /dev/null +++ b/presto-delta/src/test/resources/time-travel-partition-changes-b/_delta_log/00000000000000000001.json @@ -0,0 +1,16 @@ +{"commitInfo":{"timestamp":1603724028432,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[\"part2\"]"},"readVersion":0,"isBlindAppend":false}} +{"metaData":{"id":"9ce7bb6f-507b-4925-a820-f33601e5d700","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"part2\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["part2"],"configuration":{},"createdTime":1603724025794}} +{"add":{"path":"part2=0/part-00000-7bce012e-f358-4a97-91da-55c4d3266fbe.c000.snappy.parquet","partitionValues":{"part2":"0"},"size":442,"modificationTime":1603724028000,"dataChange":true}} +{"add":{"path":"part2=1/part-00000-82368d1d-588b-487a-be01-16dc85260296.c000.snappy.parquet","partitionValues":{"part2":"1"},"size":437,"modificationTime":1603724028000,"dataChange":true}} +{"add":{"path":"part2=0/part-00001-2a830e69-78f3-4d09-9b2c-3bfd9debc2f0.c000.snappy.parquet","partitionValues":{"part2":"0"},"size":437,"modificationTime":1603724028000,"dataChange":true}} +{"add":{"path":"part2=1/part-00001-0a72544a-fb83-4eaa-8d62-9e6ab59afa8b.c000.snappy.parquet","partitionValues":{"part2":"1"},"size":442,"modificationTime":1603724028000,"dataChange":true}} +{"remove":{"path":"part5=0/part-00000-67b6882e-f49f-4df5-9850-b5e8a72f4917.c000.snappy.parquet","deletionTimestamp":1603724028432,"dataChange":true}} +{"remove":{"path":"part5=0/part-00001-4f02a740-31dc-46c6-bc0e-c19d164ac82d.c000.snappy.parquet","deletionTimestamp":1603724028432,"dataChange":true}} +{"remove":{"path":"part5=1/part-00001-3dcad520-b001-4829-a6e5-3d578b0964f4.c000.snappy.parquet","deletionTimestamp":1603724028432,"dataChange":true}} +{"remove":{"path":"part5=2/part-00000-ec6e3a2e-ecbf-4d39-9076-37e523cd62f1.c000.snappy.parquet","deletionTimestamp":1603724028432,"dataChange":true}} +{"remove":{"path":"part5=2/part-00001-e20bae81-3f27-4c5c-aeca-5cfa6b38615c.c000.snappy.parquet","deletionTimestamp":1603724028432,"dataChange":true}} +{"remove":{"path":"part5=4/part-00000-ce66c2ca-8fdf-48d3-a6e7-5980a370461a.c000.snappy.parquet","deletionTimestamp":1603724028432,"dataChange":true}} +{"remove":{"path":"part5=1/part-00000-8a40c3d2-f658-4131-a17f-388265ab04b7.c000.snappy.parquet","deletionTimestamp":1603724028432,"dataChange":true}} +{"remove":{"path":"part5=3/part-00000-eaf1edf4-b9da-4df8-b957-08583e2a1d1b.c000.snappy.parquet","deletionTimestamp":1603724028432,"dataChange":true}} +{"remove":{"path":"part5=3/part-00001-b9c6b926-a274-4d8e-b882-31c4aac05038.c000.snappy.parquet","deletionTimestamp":1603724028432,"dataChange":true}} +{"remove":{"path":"part5=4/part-00001-5705917d-d837-4d7f-b8c4-f0ada8cf9663.c000.snappy.parquet","deletionTimestamp":1603724028432,"dataChange":true}} diff --git a/presto-delta/src/test/resources/time-travel-partition-changes-b/part2=0/.part-00000-7bce012e-f358-4a97-91da-55c4d3266fbe.c000.snappy.parquet.crc b/presto-delta/src/test/resources/time-travel-partition-changes-b/part2=0/.part-00000-7bce012e-f358-4a97-91da-55c4d3266fbe.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..665b0c6fadab49951db19832a39ec4a980225572 GIT binary patch literal 12 TcmYc;N@ieSU}De~^{E2@4}k(j literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/time-travel-partition-changes-b/part2=0/.part-00001-2a830e69-78f3-4d09-9b2c-3bfd9debc2f0.c000.snappy.parquet.crc b/presto-delta/src/test/resources/time-travel-partition-changes-b/part2=0/.part-00001-2a830e69-78f3-4d09-9b2c-3bfd9debc2f0.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..198c12d32b4002da96c7264d8733b6c00a91f78c GIT binary patch literal 12 TcmYc;N@ieSU}6X?cKZeZ5uXD) literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/time-travel-partition-changes-b/part2=0/part-00000-7bce012e-f358-4a97-91da-55c4d3266fbe.c000.snappy.parquet b/presto-delta/src/test/resources/time-travel-partition-changes-b/part2=0/part-00000-7bce012e-f358-4a97-91da-55c4d3266fbe.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..8b95fb6fe9e960129c011edef36fe347f0bd8d7e GIT binary patch literal 442 zcmb7BO-sW-5S_#lV=3M?WMPr8U}+&PiKS^1dhsTnM8u1TY`U8o%txCKMN0pb`1{;y zdh_BQhMD)~&Aef6ukL&VXn`UZ87eW5No?~JexGgjyUc>oAE+YFh_2$%aUV{!#h~jx z=#h@y>p{Kd)$4P;e33avI-uUUZtJN4%fJMap-E5=PfalJ9!0ukSYMj;sSVA=Kb&Bq z+=Chqu$LQvi9If>72|8JHnSfStlnfyx!JU`VO9Cga@laf8*V>rY4fp`G@^A=wabRO zG)ZM9Y8pLmX~FY7l?nxSZCP2YR8_QD#^X$}N+w-;7)Yb-ZuhhU`=(oWkyp-}tm?EZ z9H03f^PN%LrkQZ~JQT@13B$={9LOXHCKKTWUJ%Nuw+!Q1IP+o|3xA9WQPk1 z_;+W`TGVeZP5P<|flY+1OVc%Ti5!biLI*a*7Ma+nPYx(H9n1MNoG)Ww-~7V_p(ReI_D2QS;3DP*3#?v5Ci3~@CQn+quAhduF`v!u+ BS5^Q3 literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/time-travel-partition-changes-b/part2=1/.part-00000-82368d1d-588b-487a-be01-16dc85260296.c000.snappy.parquet.crc b/presto-delta/src/test/resources/time-travel-partition-changes-b/part2=1/.part-00000-82368d1d-588b-487a-be01-16dc85260296.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..39dc81d6349f138cfa766161ecbf3d8c20a36542 GIT binary patch literal 12 TcmYc;N@ieSU}6YkV!I0f561#_ literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/time-travel-partition-changes-b/part2=1/.part-00001-0a72544a-fb83-4eaa-8d62-9e6ab59afa8b.c000.snappy.parquet.crc b/presto-delta/src/test/resources/time-travel-partition-changes-b/part2=1/.part-00001-0a72544a-fb83-4eaa-8d62-9e6ab59afa8b.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..32ba636ce5f582c12c7a560bd2a3d9b35cdb0fd9 GIT binary patch literal 12 TcmYc;N@ieSU}8uVytx(t5taip literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/time-travel-partition-changes-b/part2=1/part-00000-82368d1d-588b-487a-be01-16dc85260296.c000.snappy.parquet b/presto-delta/src/test/resources/time-travel-partition-changes-b/part2=1/part-00000-82368d1d-588b-487a-be01-16dc85260296.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9f6f4ad0e0c6afd40150a3f2693ecfc391d66f7a GIT binary patch literal 437 zcmb7BO-sW-5Z$Dq1k~e(EbJjHSXzinVrkkGFW$sck>W)}HrY)LZjv@16)F8Y9{iO~ zwY_<955vrR^Jd<#i`$0~BkbeA#}*`gY*W|1K)=tm&yJachJTbI9BeN~0PCc@UG>6*DjjzuV;1Dj%tOl;IA2Nau*<$M~>mocz!{$YYpFpgk6 zxBPVwugdI38aok;sX^^Nyh9g2LT(>k3TEK^W13Kywti$WV!aOk$g-@cV4D-(?mI|3DRaMsyVq4*PJV%}=`S zy&mbTTTGB641;fVQ*PF+Y*8`^xVB#mg@RPC~% zE=^LIiJC?a8(Q#uN2Nl+U0YTb&#EfgEaP#eSS6D#-4CSEX1jgdf_>AiyT~i&O;&YU z7LLz+kNM6pZqrOSd=`pimW1K>G7e-C1mm&r0xt;V#9M~(G@N>|jDLt z8@p%@;pKhb``$}3zqtxdjQ{DXzIvnveyiFJ?=O#R!_W+m&l zmuB&*3rR2{y>sm1Tp(<>eC%du!Eg1%lWnj_Vv;7i2gCB3<4tND z8@p%@;eGG>-uGU}?D{Swh%n;$1Ow<2o4NKe`aK8ap#C2$Or2h0xR0#8gkb95oYYHM zy*}6T7oAIj5$T*^7iS7Nm@>+GHX|K6vQd|AS%Mwxd|J+z)w8b-mIw5YIb{$S!Fq(Y za*QZ*2W7Poaw)-C{Dcto$_PlaYIP&3@?GS*kxDkweOUA6W2yO+*G<*V8}9Qo(?->N z`ncwW%(tyf0lgbDWwC&&XpNDHfiB3j&-V-YbiLU;ZIFArvpdf#?@d>ARu*0;!a#&x yKWQ_gJUNL~I!WVrc#%XpjiTXD1yK;idKAp#WE_u!L?9U literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/time-travel-partition-changes-b/part5=1/part-00000-8a40c3d2-f658-4131-a17f-388265ab04b7.c000.snappy.parquet b/presto-delta/src/test/resources/time-travel-partition-changes-b/part5=1/part-00000-8a40c3d2-f658-4131-a17f-388265ab04b7.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..ebf0901460f24366a1fcd1e2efd6811444cd7d42 GIT binary patch literal 429 zcmZ`$&r8EF7)|3CJ&3oI2689?#|AaD&UNeX;!Qja5icT=F6k_$?baU?8T&Up_}BVP zH+In+!pr-<_q~^7c5@#RL>TdWf&p}i&0O0;zw3w`)c=EpuCpr)`-yds5KR5s(|ReZ zx0ibUs&h#&BAs*W;!Gh2Q$|_OW~4($HtNzHOR$5TPs{nTdiM3v@`(O1rwjrk7?03S zjuB<Lm8 yCv9ewCnvE=CutlHFOx{8Q8XN?APS;bkAiuejN@^T=tPAB${5`40Umb-AN398aaJ<` literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/time-travel-partition-changes-b/part5=1/part-00001-3dcad520-b001-4829-a6e5-3d578b0964f4.c000.snappy.parquet b/presto-delta/src/test/resources/time-travel-partition-changes-b/part5=1/part-00001-3dcad520-b001-4829-a6e5-3d578b0964f4.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a1caecfbace73080e53a4875572a74b87c91aca7 GIT binary patch literal 429 zcmZ`$&r8EF7)|3CJ&3oI2689?#|AaD&UNeX;!Qja5icT=F6k_$?baU?8T&Up_}BVP zH+In+!u#I$z3;t{+0A`O5Mjjg2?o$5HgoMR`aMVFp#C2$Or2d}xKFHugkb95p4LlQ zy}i`)SDj0O5$T*`7iS7Nm@>+GHX|K6vQd}rSb`nwd|J+z)w8dUmPhoDIb{$S!Fq&t za*QZ*2W7Poaw)-C{Dcto$_PlaYIP&3@?GS*kxDkwecbToW2yO+*G<*V8}9Qo(?->N z`n2JN%y+F!0lgnHWwC&&XpNDHfiB3j&kqavbhF()Z;^Yqx4XzIZ>_63D+@0aVIV@U ypR}1#o}9!gouqL*yi6jUM$vGnf+&b$JqqS=GLFYVq7xMkC}Z%t2MeJYeAG7|$W~wg literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/time-travel-partition-changes-b/part5=2/.part-00000-ec6e3a2e-ecbf-4d39-9076-37e523cd62f1.c000.snappy.parquet.crc b/presto-delta/src/test/resources/time-travel-partition-changes-b/part5=2/.part-00000-ec6e3a2e-ecbf-4d39-9076-37e523cd62f1.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..3a4bed33c582f3176c8280f0b973658d166a8651 GIT binary patch literal 12 TcmYc;N@ieSU}9+5;QAQ=6CDGm literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/time-travel-partition-changes-b/part5=2/.part-00001-e20bae81-3f27-4c5c-aeca-5cfa6b38615c.c000.snappy.parquet.crc b/presto-delta/src/test/resources/time-travel-partition-changes-b/part5=2/.part-00001-e20bae81-3f27-4c5c-aeca-5cfa6b38615c.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..0cd5190c2cd66d480bcd0ab44dfc07108ce4da02 GIT binary patch literal 12 TcmYc;N@ieSU}A7p?>h_t5a}6HI5w!Eb*`?%i#PE!M7)Sdx}>w1wp)KFGWKtH@UJ!1 zja@XSm-oH*y%#dSz6&v6iaj4A1Nzveu8rXL9FPOUKUkwWx#+}4*4}~<@o$dnm8f2y z>&1)81xA2-r^rQ_1P2jH=)k79M$T>l&AL?$ugZ6xt42uC2=`&bnva!YGgdcMyJ(ot(oAVtv)SW@ z6(a9OnKblHrpsb!ZbhrLNVJ(6oATLyA)9Ts+ovtKx4XEryz<^uRcB@4g**&+=na!L w)6x^uSfqRoq|zuFk7W=AQLHAxB2F&j%OFvS3`c}g^SXNrU~WF@8}S)dJOBUy literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/time-travel-partition-changes-b/part5=2/part-00001-e20bae81-3f27-4c5c-aeca-5cfa6b38615c.c000.snappy.parquet b/presto-delta/src/test/resources/time-travel-partition-changes-b/part5=2/part-00001-e20bae81-3f27-4c5c-aeca-5cfa6b38615c.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..1cd670d1938d11793908ad9575052e04c257dcbe GIT binary patch literal 429 zcmZ`$&r8EF7)|4tJ&3oI2689?#|AaD&UNeX;!Qja5icT=F6k_$?baWPjQtxP{A>ND z8@p%@;eGG>-uGU}?D{Swh%n;$1Ow<2o4NKe`aK8ap#C2$Or2h0xR0#8gkb95oYYHM zy*}6T7oAIj5$T*^7iS7Nm@>+GHX|K6vQd|AS%Mwxd|J+z)w8b-mIw5YIb{$S!Fq(Y za*QZ*2W7Poaw)-C{Dcto$_PlaYIP&3@?GS*kxDkweOUA6W2yO+*G<*V8}9Qo(?->N z`ncwW%(tyf0lgbDWwC&&XpNDHfiB3j&-V-YbiLU;ZIFArvpdf#?@d>ARu*0;!a#&x yKWQ_gJUNL~I!WVrc#%XpjiTXD1yK;idKAp#WE_u!L?7~7B&Q| literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/time-travel-partition-changes-b/part5=3/part-00000-eaf1edf4-b9da-4df8-b957-08583e2a1d1b.c000.snappy.parquet b/presto-delta/src/test/resources/time-travel-partition-changes-b/part5=3/part-00000-eaf1edf4-b9da-4df8-b957-08583e2a1d1b.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..ec63649165cdab46a6642fe3e32f8be13979b618 GIT binary patch literal 429 zcmZ`$&r8EF82y@!(SvwPX&{FZaBNUR>s+@EFW$t{5b+`+>5|T3+HU=!$k@N(!N1l_ zH+In+!u#I$z3;t{`So3h0aNVx7#YyRHg)X|{GKCnVDJZPrcN)BxjWWDf)Vj=PU@AY zUZ3m5i^>H?fV*eNMVSN#5lU#^rnpPSHuT7j#mGU_gD8k%H4YYWGKnWaq7oSn38m(B_ZGn1eAGAm!B#~8 literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/time-travel-partition-changes-b/part5=3/part-00001-b9c6b926-a274-4d8e-b882-31c4aac05038.c000.snappy.parquet b/presto-delta/src/test/resources/time-travel-partition-changes-b/part5=3/part-00001-b9c6b926-a274-4d8e-b882-31c4aac05038.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..8a4536406db178ef9eeb5aba94b67e03c121550b GIT binary patch literal 429 zcmZ`$&r8EF82uW@=s~=tG>}6HI5w!Eb*@{77jNQeh*fmWoYT-Bj(oVLnSUrDe^g zPa9T!>8U^z>8OP%wQHcx(gi`am2Mb_kKI$7c7*=us literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/time-travel-partition-changes-b/part5=4/.part-00000-ce66c2ca-8fdf-48d3-a6e7-5980a370461a.c000.snappy.parquet.crc b/presto-delta/src/test/resources/time-travel-partition-changes-b/part5=4/.part-00000-ce66c2ca-8fdf-48d3-a6e7-5980a370461a.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..4c990a139198109596705fc833bb98d76463bfac GIT binary patch literal 12 TcmYc;N@ieSU}D%@W%>pH6A%N5 literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/time-travel-partition-changes-b/part5=4/.part-00001-5705917d-d837-4d7f-b8c4-f0ada8cf9663.c000.snappy.parquet.crc b/presto-delta/src/test/resources/time-travel-partition-changes-b/part5=4/.part-00001-5705917d-d837-4d7f-b8c4-f0ada8cf9663.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..551c8f5742f5d4ae1907f4bd4ca5cab1c1b2fca0 GIT binary patch literal 12 TcmYc;N@ieSU}EUI+GHX|K6vQd}rSb`nwd|J+z)w8dUmPhoDIb{$S!Fq&t za*QZ*2W7Poaw)-C{Dcto$_PlaYIP&3@?GS*kxDkwecbToW2yO+*G<*V8}9Qo(?->N z`n2JN%y+F!0lgnHWwC&&XpNDHfiB3j&kqavbhF()Z;^Yqx4XzIZ>_63D+@0aVIV@U ypR}1#o}9!gouqL*yi6jUM$vGnf+&b$JqqS=GLFYVq7xMkC}Z%t2MeJYeAG7xcvemT literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/time-travel-partition-changes-b/part5=4/part-00001-5705917d-d837-4d7f-b8c4-f0ada8cf9663.c000.snappy.parquet b/presto-delta/src/test/resources/time-travel-partition-changes-b/part5=4/part-00001-5705917d-d837-4d7f-b8c4-f0ada8cf9663.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..533801e17f8f556a8e034d9bd7f7411d33ed8dfe GIT binary patch literal 429 zcmZ`$&r8EF7)|3C6~tRg138p{V}lx6=el)x@g|;zh!+t_mvk1>cIyvC#{LZt{TdWf&p}i&0PB!{hkAIQ2!4WrcN(0+(*`4LNN7jPU@ws zUZ3mvi_Rs%h;+`di!+5BOc`Z8n~@G3*{DmmEWr+TJ}u|V>e*KZ%LDqyoH7WEU_C-x zIYyMZgR)u(xs>26enN!xbw4flDPX`^aB zeO&WG=G#`LfZmOnvRFV>w8qH9Ko?}%=lg|xy54M_Hpso**`4Q=_ok~lD+@0aVIV@U ypR}1#o}9!gouqL*yhtLQM$vGnf+&b$JqqS=GLFYVq7xMkC}Z%tdkdi%eAG8Wz*c(z literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/time-travel-schema-changes-b/.part-00000-83680aa8-547c-40bc-8ca9-5c10997e307b-c000.snappy.parquet.crc b/presto-delta/src/test/resources/time-travel-schema-changes-b/.part-00000-83680aa8-547c-40bc-8ca9-5c10997e307b-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..919ac1ceb47d5109e0d7af71a405e69a52b689e1 GIT binary patch literal 12 TcmYc;N@ieSU}9MHjoBOk6GsCT literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/time-travel-schema-changes-b/.part-00000-a830a49c-6cc8-4caf-80a5-7ff8a959bd53-c000.snappy.parquet.crc b/presto-delta/src/test/resources/time-travel-schema-changes-b/.part-00000-a830a49c-6cc8-4caf-80a5-7ff8a959bd53-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..d586caeb413ebdfe1c71960bb5c844ed9c485d9b GIT binary patch literal 16 XcmYc;N@ieSU}DfKy%wAM-C{QYBd7&v literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/time-travel-schema-changes-b/.part-00001-3c1f89ce-a996-4d44-a79c-21a6f3d53138-c000.snappy.parquet.crc b/presto-delta/src/test/resources/time-travel-schema-changes-b/.part-00001-3c1f89ce-a996-4d44-a79c-21a6f3d53138-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..03adb57481dcfd3e91f4086a2a83786c91be40ec GIT binary patch literal 12 TcmYc;N@ieSU}6xqo52PE54-{@ literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/time-travel-schema-changes-b/.part-00001-5fdfd303-d5e8-4e77-9b5d-4e831fa723e1-c000.snappy.parquet.crc b/presto-delta/src/test/resources/time-travel-schema-changes-b/.part-00001-5fdfd303-d5e8-4e77-9b5d-4e831fa723e1-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..d104b14b1499cda6dcb5b83565ae43608faa8303 GIT binary patch literal 16 XcmYc;N@ieSU}9+5vhQH-cZ=NsDRl-( literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/time-travel-schema-changes-b/_delta_log/00000000000000000000.json b/presto-delta/src/test/resources/time-travel-schema-changes-b/_delta_log/00000000000000000000.json new file mode 100644 index 000000000000..43ec00a86ab3 --- /dev/null +++ b/presto-delta/src/test/resources/time-travel-schema-changes-b/_delta_log/00000000000000000000.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1603724023478,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isBlindAppend":true}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"37664cd7-239f-4dbc-a56b-d47437be8ddb","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724023419}} +{"add":{"path":"part-00000-83680aa8-547c-40bc-8ca9-5c10997e307b-c000.snappy.parquet","partitionValues":{},"size":449,"modificationTime":1603724023000,"dataChange":true}} +{"add":{"path":"part-00001-3c1f89ce-a996-4d44-a79c-21a6f3d53138-c000.snappy.parquet","partitionValues":{},"size":451,"modificationTime":1603724023000,"dataChange":true}} diff --git a/presto-delta/src/test/resources/time-travel-schema-changes-b/_delta_log/00000000000000000001.json b/presto-delta/src/test/resources/time-travel-schema-changes-b/_delta_log/00000000000000000001.json new file mode 100644 index 000000000000..e7ab72b3d822 --- /dev/null +++ b/presto-delta/src/test/resources/time-travel-schema-changes-b/_delta_log/00000000000000000001.json @@ -0,0 +1,4 @@ +{"commitInfo":{"timestamp":1603724024783,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isBlindAppend":true}} +{"metaData":{"id":"37664cd7-239f-4dbc-a56b-d47437be8ddb","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"part\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724023419}} +{"add":{"path":"part-00000-a830a49c-6cc8-4caf-80a5-7ff8a959bd53-c000.snappy.parquet","partitionValues":{},"size":711,"modificationTime":1603724024000,"dataChange":true}} +{"add":{"path":"part-00001-5fdfd303-d5e8-4e77-9b5d-4e831fa723e1-c000.snappy.parquet","partitionValues":{},"size":711,"modificationTime":1603724024000,"dataChange":true}} diff --git a/presto-delta/src/test/resources/time-travel-schema-changes-b/part-00000-83680aa8-547c-40bc-8ca9-5c10997e307b-c000.snappy.parquet b/presto-delta/src/test/resources/time-travel-schema-changes-b/part-00000-83680aa8-547c-40bc-8ca9-5c10997e307b-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..f3512340b40753e8ac6232d7d926b78bfe507dbe GIT binary patch literal 449 zcmb7B%SyvQ6un7H3BfKWWMIHBU}+%^iKS^%x^Y)r6%jWoGU-feFpoA5MM{6pkMMh( zYPxgdEQWi}nRD*B%j=d!bB|r${ye-HvtoSRFoUWw_HtTKOtDPOPNZuYh=yJ;+AcCZN{Za^neQ>*8AeT#3Wv`^5zpc<9A8C&jDui27GB^5p`3W@Fq(!_FOre)N0<<` L-JJ%oQlIw?qas)g literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/time-travel-schema-changes-b/part-00000-a830a49c-6cc8-4caf-80a5-7ff8a959bd53-c000.snappy.parquet b/presto-delta/src/test/resources/time-travel-schema-changes-b/part-00000-a830a49c-6cc8-4caf-80a5-7ff8a959bd53-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..5829df4c477600ddb1950777a0030f521092cc75 GIT binary patch literal 711 zcmcgq!A`Mrj<}A;o{9055%h)v!z>YYzwqSV+cRtclaG1 zIr=BwU7)B3FL-dX+04$H_h#Q@ho_e|0(6ZoOr-FjjATQT4`8!2St%kFn!jW@NDPsm zr9%ZO0_<1zB&gQpL9L)|h5n<3s1MGn*@UJKx7p~P#S}?^q$5!iLw^%U9LzygbVD~A zvVkPMBSTqV6(CVn)OSIB6B_bhhXLpYw+3i6gT_+$mRz^`A-WX)sH(n}R0VWHYmL)8 zLMN14BWsr=n?{5?n$B2G((x0CSx$YLQ|(2Vhv z-4^Jcz&UJW2P33o=jymz^WycnT)xPZBORdDnQrN^0H%QnCVfay3y&e_;5~|T(=fj@ z^Hb}?`9C6IqSgS)9)P{v0GL?AyjU^5=4vwg3Bk%u!c>|~Eh|>!?<|!S7rf%u{gzfA zYe{FctcrSBQHMscOhid%4_lh?bWg=X!OgQI&sM6->LlS|qF5!94m}vs*><;k+5zjj z8F!u*_M0rqIL~a4c`ozpL0HF$u=z9)(KHHz(M9OX$oEGh;rgy0$g#T&!bvc3Lm3Kh PhzU{K-Dv{FzTN3Q;$i4j)Z7uX4Ekvz%ww;b>@^G6D?pa8Y1V}m(RnhmBfyBWKl(MF4 zMord{q_t!yYV!glDrMzeP~L=^+}mIPn$E2On$Ms%6TT%Et$v8kgg>e(uLY$HnyxnD zN zhI7dGczRV9#L*qUMR^cVH{d9zF$1p^ad$GE8eh!$7&qhIOhG+gz4bUus*fy5{Wz*x z#4?Fht-HA&cvaeQykTePIL#B+W<%R7 literal 0 HcmV?d00001 From 7eb7c9afb019217f5eb09c3bebf1165e2e7f0f03 Mon Sep 17 00:00:00 2001 From: Venki Korukanti Date: Mon, 27 Sep 2021 14:03:25 -0700 Subject: [PATCH 4/5] Delta lake connector - reader --- pom.xml | 14 + presto-delta-driver/pom.xml | 6 +- presto-delta/pom.xml | 273 +++++++++++++++ .../java/io/prestosql/delta/DeltaClient.java | 189 +++++++++++ .../java/io/prestosql/delta/DeltaColumn.java | 99 ++++++ .../io/prestosql/delta/DeltaColumnHandle.java | 112 ++++++ .../java/io/prestosql/delta/DeltaConfig.java | 76 +++++ .../delta/DeltaConnectionHandleResolver.java | 48 +++ .../io/prestosql/delta/DeltaConnector.java | 102 ++++++ .../delta/DeltaConnectorFactory.java | 87 +++++ .../io/prestosql/delta/DeltaConnectorId.java | 54 +++ .../io/prestosql/delta/DeltaErrorCode.java | 48 +++ .../prestosql/delta/DeltaExpressionUtils.java | 243 +++++++++++++ .../io/prestosql/delta/DeltaMetadata.java | 314 +++++++++++++++++ .../java/io/prestosql/delta/DeltaModule.java | 117 +++++++ .../io/prestosql/delta/DeltaPageSource.java | 145 ++++++++ .../delta/DeltaPageSourceProvider.java | 318 ++++++++++++++++++ .../java/io/prestosql/delta/DeltaPlugin.java | 28 ++ .../delta/DeltaSessionProperties.java | 74 ++++ .../java/io/prestosql/delta/DeltaSplit.java | 131 ++++++++ .../io/prestosql/delta/DeltaSplitManager.java | 121 +++++++ .../java/io/prestosql/delta/DeltaTable.java | 118 +++++++ .../io/prestosql/delta/DeltaTableHandle.java | 120 +++++++ .../io/prestosql/delta/DeltaTableName.java | 143 ++++++++ .../delta/DeltaTransactionHandle.java | 22 ++ .../io/prestosql/delta/DeltaTypeUtils.java | 229 +++++++++++++ ...AbstractDeltaDistributedQueryTestBase.java | 155 +++++++++ .../delta/TestDeltaColumnHandle.java | 74 ++++ .../io/prestosql/delta/TestDeltaConfig.java | 52 +++ .../prestosql/delta/TestDeltaIntegration.java | 239 +++++++++++++ .../delta/TestDeltaOptimizationsTests.java | 254 ++++++++++++++ .../io/prestosql/delta/TestDeltaSplit.java | 56 +++ .../prestosql/delta/TestDeltaTableHandle.java | 130 +++++++ .../prestosql/delta/TestDeltaTableName.java | 95 ++++++ .../prestosql/delta/TestDeltaTypeUtils.java | 82 +++++ .../etc/catalog/delta.properties | 2 + presto-server-main/etc/config.properties | 3 +- presto-server/src/main/provisio/presto.xml | 6 + 38 files changed, 4374 insertions(+), 5 deletions(-) create mode 100644 presto-delta/pom.xml create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaClient.java create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaColumn.java create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaColumnHandle.java create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaConfig.java create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaConnectionHandleResolver.java create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaConnector.java create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaConnectorFactory.java create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaConnectorId.java create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaErrorCode.java create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaExpressionUtils.java create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaMetadata.java create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaModule.java create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaPageSource.java create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaPageSourceProvider.java create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaPlugin.java create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaSessionProperties.java create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaSplit.java create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaSplitManager.java create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaTable.java create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaTableHandle.java create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaTableName.java create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaTransactionHandle.java create mode 100644 presto-delta/src/main/java/io/prestosql/delta/DeltaTypeUtils.java create mode 100644 presto-delta/src/test/java/io/prestosql/delta/AbstractDeltaDistributedQueryTestBase.java create mode 100644 presto-delta/src/test/java/io/prestosql/delta/TestDeltaColumnHandle.java create mode 100644 presto-delta/src/test/java/io/prestosql/delta/TestDeltaConfig.java create mode 100644 presto-delta/src/test/java/io/prestosql/delta/TestDeltaIntegration.java create mode 100644 presto-delta/src/test/java/io/prestosql/delta/TestDeltaOptimizationsTests.java create mode 100644 presto-delta/src/test/java/io/prestosql/delta/TestDeltaSplit.java create mode 100644 presto-delta/src/test/java/io/prestosql/delta/TestDeltaTableHandle.java create mode 100644 presto-delta/src/test/java/io/prestosql/delta/TestDeltaTableName.java create mode 100644 presto-delta/src/test/java/io/prestosql/delta/TestDeltaTypeUtils.java create mode 100644 presto-server-main/etc/catalog/delta.properties diff --git a/pom.xml b/pom.xml index feb25862b581..94be2342f30e 100644 --- a/pom.xml +++ b/pom.xml @@ -148,6 +148,7 @@ presto-oracle presto-druid presto-delta-driver + presto-delta @@ -1200,6 +1201,19 @@ 0.2.0 + + io.prestosql.delta + presto-delta-driver + shaded + ${project.version} + + + + io.prestosql + presto-delta + ${project.version} + + org.javassist diff --git a/presto-delta-driver/pom.xml b/presto-delta-driver/pom.xml index 4e03a01b8ae8..bb5d6e408864 100644 --- a/presto-delta-driver/pom.xml +++ b/presto-delta-driver/pom.xml @@ -1,12 +1,10 @@ - + 4.0.0 io.prestosql presto-root - 0.264-SNAPSHOT + 339 diff --git a/presto-delta/pom.xml b/presto-delta/pom.xml new file mode 100644 index 000000000000..06616d0d9f52 --- /dev/null +++ b/presto-delta/pom.xml @@ -0,0 +1,273 @@ + + + 4.0.0 + + io.prestosql + presto-root + 339 + + + presto-delta + Presto - Delta Table Connector + presto-plugin + + + ${project.parent.basedir} + true + + + + + io.prestosql.delta + presto-delta-driver + shaded + ${project.version} + + + io.delta + delta-standalone_2.12 + + + org.scala-lang + scala-library + + + commons-io + commons-io + + + com.fasterxml.jackson.module + jackson-module-scala_2.12 + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.core + jackson-databind + + + + + + io.airlift + bootstrap + + + + io.airlift + json + + + + io.airlift + log + + + + io.airlift + concurrent + + + + io.prestosql + presto-plugin-toolkit + + + + io.airlift + configuration + + + + com.google.guava + guava + + + + com.google.inject + guice + + + + javax.validation + validation-api + + + + javax.inject + javax.inject + + + + com.fasterxml.jackson.core + jackson-databind + + + + io.prestosql + presto-spi + provided + + + + io.airlift + slice + provided + + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + + io.prestosql.hive + hive-apache + + + + io.prestosql + presto-parquet + + + + io.prestosql + presto-memory-context + + + + org.weakref + jmxutils + + + + org.openjdk.jol + jol-core + provided + + + + io.prestosql + presto-hive + compile + + + io.airlift + discovery + + + io.airlift + event + + + + + + + io.prestosql + presto-tests + test + + + io.airlift + discovery + + + io.airlift + event + + + + + + io.prestosql + presto-testing + test + + + + io.prestosql.hadoop + hadoop-apache + compile + + + + io.prestosql + presto-tpch + test + + + + io.prestosql + presto-main + test-jar + test + + + + io.prestosql + presto-spi + test-jar + test + + + + io.prestosql + presto-main + test + + + + org.testng + testng + test + + + + io.airlift + testing + test + + + + io.airlift + http-server + test + + + + io.airlift + node + test + + + io.airlift + discovery + + + io.airlift + event + + + + + + javax.servlet + javax.servlet-api + test + + + + io.prestosql + presto-hive-hadoop2 + test + + + diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaClient.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaClient.java new file mode 100644 index 000000000000..c50f349ead72 --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaClient.java @@ -0,0 +1,189 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import io.delta.standalone.DeltaLog; +import io.delta.standalone.Snapshot; +import io.delta.standalone.actions.AddFile; +import io.delta.standalone.actions.Metadata; +import io.prestosql.plugin.hive.HdfsEnvironment; +import io.prestosql.plugin.hive.HdfsEnvironment.HdfsContext; +import io.prestosql.spi.PrestoException; +import io.prestosql.spi.connector.ConnectorSession; +import io.prestosql.spi.connector.SchemaTableName; +import io.prestosql.spi.type.TypeSignature; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import javax.inject.Inject; + +import java.io.IOException; +import java.time.Instant; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static io.prestosql.delta.DeltaErrorCode.DELTA_UNSUPPORTED_DATA_FORMAT; +import static io.prestosql.delta.DeltaTypeUtils.convertDeltaDataTypePrestoDataType; +import static io.prestosql.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static io.prestosql.spi.StandardErrorCode.NOT_FOUND; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +/** + * Class to interact with Delta lake table APIs. + */ +public class DeltaClient +{ + private final HdfsEnvironment hdfsEnvironment; + + @Inject + public DeltaClient(HdfsEnvironment hdfsEnvironment) + { + this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); + } + + /** + * Load the delta table. + * + * @param session Current user session + * @param schemaTableName Schema and table name referred to as in the query + * @param tableLocation Location of the Delta table on storage + * @param snapshotId Id of the snapshot to read from the Delta table + * @param snapshotAsOfTimestampMillis Latest snapshot as of given timestamp + * @return If the table is found return {@link DeltaTable}. + */ + public Optional getTable( + ConnectorSession session, + SchemaTableName schemaTableName, + String tableLocation, + Optional snapshotId, + Optional snapshotAsOfTimestampMillis) + { + Optional deltaLog = loadDeltaTableLog(session, new Path(tableLocation)); + if (!deltaLog.isPresent()) { + return Optional.empty(); + } + + // Fetch the snapshot info for given snapshot version. If no snapshot version is given, get the latest snapshot info. + // Lock the snapshot version here and use it later in the rest of the query (such as fetching file list etc.). + // If we don't lock the snapshot version here, the query may end up with schema from one version and data files from another + // version when the underlying delta table is changing while the query is running. + Snapshot snapshot; + if (snapshotId.isPresent()) { + try { + snapshot = deltaLog.get().getSnapshotForVersionAsOf(snapshotId.get()); + } + catch (IllegalArgumentException iae) { + throw new PrestoException( + NOT_FOUND, + format("Snapshot version %d does not exist in Delta table '%s'.", snapshotId.get(), schemaTableName), + iae); + } + } + else if (snapshotAsOfTimestampMillis.isPresent()) { + try { + snapshot = deltaLog.get().getSnapshotForTimestampAsOf(snapshotAsOfTimestampMillis.get()); + } + catch (IllegalArgumentException iae) { + throw new PrestoException( + NOT_FOUND, + format( + "There is no snapshot exists in Delta table '%s' that is created on or before '%s'", + schemaTableName, + Instant.ofEpochMilli(snapshotAsOfTimestampMillis.get())), + iae); + } + } + else { + snapshot = deltaLog.get().snapshot(); + } + + Metadata metadata = snapshot.getMetadata(); + String format = metadata.getFormat().getProvider(); + if (!"parquet".equalsIgnoreCase(format)) { + throw new PrestoException(DELTA_UNSUPPORTED_DATA_FORMAT, + format("Delta table %s has unsupported data format: %s. Currently only Parquet data format is supported", schemaTableName, format)); + } + + return Optional.of(new DeltaTable( + schemaTableName.getSchemaName(), + schemaTableName.getTableName(), + tableLocation, + Optional.of(snapshot.getVersion()), // lock the snapshot version + getSchema(schemaTableName, metadata))); + } + + /** + * Get the list of files corresponding to the given Delta table. + * + * @return + */ + public Iterator listFiles(ConnectorSession session, DeltaTable deltaTable) + { + Optional deltaLog = loadDeltaTableLog(session, new Path(deltaTable.getTableLocation())); + if (!deltaLog.isPresent()) { + throw new PrestoException(NOT_FOUND, + format("Delta table (%s.%s) no longer exists.", deltaTable.getSchemaName(), deltaTable.getTableName())); + } + + return deltaLog.get() + .getSnapshotForVersionAsOf(deltaTable.getSnapshotId().get()) + .getAllFiles() + .iterator(); + } + + private Optional loadDeltaTableLog(ConnectorSession session, Path tableLocation) + { + try { + HdfsContext hdfsContext = new HdfsContext(session.getIdentity()); + Configuration hdfsConf = hdfsEnvironment.getConfiguration(hdfsContext, tableLocation); + FileSystem fileSystem = hdfsEnvironment.getFileSystem(hdfsContext, tableLocation); + if (!fileSystem.isDirectory(tableLocation)) { + Optional.empty(); + } + return Optional.of(DeltaLog.forTable(hdfsConf, tableLocation)); + } + catch (IOException io) { + throw new PrestoException(GENERIC_INTERNAL_ERROR, "Failed to load Delta table: " + io.getMessage(), io); + } + } + + /** + * Utility method that returns the columns in given Delta metadata. Returned columns include regular and partition types. + * Data type from Delta is mapped to appropriate Presto data type. + */ + private List getSchema(SchemaTableName tableName, Metadata metadata) + { + Set partitionColumns = metadata.getPartitionColumns().stream() + .map(String::toLowerCase) + .collect(Collectors.toSet()); + + return Arrays.stream(metadata.getSchema().getFields()) + .map(field -> { + String columnName = field.getName().toLowerCase(Locale.US); + TypeSignature prestoType = convertDeltaDataTypePrestoDataType(tableName, columnName, field.getDataType()); + return new DeltaColumn( + columnName, + prestoType, + field.isNullable(), + partitionColumns.contains(columnName)); + }).collect(Collectors.toList()); + } +} diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaColumn.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaColumn.java new file mode 100644 index 000000000000..6c3e465d04d7 --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaColumn.java @@ -0,0 +1,99 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.prestosql.spi.type.TypeSignature; + +import java.util.Objects; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Strings.isNullOrEmpty; +import static java.util.Objects.requireNonNull; + +public final class DeltaColumn +{ + private final String name; + private final TypeSignature type; + private final boolean nullable; + private final boolean partition; + + @JsonCreator + public DeltaColumn( + @JsonProperty("name") String name, + @JsonProperty("type") TypeSignature type, + @JsonProperty("nullable") boolean nullable, + @JsonProperty("partition") boolean partition) + { + checkArgument(!isNullOrEmpty(name), "name is null or is empty"); + this.name = name; + this.type = requireNonNull(type, "type is null"); + this.nullable = nullable; + this.partition = partition; + } + + @JsonProperty + public String getName() + { + return name; + } + + @JsonProperty + public TypeSignature getType() + { + return type; + } + + @JsonProperty + public boolean isNullable() + { + return nullable; + } + + @JsonProperty + public boolean isPartition() + { + return partition; + } + + @Override + public int hashCode() + { + return Objects.hash(name, type, nullable, partition); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + DeltaColumn other = (DeltaColumn) obj; + return Objects.equals(this.name, other.name) && + Objects.equals(this.type, other.type) && + Objects.equals(this.nullable, other.nullable) && + Objects.equals(this.partition, other.partition); + } + + @Override + public String toString() + { + return name + ":nullable=" + nullable + ":partition=" + partition; + } +} diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaColumnHandle.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaColumnHandle.java new file mode 100644 index 000000000000..88f2b25ac780 --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaColumnHandle.java @@ -0,0 +1,112 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.prestosql.spi.connector.ColumnHandle; +import io.prestosql.spi.type.TypeSignature; + +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public final class DeltaColumnHandle + implements ColumnHandle +{ + private final String connectorId; + private final String name; + private final TypeSignature dataType; + private final ColumnType columnType; + + public enum ColumnType + { + REGULAR, + PARTITION, + SUBFIELD, + } + + @JsonCreator + public DeltaColumnHandle( + @JsonProperty("connectorId") String connectorId, + @JsonProperty("columnName") String name, + @JsonProperty("dataType") TypeSignature dataType, + @JsonProperty("columnType") ColumnType columnType) + { + this.connectorId = requireNonNull(connectorId, "connectorId is null"); + this.name = requireNonNull(name, "columnName is null"); + this.dataType = requireNonNull(dataType, "dataType is null"); + this.columnType = requireNonNull(columnType, "columnType is null"); + } + + @JsonProperty + public String getConnectorId() + { + return connectorId; + } + + @JsonProperty + public String getName() + { + return name; + } + + @JsonProperty + public TypeSignature getDataType() + { + return dataType; + } + + @JsonProperty + public ColumnType getColumnType() + { + return columnType; + } + + @Override + public String toString() + { + final StringBuilder sb = new StringBuilder("DeltaColumnHandle{"); + sb.append("connectorId='").append(connectorId).append('\''); + sb.append(", name='").append(name).append('\''); + sb.append(", dataType=").append(dataType); + sb.append(", columnType=").append(columnType); + sb.append('}'); + return sb.toString(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + DeltaColumnHandle that = (DeltaColumnHandle) o; + return connectorId.equals(that.connectorId) && + name.equals(that.name) && + dataType.equals(that.dataType) && + columnType == that.columnType; + } + + @Override + public int hashCode() + { + return Objects.hash(connectorId, name, dataType, columnType); + } +} diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaConfig.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaConfig.java new file mode 100644 index 000000000000..7aa8131bd742 --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaConfig.java @@ -0,0 +1,76 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import io.airlift.configuration.Config; + +import javax.validation.constraints.NotNull; + +public class DeltaConfig +{ + private boolean partitionPruningEnabled = true; + private boolean filterPushdownEnabled = true; + private boolean projectionPushdownEnabled = true; + private int maxSplitsBatchSize = 200; + + @NotNull + public boolean isPartitionPruningEnabled() + { + return partitionPruningEnabled; + } + + @Config("delta.partition-pruning-enabled") + public DeltaConfig setPartitionPruningEnabled(boolean partitionPruningEnabled) + { + this.partitionPruningEnabled = partitionPruningEnabled; + return this; + } + + @NotNull + public boolean isFilterPushdownEnabled() + { + return filterPushdownEnabled; + } + + @Config("delta.filter-pushdown-enabled") + public DeltaConfig setFilterPushdownEnabled(boolean filterPushdownEnabled) + { + this.filterPushdownEnabled = filterPushdownEnabled; + return this; + } + + public boolean isProjectionPushdownEnabled() + { + return projectionPushdownEnabled; + } + + @Config("delta.projection-pushdown-enabled") + public DeltaConfig setProjectionPushdownEnabled(boolean projectionPushdownEnabled) + { + this.projectionPushdownEnabled = projectionPushdownEnabled; + return this; + } + + public int getMaxSplitsBatchSize() + { + return maxSplitsBatchSize; + } + + @Config("delta.max-splits-batch-size") + public DeltaConfig setMaxSplitsBatchSize(int maxSplitsBatchSize) + { + this.maxSplitsBatchSize = maxSplitsBatchSize; + return this; + } +} diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaConnectionHandleResolver.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaConnectionHandleResolver.java new file mode 100644 index 000000000000..bea181d27992 --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaConnectionHandleResolver.java @@ -0,0 +1,48 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import io.prestosql.spi.connector.ColumnHandle; +import io.prestosql.spi.connector.ConnectorHandleResolver; +import io.prestosql.spi.connector.ConnectorSplit; +import io.prestosql.spi.connector.ConnectorTableHandle; +import io.prestosql.spi.connector.ConnectorTransactionHandle; + +public class DeltaConnectionHandleResolver + implements ConnectorHandleResolver +{ + @Override + public Class getTableHandleClass() + { + return DeltaTableHandle.class; + } + + @Override + public Class getColumnHandleClass() + { + return DeltaColumnHandle.class; + } + + @Override + public Class getSplitClass() + { + return DeltaSplit.class; + } + + @Override + public Class getTransactionHandleClass() + { + return DeltaTransactionHandle.class; + } +} diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaConnector.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaConnector.java new file mode 100644 index 000000000000..0fbf1b4b40d0 --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaConnector.java @@ -0,0 +1,102 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import io.airlift.bootstrap.LifeCycleManager; +import io.airlift.log.Logger; +import io.prestosql.plugin.base.classloader.ClassLoaderSafeConnectorMetadata; +import io.prestosql.plugin.base.classloader.ClassLoaderSafeConnectorPageSourceProvider; +import io.prestosql.plugin.base.classloader.ClassLoaderSafeConnectorSplitManager; +import io.prestosql.spi.connector.Connector; +import io.prestosql.spi.connector.ConnectorMetadata; +import io.prestosql.spi.connector.ConnectorPageSourceProvider; +import io.prestosql.spi.connector.ConnectorSplitManager; +import io.prestosql.spi.connector.ConnectorTransactionHandle; +import io.prestosql.spi.session.PropertyMetadata; +import io.prestosql.spi.transaction.IsolationLevel; + +import javax.inject.Inject; + +import java.util.List; + +import static io.prestosql.delta.DeltaTransactionHandle.INSTANCE; +import static java.util.Objects.requireNonNull; + +public class DeltaConnector + implements Connector +{ + private static final Logger log = Logger.get(DeltaConnector.class); + + private final LifeCycleManager lifeCycleManager; + private final DeltaMetadata metadata; + private final DeltaSplitManager splitManager; + private final DeltaSessionProperties sessionProperties; + private final DeltaPageSourceProvider pageSourceProvider; + + @Inject + public DeltaConnector( + LifeCycleManager lifeCycleManager, + DeltaMetadata metadata, + DeltaSplitManager splitManager, + DeltaSessionProperties sessionProperties, + DeltaPageSourceProvider pageSourceProvider) + { + this.lifeCycleManager = requireNonNull(lifeCycleManager, "lifeCycleManager is null"); + this.metadata = requireNonNull(metadata, "metadata is null"); + this.splitManager = requireNonNull(splitManager, "splitManager is null"); + this.sessionProperties = requireNonNull(sessionProperties, "sessionProperties is null"); + this.pageSourceProvider = requireNonNull(pageSourceProvider, "pageSourceProvider is null"); + } + + @Override + public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly) + { + return INSTANCE; + } + + @Override + public ConnectorMetadata getMetadata(ConnectorTransactionHandle transactionHandle) + { + return new ClassLoaderSafeConnectorMetadata(metadata, getClass().getClassLoader()); + } + + @Override + public ConnectorSplitManager getSplitManager() + { + return new ClassLoaderSafeConnectorSplitManager(splitManager, getClass().getClassLoader()); + } + + @Override + public ConnectorPageSourceProvider getPageSourceProvider() + { + return new ClassLoaderSafeConnectorPageSourceProvider(pageSourceProvider, getClass().getClassLoader()); + } + + @Override + public List> getSessionProperties() + { + return sessionProperties.getSessionProperties(); + } + + @Override + public final void shutdown() + { + try { + lifeCycleManager.stop(); + } + catch (Exception e) { + log.error(e, "Error shutting down connector"); + } + } +} diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaConnectorFactory.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaConnectorFactory.java new file mode 100644 index 000000000000..f8c5587b2435 --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaConnectorFactory.java @@ -0,0 +1,87 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import com.google.inject.Injector; +import io.airlift.bootstrap.Bootstrap; +import io.airlift.json.JsonModule; +import io.prestosql.plugin.base.CatalogName; +import io.prestosql.plugin.hive.HiveHdfsModule; +import io.prestosql.plugin.hive.NodeVersion; +import io.prestosql.plugin.hive.authentication.HiveAuthenticationModule; +import io.prestosql.plugin.hive.azure.HiveAzureModule; +import io.prestosql.plugin.hive.gcs.HiveGcsModule; +import io.prestosql.plugin.hive.metastore.HiveMetastoreModule; +import io.prestosql.plugin.hive.s3.HiveS3Module; +import io.prestosql.spi.NodeManager; +import io.prestosql.spi.connector.Connector; +import io.prestosql.spi.connector.ConnectorContext; +import io.prestosql.spi.connector.ConnectorFactory; +import io.prestosql.spi.connector.ConnectorHandleResolver; + +import java.util.Map; +import java.util.Optional; + +import static com.google.common.base.Throwables.throwIfUnchecked; +import static java.util.Objects.requireNonNull; + +public class DeltaConnectorFactory + implements ConnectorFactory +{ + @Override + public String getName() + { + return "delta"; + } + + @Override + public ConnectorHandleResolver getHandleResolver() + { + return new DeltaConnectionHandleResolver(); + } + + @Override + public Connector create(String catalogName, Map requiredConfig, ConnectorContext context) + { + requireNonNull(requiredConfig, "requiredConfig is null"); + try { + Bootstrap app = new Bootstrap( + new JsonModule(), + new DeltaModule(catalogName, context.getTypeManager()), + new HiveS3Module(), + new HiveAzureModule(), + new HiveGcsModule(), + new HiveHdfsModule(), + new HiveAuthenticationModule(), + new HiveMetastoreModule(Optional.empty()), + //new CachingHiveMetastoreModule(), + binder -> { + binder.bind(NodeVersion.class).toInstance(new NodeVersion(context.getNodeManager().getCurrentNode().getVersion())); + binder.bind(NodeManager.class).toInstance(context.getNodeManager()); + binder.bind(CatalogName.class).toInstance(new CatalogName(catalogName)); + }); + + Injector injector = app + .doNotInitializeLogging() + .setRequiredConfigurationProperties(requiredConfig) + .initialize(); + + return injector.getInstance(DeltaConnector.class); + } + catch (Exception e) { + throwIfUnchecked(e); + throw new RuntimeException(e); + } + } +} diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaConnectorId.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaConnectorId.java new file mode 100644 index 000000000000..ed40d8df6462 --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaConnectorId.java @@ -0,0 +1,54 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public final class DeltaConnectorId +{ + private final String id; + + public DeltaConnectorId(String id) + { + this.id = requireNonNull(id, "id is null"); + } + + @Override + public String toString() + { + return id; + } + + @Override + public int hashCode() + { + return Objects.hash(id); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + + DeltaConnectorId other = (DeltaConnectorId) obj; + return Objects.equals(this.id, other.id); + } +} diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaErrorCode.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaErrorCode.java new file mode 100644 index 000000000000..be4a14ab1ba4 --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaErrorCode.java @@ -0,0 +1,48 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import io.prestosql.spi.ErrorCode; +import io.prestosql.spi.ErrorCodeSupplier; +import io.prestosql.spi.ErrorType; + +import static io.prestosql.spi.ErrorType.EXTERNAL; +import static io.prestosql.spi.ErrorType.INTERNAL_ERROR; +import static io.prestosql.spi.ErrorType.USER_ERROR; + +public enum DeltaErrorCode + implements ErrorCodeSupplier +{ + DELTA_UNSUPPORTED_COLUMN_TYPE(0, USER_ERROR), + DELTA_UNSUPPORTED_DATA_FORMAT(1, USER_ERROR), + DELTA_PARQUET_SCHEMA_MISMATCH(2, EXTERNAL), + DELTA_BAD_DATA(3, EXTERNAL), + DELTA_CANNOT_OPEN_SPLIT(4, EXTERNAL), + DELTA_MISSING_DATA(5, EXTERNAL), + DELTA_READ_DATA_ERROR(6, INTERNAL_ERROR), + DELTA_INVALID_PARTITION_VALUE(7, EXTERNAL); + + private final ErrorCode errorCode; + + DeltaErrorCode(int code, ErrorType type) + { + errorCode = new ErrorCode(code + 0x1905_0000, name(), type); + } + + @Override + public ErrorCode toErrorCode() + { + return errorCode; + } +} diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaExpressionUtils.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaExpressionUtils.java new file mode 100644 index 000000000000..788742343d64 --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaExpressionUtils.java @@ -0,0 +1,243 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import com.google.common.collect.ImmutableList; +import io.airlift.slice.Slice; +import io.delta.standalone.actions.AddFile; +import io.prestosql.spi.PrestoException; +import io.prestosql.spi.connector.ColumnHandle; +import io.prestosql.spi.connector.ConnectorSession; +import io.prestosql.spi.predicate.Domain; +import io.prestosql.spi.predicate.TupleDomain; +import io.prestosql.spi.predicate.ValueSet; +import io.prestosql.spi.type.StandardTypes; +import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.TypeManager; + +import java.sql.Date; +import java.sql.Timestamp; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.stream.Collectors; + +import static io.airlift.slice.Slices.utf8Slice; +import static io.prestosql.delta.DeltaColumnHandle.ColumnType.PARTITION; +import static io.prestosql.delta.DeltaErrorCode.DELTA_INVALID_PARTITION_VALUE; +import static io.prestosql.delta.DeltaErrorCode.DELTA_UNSUPPORTED_COLUMN_TYPE; +import static io.prestosql.delta.DeltaSessionProperties.isPartitionPruningEnabled; +import static java.lang.Double.doubleToRawLongBits; +import static java.lang.Double.parseDouble; +import static java.lang.Float.floatToRawIntBits; +import static java.lang.Float.parseFloat; +import static java.lang.Long.parseLong; +import static java.lang.String.format; +import static java.util.stream.Collectors.toMap; + +public class DeltaExpressionUtils +{ + private DeltaExpressionUtils() + { + } + + /** + * Split the predicate into partition and regular column predicates + */ + public static List> splitPredicate( + TupleDomain predicate) + { + Map partitionColumnPredicates = new HashMap<>(); + Map regularColumnPredicates = new HashMap<>(); + + Optional> domains = predicate.getDomains(); + if (domains.isPresent()) { + domains.get().entrySet().stream() + .forEach(domainPair -> { + DeltaColumnHandle columnHandle = (DeltaColumnHandle) domainPair.getKey(); + if (columnHandle.getColumnType() == PARTITION) { + partitionColumnPredicates.put(domainPair.getKey(), domainPair.getValue()); + } + else { + regularColumnPredicates.put(domainPair.getKey(), domainPair.getValue()); + } + }); + } + + return ImmutableList.of( + TupleDomain.withColumnDomains(partitionColumnPredicates), + TupleDomain.withColumnDomains(regularColumnPredicates)); + } + + /** + * Utility method to + * 1) remove the null value partition values. These null values cause problems later + * when used with Guava Immutable map structures. + * 2) convert column names to lower case. Presto converts the column names to lowercase. + */ + public static Map sanitizePartitionValues(Map partitionValues) + { + return partitionValues.entrySet().stream() + .filter(entry -> entry.getValue() != null) + .collect(toMap(entry -> entry.getKey().toLowerCase(Locale.ROOT), Map.Entry::getValue)); + } + + /** + * Utility method that takes an iterator of {@link AddFile}s and a predicate and returns an iterator of {@link AddFile}s + * that satisfy the predicate (predicate evaluates to a deterministic NO) + */ + public static Iterator iterateWithPredicate( + ConnectorSession session, + Iterator inputIter, + TupleDomain predicate, + TypeManager typeManager) + { + TupleDomain partitionPredicate = extractPartitionColumnsPredicate(predicate); + if (partitionPredicate.isAll() /* no partition filter */ || !isPartitionPruningEnabled(session)) { + return inputIter; + } + + if (partitionPredicate.isNone()) { + return Collections.emptyIterator(); // nothing passes the partition predicate, return empty iterator + } + + List partitionColumns = + predicate.getColumnDomains().get().stream() + .filter(entry -> entry.getColumn().getColumnType() == PARTITION) + .map(entry -> entry.getColumn()) + .collect(Collectors.toList()); + + return new Iterator() + { + private AddFile nextItem; + + @Override + public boolean hasNext() + { + if (nextItem != null) { + return true; + } + + while (inputIter.hasNext()) { + AddFile nextFile = inputIter.next(); + if (evaluatePartitionPredicate(partitionPredicate, partitionColumns, typeManager, nextFile)) { + nextItem = nextFile; + break; + } + } + + return nextItem != null; + } + + @Override + public AddFile next() + { + if (!hasNext()) { + throw new NoSuchElementException("there are no more files"); + } + AddFile toReturn = nextItem; + nextItem = null; + return toReturn; + } + }; + } + + private static TupleDomain extractPartitionColumnsPredicate(TupleDomain predicate) + { + return predicate.transform( + deltaColumnHandle -> { + if (deltaColumnHandle.getColumnType() != PARTITION) { + return null; + } + return deltaColumnHandle.getName(); + }); + } + + private static boolean evaluatePartitionPredicate( + TupleDomain partitionPredicate, + List partitionColumns, + TypeManager typeManager, + AddFile addFile) + { + Map partitionValues = sanitizePartitionValues(addFile.getPartitionValues()); + for (DeltaColumnHandle partitionColumn : partitionColumns) { + String columnName = partitionColumn.getName(); + String partitionValue = partitionValues.get(columnName.toLowerCase(Locale.ROOT)); + Domain domain = getDomain(partitionColumn, partitionValue, typeManager, addFile.getPath()); + Domain columnPredicate = partitionPredicate.getDomains().get().get(columnName); + + if (columnPredicate == null) { + continue; // there is no predicate on this column + } + + if (columnPredicate.intersect(domain).isNone()) { + return false; + } + } + + return true; + } + + private static Domain getDomain(DeltaColumnHandle columnHandle, String partitionValue, TypeManager typeManager, String filePath) + { + Type type = typeManager.getType(columnHandle.getDataType()); + if (partitionValue == null) { + return Domain.onlyNull(type); + } + + String typeBase = columnHandle.getDataType().getBase(); + try { + switch (typeBase) { + case StandardTypes.TINYINT: + case StandardTypes.SMALLINT: + case StandardTypes.INTEGER: + case StandardTypes.BIGINT: + Long intValue = parseLong(partitionValue); + return Domain.create(ValueSet.of(type, intValue), false); + case StandardTypes.REAL: + Long realValue = (long) floatToRawIntBits(parseFloat(partitionValue)); + return Domain.create(ValueSet.of(type, realValue), false); + case StandardTypes.DOUBLE: + Long doubleValue = doubleToRawLongBits(parseDouble(partitionValue)); + return Domain.create(ValueSet.of(type, doubleValue), false); + case StandardTypes.VARCHAR: + case StandardTypes.VARBINARY: + Slice sliceValue = utf8Slice(partitionValue); + return Domain.create(ValueSet.of(type, sliceValue), false); + case StandardTypes.DATE: + Long dateValue = Date.valueOf(partitionValue).toLocalDate().toEpochDay(); + return Domain.create(ValueSet.of(type, dateValue), false); + case StandardTypes.TIMESTAMP: + Long timestampValue = Timestamp.valueOf(partitionValue).getTime(); // convert to millis + return Domain.create(ValueSet.of(type, timestampValue), false); + case StandardTypes.BOOLEAN: + Boolean booleanValue = Boolean.valueOf(partitionValue); + return Domain.create(ValueSet.of(type, booleanValue), false); + default: + throw new PrestoException(DELTA_UNSUPPORTED_COLUMN_TYPE, + format("Unsupported data type '%s' for partition column %s", columnHandle.getDataType(), columnHandle.getName())); + } + } + catch (IllegalArgumentException e) { + throw new PrestoException(DELTA_INVALID_PARTITION_VALUE, + format("Can not parse partition value '%s' of type '%s' for partition column '%s' in file '%s'", + partitionValue, columnHandle.getDataType(), columnHandle.getName(), filePath)); + } + } +} diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaMetadata.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaMetadata.java new file mode 100644 index 000000000000..f79d66851abc --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaMetadata.java @@ -0,0 +1,314 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.airlift.log.Logger; +import io.prestosql.plugin.hive.authentication.HiveIdentity; +import io.prestosql.plugin.hive.metastore.HiveMetastore; +import io.prestosql.plugin.hive.metastore.Storage; +import io.prestosql.plugin.hive.metastore.Table; +import io.prestosql.spi.connector.ColumnHandle; +import io.prestosql.spi.connector.ColumnMetadata; +import io.prestosql.spi.connector.ConnectorMetadata; +import io.prestosql.spi.connector.ConnectorSession; +import io.prestosql.spi.connector.ConnectorTableHandle; +import io.prestosql.spi.connector.ConnectorTableLayout; +import io.prestosql.spi.connector.ConnectorTableLayoutHandle; +import io.prestosql.spi.connector.ConnectorTableMetadata; +import io.prestosql.spi.connector.ConnectorTableProperties; +import io.prestosql.spi.connector.Constraint; +import io.prestosql.spi.connector.ConstraintApplicationResult; +import io.prestosql.spi.connector.ProjectionApplicationResult; +import io.prestosql.spi.connector.SchemaTableName; +import io.prestosql.spi.connector.SchemaTablePrefix; +import io.prestosql.spi.expression.ConnectorExpression; +import io.prestosql.spi.predicate.TupleDomain; +import io.prestosql.spi.type.TypeManager; + +import javax.inject.Inject; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import static com.google.common.base.Preconditions.checkArgument; +import static io.prestosql.delta.DeltaColumnHandle.ColumnType.PARTITION; +import static io.prestosql.delta.DeltaColumnHandle.ColumnType.REGULAR; +import static io.prestosql.delta.DeltaExpressionUtils.splitPredicate; +import static io.prestosql.delta.DeltaSessionProperties.isFilterPushdownEnabled; +import static io.prestosql.delta.DeltaSessionProperties.isProjectionPushdownEnabled; +import static java.util.Collections.singletonList; +import static java.util.Objects.requireNonNull; + +public class DeltaMetadata + implements ConnectorMetadata +{ + private static final Logger log = Logger.get(DeltaMetadata.class); + + /** + * Special schema used when querying a Delta table by storage location. + * Ex. SELECT * FROM delta."$PATH$"."s3://bucket/path/to/table". User is not able to list any tables + * in this schema. It is just used to query a Delta table by storage location. + */ + private static final String PATH_SCHEMA = "$PATH$"; + + private final String connectorId; + private final DeltaClient deltaClient; + private final HiveMetastore metastore; + private final TypeManager typeManager; + private final DeltaConfig config; + + @Inject + public DeltaMetadata( + DeltaConnectorId connectorId, + DeltaClient deltaClient, + HiveMetastore metastore, + TypeManager typeManager, + DeltaConfig config) + { + this.connectorId = requireNonNull(connectorId, "connectorId is null").toString(); + this.deltaClient = requireNonNull(deltaClient, "deltaClient is null"); + this.metastore = requireNonNull(metastore, "metastore is null"); + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + this.config = requireNonNull(config, "config is null"); + } + + @Override + public List listSchemaNames(ConnectorSession session) + { + ArrayList schemas = new ArrayList<>(); + schemas.addAll(metastore.getAllDatabases()); + schemas.add(PATH_SCHEMA.toLowerCase(Locale.ROOT)); + return schemas; + } + + @Override + public DeltaTableHandle getTableHandle(ConnectorSession session, SchemaTableName schemaTableName) + { + String schemaName = schemaTableName.getSchemaName(); + String tableName = schemaTableName.getTableName(); + if (!listSchemaNames(session).contains(schemaName)) { + return null; // indicates table doesn't exist + } + + DeltaTableName deltaTableName = DeltaTableName.from(tableName); + String tableLocation; + if (PATH_SCHEMA.equalsIgnoreCase(schemaName)) { + tableLocation = deltaTableName.getTableNameOrPath(); + } + else { + Optional metastoreTable = metastore.getTable( + new HiveIdentity(session), + schemaName, + deltaTableName.getTableNameOrPath()); + if (!metastoreTable.isPresent()) { + return null; // indicates table doesn't exist + } + + Map tableParameters = metastoreTable.get().getParameters(); + Storage storage = metastoreTable.get().getStorage(); + tableLocation = storage.getLocation(); + + // Delta table written using Spark and Hive have set the table parameter + // "spark.sql.sources.provider = delta". If this property is found table + // location is found in SerDe properties with key "path". + if ("delta".equalsIgnoreCase(tableParameters.get("spark.sql.sources.provider"))) { + tableLocation = storage.getSerdeParameters().get("path"); + if (Strings.isNullOrEmpty(tableLocation)) { + log.warn("Location key ('path') is missing in SerDe properties for table %s. " + + "Using the 'location' attribute as the table location.", schemaTableName); + // fallback to using the location attribute + tableLocation = storage.getLocation(); + } + } + } + + Optional table = deltaClient.getTable( + session, + schemaTableName, + tableLocation, + deltaTableName.getSnapshotId(), + deltaTableName.getSnapshotAsOfTimestamp()); + if (table.isPresent()) { + return new DeltaTableHandle(connectorId, table.get(), TupleDomain.all(), Optional.empty()); + } + return null; + } + + @Override + public boolean usesLegacyTableLayouts() + { + return false; + } + + @Override + public ConnectorTableProperties getTableProperties(ConnectorSession session, ConnectorTableHandle tableHandle) + { + return new ConnectorTableProperties(); + } + + @Override + public Optional> applyFilter(ConnectorSession session, ConnectorTableHandle handle, Constraint constraint) + { + if (!isFilterPushdownEnabled(session)) { + return Optional.empty(); + } + + DeltaTableHandle table = (DeltaTableHandle) handle; + if (constraint.getSummary().equals(table.getPredicate())) { + return Optional.empty(); + } + + // Split the predicate into partition column predicate and other column predicates + // Only the partition column predicate is fully enforced. Other predicate is partially enforced (best effort). + List> predicate = splitPredicate(constraint.getSummary()); + TupleDomain enforcedPredicate = predicate.get(0); + TupleDomain unenforcedPredicate = predicate.get(1); + + DeltaTableHandle newDeltaTableHandle = table.withPredicate( + constraint.getSummary().transform(DeltaColumnHandle.class::cast), + constraint.getSummary().toString(session)); + + return Optional.of(new ConstraintApplicationResult<>( + newDeltaTableHandle, + unenforcedPredicate)); + } + + @Override + public Optional> applyProjection( + ConnectorSession session, ConnectorTableHandle handle, + List projections, + Map assignments) + { + if (!isProjectionPushdownEnabled(session)) { + return Optional.empty(); + } + + // TODO: implement + return Optional.empty(); + } + + @Override + public ConnectorTableLayout getTableLayout(ConnectorSession session, ConnectorTableLayoutHandle handle) + { + return new ConnectorTableLayout(handle); + } + + @Override + public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle table) + { + DeltaTableHandle deltaTableHandle = (DeltaTableHandle) table; + checkConnectorId(deltaTableHandle); + return getTableMetadata(session, deltaTableHandle.toSchemaTableName()); + } + + @Override + public List listTables(ConnectorSession session, Optional schemaName) + { + List schemaNames = schemaName.isEmpty() ? listSchemaNames(session) : ImmutableList.of(schemaName.get()); + + ImmutableList.Builder tableNames = ImmutableList.builder(); + for (String schema : schemaNames) { + for (String table : metastore.getAllTables(schema)) { + tableNames.add(new SchemaTableName(schema, table)); + } + } + return tableNames.build(); + } + + @Override + public Map getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle) + { + DeltaTableHandle deltaTableHandle = (DeltaTableHandle) tableHandle; + checkConnectorId(deltaTableHandle); + + ImmutableMap.Builder columnHandles = ImmutableMap.builder(); + for (DeltaColumn column : deltaTableHandle.getDeltaTable().getColumns()) { + columnHandles.put( + column.getName(), + new DeltaColumnHandle( + connectorId, + column.getName(), + column.getType(), + column.isPartition() ? PARTITION : REGULAR)); + } + return columnHandles.build(); + } + + @Override + public Map> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix) + { + requireNonNull(prefix, "prefix is null"); + ImmutableMap.Builder> columns = ImmutableMap.builder(); + for (SchemaTableName tableName : listTables(session, prefix)) { + ConnectorTableMetadata tableMetadata = getTableMetadata(session, tableName); + // table can disappear during listing operation + if (tableMetadata != null) { + columns.put(tableName, tableMetadata.getColumns()); + } + } + return columns.build(); + } + + private ConnectorTableMetadata getTableMetadata(ConnectorSession session, SchemaTableName tableName) + { + DeltaTableHandle tableHandle = getTableHandle(session, tableName); + if (tableHandle == null) { + return null; + } + + List columnMetadata = tableHandle.getDeltaTable().getColumns().stream() + .map(column -> getColumnMetadata(column)) + .collect(Collectors.toList()); + + return new ConnectorTableMetadata(tableName, columnMetadata); + } + + @Override + public ColumnMetadata getColumnMetadata(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle columnHandle) + { + return getColumnMetadata(columnHandle); + } + + private ColumnMetadata getColumnMetadata(ColumnHandle columnHandle) + { + DeltaColumnHandle deltaColumnHandle = (DeltaColumnHandle) columnHandle; + return new ColumnMetadata(deltaColumnHandle.getName(), typeManager.getType(deltaColumnHandle.getDataType())); + } + + private List listTables(ConnectorSession session, SchemaTablePrefix prefix) + { + if (prefix.getTable().isEmpty()) { + return listTables(session, prefix.getSchema()); + } + + return singletonList(new SchemaTableName(prefix.getSchema().get(), prefix.getTable().get())); + } + + private ColumnMetadata getColumnMetadata(DeltaColumn deltaColumn) + { + return new ColumnMetadata(deltaColumn.getName(), typeManager.getType(deltaColumn.getType())); + } + + private void checkConnectorId(DeltaTableHandle tableHandle) + { + checkArgument(tableHandle.getConnectorId().equals(connectorId), "table handle is not for this connector"); + } +} diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaModule.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaModule.java new file mode 100644 index 000000000000..3eb4c20abf86 --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaModule.java @@ -0,0 +1,117 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.Scopes; +import io.prestosql.plugin.base.CatalogName; +import io.prestosql.plugin.hive.DynamicConfigurationProvider; +import io.prestosql.plugin.hive.FileFormatDataSourceStats; +import io.prestosql.plugin.hive.HdfsConfiguration; +import io.prestosql.plugin.hive.HdfsConfigurationInitializer; +import io.prestosql.plugin.hive.HdfsEnvironment; +import io.prestosql.plugin.hive.HiveConfig; +import io.prestosql.plugin.hive.HiveHdfsConfiguration; +import io.prestosql.plugin.hive.metastore.MetastoreConfig; +import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.TypeId; +import io.prestosql.spi.type.TypeManager; + +import javax.inject.Inject; +import javax.inject.Singleton; + +import java.util.concurrent.ExecutorService; + +import static com.google.inject.multibindings.Multibinder.newSetBinder; +import static io.airlift.concurrent.Threads.daemonThreadsNamed; +import static io.airlift.configuration.ConfigBinder.configBinder; +import static io.airlift.json.JsonBinder.jsonBinder; +import static io.airlift.json.JsonCodec.listJsonCodec; +import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.Executors.newCachedThreadPool; +import static org.weakref.jmx.ObjectNames.generatedNameOf; +import static org.weakref.jmx.guice.ExportBinder.newExporter; + +public class DeltaModule + implements Module +{ + private final String connectorId; + private final TypeManager typeManager; + + public DeltaModule(String connectorId, TypeManager typeManager) + { + this.connectorId = requireNonNull(connectorId, "connector id is null"); + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + } + + @Override + public void configure(Binder binder) + { + binder.bind(TypeManager.class).toInstance(typeManager); + + binder.bind(DeltaConnector.class).in(Scopes.SINGLETON); + binder.bind(DeltaConnectorId.class).toInstance(new DeltaConnectorId(connectorId)); + binder.bind(DeltaMetadata.class).in(Scopes.SINGLETON); + binder.bind(DeltaClient.class).in(Scopes.SINGLETON); + binder.bind(DeltaSplitManager.class).in(Scopes.SINGLETON); + binder.bind(DeltaPageSourceProvider.class).in(Scopes.SINGLETON); + binder.bind(DeltaSessionProperties.class).in(Scopes.SINGLETON); + configBinder(binder).bindConfig(DeltaConfig.class); + + configBinder(binder).bindConfig(MetastoreConfig.class); + configBinder(binder).bindConfig(HiveConfig.class); + + binder.bind(HdfsConfigurationInitializer.class).in(Scopes.SINGLETON); + binder.bind(HdfsConfiguration.class).to(HiveHdfsConfiguration.class).in(Scopes.SINGLETON); + newSetBinder(binder, DynamicConfigurationProvider.class); + binder.bind(HdfsEnvironment.class).in(Scopes.SINGLETON); + + jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class); + jsonCodecBinder(binder).bindMapJsonCodec(String.class, listJsonCodec(DeltaTable.class)); + + binder.bind(FileFormatDataSourceStats.class).in(Scopes.SINGLETON); + newExporter(binder).export(FileFormatDataSourceStats.class).as(generatedNameOf(FileFormatDataSourceStats.class, connectorId)); + } + + public static final class TypeDeserializer + extends FromStringDeserializer + { + private final TypeManager typeManager; + + @Inject + public TypeDeserializer(TypeManager typeManager) + { + super(Type.class); + this.typeManager = requireNonNull(typeManager, "metadata is null"); + } + + @Override + protected Type _deserialize(String value, DeserializationContext context) + { + return typeManager.getType(TypeId.of(value)); + } + } + + @Singleton + @Provides + public ExecutorService createDeltaClientExecutor(CatalogName catalogName) + { + return newCachedThreadPool(daemonThreadsNamed("delta-" + catalogName + "-%s")); + } +} diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaPageSource.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaPageSource.java new file mode 100644 index 000000000000..dcf1f7f3add8 --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaPageSource.java @@ -0,0 +1,145 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import io.prestosql.spi.Page; +import io.prestosql.spi.PrestoException; +import io.prestosql.spi.block.Block; +import io.prestosql.spi.block.RunLengthEncodedBlock; +import io.prestosql.spi.connector.ConnectorPageSource; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +import static io.prestosql.delta.DeltaColumnHandle.ColumnType.PARTITION; +import static io.prestosql.delta.DeltaErrorCode.DELTA_READ_DATA_ERROR; +import static java.util.Objects.requireNonNull; + +/** + * {@link ConnectorPageSource} implementation for Delta tables that prefills + * partition column blocks and combines them with regular column blocks returned + * by the underlying file reader {@link ConnectorPageSource} implementation. + */ +public class DeltaPageSource + implements ConnectorPageSource +{ + private final List columnHandles; + private final ConnectorPageSource dataPageSource; + private final Map partitionValues; + + /** + * Create a DeltaPageSource + * + * @param columnHandles List of columns (includes partition and regular) in order for which data needed in output. + * @param partitionValues Partition values (partition column -> partition value map). + * @param dataPageSource Initialized underlying file reader which returns the data for regular columns. + */ + public DeltaPageSource( + List columnHandles, + Map partitionValues, + ConnectorPageSource dataPageSource) + { + this.columnHandles = requireNonNull(columnHandles, "columnHandles is null"); + this.dataPageSource = requireNonNull(dataPageSource, "dataPageSource is null"); + this.partitionValues = requireNonNull(partitionValues, "partitionValues is null"); + } + + @Override + public Page getNextPage() + { + try { + Page dataPage = dataPageSource.getNextPage(); + if (dataPage == null) { + return null; // reader is done + } + int positionCount = dataPage.getPositionCount(); + + int dataColumnIndex = 0; + int columnIndex = 0; + Block[] blocksWithPartitionColumns = new Block[columnHandles.size()]; + for (DeltaColumnHandle columnHandle : columnHandles) { + if (columnHandle.getColumnType() == PARTITION) { + Block partitionValue = partitionValues.get(columnHandle.getName()); + blocksWithPartitionColumns[columnIndex++] = new RunLengthEncodedBlock(partitionValue, positionCount); + } + else { + blocksWithPartitionColumns[columnIndex++] = (dataPage.getBlock(dataColumnIndex)); + dataColumnIndex++; + } + } + return new Page(positionCount, blocksWithPartitionColumns); + } + catch (PrestoException e) { + closeWithSuppression(e); + throw e; // already properly handled exception - throw without any additional info + } + catch (RuntimeException e) { + closeWithSuppression(e); + throw new PrestoException(DELTA_READ_DATA_ERROR, e); + } + } + + private void closeWithSuppression(Throwable throwable) + { + requireNonNull(throwable, "throwable is null"); + try { + close(); + } + catch (Exception e) { + // Self-suppression not permitted + if (e != throwable) { + throwable.addSuppressed(e); + } + } + } + + @Override + public long getCompletedBytes() + { + return dataPageSource.getCompletedBytes(); + } + + @Override + public long getReadTimeNanos() + { + return dataPageSource.getReadTimeNanos(); + } + + @Override + public boolean isFinished() + { + return dataPageSource.isFinished(); + } + + @Override + public long getSystemMemoryUsage() + { + return dataPageSource.getSystemMemoryUsage(); + } + + @Override + public void close() + throws IOException + { + dataPageSource.close(); + } + + @Override + public CompletableFuture isBlocked() + { + return dataPageSource.isBlocked(); + } +} diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaPageSourceProvider.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaPageSourceProvider.java new file mode 100644 index 000000000000..250e6a7acd4e --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaPageSourceProvider.java @@ -0,0 +1,318 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.delta; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.prestosql.memory.context.AggregatedMemoryContext; +import io.prestosql.parquet.Field; +import io.prestosql.parquet.ParquetCorruptionException; +import io.prestosql.parquet.ParquetDataSource; +import io.prestosql.parquet.ParquetReaderOptions; +import io.prestosql.parquet.RichColumnDescriptor; +import io.prestosql.parquet.predicate.Predicate; +import io.prestosql.parquet.reader.MetadataReader; +import io.prestosql.parquet.reader.ParquetReader; +import io.prestosql.plugin.hive.FileFormatDataSourceStats; +import io.prestosql.plugin.hive.HdfsEnvironment; +import io.prestosql.plugin.hive.HdfsEnvironment.HdfsContext; +import io.prestosql.plugin.hive.parquet.ParquetPageSource; +import io.prestosql.spi.PrestoException; +import io.prestosql.spi.block.Block; +import io.prestosql.spi.connector.ColumnHandle; +import io.prestosql.spi.connector.ConnectorPageSource; +import io.prestosql.spi.connector.ConnectorPageSourceProvider; +import io.prestosql.spi.connector.ConnectorSession; +import io.prestosql.spi.connector.ConnectorSplit; +import io.prestosql.spi.connector.ConnectorTableHandle; +import io.prestosql.spi.connector.ConnectorTransactionHandle; +import io.prestosql.spi.predicate.Domain; +import io.prestosql.spi.predicate.TupleDomain; +import io.prestosql.spi.predicate.Utils; +import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.TypeManager; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.AccessControlException; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.hadoop.metadata.BlockMetaData; +import org.apache.parquet.hadoop.metadata.FileMetaData; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; +import org.apache.parquet.io.MessageColumnIO; +import org.apache.parquet.schema.MessageType; + +import javax.inject.Inject; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Strings.nullToEmpty; +import static io.prestosql.delta.DeltaColumnHandle.ColumnType.PARTITION; +import static io.prestosql.delta.DeltaColumnHandle.ColumnType.REGULAR; +import static io.prestosql.delta.DeltaColumnHandle.ColumnType.SUBFIELD; +import static io.prestosql.delta.DeltaErrorCode.DELTA_BAD_DATA; +import static io.prestosql.delta.DeltaErrorCode.DELTA_CANNOT_OPEN_SPLIT; +import static io.prestosql.delta.DeltaErrorCode.DELTA_MISSING_DATA; +import static io.prestosql.delta.DeltaTypeUtils.convertPartitionValue; +import static io.prestosql.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; +import static io.prestosql.parquet.ParquetTypeUtils.getColumnIO; +import static io.prestosql.parquet.ParquetTypeUtils.getDescriptors; +import static io.prestosql.parquet.ParquetTypeUtils.getParquetTypeByName; +import static io.prestosql.parquet.ParquetTypeUtils.lookupColumnByName; +import static io.prestosql.parquet.predicate.PredicateUtils.buildPredicate; +import static io.prestosql.parquet.predicate.PredicateUtils.predicateMatches; +import static io.prestosql.plugin.hive.parquet.HdfsParquetDataSource.buildHdfsParquetDataSource; +import static io.prestosql.plugin.hive.parquet.ParquetColumnIOConverter.constructField; +import static io.prestosql.spi.StandardErrorCode.PERMISSION_DENIED; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.toMap; + +public class DeltaPageSourceProvider + implements ConnectorPageSourceProvider +{ + private final HdfsEnvironment hdfsEnvironment; + private final TypeManager typeManager; + private final FileFormatDataSourceStats fileFormatDataSourceStats; + + @Inject + public DeltaPageSourceProvider( + HdfsEnvironment hdfsEnvironment, + TypeManager typeManager, + FileFormatDataSourceStats fileFormatDataSourceStats) + { + this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + this.fileFormatDataSourceStats = requireNonNull(fileFormatDataSourceStats, "fileFormatDataSourceStats is null"); + } + + @Override + public ConnectorPageSource createPageSource( + ConnectorTransactionHandle transaction, + ConnectorSession session, + ConnectorSplit split, + ConnectorTableHandle table, + List columns, + TupleDomain dynamicFilter) + { + DeltaSplit deltaSplit = (DeltaSplit) split; + DeltaTableHandle deltaTableHandle = (DeltaTableHandle) table; + + HdfsContext hdfsContext = new HdfsContext(session.getIdentity()); + Path filePath = new Path(deltaSplit.getFilePath()); + List deltaColumnHandles = columns.stream() + .map(DeltaColumnHandle.class::cast) + .collect(Collectors.toList()); + + List regularColumnHandles = deltaColumnHandles.stream() + .filter(columnHandle -> columnHandle.getColumnType() != PARTITION) + .collect(Collectors.toList()); + + ConnectorPageSource dataPageSource = createParquetPageSource( + hdfsEnvironment, + session.getUser(), + hdfsEnvironment.getConfiguration(hdfsContext, filePath), + filePath, + deltaSplit.getStart(), + deltaSplit.getLength(), + deltaSplit.getFileSize(), + regularColumnHandles, + typeManager, + deltaTableHandle.getPredicate(), + fileFormatDataSourceStats); + + return new DeltaPageSource( + deltaColumnHandles, + convertPartitionValues(deltaColumnHandles, deltaSplit.getPartitionValues()), + dataPageSource); + } + + /** + * Go through all the output columns, identify the partition columns and convert the partition values to Presto internal format. + */ + private Map convertPartitionValues( + List allColumns, + Map partitionValues) + { + return allColumns.stream() + .filter(columnHandle -> columnHandle.getColumnType() == PARTITION) + .collect(toMap( + DeltaColumnHandle::getName, + columnHandle -> + Utils.nativeValueToBlock( + typeManager.getType(columnHandle.getDataType()), + convertPartitionValue( + columnHandle.getName(), + partitionValues.get(columnHandle.getName()), + columnHandle.getDataType())))); + } + + private static ConnectorPageSource createParquetPageSource( + HdfsEnvironment hdfsEnvironment, + String user, + Configuration configuration, + Path path, + long start, + long length, + long fileSize, + List columns, + TypeManager typeManager, + TupleDomain effectivePredicate, + FileFormatDataSourceStats stats) + { + AggregatedMemoryContext systemMemoryContext = newSimpleAggregatedMemoryContext(); + + ParquetDataSource dataSource = null; + try { + FSDataInputStream inputStream = hdfsEnvironment + .getFileSystem(user, path, configuration) + .open(path); + ParquetReaderOptions readerOptions = new ParquetReaderOptions(); + dataSource = buildHdfsParquetDataSource(inputStream, path, fileSize, stats, readerOptions); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(inputStream, path, fileSize); + + FileMetaData fileMetaData = parquetMetadata.getFileMetaData(); + MessageType fileSchema = fileMetaData.getSchema(); + + Optional message = columns.stream() + .filter(column -> column.getColumnType() == REGULAR) + .map(column -> getParquetType(fileSchema, column)) + .filter(Optional::isPresent) + .map(Optional::get) + .map(type -> new MessageType(fileSchema.getName(), type)) + .reduce(MessageType::union); + + MessageType requestedSchema = message + .orElse(new MessageType(fileSchema.getName(), ImmutableList.of())); + + ImmutableList.Builder footerBlocks = ImmutableList.builder(); + for (BlockMetaData block : parquetMetadata.getBlocks()) { + long firstDataPage = block.getColumns().get(0).getFirstDataPageOffset(); + if (firstDataPage >= start && firstDataPage < start + length) { + footerBlocks.add(block); + } + } + + Map, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, requestedSchema); + TupleDomain parquetTupleDomain = getParquetTupleDomain(descriptorsByPath, effectivePredicate); + Predicate parquetPredicate = buildPredicate(requestedSchema, parquetTupleDomain, descriptorsByPath); + final ParquetDataSource finalDataSource = dataSource; + ImmutableList.Builder blocks = ImmutableList.builder(); + for (BlockMetaData block : footerBlocks.build()) { + if (predicateMatches( + parquetPredicate, + block, + finalDataSource, + descriptorsByPath, + parquetTupleDomain, + readerOptions.isFailOnCorruptedStatistics())) { + blocks.add(block); + } + } + MessageColumnIO messageColumnIO = getColumnIO(fileSchema, requestedSchema); + ParquetReader parquetReader = new ParquetReader( + Optional.ofNullable(fileMetaData.getCreatedBy()), + messageColumnIO, + blocks.build(), + dataSource, + systemMemoryContext, + readerOptions); + + ImmutableList.Builder namesBuilder = ImmutableList.builder(); + ImmutableList.Builder typesBuilder = ImmutableList.builder(); + ImmutableList.Builder> fieldsBuilder = ImmutableList.builder(); + for (DeltaColumnHandle column : columns) { + checkArgument(column.getColumnType() == REGULAR || column.getColumnType() == SUBFIELD, + "column type must be regular or subfield column"); + + String name = column.getName(); + Type type = typeManager.getType(column.getDataType()); + + namesBuilder.add(name); + typesBuilder.add(type); + + if (getParquetType(fileSchema, column).isPresent()) { + String columnName = name; + fieldsBuilder.add(constructField(type, lookupColumnByName(messageColumnIO, columnName))); + } + else { + fieldsBuilder.add(Optional.empty()); + } + } + return new ParquetPageSource( + parquetReader, + typesBuilder.build(), + fieldsBuilder.build()); + } + catch (Exception e) { + try { + if (dataSource != null) { + dataSource.close(); + } + } + catch (IOException ignored) { + } + if (e instanceof PrestoException) { + throw (PrestoException) e; + } + if (e instanceof ParquetCorruptionException) { + throw new PrestoException(DELTA_BAD_DATA, e); + } + if (e instanceof AccessControlException) { + throw new PrestoException(PERMISSION_DENIED, e.getMessage(), e); + } + if (nullToEmpty(e.getMessage()).trim().equals("Filesystem closed") || e instanceof FileNotFoundException) { + throw new PrestoException(DELTA_CANNOT_OPEN_SPLIT, e); + } + String message = format("Error opening Hive split %s (offset=%s, length=%s): %s", path, start, length, e.getMessage()); + if (e.getClass().getSimpleName().equals("BlockMissingException")) { + throw new PrestoException(DELTA_MISSING_DATA, message, e); + } + throw new PrestoException(DELTA_CANNOT_OPEN_SPLIT, message, e); + } + } + + public static TupleDomain getParquetTupleDomain(Map, RichColumnDescriptor> descriptorsByPath, TupleDomain effectivePredicate) + { + if (effectivePredicate.isNone()) { + return TupleDomain.none(); + } + + ImmutableMap.Builder predicate = ImmutableMap.builder(); + for (Map.Entry entry : effectivePredicate.getDomains().get().entrySet()) { + DeltaColumnHandle columnHandle = entry.getKey(); + + RichColumnDescriptor descriptor = descriptorsByPath.get(ImmutableList.of(columnHandle.getName())); + if (descriptor != null) { + predicate.put(descriptor, entry.getValue()); + } + } + return TupleDomain.withColumnDomains(predicate.build()); + } + + public static Optional getParquetType( + MessageType messageType, + DeltaColumnHandle column) + { + org.apache.parquet.schema.Type type = getParquetTypeByName(column.getName(), messageType); + return Optional.of(type); + } +} diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaPlugin.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaPlugin.java new file mode 100644 index 000000000000..352618cf4399 --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaPlugin.java @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import com.google.common.collect.ImmutableList; +import io.prestosql.spi.Plugin; +import io.prestosql.spi.connector.ConnectorFactory; + +public class DeltaPlugin + implements Plugin +{ + @Override + public Iterable getConnectorFactories() + { + return ImmutableList.of(new DeltaConnectorFactory()); + } +} diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaSessionProperties.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaSessionProperties.java new file mode 100644 index 000000000000..b7d744114c86 --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaSessionProperties.java @@ -0,0 +1,74 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import com.google.common.collect.ImmutableList; +import io.prestosql.spi.connector.ConnectorSession; +import io.prestosql.spi.session.PropertyMetadata; + +import javax.inject.Inject; + +import java.util.List; + +import static io.prestosql.spi.session.PropertyMetadata.booleanProperty; + +public final class DeltaSessionProperties +{ + public static final String PARTITION_PRUNING_ENABLED = "partition_pruning_enabled"; + public static final String FILTER_PUSHDOWN_ENABLED = "filter_pushdown_enabled"; + public static final String PROJECTION_PUSHDOWN_ENABLED = "projection_pushdown_enabled"; + + private final List> sessionProperties; + + @Inject + public DeltaSessionProperties(DeltaConfig deltaConfig) + { + sessionProperties = ImmutableList.of( + booleanProperty( + PARTITION_PRUNING_ENABLED, + "Enable partition pruning in Delta tables", + deltaConfig.isPartitionPruningEnabled(), + false), + booleanProperty( + FILTER_PUSHDOWN_ENABLED, + "Enable filter pushdown into Delta tables", + deltaConfig.isFilterPushdownEnabled(), + false), + booleanProperty( + PROJECTION_PUSHDOWN_ENABLED, + "Enable project pushdown into Delta tables", + deltaConfig.isProjectionPushdownEnabled(), + false)); + } + + public List> getSessionProperties() + { + return sessionProperties; + } + + public static boolean isPartitionPruningEnabled(ConnectorSession session) + { + return session.getProperty(PARTITION_PRUNING_ENABLED, Boolean.class); + } + + public static boolean isFilterPushdownEnabled(ConnectorSession session) + { + return session.getProperty(FILTER_PUSHDOWN_ENABLED, Boolean.class); + } + + public static boolean isProjectionPushdownEnabled(ConnectorSession session) + { + return session.getProperty(PROJECTION_PUSHDOWN_ENABLED, Boolean.class); + } +} diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaSplit.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaSplit.java new file mode 100644 index 000000000000..8f6ba34066b4 --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaSplit.java @@ -0,0 +1,131 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.prestosql.spi.HostAddress; +import io.prestosql.spi.connector.ConnectorSplit; + +import java.util.List; +import java.util.Map; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class DeltaSplit + implements ConnectorSplit +{ + private final String connectorId; + private final String schema; + private final String table; + private final String filePath; + private final long start; + private final long length; + private final long fileSize; + private final Map partitionValues; + + @JsonCreator + public DeltaSplit( + @JsonProperty("connectorId") String connectorId, + @JsonProperty("schemaName") String schema, + @JsonProperty("tableName") String table, + @JsonProperty("filePath") String filePath, + @JsonProperty("start") long start, + @JsonProperty("length") long length, + @JsonProperty("fileSize") long fileSize, + @JsonProperty("partitionValues") Map partitionValues) + { + checkArgument(start >= 0, "start must be positive"); + checkArgument(length >= 0, "length must be positive"); + checkArgument(fileSize >= 0, "fileSize must be positive"); + + this.connectorId = requireNonNull(connectorId, "connector id is null"); + this.schema = requireNonNull(schema, "schema name is null"); + this.table = requireNonNull(table, "table name is null"); + this.filePath = requireNonNull(filePath, "filePath name is null"); + this.start = start; + this.length = length; + this.fileSize = fileSize; + this.partitionValues = ImmutableMap.copyOf(requireNonNull(partitionValues, "partitionValues id is null")); + } + + @JsonProperty + public String getConnectorId() + { + return connectorId; + } + + @JsonProperty + public String getSchema() + { + return schema; + } + + @JsonProperty + public String getTable() + { + return table; + } + + @JsonProperty + public String getFilePath() + { + return filePath; + } + + @JsonProperty + public long getStart() + { + return start; + } + + @JsonProperty + public long getLength() + { + return length; + } + + @JsonProperty + public long getFileSize() + { + return fileSize; + } + + @JsonProperty + public Map getPartitionValues() + { + return partitionValues; + } + + @Override + public boolean isRemotelyAccessible() + { + return true; + } + + @Override + public List getAddresses() + { + return ImmutableList.of(); + } + + @Override + public Object getInfo() + { + return this; + } +} diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaSplitManager.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaSplitManager.java new file mode 100644 index 000000000000..db71c0140114 --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaSplitManager.java @@ -0,0 +1,121 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import io.delta.standalone.actions.AddFile; +import io.prestosql.spi.connector.ConnectorPartitionHandle; +import io.prestosql.spi.connector.ConnectorSession; +import io.prestosql.spi.connector.ConnectorSplit; +import io.prestosql.spi.connector.ConnectorSplitManager; +import io.prestosql.spi.connector.ConnectorSplitSource; +import io.prestosql.spi.connector.ConnectorTableHandle; +import io.prestosql.spi.connector.ConnectorTransactionHandle; +import io.prestosql.spi.type.TypeManager; +import org.apache.hadoop.fs.Path; + +import javax.inject.Inject; + +import java.net.URI; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static io.prestosql.delta.DeltaExpressionUtils.sanitizePartitionValues; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.CompletableFuture.completedFuture; + +public class DeltaSplitManager + implements ConnectorSplitManager +{ + private final String connectorId; + private final DeltaConfig deltaConfig; + private final DeltaClient deltaClient; + private final TypeManager typeManager; + + @Inject + public DeltaSplitManager( + DeltaConnectorId connectorId, + DeltaConfig deltaConfig, + DeltaClient deltaClient, + TypeManager typeManager) + { + this.connectorId = requireNonNull(connectorId, "connectorId is null").toString(); + this.deltaConfig = requireNonNull(deltaConfig, "deltaConfig is null"); + this.deltaClient = requireNonNull(deltaClient, "deltaClient is null"); + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + } + + @Override + public ConnectorSplitSource getSplits( + ConnectorTransactionHandle transaction, + ConnectorSession session, + ConnectorTableHandle table, + SplitSchedulingStrategy splitSchedulingStrategy) + { + DeltaTableHandle tableHandle = (DeltaTableHandle) table; + return new DeltaSplitSource(session, tableHandle); + } + + private class DeltaSplitSource + implements ConnectorSplitSource + { + private final DeltaTable deltaTable; + private final Iterator fileIter; + private final int maxBatchSize; + + DeltaSplitSource(ConnectorSession session, DeltaTableHandle deltaTableHandle) + { + this.deltaTable = deltaTableHandle.getDeltaTable(); + this.fileIter = DeltaExpressionUtils.iterateWithPredicate( + session, + deltaClient.listFiles(session, deltaTable), + deltaTableHandle.getPredicate(), + typeManager); + this.maxBatchSize = deltaConfig.getMaxSplitsBatchSize(); + } + + @Override + public CompletableFuture getNextBatch(ConnectorPartitionHandle partitionHandle, int maxSize) + { + List splits = new ArrayList<>(); + while (fileIter.hasNext() && splits.size() < maxSize && splits.size() < maxBatchSize) { + AddFile file = fileIter.next(); + Path filePath = new Path(deltaTable.getTableLocation(), URI.create(file.getPath()).getPath()); + splits.add(new DeltaSplit( + connectorId, + deltaTable.getSchemaName(), + deltaTable.getTableName(), + filePath.toString(), + 0, /* start */ + file.getSize() /* split length - read the entire file in one split */, + file.getSize(), + sanitizePartitionValues(file.getPartitionValues()))); + } + + return completedFuture(new ConnectorSplitBatch(splits, !fileIter.hasNext())); + } + + @Override + public void close() + { + } + + @Override + public boolean isFinished() + { + return !fileIter.hasNext(); + } + } +} diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaTable.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaTable.java new file mode 100644 index 000000000000..f5ae110ccb75 --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaTable.java @@ -0,0 +1,118 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Strings.isNullOrEmpty; +import static java.util.Objects.requireNonNull; + +public class DeltaTable +{ + private final String schemaName; + private final String tableName; + private final String tableLocation; + private final Optional snapshotId; + private final List columns; + + @JsonCreator + public DeltaTable( + @JsonProperty("schemaName") String schemaName, + @JsonProperty("tableName") String tableName, + @JsonProperty("tableLocation") String tableLocation, + @JsonProperty("snapshotId") Optional snapshotId, + @JsonProperty("columns") List columns) + { + checkArgument(!isNullOrEmpty(schemaName), "schemaName is null or is empty"); + checkArgument(!isNullOrEmpty(tableName), "tableName is null or is empty"); + this.schemaName = schemaName; + this.tableName = tableName; + this.tableLocation = requireNonNull(tableLocation, "tableLocation is null"); + this.snapshotId = requireNonNull(snapshotId, "snapshotId is null"); + this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null")); + } + + @JsonProperty + public String getSchemaName() + { + return schemaName; + } + + @JsonProperty + public String getTableName() + { + return tableName; + } + + @JsonProperty + public String getTableLocation() + { + return tableLocation; + } + + @JsonProperty + public Optional getSnapshotId() + { + return snapshotId; + } + + @JsonProperty + public List getColumns() + { + return columns; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + DeltaTable that = (DeltaTable) o; + return Objects.equals(schemaName, that.schemaName) && + Objects.equals(tableName, that.tableName) && + Objects.equals(tableLocation, that.tableLocation) && + Objects.equals(snapshotId, that.snapshotId) && + Objects.equals(columns, that.columns); + } + + @Override + public int hashCode() + { + return Objects.hash(schemaName, tableName, tableLocation, snapshotId, columns); + } + + @Override + public String toString() + { + final StringBuilder sb = new StringBuilder("DeltaTableHandle{"); + sb.append("table='").append(schemaName + "." + tableName).append('\''); + sb.append(", location=").append(tableLocation); + sb.append(", snapshotId=").append(snapshotId); + sb.append(", columns=").append(columns); + sb.append('}'); + return sb.toString(); + } +} diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaTableHandle.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaTableHandle.java new file mode 100644 index 000000000000..683571d212de --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaTableHandle.java @@ -0,0 +1,120 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.prestosql.spi.connector.ConnectorTableHandle; +import io.prestosql.spi.connector.SchemaTableName; +import io.prestosql.spi.predicate.TupleDomain; + +import java.util.Objects; +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + +public final class DeltaTableHandle + implements ConnectorTableHandle +{ + private final String connectorId; + private final DeltaTable deltaTable; + private final TupleDomain predicate; + private final Optional predicateString; + + @JsonCreator + public DeltaTableHandle( + @JsonProperty("connectorId") String connectorId, + @JsonProperty("deltaTable") DeltaTable deltaTable, + @JsonProperty("predicate") TupleDomain predicate, + @JsonProperty("predicateString") Optional predicateString) + { + this.connectorId = requireNonNull(connectorId, "connectorId is null"); + this.deltaTable = requireNonNull(deltaTable, "deltaTable is null"); + this.predicate = requireNonNull(predicate, "predicate is null"); + this.predicateString = requireNonNull(predicateString, "predicateString is null"); + } + + DeltaTableHandle withPredicate(TupleDomain predicate, String predicateString) + { + checkState(this.predicate.isAll(), "There is already a predicate."); + return new DeltaTableHandle( + connectorId, + deltaTable, + predicate, + Optional.of(predicateString)); + } + + @JsonProperty + public String getConnectorId() + { + return connectorId; + } + + @JsonProperty + public DeltaTable getDeltaTable() + { + return deltaTable; + } + + @JsonProperty + public TupleDomain getPredicate() + { + return predicate; + } + + @JsonProperty + public Optional getPredicateString() + { + return predicateString; + } + + public SchemaTableName toSchemaTableName() + { + return new SchemaTableName(deltaTable.getSchemaName(), deltaTable.getTableName()); + } + + @Override + public int hashCode() + { + return Objects.hash(connectorId, deltaTable, predicate); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + + DeltaTableHandle other = (DeltaTableHandle) obj; + return Objects.equals(this.connectorId, other.connectorId) && + Objects.equals(this.deltaTable, other.deltaTable) && + Objects.equals(this.predicate, other.predicate); + } + + @Override + public String toString() + { + final StringBuilder sb = new StringBuilder("DeltaTableHandle{"); + sb.append("connectorId='").append(connectorId).append('\''); + sb.append(", table='").append(toSchemaTableName()).append('\''); + sb.append(", predicate=").append(predicateString); + sb.append('}'); + return sb.toString(); + } +} diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaTableName.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaTableName.java new file mode 100644 index 000000000000..de504ac45709 --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaTableName.java @@ -0,0 +1,143 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import io.prestosql.spi.PrestoException; + +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.format.DateTimeParseException; +import java.time.temporal.ChronoField; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +/** + * Delta table name. Supported table name formats: + *
    + *
  • table name. E.g. SELECT * FROM delta.db.sales
    • + *
  • path to the Delta table location. E.g. SELECT * FROM "s3://bucket/path/delta-table"
    • + *
+ *

+ * Table name can have a suffix to indicate reading a particular snapshot of the table + *

    + *
  • Snapshot version. E.g. SELECT * FROM delta.db."sales@v123" -- to read 123 snapshot version
    • + *
  • Latest snapshot as of given timestamp. E.g. SELECT * FROM delta.db."sales@t2021-11-18 10:30:00" + * -- to read latest snapshot on or before 2021-11-18 10:30:00
    • + *
+ */ +public class DeltaTableName +{ + private static final Pattern TABLE_PATTERN = Pattern.compile( + "(?
[^@]+)" + /* matches table name that doesn't contain `@` character */ + "(?:@v(?[0-9]+))?" + + "(?:@t(?[0-9.\\-: ]+))?"); + + private static final DateTimeFormatter TIMESTAMP_PARSER = + new DateTimeFormatterBuilder().appendPattern("yyyy[-MM[-dd[ HH[:mm[:ss]]]]]") + .parseDefaulting(ChronoField.MONTH_OF_YEAR, 1) + .parseDefaulting(ChronoField.DAY_OF_MONTH, 1) + .parseDefaulting(ChronoField.HOUR_OF_DAY, 0) + .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0) + .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0) + .toFormatter() + .withZone(ZoneId.of("UTC")); + + private final String tableNameOrPath; + private final Optional snapshotId; + private final Optional snapshotAsOfTimestamp; + + public DeltaTableName(String tableNameOrPath, Optional snapshotId, Optional snapshotAsOfTimestamp) + { + this.tableNameOrPath = requireNonNull(tableNameOrPath, "tableNameOrPath is null"); + this.snapshotId = requireNonNull(snapshotId, "snapshotId is null"); + this.snapshotAsOfTimestamp = requireNonNull(snapshotAsOfTimestamp, "snapshotAsOfTimestamp is null"); + } + + public String getTableNameOrPath() + { + return tableNameOrPath; + } + + public Optional getSnapshotId() + { + return snapshotId; + } + + public Optional getSnapshotAsOfTimestamp() + { + return snapshotAsOfTimestamp; + } + + @Override + public String toString() + { + StringBuilder builder = new StringBuilder() + .append("Table[" + tableNameOrPath + "]"); + snapshotId.map(id -> builder.append("@v" + id)); + snapshotAsOfTimestamp.map(ts -> builder.append("@t" + new Timestamp(ts))); + return builder.toString(); + } + + public static DeltaTableName from(String tableName) + { + Matcher match = TABLE_PATTERN.matcher(tableName); + if (!match.matches()) { + throw new PrestoException(NOT_SUPPORTED, "Invalid Delta table name: " + tableName + + ", Expected table name form 'tableName[@v][@t]'. " + + "The table can have either a particular snapshot identifier or a timestamp of the snapshot. " + + "If timestamp is given the latest snapshot of the table that was generated at or " + + "before the given timestamp is read"); + } + + String tableNameOrPath = match.group("table"); + String snapshotValue = match.group("snapshotId"); + Optional snapshot = Optional.empty(); + if (snapshotValue != null) { + snapshot = Optional.of(Long.parseLong(snapshotValue)); + } + + Optional timestampMillisUtc = Optional.empty(); + String timestampValue = match.group("timestamp"); + if (timestampValue != null) { + try { + timestampMillisUtc = Optional.of( + LocalDateTime.from(TIMESTAMP_PARSER.parse(timestampValue)) + .toInstant(ZoneOffset.UTC) + .toEpochMilli()); + } + catch (IllegalArgumentException | DateTimeParseException ex) { + throw new PrestoException(NOT_SUPPORTED, + format("Invalid Delta table name: %s, given snapshot timestamp (%s) format is not valid. " + + "Expected timestamp format 'YYYY-MM-DD HH:mm:ss'", tableName, timestampValue)); + } + } + + if (snapshot.isPresent() && timestampMillisUtc.isPresent()) { + throw new PrestoException(NOT_SUPPORTED, "Invalid Delta table name: " + tableName + + ", Table suffix contains both snapshot id and timestamp of snapshot to read. " + + "Only one of them is supported."); + } + + return new DeltaTableName(tableNameOrPath, snapshot, timestampMillisUtc); + } +} diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaTransactionHandle.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaTransactionHandle.java new file mode 100644 index 000000000000..a90a940c1032 --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaTransactionHandle.java @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import io.prestosql.spi.connector.ConnectorTransactionHandle; + +public enum DeltaTransactionHandle + implements ConnectorTransactionHandle +{ + INSTANCE +} diff --git a/presto-delta/src/main/java/io/prestosql/delta/DeltaTypeUtils.java b/presto-delta/src/main/java/io/prestosql/delta/DeltaTypeUtils.java new file mode 100644 index 000000000000..85b3bac346fc --- /dev/null +++ b/presto-delta/src/main/java/io/prestosql/delta/DeltaTypeUtils.java @@ -0,0 +1,229 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import com.google.common.collect.ImmutableList; +import io.delta.standalone.types.ArrayType; +import io.delta.standalone.types.BinaryType; +import io.delta.standalone.types.BooleanType; +import io.delta.standalone.types.ByteType; +import io.delta.standalone.types.DataType; +import io.delta.standalone.types.DateType; +import io.delta.standalone.types.DecimalType; +import io.delta.standalone.types.DoubleType; +import io.delta.standalone.types.FloatType; +import io.delta.standalone.types.IntegerType; +import io.delta.standalone.types.LongType; +import io.delta.standalone.types.MapType; +import io.delta.standalone.types.ShortType; +import io.delta.standalone.types.StringType; +import io.delta.standalone.types.StructType; +import io.delta.standalone.types.TimestampType; +import io.prestosql.spi.PrestoException; +import io.prestosql.spi.connector.SchemaTableName; +import io.prestosql.spi.type.Decimals; +import io.prestosql.spi.type.NamedTypeSignature; +import io.prestosql.spi.type.RowFieldName; +import io.prestosql.spi.type.StandardTypes; +import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.TypeSignature; +import io.prestosql.spi.type.TypeSignatureParameter; + +import java.sql.Timestamp; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.Arrays; +import java.util.Locale; +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkArgument; +import static io.airlift.slice.Slices.utf8Slice; +import static io.prestosql.delta.DeltaErrorCode.DELTA_INVALID_PARTITION_VALUE; +import static io.prestosql.delta.DeltaErrorCode.DELTA_UNSUPPORTED_COLUMN_TYPE; +import static io.prestosql.spi.type.BigintType.BIGINT; +import static io.prestosql.spi.type.BooleanType.BOOLEAN; +import static io.prestosql.spi.type.DateType.DATE; +import static io.prestosql.spi.type.DecimalType.createDecimalType; +import static io.prestosql.spi.type.DoubleType.DOUBLE; +import static io.prestosql.spi.type.IntegerType.INTEGER; +import static io.prestosql.spi.type.RealType.REAL; +import static io.prestosql.spi.type.SmallintType.SMALLINT; +import static io.prestosql.spi.type.StandardTypes.ARRAY; +import static io.prestosql.spi.type.StandardTypes.MAP; +import static io.prestosql.spi.type.StandardTypes.ROW; +import static io.prestosql.spi.type.TimestampType.TIMESTAMP; +import static io.prestosql.spi.type.TinyintType.TINYINT; +import static io.prestosql.spi.type.VarbinaryType.VARBINARY; +import static io.prestosql.spi.type.VarcharType.createUnboundedVarcharType; +import static java.lang.Double.parseDouble; +import static java.lang.Float.floatToRawIntBits; +import static java.lang.Float.parseFloat; +import static java.lang.Long.parseLong; +import static java.lang.String.format; + +/** + * Contains utility methods to convert Delta data types (and data values) to Presto data types (and data values) + */ +public class DeltaTypeUtils +{ + private DeltaTypeUtils() + { + } + + /** + * Convert given Delta data type to Presto data type signature. + * + * @param tableName Used in error messages when an unsupported data type is encountered. + * @param columnName Used in error messages when an unsupported data type is encountered. + * @param deltaType Data type to convert + * @return + */ + public static TypeSignature convertDeltaDataTypePrestoDataType(SchemaTableName tableName, String columnName, DataType deltaType) + { + checkArgument(deltaType != null); + + if (deltaType instanceof StructType) { + StructType deltaStructType = (StructType) deltaType; + ImmutableList.Builder typeSignatureBuilder = ImmutableList.builder(); + Arrays.stream(deltaStructType.getFields()) + .forEach(field -> { + String rowFieldName = field.getName().toLowerCase(Locale.US); + TypeSignature rowFieldType = convertDeltaDataTypePrestoDataType( + tableName, + columnName + "." + field.getName(), + field.getDataType()); + typeSignatureBuilder.add( + TypeSignatureParameter.namedTypeParameter( + new NamedTypeSignature( + Optional.of(new RowFieldName(rowFieldName)), + rowFieldType))); + }); + return new TypeSignature(ROW, typeSignatureBuilder.build()); + } + else if (deltaType instanceof ArrayType) { + ArrayType deltaArrayType = (ArrayType) deltaType; + TypeSignature elementType = convertDeltaDataTypePrestoDataType( + tableName, + columnName, + deltaArrayType.getElementType()); + return new TypeSignature( + ARRAY, + ImmutableList.of(TypeSignatureParameter.typeParameter(elementType))); + } + else if (deltaType instanceof MapType) { + MapType deltaMapType = (MapType) deltaType; + TypeSignature keyType = convertDeltaDataTypePrestoDataType(tableName, columnName, deltaMapType.getKeyType()); + TypeSignature valueType = convertDeltaDataTypePrestoDataType(tableName, columnName, deltaMapType.getValueType()); + return new TypeSignature(MAP, ImmutableList.of( + TypeSignatureParameter.typeParameter(keyType), + TypeSignatureParameter.typeParameter(valueType))); + } + + return convertDeltaPrimitiveTypeToPrestoPrimitiveType(tableName, columnName, deltaType).getTypeSignature(); + } + + public static Object convertPartitionValue( + String partitionColumnName, + String partitionValue, + TypeSignature partitionDataType) + { + if (partitionValue == null) { + return null; + } + + String typeBase = partitionDataType.getBase(); + try { + switch (typeBase) { + case StandardTypes.TINYINT: + case StandardTypes.SMALLINT: + case StandardTypes.INTEGER: + case StandardTypes.BIGINT: + return parseLong(partitionValue); + case StandardTypes.REAL: + return (long) floatToRawIntBits(parseFloat(partitionValue)); + case StandardTypes.DOUBLE: + return parseDouble(partitionValue); + case StandardTypes.VARCHAR: + case StandardTypes.VARBINARY: + return utf8Slice(partitionValue); + case StandardTypes.DATE: + return LocalDate.parse(partitionValue, DateTimeFormatter.ISO_LOCAL_DATE).toEpochDay(); + case StandardTypes.TIMESTAMP: + return Timestamp.valueOf(partitionValue).toLocalDateTime().toEpochSecond(ZoneOffset.UTC) * 1000; + case StandardTypes.BOOLEAN: + checkArgument(partitionValue.equalsIgnoreCase("true") || partitionValue.equalsIgnoreCase("false")); + return Boolean.valueOf(partitionValue); + case StandardTypes.DECIMAL: + return Decimals.parse(partitionValue).getObject(); + default: + throw new PrestoException(DELTA_UNSUPPORTED_COLUMN_TYPE, + format("Unsupported data type '%s' for partition column %s", partitionDataType, partitionColumnName)); + } + } + catch (IllegalArgumentException | DateTimeParseException e) { + throw new PrestoException(DELTA_INVALID_PARTITION_VALUE, + format("Can not parse partition value '%s' of type '%s' for partition column '%s'", + partitionValue, partitionDataType, partitionColumnName)); + } + } + + private static Type convertDeltaPrimitiveTypeToPrestoPrimitiveType(SchemaTableName tableName, String columnName, DataType deltaType) + { + if (deltaType instanceof BinaryType) { + return VARBINARY; + } + else if (deltaType instanceof BooleanType) { + return BOOLEAN; + } + else if (deltaType instanceof ByteType) { + return TINYINT; + } + else if (deltaType instanceof DateType) { + return DATE; + } + else if (deltaType instanceof DecimalType) { + DecimalType decimalType = (DecimalType) deltaType; + return createDecimalType(decimalType.getPrecision(), decimalType.getScale()); + } + else if (deltaType instanceof DoubleType) { + return DOUBLE; + } + else if (deltaType instanceof FloatType) { + return REAL; + } + else if (deltaType instanceof IntegerType) { + return INTEGER; + } + else if (deltaType instanceof LongType) { + return BIGINT; + } + else if (deltaType instanceof ShortType) { + return SMALLINT; + } + else if (deltaType instanceof StringType) { + return createUnboundedVarcharType(); + } + else if (deltaType instanceof TimestampType) { + return TIMESTAMP; + } + + throw new PrestoException(DELTA_UNSUPPORTED_COLUMN_TYPE, + format("Column '%s' in Delta table %s contains unsupported data type: %s", + columnName, + tableName, + deltaType.getCatalogString())); + } +} diff --git a/presto-delta/src/test/java/io/prestosql/delta/AbstractDeltaDistributedQueryTestBase.java b/presto-delta/src/test/java/io/prestosql/delta/AbstractDeltaDistributedQueryTestBase.java new file mode 100644 index 000000000000..4888084970fe --- /dev/null +++ b/presto-delta/src/test/java/io/prestosql/delta/AbstractDeltaDistributedQueryTestBase.java @@ -0,0 +1,155 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import com.google.common.collect.ImmutableMap; +import io.prestosql.Session; +import io.prestosql.plugin.hive.HiveHadoop2Plugin; +import io.prestosql.plugin.tpch.TpchPlugin; +import io.prestosql.spi.type.TimeZoneKey; +import io.prestosql.testing.AbstractTestQueryFramework; +import io.prestosql.testing.DistributedQueryRunner; +import io.prestosql.testing.QueryRunner; + +import java.nio.file.Path; +import java.util.Locale; +import java.util.Map; + +import static io.prestosql.testing.TestingSession.testSessionBuilder; +import static java.lang.String.format; + +public abstract class AbstractDeltaDistributedQueryTestBase + extends AbstractTestQueryFramework +{ + public static final String DELTA_CATALOG = "delta"; + public static final String HIVE_CATALOG = "hive"; + public static final String PATH_SCHEMA = "$path$"; + public static final String DELTA_SCHEMA = "deltaTables"; // Schema in Hive which has test Delta tables + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + QueryRunner queryRunner = createDeltaQueryRunner(ImmutableMap.of()); + return queryRunner; + } + + /** + * Helper method to + * - registering Delta table given location as a Hive table and + * - run the test query, expected results query and compare the results. + */ + protected void assertDeltaQuery(String deltaTable, String testQuery, String expResultsQuery) + { + try { + registerDeltaTableInHMS(deltaTable, deltaTable); + assertQuery(testQuery, expResultsQuery); + } + finally { + unregisterDeltaTableInHMS(deltaTable); + } + } + + /** + * Helper method to + * - registering Delta table given location as a Hive table and + * - run the test query, expect the query to fail and failure contains the expected message + */ + protected void assertDeltaQueryFails(String deltaTable, String testQuery, String expFailure) + { + try { + registerDeltaTableInHMS(deltaTable, deltaTable); + assertQueryFails(testQuery, expFailure); + } + finally { + unregisterDeltaTableInHMS(deltaTable); + } + } + + protected static String goldenTablePath(String tableName) + { + return AbstractDeltaDistributedQueryTestBase.class.getClassLoader().getResource(tableName).toString(); + } + + private static DistributedQueryRunner createDeltaQueryRunner(Map extraProperties) + throws Exception + { + Session session = testSessionBuilder() + .setCatalog(DELTA_CATALOG) + .setSchema(DELTA_SCHEMA.toLowerCase(Locale.ROOT)) + .setTimeZoneKey(TimeZoneKey.UTC_KEY) + .build(); + + DistributedQueryRunner queryRunner = DistributedQueryRunner.builder(session) + .setExtraProperties(extraProperties) + .build(); + + // Install the TPCH plugin for test data (not in Delta format) + queryRunner.installPlugin(new TpchPlugin()); + queryRunner.createCatalog("tpch", "tpch"); + + Path dataDir = queryRunner.getCoordinator().getBaseDataDir().resolve("delta_metadata"); + Path catalogDir = dataDir.getParent().resolve("catalog"); + + // Install a Delta connector catalog + queryRunner.installPlugin(new DeltaPlugin()); + Map deltaProperties = ImmutableMap.builder() + .put("hive.metastore", "file") + .put("hive.metastore.catalog.dir", catalogDir.toFile().toURI().toString()) + .build(); + queryRunner.createCatalog(DELTA_CATALOG, "delta", deltaProperties); + + // Install a Hive connector catalog that uses the same metastore as Delta + // This catalog will be used to create tables in metastore as the Delta connector doesn't + // support creating tables yet. + queryRunner.installPlugin(new HiveHadoop2Plugin()); + Map hiveProperties = ImmutableMap.builder() + .put("hive.metastore", "file") + .put("hive.metastore.catalog.dir", catalogDir.toFile().toURI().toString()) + .put("hive.allow-drop-table", "true") + .put("hive.security", "legacy") + .build(); + queryRunner.createCatalog(HIVE_CATALOG, "hive-hadoop2", hiveProperties); + queryRunner.execute(format("CREATE SCHEMA %s.%s", HIVE_CATALOG, DELTA_SCHEMA)); + + return queryRunner; + } + + /** + * Register the given deltaTableName as hiveTableName in HMS using the Hive storage catalog. + * Hive and Delta catalogs share the same HMS in this test. Hive is used to register the tables as Delta + * connector doesn't have the write support yet. + * + * @param deltaTableName Name of the delta table which is on the classpath. + * @param hiveTableName Name of the Hive table that the Delta table is to be registered as in HMS + */ + protected void registerDeltaTableInHMS(String deltaTableName, String hiveTableName) + { + getQueryRunner().execute(format( + "CREATE TABLE %s.\"%s\".\"%s\" (dummyColumn INT) WITH (external_location = '%s')", + HIVE_CATALOG, + DELTA_SCHEMA, + hiveTableName, + goldenTablePath(deltaTableName))); + } + + /** + * Drop the given table from HMS + */ + protected void unregisterDeltaTableInHMS(String hiveTableName) + { + getQueryRunner().execute( + format("DROP TABLE IF EXISTS %s.\"%s\".\"%s\"", HIVE_CATALOG, DELTA_SCHEMA, hiveTableName)); + } +} diff --git a/presto-delta/src/test/java/io/prestosql/delta/TestDeltaColumnHandle.java b/presto-delta/src/test/java/io/prestosql/delta/TestDeltaColumnHandle.java new file mode 100644 index 000000000000..bf4a0ad8e49c --- /dev/null +++ b/presto-delta/src/test/java/io/prestosql/delta/TestDeltaColumnHandle.java @@ -0,0 +1,74 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import com.google.common.collect.ImmutableMap; +import io.airlift.json.JsonCodec; +import io.airlift.json.JsonCodecFactory; +import io.airlift.json.ObjectMapperProvider; +import io.prestosql.spi.type.TypeSignature; +import io.prestosql.type.TypeSignatureDeserializer; +import org.testng.annotations.Test; + +import static io.prestosql.delta.DeltaColumnHandle.ColumnType.PARTITION; +import static io.prestosql.delta.DeltaColumnHandle.ColumnType.REGULAR; +import static io.prestosql.spi.type.DoubleType.DOUBLE; +import static org.testng.Assert.assertEquals; + +/** + * Test {@link DeltaColumnHandle} is created with correct parameters and round trip JSON SerDe works. + */ +public class TestDeltaColumnHandle +{ + private final JsonCodec codec = + JsonCodec.jsonCodec(DeltaColumnHandle.class); + + @Test + public void testRegularColumn() + { + DeltaColumnHandle expectedPartitionColumn = new DeltaColumnHandle( + "name", + "partCol", + DOUBLE.getTypeSignature(), + PARTITION); + testRoundTrip(expectedPartitionColumn); + } + + @Test + public void testPartitionKeyColumn() + { + DeltaColumnHandle expectedRegularColumn = new DeltaColumnHandle( + "name", + "regularCol", + DOUBLE.getTypeSignature(), + REGULAR); + testRoundTrip(expectedRegularColumn); + } + + private void testRoundTrip(DeltaColumnHandle expected) + { + ObjectMapperProvider provider = new ObjectMapperProvider(); + provider.setJsonDeserializers(ImmutableMap.of( + TypeSignature.class, new TypeSignatureDeserializer())); + JsonCodec codec = new JsonCodecFactory(provider) + .jsonCodec(DeltaColumnHandle.class); + String json = codec.toJson(expected); + DeltaColumnHandle actual = codec.fromJson(json); + + assertEquals(actual.getConnectorId(), expected.getConnectorId()); + assertEquals(actual.getName(), expected.getName()); + assertEquals(actual.getColumnType(), expected.getColumnType()); + assertEquals(actual.getDataType(), expected.getDataType()); + } +} diff --git a/presto-delta/src/test/java/io/prestosql/delta/TestDeltaConfig.java b/presto-delta/src/test/java/io/prestosql/delta/TestDeltaConfig.java new file mode 100644 index 000000000000..b1903922c7f0 --- /dev/null +++ b/presto-delta/src/test/java/io/prestosql/delta/TestDeltaConfig.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import com.google.common.collect.ImmutableMap; +import io.airlift.configuration.testing.ConfigAssertions; +import org.testng.annotations.Test; + +import java.util.Map; + +public class TestDeltaConfig +{ + @Test + public void testDefaults() + { + ConfigAssertions.assertRecordedDefaults(ConfigAssertions.recordDefaults(DeltaConfig.class) + .setPartitionPruningEnabled(true) + .setFilterPushdownEnabled(true) + .setMaxSplitsBatchSize(200) + .setProjectionPushdownEnabled(true)); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("delta.partition-pruning-enabled", "false") + .put("delta.filter-pushdown-enabled", "false") + .put("delta.max-splits-batch-size", "400") + .put("delta.projection-pushdown-enabled", "false") + .build(); + + DeltaConfig expected = new DeltaConfig() + .setFilterPushdownEnabled(false) + .setPartitionPruningEnabled(false) + .setMaxSplitsBatchSize(400) + .setProjectionPushdownEnabled(false); + + ConfigAssertions.assertFullMapping(properties, expected); + } +} diff --git a/presto-delta/src/test/java/io/prestosql/delta/TestDeltaIntegration.java b/presto-delta/src/test/java/io/prestosql/delta/TestDeltaIntegration.java new file mode 100644 index 000000000000..67ae1cbe1840 --- /dev/null +++ b/presto-delta/src/test/java/io/prestosql/delta/TestDeltaIntegration.java @@ -0,0 +1,239 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import com.google.common.base.Joiner; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.List; + +import static java.lang.String.format; + +/** + * Integration tests for reading Delta tables. + */ +public class TestDeltaIntegration + extends AbstractDeltaDistributedQueryTestBase +{ + @Test + public void readPrimitiveTypeData() + { + // Test reading following primitive types from a Delta table (all ints, float, double, decimal, boolean, varchar, varbinary) + String testQuery = + format("SELECT * FROM \"%s\".\"%s\"", PATH_SCHEMA, goldenTablePath("data-reader-primitives")); + String expResultsQuery = getPrimitiveTypeTableData(); + assertQuery(testQuery, expResultsQuery); + } + + @Test + public void readArrayTypeData() + { + // Test reading following array elements with type (all ints, float, double, decimal, boolean, varchar, varbinary) + String testQuery = + format("SELECT * FROM \"%s\".\"%s\"", PATH_SCHEMA, goldenTablePath("data-reader-array-primitives")); + + // Create query for the expected results. + List expRows = new ArrayList<>(); + for (byte i = 0; i < 10; i++) { + expRows.add(format("SELECT " + + " array[cast(%s as integer)]," + + " array[cast(%s as bigint)]," + + " array[cast(%s as tinyint)]," + + " array[cast(%s as smallint)]," + + " array[%s]," + + " array[cast(%s as real)]," + + " array[cast(%s as double)], " + + " array['%s'], " + + " array[cast(X'0%s0%s' as varbinary)], " + + " array[cast(%s as decimal)]", i, i, i, i, (i % 2 == 0 ? "true" : "false"), i, i, i, i, i, i)); + } + String expResultsQuery = Joiner.on(" UNION ").join(expRows); + + assertQuery(testQuery, expResultsQuery); + } + + @Test + public void readMapTypeData() + { + // Test reading MAP data type columns from Delta table + String testQuery = + format("SELECT map_keys(a), map_values(e) FROM \"%s\".\"%s\"", PATH_SCHEMA, goldenTablePath("data-reader-map")); + + List expRows = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + expRows.add("SELECT " + + " ARRAY[cast(" + i + " as integer)]," + + " ARRAY[cast(" + i + " as decimal)]"); + } + String expResultsQuery = Joiner.on(" UNION ").join(expRows); + + assertQuery(testQuery, expResultsQuery); + } + + @Test + void readTableRegisteredInHMS() + { + String deltaTable = "data-reader-primitives"; + String testQuery = format("SELECT * FROM \"%s\"", deltaTable); + String expResultsQuery = getPrimitiveTypeTableData(); + assertDeltaQuery(deltaTable, testQuery, expResultsQuery); + } + + @Test + void readSpecificSnapshotVersion() + { + String deltaTable = "snapshot-data3"; + String testQueryTemplate = "SELECT * FROM \"%s@%s\" WHERE col1 = 0"; + + // read snapshot version 2 + String testQueryV2 = format(testQueryTemplate, deltaTable, "v2"); + String expResultsQueryV2 = "SELECT * FROM VALUES(0, 'data-2-0')"; + assertDeltaQuery(deltaTable, testQueryV2, expResultsQueryV2); + + // read snapshot version 3 + String testQueryV3 = format(testQueryTemplate, deltaTable, "v3"); + String expResultsQueryV3 = "SELECT * FROM VALUES(0, 'data-2-0'), (0, 'data-3-0')"; + assertDeltaQuery(deltaTable, testQueryV3, expResultsQueryV3); + + // All snapshots are created at 2020-10-26 02:52:47.515 + // read snapshot as of 2020-10-26 02:50:00 - this should fail as there are no snapshots before this timestamp + String testQueryTs1 = format(testQueryTemplate, deltaTable, "t2020-10-27 02:50:00"); + assertDeltaQueryFails( + deltaTable, + testQueryTs1, + "There is no snapshot exists in Delta table 'deltatables.snapshot-data3@t2020-10-27 02:50:00' " + + "that is created on or before '2020-10-27T02:50:00Z'"); + + // All snapshots are created at 2020-10-26 02:52:47.515 + // read snapshot as of 2020-10-26 02:52:48 - this should read the latest data + String testQueryTs2 = format(testQueryTemplate, deltaTable, "t2020-10-26 02:52:48"); + String expResultsQueryTs2 = "SELECT * FROM VALUES(0, 'data-2-0'), (0, 'data-3-0')"; + assertDeltaQuery(deltaTable, testQueryTs2, expResultsQueryTs2); + } + + @Test + void readCheckpointedDeltaTable() + { + // Delta table commits are periodically checkpointed into a parquet file. + // Test Delta connector is able to read the checkpointed commits in a parquet file. + // Test table has commit files (0-10) deleted. So it has to rely on reading the Parquet file + // to fetch the files latest commit (i.e > 10). + String deltaTable = "checkpointed-delta-table"; + String testQueryTemplate = "SELECT * FROM \"%s%s\" WHERE col1 in (0, 10, 15)"; + + // read snapshot version 3 - expect can't time travel error + String testQueryV3 = format(testQueryTemplate, deltaTable, "@v3"); + assertDeltaQueryFails( + deltaTable, + testQueryV3, + "Can not find snapshot \\(3\\) in Delta table 'deltatables.checkpointed-delta-table\\@v3': No reproducible commits found at .*"); + + // read latest data + String testQueryLatest = format(testQueryTemplate, deltaTable, ""); + String expResultsQueryLatest = "SELECT * FROM VALUES(0, 'data-0-0'), (10, 'data-0-10'), (15, 'data-0-15')"; + assertDeltaQuery(deltaTable, testQueryLatest, expResultsQueryLatest); + + // read snapshot version 13 + String testQueryV13 = format(testQueryTemplate, deltaTable, "@v13"); + String expResultsQueryV13 = "SELECT * FROM VALUES(0, 'data-0-0'), (10, 'data-0-10')"; + assertDeltaQuery(deltaTable, testQueryV13, expResultsQueryV13); + } + + @Test + public void readPartitionedTable() + { + String deltaTable = "time-travel-partition-changes-b"; + String testQuery1 = format("SELECT * FROM \"%s\" WHERE id in (10, 15, 12, 13)", deltaTable); + String expResultsQuery1 = "SELECT * FROM VALUES(10, 0),(15, 1),(12, 0),(13, 1)"; + assertDeltaQuery(deltaTable, testQuery1, expResultsQuery1); + + // reorder the columns in output and query the partitioned table + String testQuery2 = format("SELECT part2, id FROM \"%s\" WHERE id in (16, 14, 19)", deltaTable); + String expResultsQuery2 = "SELECT * FROM VALUES(0, 16),(0, 14),(1, 19)"; + assertDeltaQuery(deltaTable, testQuery2, expResultsQuery2); + } + + @Test + public void readPartitionedTableAllDataTypes() + { + String deltaTable = "data-reader-partition-values"; + String testQuery = format("SELECT * FROM \"%s\"", deltaTable); + String expResultsQuery = "SELECT * FROM VALUES" + + "( 0," + + " cast(0 as bigint)," + + " cast(0 as smallint), " + + " cast(0 as tinyint), " + + " true, " + + " 0.0, " + + " cast(0.0 as double), " + + " '0', " + + " DATE '2021-09-08', " + + " TIMESTAMP '2021-09-08 11:11:11', " + + " cast(0 as decimal)," + + " '0'" + // regular column + "), " + + "( 1," + + " cast(1 as bigint)," + + " cast(1 as smallint), " + + " cast(1 as tinyint), " + + " false, " + + " 1.0, " + + " cast(1.0 as double), " + + " '1', " + + " DATE '2021-09-08', " + + " TIMESTAMP '2021-09-08 11:11:11', " + + " cast(1 as decimal), " + + " '1'" + // regular column + "), " + + "( null," + + " null," + + " null, " + + " null, " + + " null, " + + " null, " + + " null, " + + " null, " + + " null, " + + " null, " + + " null, " + + " '2'" + // regular column + ")"; + assertDeltaQuery(deltaTable, testQuery, expResultsQuery); + } + + /** + * Expected results for table "data-reader-primitives" + */ + private static String getPrimitiveTypeTableData() + { + // Create query for the expected results. + List expRows = new ArrayList<>(); + for (byte i = 0; i < 10; i++) { + expRows.add(format("SELECT " + + " cast(%s as integer)," + + " cast(%s as bigint)," + + " cast(%s as tinyint)," + + " cast(%s as smallint)," + + " %s," + + " cast(%s as real)," + + " cast(%s as double), " + + " '%s', " + + " cast(X'0%s0%s' as varbinary), " + + " cast(%s as decimal)", i, i, i, i, (i % 2 == 0 ? "true" : "false"), i, i, i, i, i, i)); + } + expRows.add("SELECT null, null, null, null, null, null, null, null, null, null"); + return Joiner.on(" UNION ").join(expRows); + } +} diff --git a/presto-delta/src/test/java/io/prestosql/delta/TestDeltaOptimizationsTests.java b/presto-delta/src/test/java/io/prestosql/delta/TestDeltaOptimizationsTests.java new file mode 100644 index 000000000000..1af4d01b2189 --- /dev/null +++ b/presto-delta/src/test/java/io/prestosql/delta/TestDeltaOptimizationsTests.java @@ -0,0 +1,254 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.prestosql.Session; +import io.prestosql.cost.PlanNodeStatsEstimate; +import io.prestosql.cost.StatsProvider; +import io.prestosql.execution.warnings.WarningCollector; +import io.prestosql.metadata.Metadata; +import io.prestosql.spi.predicate.Domain; +import io.prestosql.spi.predicate.Range; +import io.prestosql.spi.predicate.SortedRangeSet; +import io.prestosql.sql.planner.Plan; +import io.prestosql.sql.planner.assertions.MatchResult; +import io.prestosql.sql.planner.assertions.Matcher; +import io.prestosql.sql.planner.assertions.PlanMatchPattern; +import io.prestosql.sql.planner.assertions.SymbolAliases; +import io.prestosql.sql.planner.plan.PlanNode; +import io.prestosql.sql.planner.plan.TableScanNode; +import org.testng.annotations.Test; + +import java.util.Collections; +import java.util.Map; + +import static io.airlift.slice.Slices.utf8Slice; +import static io.prestosql.spi.predicate.Domain.multipleValues; +import static io.prestosql.spi.predicate.Domain.notNull; +import static io.prestosql.spi.predicate.Domain.onlyNull; +import static io.prestosql.spi.predicate.Domain.singleValue; +import static io.prestosql.spi.type.IntegerType.INTEGER; +import static io.prestosql.spi.type.VarcharType.VARCHAR; +import static io.prestosql.sql.planner.assertions.MatchResult.NO_MATCH; +import static io.prestosql.sql.planner.assertions.MatchResult.match; +import static io.prestosql.sql.planner.assertions.PlanAssert.assertPlan; +import static io.prestosql.sql.planner.assertions.PlanMatchPattern.anyTree; +import static io.prestosql.transaction.TransactionBuilder.transaction; +import static java.lang.String.format; + +/** + * Integrations tests for various optimization (such as filter pushdown, nested column project/filter pushdown etc) + * that speed up reading data from Delta tables. + */ +public class TestDeltaOptimizationsTests + extends AbstractDeltaDistributedQueryTestBase +{ + @Test + public void filterOnRegularColumn() + { + String tableName = "data-reader-primitives"; + String testQuery = format("SELECT as_int, as_string FROM \"%s\" WHERE as_int = 1", tableName); + String expResultsQuery = "SELECT 1, cast('1' as varchar)"; + + assertDeltaQueryOptimized( + tableName, + testQuery, + expResultsQuery, + ImmutableMap.of("as_int", singleValue(INTEGER, 1L)), + Collections.emptyMap()); + } + + @Test + public void filterOnPartitionColumn() + { + String tableName = "deltatbl-partition-prune"; + String testQuery = format("SELECT date, name, city, cnt FROM \"%s\" WHERE city in ('sh', 'sz')", tableName); + String expResultsQuery = "SELECT * FROM VALUES('20180512', 'Jay', 'sh', 4),('20181212', 'Linda', 'sz', 8)"; + + assertDeltaQueryOptimized( + tableName, + testQuery, + expResultsQuery, + ImmutableMap.of("city", multipleValues(VARCHAR, ImmutableList.of(utf8Slice("sh"), utf8Slice("sz")))), + ImmutableMap.of("city", multipleValues(VARCHAR, ImmutableList.of(utf8Slice("sh"), utf8Slice("sz"))))); + } + + @Test + public void filterOnMultiplePartitionColumns() + { + String tableName = "deltatbl-partition-prune"; + String testQuery = + format("SELECT date, name, city, cnt FROM \"%s\" WHERE city in ('sh', 'sz') AND \"date\" = '20180512'", tableName); + String expResultsQuery = "SELECT * FROM VALUES('20180512', 'Jay', 'sh', 4)"; + + assertDeltaQueryOptimized( + tableName, + testQuery, + expResultsQuery, + ImmutableMap.of( + "city", multipleValues(VARCHAR, ImmutableList.of(utf8Slice("sh"), utf8Slice("sz"))), + "date", singleValue(VARCHAR, utf8Slice("20180512"))), + ImmutableMap.of( + "city", multipleValues(VARCHAR, ImmutableList.of(utf8Slice("sh"), utf8Slice("sz"))), + "date", singleValue(VARCHAR, utf8Slice("20180512")))); + } + + @Test + public void filterOnPartitionColumnAndRegularColumns() + { + String tableName = "deltatbl-partition-prune"; + String testQuery = format("SELECT date, name, city, cnt FROM \"%s\" WHERE city in ('sh', 'sz') AND name = 'Linda'", tableName); + String expResultsQuery = "SELECT * FROM VALUES('20181212', 'Linda', 'sz', 8)"; + + assertDeltaQueryOptimized( + tableName, + testQuery, + expResultsQuery, + ImmutableMap.of( + "city", multipleValues(VARCHAR, ImmutableList.of(utf8Slice("sh"), utf8Slice("sz"))), + "name", singleValue(VARCHAR, utf8Slice("Linda"))), + ImmutableMap.of("city", multipleValues(VARCHAR, ImmutableList.of(utf8Slice("sh"), utf8Slice("sz"))))); + } + + @Test + public void nullPartitionFilter() + { + String tableName = "data-reader-partition-values"; + String testQuery = + format("SELECT value, as_boolean FROM \"%s\" WHERE as_int is null and value is not null", tableName); + String expResultsQuery = "SELECT * FROM VALUES('2', null)"; + + assertDeltaQueryOptimized( + tableName, + testQuery, + expResultsQuery, + ImmutableMap.of( + "as_int", onlyNull(INTEGER), + "value", notNull(VARCHAR)), + ImmutableMap.of("as_int", onlyNull(INTEGER))); + } + + @Test + public void notNullPartitionFilter() + { + String tableName = "data-reader-partition-values"; + String testQuery = format("SELECT value, as_boolean FROM \"%s\" WHERE as_int is not null and value = '1'", tableName); + String expResultsQuery = "SELECT * FROM VALUES('1', false)"; + + assertDeltaQueryOptimized( + tableName, + testQuery, + expResultsQuery, + ImmutableMap.of( + "as_int", notNull(INTEGER), + "value", singleValue(VARCHAR, utf8Slice("1"))), + ImmutableMap.of("as_int", notNull(INTEGER))); + } + + @Test + public void nestedColumnFilter() + { + String tableName = "data-reader-nested-struct"; + String testQuery = format("SELECT a.aa, a.ac.aca FROM \"%s\" WHERE a.aa in ('8', '9') AND a.ac.aca > 6", tableName); + String expResultsQuery = "SELECT * FROM VALUES('8', 8),('9', 9)"; + + assertDeltaQueryOptimized( + tableName, + testQuery, + expResultsQuery, + ImmutableMap.of( + "a$_$_$aa", multipleValues(VARCHAR, ImmutableList.of(utf8Slice("8"), utf8Slice("9"))), + "a$_$_$ac$_$_$aca", Domain.create( + SortedRangeSet.copyOf( + INTEGER, + ImmutableList.of(Range.greaterThan(INTEGER, 6L))), + false)), + ImmutableMap.of()); + } + + protected void assertDeltaQueryOptimized( + String tableName, + String testQuery, + String expResultsQuery, + Map expectedConstraint, + Map expectedEnforcedConstraint) + { + try { + registerDeltaTableInHMS(tableName, tableName); + + // verify the query returns correct results + assertQuery(testQuery, expResultsQuery); + + // verify the plan contains filter pushed down into scan appropriately + transaction(getQueryRunner().getTransactionManager(), getQueryRunner().getAccessControl()) + .execute(getSession(), session -> { + Plan plan = getQueryRunner().createPlan(session, testQuery, WarningCollector.NOOP); + assertPlan( + session, + getQueryRunner().getMetadata(), + (node, sourceStats, lookup, ignore, types) -> PlanNodeStatsEstimate.unknown(), + plan, + anyTree(tableScanWithConstraints( + tableName, + expectedConstraint, + expectedEnforcedConstraint))); + }); + } + finally { + unregisterDeltaTableInHMS(tableName); + } + } + + /** + * Utility plan verification method that checks whether the table scan node has given constraint. + */ + private static PlanMatchPattern tableScanWithConstraints( + String tableName, + Map expectedConstraint, + Map expectedEnforcedConstraint) + { + return PlanMatchPattern.tableScan(tableName).with(new Matcher() + { + @Override + public boolean shapeMatches(PlanNode node) + { + return node instanceof TableScanNode; + } + + @Override + public MatchResult detailMatches(PlanNode node, StatsProvider stats, Session session, Metadata metadata, SymbolAliases symbolAliases) + { + TableScanNode tableScan = (TableScanNode) node; + Map actualEnforcedConstraint = tableScan.getEnforcedConstraint() + .transform(DeltaColumnHandle.class::cast) + .transform(DeltaColumnHandle::getName) + .getDomains() + .get(); + DeltaTableHandle deltaTableHandle = (DeltaTableHandle) tableScan.getTable().getConnectorHandle(); + Map actualConstraint = deltaTableHandle.getPredicate() + .transform(DeltaColumnHandle::getName) + .getDomains() + .get(); + + if (!expectedConstraint.equals(actualConstraint) || !expectedEnforcedConstraint.equals(actualEnforcedConstraint)) { + return NO_MATCH; + } + + return match(); + } + }); + } +} diff --git a/presto-delta/src/test/java/io/prestosql/delta/TestDeltaSplit.java b/presto-delta/src/test/java/io/prestosql/delta/TestDeltaSplit.java new file mode 100644 index 000000000000..f2a7ea85d48c --- /dev/null +++ b/presto-delta/src/test/java/io/prestosql/delta/TestDeltaSplit.java @@ -0,0 +1,56 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.airlift.json.JsonCodec; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; + +/** + * Test {@link DeltaSplit} is created correctly with given arguments and JSON serialization/deserialization works. + */ +public class TestDeltaSplit +{ + private final JsonCodec codec = JsonCodec.jsonCodec(DeltaSplit.class); + + @Test + public void testJsonRoundTrip() + { + DeltaSplit expected = new DeltaSplit( + "delta", + "database", + "table", + "s3://bucket/path/to/delta/table/file1.parquet", + 0, + 200, + 500, + ImmutableMap.of("part1", "part1Val")); + + String json = codec.toJson(expected); + DeltaSplit actual = codec.fromJson(json); + + assertEquals(actual.getConnectorId(), expected.getConnectorId()); + assertEquals(actual.getSchema(), expected.getSchema()); + assertEquals(actual.getTable(), expected.getTable()); + assertEquals(actual.getFilePath(), expected.getFilePath()); + assertEquals(actual.getStart(), expected.getStart()); + assertEquals(actual.getLength(), expected.getLength()); + assertEquals(actual.getFileSize(), expected.getFileSize()); + assertEquals(actual.getAddresses(), ImmutableList.of()); + assertEquals(actual.getPartitionValues(), expected.getPartitionValues()); + } +} diff --git a/presto-delta/src/test/java/io/prestosql/delta/TestDeltaTableHandle.java b/presto-delta/src/test/java/io/prestosql/delta/TestDeltaTableHandle.java new file mode 100644 index 000000000000..67aed9061239 --- /dev/null +++ b/presto-delta/src/test/java/io/prestosql/delta/TestDeltaTableHandle.java @@ -0,0 +1,130 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.Module; +import io.airlift.bootstrap.Bootstrap; +import io.airlift.json.JsonBinder; +import io.airlift.json.JsonCodec; +import io.airlift.json.JsonModule; +import io.prestosql.block.BlockJsonSerde; +import io.prestosql.metadata.HandleJsonModule; +import io.prestosql.metadata.HandleResolver; +import io.prestosql.spi.block.Block; +import io.prestosql.spi.block.TestingBlockEncodingSerde; +import io.prestosql.spi.predicate.Domain; +import io.prestosql.spi.predicate.Range; +import io.prestosql.spi.predicate.SortedRangeSet; +import io.prestosql.spi.predicate.TupleDomain; +import io.prestosql.spi.type.TestingTypeDeserializer; +import io.prestosql.spi.type.TestingTypeManager; +import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.TypeSignature; +import io.prestosql.type.TypeSignatureDeserializer; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.Optional; + +import static io.airlift.json.JsonBinder.jsonBinder; +import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; +import static io.prestosql.delta.DeltaColumnHandle.ColumnType.PARTITION; +import static io.prestosql.delta.DeltaColumnHandle.ColumnType.REGULAR; +import static io.prestosql.spi.type.DateType.DATE; +import static io.prestosql.spi.type.DoubleType.DOUBLE; +import static io.prestosql.spi.type.IntegerType.INTEGER; +import static io.prestosql.spi.type.RealType.REAL; +import static org.testng.Assert.assertEquals; + +/** + * Test {@link DeltaTableHandle} is created correctly with given arguments and JSON serialization/deserialization works. + */ +public class TestDeltaTableHandle +{ + @Test + public void testJsonRoundTrip() + { + List columns = ImmutableList.of( + new DeltaColumn("c1", REAL.getTypeSignature(), true, true), + new DeltaColumn("c2", INTEGER.getTypeSignature(), false, true), + new DeltaColumn("c3", DOUBLE.getTypeSignature(), false, false), + new DeltaColumn("c4", DATE.getTypeSignature(), true, false)); + + DeltaTable deltaTable = new DeltaTable( + "schema", + "table", + "s3:/bucket/table/location", + Optional.of(1L), + columns); + + DeltaColumnHandle c3ColumnHandle = new DeltaColumnHandle( + "delta", + columns.get(2).getName(), + columns.get(2).getType(), + columns.get(2).isPartition() ? PARTITION : REGULAR); + + TupleDomain predicate = TupleDomain.withColumnDomains(ImmutableMap.of( + c3ColumnHandle, Domain.create(SortedRangeSet.copyOf(DOUBLE, + ImmutableList.of( + Range.equal(DOUBLE, (double) (100.0f + 0)), + Range.equal(DOUBLE, (double) (100.008f + 0)), + Range.equal(DOUBLE, (double) (100.0f + 14)))), + false))); + + DeltaTableHandle expected = new DeltaTableHandle( + "delta", + deltaTable, + predicate, + Optional.of("predicateString")); + + String json = getJsonCodec().toJson(expected); + DeltaTableHandle actual = getJsonCodec().fromJson(json); + + assertEquals(actual.getDeltaTable(), expected.getDeltaTable()); + assertEquals(actual.getPredicate(), expected.getPredicate()); + assertEquals(actual.getPredicateString(), expected.getPredicateString()); + } + + private JsonCodec getJsonCodec() + { + Module module = binder -> { + binder.install(new JsonModule()); + binder.install(new HandleJsonModule()); + + TestingBlockEncodingSerde blockEncodingSerde = new TestingBlockEncodingSerde(); + + JsonBinder jsonBinder = jsonBinder(binder); + jsonBinder.addDeserializerBinding(Type.class).toInstance(new TestingTypeDeserializer(new TestingTypeManager())); + jsonBinder.addSerializerBinding(Block.class).toInstance(new BlockJsonSerde.Serializer(blockEncodingSerde)); + jsonBinder.addDeserializerBinding(Block.class).toInstance(new BlockJsonSerde.Deserializer(blockEncodingSerde)); + jsonBinder.addDeserializerBinding(TypeSignature.class).to(TypeSignatureDeserializer.class); + + jsonCodecBinder(binder).bindJsonCodec(DeltaTableHandle.class); + }; + Bootstrap app = new Bootstrap(ImmutableList.of(module)); + Injector injector = app + .doNotInitializeLogging() + .quiet() + .initialize(); + HandleResolver handleResolver = injector.getInstance(HandleResolver.class); + handleResolver.addCatalogHandleResolver("delta", new DeltaConnectionHandleResolver()); + return injector.getInstance(new Key<>() + { + }); + } +} diff --git a/presto-delta/src/test/java/io/prestosql/delta/TestDeltaTableName.java b/presto-delta/src/test/java/io/prestosql/delta/TestDeltaTableName.java new file mode 100644 index 000000000000..12e19e268e02 --- /dev/null +++ b/presto-delta/src/test/java/io/prestosql/delta/TestDeltaTableName.java @@ -0,0 +1,95 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import io.prestosql.spi.PrestoException; +import org.testng.annotations.Test; + +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Optional; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +public class TestDeltaTableName +{ + private static final DateTimeFormatter TIMESTAMP_PARSER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + @Test + public void parsingValidTableNames() + { + assertValid("tbl", "tbl"); + assertValid("tbl@v2", "tbl", 2L); + assertValid("s3://bucket/path/tbl", "s3://bucket/path/tbl"); + assertValid("s3://bucket/path/tbl@v5", "s3://bucket/path/tbl", 5L); + assertValid("tbl@t2021", "tbl", "2021-01-01 00:00:00"); + assertValid("tbl@t2021-11", "tbl", "2021-11-01 00:00:00"); + assertValid("tbl@t2021-11-18", "tbl", "2021-11-18 00:00:00"); + assertValid("tbl@t2021-11-18 10", "tbl", "2021-11-18 10:00:00"); + assertValid("tbl@t2021-11-18 10:30", "tbl", "2021-11-18 10:30:00"); + assertValid("tbl@t2021-11-18 10:30:03", "tbl", "2021-11-18 10:30:03"); + } + + @Test + public void parsingInvalidTableNames() + { + assertInvalid("sales_table@v-1", + "Invalid Delta table name: sales_table@v-1, " + + "Expected table name form 'tableName[@v][@t]'."); + + assertInvalid("sales_table@t2021-14", + "Invalid Delta table name: sales_table@t2021-14, given snapshot timestamp (2021-14) format is not valid"); + } + + private static void assertValid(String inputTableName, String expectedTableName) + { + DeltaTableName deltaTableName = DeltaTableName.from(inputTableName); + assertEquals(deltaTableName.getTableNameOrPath(), expectedTableName); + assertEquals(deltaTableName.getSnapshotId(), Optional.empty()); + assertEquals(deltaTableName.getSnapshotAsOfTimestamp(), Optional.empty()); + } + + private static void assertValid(String inputTableName, String expectedTableName, Long expectedSnapshotId) + { + DeltaTableName deltaTableName = DeltaTableName.from(inputTableName); + assertEquals(deltaTableName.getTableNameOrPath(), expectedTableName); + assertEquals(deltaTableName.getSnapshotId(), Optional.of(expectedSnapshotId)); + assertEquals(deltaTableName.getSnapshotAsOfTimestamp(), Optional.empty()); + } + + private static void assertValid(String inputTableName, String expectedTableName, String expectedTimestamp) + { + DeltaTableName deltaTableName = DeltaTableName.from(inputTableName); + assertEquals(deltaTableName.getTableNameOrPath(), expectedTableName); + assertEquals(deltaTableName.getSnapshotId(), Optional.empty()); + Long expectedTimestampEpochMillis = LocalDateTime.from(TIMESTAMP_PARSER.parse(expectedTimestamp)) + .toInstant(ZoneOffset.UTC) + .toEpochMilli(); + assertEquals(deltaTableName.getSnapshotAsOfTimestamp(), Optional.of(expectedTimestampEpochMillis)); + } + + private static void assertInvalid(String inputTableName, String expErrorMessage) + { + try { + DeltaTableName.from(inputTableName); + fail("expected the above call to fail"); + } + catch (PrestoException ex) { + assertTrue(ex.getMessage().contains(expErrorMessage), ex.getMessage()); + } + } +} diff --git a/presto-delta/src/test/java/io/prestosql/delta/TestDeltaTypeUtils.java b/presto-delta/src/test/java/io/prestosql/delta/TestDeltaTypeUtils.java new file mode 100644 index 000000000000..7c318d62ac83 --- /dev/null +++ b/presto-delta/src/test/java/io/prestosql/delta/TestDeltaTypeUtils.java @@ -0,0 +1,82 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.delta; + +import io.airlift.slice.Slices; +import io.prestosql.spi.PrestoException; +import io.prestosql.spi.type.Type; +import org.testng.annotations.Test; + +import java.math.BigInteger; + +import static io.prestosql.delta.DeltaErrorCode.DELTA_INVALID_PARTITION_VALUE; +import static io.prestosql.delta.DeltaTypeUtils.convertPartitionValue; +import static io.prestosql.spi.type.BigintType.BIGINT; +import static io.prestosql.spi.type.BooleanType.BOOLEAN; +import static io.prestosql.spi.type.DateType.DATE; +import static io.prestosql.spi.type.DecimalType.createDecimalType; +import static io.prestosql.spi.type.Decimals.encodeUnscaledValue; +import static io.prestosql.spi.type.DoubleType.DOUBLE; +import static io.prestosql.spi.type.IntegerType.INTEGER; +import static io.prestosql.spi.type.RealType.REAL; +import static io.prestosql.spi.type.SmallintType.SMALLINT; +import static io.prestosql.spi.type.TimestampType.TIMESTAMP; +import static io.prestosql.spi.type.TinyintType.TINYINT; +import static io.prestosql.spi.type.VarcharType.VARCHAR; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +public class TestDeltaTypeUtils +{ + @Test + public void partitionValueParsing() + { + assertPartitionValue("str", VARCHAR, Slices.utf8Slice("str")); + assertPartitionValue("3", TINYINT, 3L); + assertPartitionValue("344", SMALLINT, 344L); + assertPartitionValue("323243", INTEGER, 323243L); + assertPartitionValue("234234234233", BIGINT, 234234234233L); + assertPartitionValue("3.234234", REAL, 1078918577L); + assertPartitionValue("34534.23423423423", DOUBLE, 34534.23423423423); + assertPartitionValue("2021-11-18", DATE, 18949L); + assertPartitionValue("2021-11-18 05:23:43", TIMESTAMP, 1637213023000L); + assertPartitionValue("true", BOOLEAN, true); + assertPartitionValue("faLse", BOOLEAN, false); + assertPartitionValue("234.5", createDecimalType(6, 3), 2345L); + assertPartitionValue("12345678901234567890123.5", createDecimalType(23, 1), encodeUnscaledValue(new BigInteger("123456789012345678901235"))); + + invalidPartitionValue("sdfsdf", BOOLEAN); + invalidPartitionValue("sdfsdf", DATE); + invalidPartitionValue("sdfsdf", TIMESTAMP); + } + + private void assertPartitionValue(String value, Type type, Object expected) + { + Object actual = convertPartitionValue("p1", value, type.getTypeSignature()); + assertEquals(actual, expected); + } + + private void invalidPartitionValue(String value, Type type) + { + try { + convertPartitionValue("p1", value, type.getTypeSignature()); + fail("expected to fail"); + } + catch (PrestoException e) { + assertEquals(e.getErrorCode(), DELTA_INVALID_PARTITION_VALUE.toErrorCode()); + assertTrue(e.getMessage().matches("Can not parse partition value .* of type .* for partition column 'p1'")); + } + } +} diff --git a/presto-server-main/etc/catalog/delta.properties b/presto-server-main/etc/catalog/delta.properties new file mode 100644 index 000000000000..92e040f1fb5e --- /dev/null +++ b/presto-server-main/etc/catalog/delta.properties @@ -0,0 +1,2 @@ +connector.name=delta +hive.metastore.uri=thrift://localhost:9083 diff --git a/presto-server-main/etc/config.properties b/presto-server-main/etc/config.properties index c013971c221f..e188ecfef61c 100644 --- a/presto-server-main/etc/config.properties +++ b/presto-server-main/etc/config.properties @@ -48,6 +48,7 @@ plugin.bundles=\ ../presto-thrift/pom.xml, \ ../presto-tpcds/pom.xml, \ ../presto-google-sheets/pom.xml, \ - ../presto-druid/pom.xml + ../presto-druid/pom.xml, \ + ../presto-delta/pom.xml node-scheduler.include-coordinator=true diff --git a/presto-server/src/main/provisio/presto.xml b/presto-server/src/main/provisio/presto.xml index 1ef267f816cc..1ff4d94793b0 100644 --- a/presto-server/src/main/provisio/presto.xml +++ b/presto-server/src/main/provisio/presto.xml @@ -252,4 +252,10 @@ + + + + + + From 33501fe3d3d9b01f42de923eff7050f64486a17e Mon Sep 17 00:00:00 2001 From: Venki Korukanti Date: Tue, 2 Nov 2021 16:57:04 -0700 Subject: [PATCH 5/5] Upgrade hadoop version containing Azure FileStatus equals bug fix --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 94be2342f30e..04600eb2f7ec 100644 --- a/pom.xml +++ b/pom.xml @@ -445,7 +445,7 @@ io.prestosql.hadoop hadoop-apache - 3.2.0-9 + 3.2.0-11-delta