Switch Julia bindings to the new async API based on RustyObjectStore (#3)

Adds support for the async interface provided by iceberg_rust_ffi in
RelationalAI/iceberg_rust_ffi#3

---------

Co-authored-by: Gerald Berger <gberger.work@gmail.com>

Author: vustef

.github/workflows/CI.yml

@@ -14,21 +14,51 @@ jobs:
       JULIA_NUM_THREADS: 2
       # TODO iceberg_rust_ffi requires this and panics otherwise, we should fix that (test
       # bucket is publicly accessible).
-      AWS_ACCESS_KEY_ID: "<AWS_ACCESS_KEY_ID>"
+      AWS_ACCESS_KEY_ID: root
+      AWS_SECRET_ACCESS_KEY: password
+      AWS_ENDPOINT_URL: http://localhost:9000
+      AWS_REGION: us-east-1
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        version:
-          - '1.10'
-          - '1.11'
-        os:
-          - ubuntu-latest
-          - macOS-latest
-        arch:
-          - x64
+        include:
+          # Ubuntu on x64
+          - version: '1.10'
+            os: ubuntu-latest
+            arch: x64
+          - version: '1.11'
+            os: ubuntu-latest
+            arch: x64
     steps:
       - uses: actions/checkout@v3.5.0
+      - name: Initialize containers
+        uses: gacts/run-and-post-run@v1
+        with:
+          run: |
+            docker network create iceberg_net
+            docker run -d --name minio \
+              --network=iceberg_net -p 9000:9000 \
+              -e MINIO_ROOT_USER=root \
+              -e MINIO_ROOT_PASSWORD=password \
+              -e MINIO_DOMAIN=minio \
+              -v ${{ github.workspace }}/assets/tpch:/input minio/minio:latest server /data/
+            until (docker exec minio mc alias set minio http://localhost:9000 root password) do echo '... waiting ...' && sleep 1; done;
+            docker exec minio mc mb minio/warehouse
+            docker exec minio mc cp -r /input/tpch.sf01/ minio/warehouse/tpch.sf01/
+            docker run -d --name rest \
+              --network=iceberg_net \
+              -p 8181:8181 \
+              -v ${{ github.workspace }}/assets/rest:/tmp \
+              -e AWS_ACCESS_KEY_ID=root \
+              -e AWS_SECRET_ACCESS_KEY=password \
+              -e AWS_ENDPOINT_URL=http://minio:9000 \
+              -e AWS_REGION=us-east-1 \
+              -e CATALOG_S3_ENDPOINT=http://minio:9000 \
+              -e CATALOG_WAREHOUSE=s3://warehouse \
+              -e CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO apache/iceberg-rest-fixture
+          post:
+            docker stop rest minio && docker rm rest minio && docker network rm iceberg_net
       - uses: julia-actions/setup-julia@v1
         with:
           version: ${{ matrix.version }}

Manifest.toml

@@ -2,7 +2,7 @@
 
 julia_version = "1.10.2"
 manifest_format = "2.0"
-project_hash = "99f2d569c75339b191aacd71e0f3e72c8ea9f4a5"
+project_hash = "4ebb16a8b4e4dee68434289ad64fac66757cf7e1"
 
 [[deps.ArgTools]]
 uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
@@ -26,6 +26,12 @@ uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
 [[deps.Base64]]
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
+[[deps.BenchmarkTools]]
+deps = ["Compat", "JSON", "Logging", "Printf", "Profile", "Statistics", "UUIDs"]
+git-tree-sha1 = "e38fbc49a620f5d0b660d7f543db1009fe0f8336"
+uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+version = "1.6.0"
+
 [[deps.BitIntegers]]
 deps = ["Random"]
 git-tree-sha1 = "f98cfeaba814d9746617822032d50a31c9926604"
@@ -46,16 +52,18 @@ version = "0.8.6"
 
 [[deps.Compat]]
 deps = ["TOML", "UUIDs"]
-git-tree-sha1 = "0037835448781bb46feb39866934e243886d756a"
+git-tree-sha1 = "9d8a54ce4b17aa5bdce0ea5c34bc5e7c340d16ad"
 uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
-version = "4.18.0"
+version = "4.18.1"
+weakdeps = ["Dates", "LinearAlgebra"]
 
     [deps.Compat.extensions]
     CompatLinearAlgebraExt = "LinearAlgebra"
 
-    [deps.Compat.weakdeps]
-    Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
-    LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+[[deps.CompilerSupportLibraries_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
+version = "1.1.0+0"
 
 [[deps.ConcurrentUtilities]]
 deps = ["Serialization", "Sockets"]
@@ -100,18 +108,15 @@ deps = ["Random"]
 uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820"
 
 [[deps.InlineStrings]]
-git-tree-sha1 = "8594fac023c5ce1ef78260f24d1ad18b4327b420"
+git-tree-sha1 = "8f3d257792a522b4601c24a577954b0a8cd7334d"
 uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48"
-version = "1.4.4"
+version = "1.4.5"
+weakdeps = ["ArrowTypes", "Parsers"]
 
     [deps.InlineStrings.extensions]
     ArrowTypesExt = "ArrowTypes"
     ParsersExt = "Parsers"
 
-    [deps.InlineStrings.weakdeps]
-    ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd"
-    Parsers = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
-
 [[deps.InteractiveUtils]]
 deps = ["Markdown"]
 uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
@@ -127,6 +132,12 @@ git-tree-sha1 = "0533e564aae234aff59ab625543145446d8b6ec2"
 uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
 version = "1.7.1"
 
+[[deps.JSON]]
+deps = ["Dates", "Mmap", "Parsers", "Unicode"]
+git-tree-sha1 = "31e996f0a15c7b280ba9f76636b3ff9e2ae58c9a"
+uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+version = "0.21.4"
+
 [[deps.LibCURL]]
 deps = ["LibCURL_jll", "MozillaCACerts_jll"]
 uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
@@ -145,6 +156,10 @@ version = "1.11.0+1"
 [[deps.Libdl]]
 uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 
+[[deps.LinearAlgebra]]
+deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"]
+uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+
 [[deps.Logging]]
 uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
 
@@ -180,17 +195,40 @@ version = "2023.1.10"
 uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
 version = "1.2.0"
 
+[[deps.OpenBLAS_jll]]
+deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
+uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
+version = "0.3.23+4"
+
+[[deps.OpenSSL_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl"]
+git-tree-sha1 = "f19301ae653233bc88b1810ae908194f07f8db9d"
+uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95"
+version = "3.5.4+0"
+
 [[deps.OrderedCollections]]
 git-tree-sha1 = "05868e21324cede2207c6f0f466b4bfef6d5e7ee"
 uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 version = "1.8.1"
 
+[[deps.Parsers]]
+deps = ["Dates", "PrecompileTools", "UUIDs"]
+git-tree-sha1 = "7d2f8f21da5db6a806faf7b9b292296da42b2810"
+uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
+version = "2.8.3"
+
 [[deps.PooledArrays]]
 deps = ["DataAPI", "Future"]
 git-tree-sha1 = "36d8b4b899628fb92c2749eb488d884a926614d3"
 uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720"
 version = "1.4.3"
 
+[[deps.PrecompileTools]]
+deps = ["Preferences"]
+git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f"
+uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
+version = "1.2.1"
+
 [[deps.Preferences]]
 deps = ["TOML"]
 git-tree-sha1 = "0f27480397253da18fe2c12a4ba4eb9eb208bf3d"
@@ -201,6 +239,10 @@ version = "1.5.0"
 deps = ["Unicode"]
 uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 
+[[deps.Profile]]
+deps = ["Printf"]
+uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"
+
 [[deps.Random]]
 deps = ["SHA"]
 uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
@@ -227,10 +269,25 @@ uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 [[deps.Sockets]]
 uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
 
+[[deps.SparseArrays]]
+deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"]
+uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+version = "1.10.0"
+
+[[deps.Statistics]]
+deps = ["LinearAlgebra", "SparseArrays"]
+uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+version = "1.10.0"
+
 [[deps.StringViews]]
-git-tree-sha1 = "ec4bf39f7d25db401bcab2f11d2929798c0578e5"
+git-tree-sha1 = "d3c7a06c80622b8404b1105886c732abcb25cc2b"
 uuid = "354b36f9-a18e-4713-926e-db85100087ba"
-version = "1.3.4"
+version = "1.3.5"
+
+[[deps.SuiteSparse_jll]]
+deps = ["Artifacts", "Libdl", "libblastrampoline_jll"]
+uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c"
+version = "7.2.1+1"
 
 [[deps.TOML]]
 deps = ["Dates"]
@@ -261,9 +318,9 @@ uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [[deps.TimeZones]]
 deps = ["Artifacts", "Dates", "Downloads", "InlineStrings", "Mocking", "Printf", "Scratch", "TZJData", "Unicode", "p7zip_jll"]
-git-tree-sha1 = "1f9a3f379a2ce2a213a0f606895567a08a1a2d08"
+git-tree-sha1 = "06f4f1f3e8ff09e42e59b043a747332e88e01aba"
 uuid = "f269a46b-ccf7-5d73-abea-4c690281aa53"
-version = "1.22.0"
+version = "1.22.1"
 
     [deps.TimeZones.extensions]
     TimeZonesRecipesBaseExt = "RecipesBase"
@@ -295,10 +352,15 @@ uuid = "3161d3a3-bdf6-5164-811a-617609db77b4"
 version = "1.5.7+1"
 
 [[deps.iceberg_rust_ffi_jll]]
-deps = ["Artifacts", "JLLWrappers", "Libdl"]
-git-tree-sha1 = "f92cda6a47dc404e19e1df5e2f01915baeb0591a"
+deps = ["Artifacts", "JLLWrappers", "Libdl", "OpenSSL_jll"]
+git-tree-sha1 = "c0054a4811aa38d441839c919469d1262bbbd417"
 uuid = "6bd5c94f-693c-53e3-983d-a09fad412f22"
-version = "0.1.0+0"
+version = "0.2.0+0"
+
+[[deps.libblastrampoline_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"
+version = "5.8.0+1"
 
 [[deps.nghttp2_jll]]
 deps = ["Artifacts", "Libdl"]

Project.toml

@@ -1,10 +1,11 @@
 name = "RustyIceberg"
 uuid = "390bdf5b-b624-43dc-a846-0ef7a3405804"
 authors = ["Vukasin Stefanovic <vukasin.stefanovic92@gmail.com>"]
-version = "0.1.0"
+version = "0.2.0"
 
 [deps]
 Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45"
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 iceberg_rust_ffi_jll = "6bd5c94f-693c-53e3-983d-a09fad412f22"

assets/.gitignore

@@ -0,0 +1,4 @@
+**/.minio*
+rest/*.so
+rest/*.so.lck
+rest/hsperfdata*

assets/rest/iceberg_catalog.db

@@ -0,0 +1 @@
+{"format-version":2,"table-uuid":"51667c9a-d3d1-4389-8461-d18626ab0e60","location":"s3://warehouse/tpch.sf01/customer","last-sequence-number":0,"last-updated-ms":1757415286085,"last-column-id":8,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"identifier-field-ids":[1],"fields":[{"id":1,"name":"c_custkey","required":true,"type":"long"},{"id":2,"name":"c_name","required":false,"type":"string"},{"id":3,"name":"c_address","required":false,"type":"string"},{"id":4,"name":"c_nationkey","required":false,"type":"int"},{"id":5,"name":"c_phone","required":false,"type":"string"},{"id":6,"name":"c_acctbal","required":false,"type":"decimal(15, 2)"},{"id":7,"name":"c_mktsegment","required":false,"type":"string"},{"id":8,"name":"c_comment","required":false,"type":"string"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"write.format.default":"parquet","write.metadata.compression-codec":"none","write.parquet.compression-codec":"zstd"},"current-snapshot-id":-1,"refs":{},"snapshots":[],"statistics":[],"partition-statistics":[],"snapshot-log":[],"metadata-log":[]}

assets/tpch/tpch.sf01/customer/data/data_customer-00000.parquet

@@ -0,0 +1 @@
+{"format-version":2,"table-uuid":"51667c9a-d3d1-4389-8461-d18626ab0e60","location":"s3://warehouse/tpch.sf01/customer","last-sequence-number":1,"last-updated-ms":1757415286199,"last-column-id":8,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"identifier-field-ids":[1],"fields":[{"id":1,"name":"c_custkey","required":true,"type":"long"},{"id":2,"name":"c_name","required":false,"type":"string"},{"id":3,"name":"c_address","required":false,"type":"string"},{"id":4,"name":"c_nationkey","required":false,"type":"int"},{"id":5,"name":"c_phone","required":false,"type":"string"},{"id":6,"name":"c_acctbal","required":false,"type":"decimal(15, 2)"},{"id":7,"name":"c_mktsegment","required":false,"type":"string"},{"id":8,"name":"c_comment","required":false,"type":"string"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"write.format.default":"parquet","write.metadata.compression-codec":"none","write.parquet.compression-codec":"zstd"},"current-snapshot-id":3441867730092225551,"refs":{"main":{"snapshot-id":3441867730092225551,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":3441867730092225551,"timestamp-ms":1757415286199,"summary":{"operation":"append","total-files-size":"0","total-position-deletes":"0","total-data-files":"0","total-delete-files":"0","total-records":"0","total-equality-deletes":"0"},"manifest-list":"s3://warehouse/tpch.sf01/customer/metadata/snap-3441867730092225551-0-01992e1d-25b3-7111-9dfc-38f55f99f91b.avro","schema-id":0}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1757415286199,"snapshot-id":3441867730092225551}],"metadata-log":[{"timestamp-ms":1757415286085,"metadata-file":"s3://warehouse/tpch.sf01/customer/metadata/00000-462e1567-8801-45a5-978c-b3beb53e7a2e.metadata.json"}]}