Java iceberg table tests

Author: platypii

src/avro.data.js

@@ -31,7 +31,7 @@ export function avroData({ reader, metadata, syncMarker }) {
     reader.offset += 16
     for (let i = 0; i < 16; i++) {
       if (blockSync[i] !== syncMarker[i]) {
-        throw new Error('Sync marker does not match')
+        throw new Error('sync marker does not match')
       }
     }
     const codec = metadata['avro.codec']
@@ -40,7 +40,7 @@ export function avroData({ reader, metadata, syncMarker }) {
     if (codec === 'deflate') {
       data = gunzip(data)
     } else if (codec !== 'null') {
-      throw new Error(`Unsupported codec: ${codec}`)
+      throw new Error(`unsupported codec: ${codec}`)
     }
 
     // Decode according to binary or json encoding
@@ -61,7 +61,7 @@ export function avroData({ reader, metadata, syncMarker }) {
 }
 
 /**
- * @import {AvroType} from './types.js'
+ * @import {AvroType} from '../src/types.js'
  * @param {DataReader} reader
  * @param {AvroType} type
  * @returns {any}
@@ -123,6 +123,6 @@ function readType(reader, type) {
     return text
   } else {
     // enum, fixed, null, map
-    throw new Error(`Unsupported type: ${type}`)
+    throw new Error(`unsupported type: ${type}`)
   }
 }

src/iceberg.fetch.js

@@ -28,7 +28,7 @@ export function translateS3Url(url) {
  * Position deletes are grouped by target data file.
  * Equality deletes are grouped by sequence number.
  *
- * @import {ManifestEntry} from './types.js'
+ * @import {ManifestEntry} from '../src/types.js'
  * @param {ManifestEntry[]} deleteEntries
  * @returns {Promise<{positionDeletesMap: Map<string, Set<bigint>>, equalityDeletesMap: Map<bigint, Record<string, any>[]>}>}
  */

src/iceberg.js

@@ -16,7 +16,7 @@ export { avroData } from './avro.data.js'
  * TODO:
  *   - Sequence number checks when filtering deletes
  *
- * @import {IcebergMetadata} from './types.js'
+ * @import {IcebergMetadata} from '../src/types.js'
  * @param {object} options
  * @param {string} options.tableUrl - Base S3 URL of the table.
  * @param {number} [options.rowStart] - The starting global row index to fetch (inclusive).

src/iceberg.manifest.js

@@ -3,7 +3,7 @@ import { fetchAvroRecords } from './iceberg.fetch.js'
 /**
  * Returns manifest entries for the current snapshot.
  *
- * @import {IcebergMetadata, Manifest, ManifestEntry} from './types.js'
+ * @import {IcebergMetadata, Manifest, ManifestEntry} from '../src/types.js'
  * @typedef {{ url: string, entries: ManifestEntry[] }[]} ManifestList
  * @param {IcebergMetadata} metadata
  * @returns {Promise<ManifestList>}

src/iceberg.metadata.js

@@ -17,7 +17,7 @@ export function icebergLatestVersion(tableUrl) {
  * Fetches the Iceberg table metadata.
  * If metadataFileName is not privided, uses icebergLatestVersion to get the version hint.
  *
- * @import {IcebergMetadata} from './types.js'
+ * @import {IcebergMetadata} from '../src/types.js'
  * @param {string} tableUrl - Base URL of the table (e.g. "s3://my-bucket/path/to/table")
  * @param {string} [metadataFileName] - Name of the metadata JSON file
  * @returns {Promise<IcebergMetadata>} The table metadata as a JSON object

test/iceberg.java.test.js

@@ -0,0 +1,87 @@
+import { describe, expect, it } from 'vitest'
+import { icebergRead } from '../src/iceberg.js'
+
+describe.concurrent('icebergRead from java iceberg table', () => {
+  const tableUrl = 'https://s3.amazonaws.com/hyperparam-iceberg/java/bunnies'
+
+  it('reads data from iceberg table', async () => {
+    const data = await icebergRead({ tableUrl, metadataFileName: 'v2.metadata.json' })
+
+    // Verify we got correct number of rows
+    expect(data).toBeInstanceOf(Array)
+    expect(data.length).toBe(21)
+
+    // Verify first row has expected structure
+    expect(data[0]).toEqual({
+      'Breed Name': 'Holland Lop',
+      'Average Weight': 1.8,
+      'Fur Length': 3,
+      Lifespan: 7n,
+      'Origin Country': 'The Netherlands',
+      'Ear Type': 'Lop',
+      Temperament: 'Friendly',
+      'Popularity Rank': 1n,
+    })
+
+    // Check we have all expected properties
+    const expectedProperties = [
+      'Breed Name',
+      'Average Weight',
+      'Fur Length',
+      'Lifespan',
+      'Origin Country',
+      'Ear Type',
+      'Temperament',
+      'Popularity Rank',
+    ]
+    data.forEach(row => {
+      expectedProperties.forEach(prop => {
+        expect(row).toHaveProperty(prop)
+      })
+    })
+  })
+
+  it('reads data v3 with added column', async () => {
+    const data = await icebergRead({ tableUrl, metadataFileName: 'v3.metadata.json' })
+
+    expect(data.length).toBe(21)
+    expect(data[2]).toEqual({
+      'Breed Name': 'Flemish Giant',
+      'Average Weight': 4.5,
+      'Fur Length': 4,
+      Lifespan: 5n,
+      'Origin Country': 'Belgium',
+      'Ear Type': 'Lop',
+      Temperament: 'Calm',
+      'Popularity Rank': 3n,
+      __happy__: undefined,
+    })
+  })
+
+  it('reads data v4 with deleted rows', async () => {
+    const data = await icebergRead({ tableUrl, metadataFileName: 'v4.metadata.json' })
+
+    expect(data.length).toBe(15)
+    expect(data[2]).toEqual({
+      'Breed Name': 'American Fuzzy Lop',
+      'Average Weight': 1.4,
+      'Fur Length': 5,
+      Lifespan: 8n,
+      'Origin Country': 'USA',
+      'Ear Type': 'Lop',
+      Temperament: 'Sociable',
+      'Popularity Rank': 8n,
+    })
+    const newZealandRow = data.find(row => row['Breed Name'] === 'New Zealand')
+    expect(newZealandRow).toEqual({
+      'Breed Name': 'New Zealand',
+      'Average Weight': 4,
+      'Fur Length': 2.7,
+      Lifespan: 8n,
+      'Origin Country': 'New Zealand',
+      'Ear Type': 'Erect',
+      Temperament: 'Affectionate',
+      'Popularity Rank': 21n,
+    })
+  })
+})