Split out iceberg metadata functions

Author: platypii

package.json

@@ -39,10 +39,10 @@
   },
   "devDependencies": {
     "@types/node": "22.13.10",
-    "@vitest/coverage-v8": "3.0.8",
+    "@vitest/coverage-v8": "3.0.9",
     "eslint": "9.22.0",
-    "eslint-plugin-jsdoc": "50.6.6",
+    "eslint-plugin-jsdoc": "50.6.8",
     "typescript": "5.8.2",
-    "vitest": "3.0.8"
+    "vitest": "3.0.9"
   }
 }

src/iceberg.fetch.js

@@ -21,33 +21,6 @@ export function translateS3Url(url) {
   return url
 }
 
-/**
- * Fetches the Iceberg table snapshot version from S3 using the version hint file.
- *
- * @param {string} tableBaseUrl - Base S3 URL of the table (e.g. "s3://my-bucket/path/to/table")
- * @returns {Promise<number>} The snapshot version
- */
-export function fetchSnapshotVersion(tableBaseUrl) {
-  const url = `${tableBaseUrl}/metadata/version-hint.text`
-  const safeUrl = translateS3Url(url)
-  // TODO: If version-hint is not found, try listing or binary search.
-  return fetch(safeUrl).then(res => res.text()).then(text => parseInt(text))
-}
-
-/**
- * Fetches the Iceberg table metadata JSON from S3
- *
- * @import {IcebergMetadata} from './types.js'
- * @param {string} tableBaseUrl - Base S3 URL of the table (e.g. "s3://my-bucket/path/to/table")
- * @param {string} metadataFileName - Name of the metadata JSON file
- * @returns {Promise<IcebergMetadata>} The table metadata as a JSON object
- */
-export function fetchIcebergMetadata(tableBaseUrl, metadataFileName) {
-  const url = `${tableBaseUrl}/metadata/${metadataFileName}`
-  const safeUrl = translateS3Url(url)
-  return fetch(safeUrl).then(res => res.json())
-}
-
 /**
  * Fetches data files information from multiple manifest file URLs.
  *

src/iceberg.js

@@ -1,8 +1,9 @@
 import { asyncBufferFromUrl, parquetReadObjects } from 'hyparquet'
 import { decompress as ZSTD } from 'fzstd'
-import {
-  fetchAvroRecords, fetchDataFilesFromManifests, fetchIcebergMetadata, fetchSnapshotVersion, translateS3Url,
-} from './iceberg.fetch.js'
+import { fetchAvroRecords, fetchDataFilesFromManifests, translateS3Url } from './iceberg.fetch.js'
+import { fetchIcebergMetadata } from './iceberg.metadata.js'
+
+export { fetchIcebergMetadata }
 
 /**
  * Returns manifest URLs for the current snapshot separated into data and delete manifests.
@@ -101,27 +102,22 @@ function equalityMatch(row, deletePredicate) {
  * @returns {Promise<Array<Record<string, any>>>} Array of data records.
  */
 export async function icebergRead({ tableUrl, rowStart, rowEnd, metadataFileName }) {
-  // Find the latest snapshot version.
-  if (!metadataFileName) {
-    const version = await fetchSnapshotVersion(tableUrl)
-    metadataFileName = `v${version}.metadata.json`
-  }
-  // Fetch table metadata and validate key fields.
+  // Fetch table metadata
   const metadata = await fetchIcebergMetadata(tableUrl, metadataFileName)
 
-  // Get manifest URLs for data and delete files.
+  // Get manifest URLs for data and delete files
   const { dataManifestUrls, deleteManifestUrls } = await getManifestUrls(metadata)
   if (dataManifestUrls.length === 0) {
     throw new Error('No data manifest files found for current snapshot')
   }
 
-  // Read data file info from data manifests.
+  // Read data file info from data manifests
   const dataFiles = await fetchDataFilesFromManifests(dataManifestUrls)
   if (dataFiles.length === 0) {
     throw new Error('No data files found in manifests (table may be empty)')
   }
 
-  // Read delete file info from delete manifests (if any).
+  // Read delete file info from delete manifests (if any)
   const deleteFiles = deleteManifestUrls.length > 0
     ? await fetchDataFilesFromManifests(deleteManifestUrls)
     : []

src/iceberg.metadata.js

@@ -0,0 +1,33 @@
+import { translateS3Url } from './iceberg.fetch.js'
+
+/**
+ * Fetches the Iceberg table snapshot version from S3 using the version hint file.
+ *
+ * @param {string} tableUrl - Base S3 URL of the table (e.g. "s3://my-bucket/path/to/table")
+ * @returns {Promise<number>} The snapshot version
+ */
+export function fetchSnapshotVersion(tableUrl) {
+  const url = `${tableUrl}/metadata/version-hint.text`
+  const safeUrl = translateS3Url(url)
+  // TODO: If version-hint is not found, try listing or binary search.
+  return fetch(safeUrl).then(res => res.text()).then(text => parseInt(text))
+}
+
+/**
+ * Fetches the Iceberg table metadata JSON from S3.
+ * If metadataFileName is not privided, uses fetchSnapshotVersion to get the version hint.
+ *
+ * @import {IcebergMetadata} from './types.js'
+ * @param {string} tableUrl - Base S3 URL of the table (e.g. "s3://my-bucket/path/to/table")
+ * @param {string} [metadataFileName] - Name of the metadata JSON file
+ * @returns {Promise<IcebergMetadata>} The table metadata as a JSON object
+ */
+export async function fetchIcebergMetadata(tableUrl, metadataFileName) {
+  if (!metadataFileName) {
+    const version = await fetchSnapshotVersion(tableUrl)
+    metadataFileName = `v${version}.metadata.json`
+  }
+  const url = `${tableUrl}/metadata/${metadataFileName}`
+  const safeUrl = translateS3Url(url)
+  return fetch(safeUrl).then(res => res.json())
+}

src/types.d.ts

@@ -7,12 +7,12 @@ export interface IcebergMetadata {
   'last-updated-ms': number
   'last-column-id': number
   'current-schema-id': number
-  schemas: any[]
+  schemas: Schema[]
   'default-spec-id': number
   'partition-specs': any[]
   'last-partition-id': number
   'default-sort-order-id': number
-  'sort-orders': any[]
+  'sort-orders': SortOrder[]
   properties: object
   'current-snapshot-id': number
   refs: object
@@ -22,7 +22,25 @@ export interface IcebergMetadata {
   'metadata-log': any[]
 }
 
-interface Snapshot {
+export interface Schema {
+  type: string
+  "schema-id": number
+  fields: Field[]
+}
+
+interface Field {
+  id: number
+  name: string
+  required: boolean
+  type: string
+}
+
+interface SortOrder {
+  "order-id": number
+  "fields": any[]
+}
+
+export interface Snapshot {
   'sequence-number': number
   'snapshot-id': number
   'timestamp-ms': number
@@ -45,7 +63,7 @@ interface Snapshot {
   'schema-id': number
 }
 
-interface Manifest {
+export interface Manifest {
   manifest_path: string
   manifest_length: bigint
   partition_spec_id: number

test/iceberg.fetch.test.js

@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest'
-import { fetchSnapshotVersion } from '../src/iceberg.fetch.js'
+import { fetchSnapshotVersion } from '../src/iceberg.metadata.js'
 
 describe('Iceberg fetch utils', () => {
   const tableUrl = 'https://s3.amazonaws.com/hyperparam-iceberg/warehouse/bunnies'