From ef08907dda612a399cd44e97f1867be261944c4e Mon Sep 17 00:00:00 2001 From: Shane Loeffler Date: Wed, 22 Oct 2025 16:05:12 -0500 Subject: [PATCH 01/14] make v2 loaders lazy --- src/initialize-store.js | 121 +++++++++++++++++++++++++++++++--------- 1 file changed, 96 insertions(+), 25 deletions(-) diff --git a/src/initialize-store.js b/src/initialize-store.js index c9d56c7..5c9593f 100644 --- a/src/initialize-store.js +++ b/src/initialize-store.js @@ -14,36 +14,107 @@ const initializeStore = async (source, version, variable, coordinateKeys) => { const coordinates = {} switch (version) { case 'v2': - await new Promise((resolve) => - zarr(window.fetch, version).openGroup(source, (err, l, m) => { - loaders = l - metadata = m - resolve() + try { + // Fetch consolidated metadata directly + const zmetadata = await fetch(`${source}/.zmetadata`).then((res) => + res.json() + ) + metadata = { metadata: zmetadata.metadata } + const rootAttrs = zmetadata.metadata['.zattrs'] + ;({ levels, maxZoom, tileSize, crs } = getPyramidMetadata( + rootAttrs.multiscales + )) + + const zattrs = metadata.metadata[`${levels[0]}/${variable}/.zattrs`] + const zarray = metadata.metadata[`${levels[0]}/${variable}/.zarray`] + dimensions = zattrs['_ARRAY_DIMENSIONS'] + shape = zarray.shape + chunks = zarray.chunks + fill_value = zarray.fill_value + dtype = zarray.dtype + + const loadersCache = {} + + const loadLevel = (key) => { + if (loadersCache[key]) { + return Promise.resolve(loadersCache[key]) + } + + const promise = new Promise((resolve) => { + const arrayMeta = metadata.metadata[`${key}/.zarray`] + zarr(window.fetch, version).open( + `${source}/${key}`, + (err, get) => { + loadersCache[key] = get + resolve(get) + }, + arrayMeta + ) + }) + + loadersCache[key] = promise + return promise + } + + await Promise.all( + coordinateKeys.map((key) => { + const coordKey = `${levels[0]}/${key}` + return loadLevel(coordKey).then( + (get) => + new Promise((resolve) => { + get([0], (err, chunk) => { + coordinates[key] = Array.from(chunk.data) + resolve() + }) + }) + ) + }) + ) + + loaders = {} + levels.forEach((level) => { + const key = `${level}/${variable}` + loaders[key] = (...args) => { + return loadLevel(key).then((loader) => loader(...args)) + } }) - ) - ;({ levels, maxZoom, tileSize, crs } = getPyramidMetadata( - metadata.metadata['.zattrs'].multiscales - )) - const zattrs = metadata.metadata[`${levels[0]}/${variable}/.zattrs`] - const zarray = metadata.metadata[`${levels[0]}/${variable}/.zarray`] - dimensions = zattrs['_ARRAY_DIMENSIONS'] - shape = zarray.shape - chunks = zarray.chunks - fill_value = zarray.fill_value - dtype = zarray.dtype + coordinateKeys.forEach((key) => { + loaders[`${levels[0]}/${key}`] = loadersCache[`${levels[0]}/${key}`] + }) + } catch (e) { + // Fallback to openGroup + await new Promise((resolve) => + zarr(window.fetch, version).openGroup(source, (err, l, m) => { + loaders = l + metadata = m + resolve() + }) + ) + ;({ levels, maxZoom, tileSize, crs } = getPyramidMetadata( + metadata.metadata['.zattrs'].multiscales + )) + + const zattrs = metadata.metadata[`${levels[0]}/${variable}/.zattrs`] + const zarray = metadata.metadata[`${levels[0]}/${variable}/.zarray`] + dimensions = zattrs['_ARRAY_DIMENSIONS'] + shape = zarray.shape + chunks = zarray.chunks + fill_value = zarray.fill_value + dtype = zarray.dtype - await Promise.all( - coordinateKeys.map( - (key) => - new Promise((resolve) => { - loaders[`${levels[0]}/${key}`]([0], (err, chunk) => { - coordinates[key] = Array.from(chunk.data) - resolve() + await Promise.all( + coordinateKeys.map( + (key) => + new Promise((resolve) => { + loaders[`${levels[0]}/${key}`]([0], (err, chunk) => { + coordinates[key] = Array.from(chunk.data) + resolve() + }) }) - }) + ) ) - ) + } break case 'v3': From 72a8181fa6122a0f752ff0e0577f585286e9ea58 Mon Sep 17 00:00:00 2001 From: Shane Loeffler Date: Thu, 23 Oct 2025 16:45:29 -0500 Subject: [PATCH 02/14] refactor to expect promises throughout --- src/initialize-store.js | 79 ++++++++++++++++++++--------------------- src/tile.js | 19 ++++++---- 2 files changed, 52 insertions(+), 46 deletions(-) diff --git a/src/initialize-store.js b/src/initialize-store.js index 5c9593f..e962728 100644 --- a/src/initialize-store.js +++ b/src/initialize-store.js @@ -33,60 +33,53 @@ const initializeStore = async (source, version, variable, coordinateKeys) => { fill_value = zarray.fill_value dtype = zarray.dtype - const loadersCache = {} + const getCache = new Map() - const loadLevel = (key) => { - if (loadersCache[key]) { - return Promise.resolve(loadersCache[key]) - } + const callGet = async (key, chunkIndices) => { + let getPromise = getCache.get(key) - const promise = new Promise((resolve) => { + if (!getPromise) { const arrayMeta = metadata.metadata[`${key}/.zarray`] - zarr(window.fetch, version).open( - `${source}/${key}`, - (err, get) => { - loadersCache[key] = get - resolve(get) - }, - arrayMeta - ) - }) + getPromise = new Promise((resolve, reject) => { + zarr(window.fetch, version).open( + `${source}/${key}`, + (err, get) => (err ? reject(err) : resolve(get)), + arrayMeta + ) + }) + getCache.set(key, getPromise) + } - loadersCache[key] = promise - return promise + const get = await getPromise + return new Promise((resolve, reject) => { + get(chunkIndices, (err, out) => (err ? reject(err) : resolve(out))) + }) } await Promise.all( - coordinateKeys.map((key) => { + coordinateKeys.map(async (key) => { const coordKey = `${levels[0]}/${key}` - return loadLevel(coordKey).then( - (get) => - new Promise((resolve) => { - get([0], (err, chunk) => { - coordinates[key] = Array.from(chunk.data) - resolve() - }) - }) - ) + const chunk = await callGet(coordKey, [0]) + coordinates[key] = Array.from(chunk.data) }) ) loaders = {} levels.forEach((level) => { const key = `${level}/${variable}` - loaders[key] = (...args) => { - return loadLevel(key).then((loader) => loader(...args)) - } + loaders[key] = (chunkIndices) => callGet(key, chunkIndices) }) coordinateKeys.forEach((key) => { - loaders[`${levels[0]}/${key}`] = loadersCache[`${levels[0]}/${key}`] + const coordKey = `${levels[0]}/${key}` + loaders[coordKey] = (chunkIndices) => callGet(coordKey, chunkIndices) }) } catch (e) { // Fallback to openGroup + let rawLoaders await new Promise((resolve) => zarr(window.fetch, version).openGroup(source, (err, l, m) => { - loaders = l + rawLoaders = l metadata = m resolve() }) @@ -104,16 +97,22 @@ const initializeStore = async (source, version, variable, coordinateKeys) => { dtype = zarray.dtype await Promise.all( - coordinateKeys.map( - (key) => - new Promise((resolve) => { - loaders[`${levels[0]}/${key}`]([0], (err, chunk) => { - coordinates[key] = Array.from(chunk.data) - resolve() - }) + coordinateKeys.map((key) => { + const coordKey = `${levels[0]}/${key}` + return new Promise((resolve, reject) => { + rawLoaders[coordKey]([0], (err, chunk) => { + if (err) return reject(err) + coordinates[key] = Array.from(chunk.data) + resolve() }) - ) + }) + }) ) + + loaders = {} + Object.keys(rawLoaders).forEach((key) => { + loaders[key] = wrapGet(rawLoaders[key]) + }) } break diff --git a/src/tile.js b/src/tile.js index b5e3a39..c30d09e 100644 --- a/src/tile.js +++ b/src/tile.js @@ -55,12 +55,19 @@ class Tile { } else { this._loading[key] = true this._ready[key] = new Promise((innerResolve) => { - this._loader(chunk, (err, data) => { - this.chunkedData[key] = data - this._loading[key] = false - innerResolve(true) - resolve(true) - }) + this._loader(chunk) + .then((data) => { + this.chunkedData[key] = data + this._loading[key] = false + innerResolve(true) + resolve(true) + }) + .catch((err) => { + console.error('Error loading chunk', key, err) + this._loading[key] = false + innerResolve(false) + resolve(false) + }) }) } }) From feb4772b0a0950683435c1ce03a18cfea6218d0a Mon Sep 17 00:00:00 2001 From: Shane Loeffler Date: Thu, 23 Oct 2025 16:45:54 -0500 Subject: [PATCH 03/14] make v3 lazy too --- src/initialize-store.js | 79 +++++++++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 30 deletions(-) diff --git a/src/initialize-store.js b/src/initialize-store.js index e962728..32b9034 100644 --- a/src/initialize-store.js +++ b/src/initialize-store.js @@ -2,6 +2,14 @@ import zarr from 'zarr-js' import { getPyramidMetadata } from './utils' +// return promises for all for consistency +const wrapGet = (getFn) => { + return (chunkIndices) => + new Promise((resolve, reject) => { + getFn(chunkIndices, (err, out) => (err ? reject(err) : resolve(out))) + }) +} + const initializeStore = async (source, version, variable, coordinateKeys) => { let metadata let loaders @@ -135,40 +143,51 @@ const initializeStore = async (source, version, variable, coordinateKeys) => { fill_value = arrayMetadata.fill_value // dtype = arrayMetadata.data_type + const getCache = new Map() + + const callGet = async (key, chunkIndices, meta = null) => { + let getPromise = getCache.get(key) + + if (!getPromise) { + getPromise = new Promise((resolve, reject) => { + zarr(window.fetch, version).open( + `${source}/${key}`, + (err, get) => (err ? reject(err) : resolve(get)), + meta + ) + }) + getCache.set(key, getPromise) + } + + const get = await getPromise + return new Promise((resolve, reject) => { + get(chunkIndices, (err, out) => (err ? reject(err) : resolve(out))) + }) + } + + await Promise.all( + coordinateKeys.map(async (key) => { + const coordKey = `${levels[0]}/${key}` + const chunk = await callGet(coordKey, [0]) + coordinates[key] = Array.from(chunk.data) + }) + ) + loaders = {} - await Promise.all([ - ...levels.map( - (level) => - new Promise((resolve) => { - zarr(window.fetch, version).open( - `${source}/${level}/${variable}`, - (err, get) => { - loaders[`${level}/${variable}`] = get - resolve() - }, - level === 0 ? arrayMetadata : null - ) - }) - ), - ...coordinateKeys.map( - (key) => - new Promise((resolve) => { - zarr(window.fetch, version).open( - `${source}/${levels[0]}/${key}`, - (err, get) => { - get([0], (err, chunk) => { - coordinates[key] = Array.from(chunk.data) - resolve() - }) - } - ) - }) - ), - ]) + levels.forEach((level) => { + const key = `${level}/${variable}` + const meta = level === 0 ? arrayMetadata : null + loaders[key] = (chunkIndices) => callGet(key, chunkIndices, meta) + }) + + coordinateKeys.forEach((key) => { + const coordKey = `${levels[0]}/${key}` + loaders[coordKey] = (chunkIndices) => callGet(coordKey, chunkIndices) + }) break default: throw new Error( - `Unexpected Zarr version: ${version}. Must be one of 'v1', 'v2'.` + `Unexpected Zarr version: ${version}. Must be one of 'v2', 'v3'.` ) } From b3a2455cf87d91584d827ef8dea12daf02c568a2 Mon Sep 17 00:00:00 2001 From: Shane Loeffler Date: Mon, 27 Oct 2025 12:09:47 -0500 Subject: [PATCH 04/14] update tests for promises --- src/tile.test.js | 55 ++++++++++++++++++------------------------------ 1 file changed, 20 insertions(+), 35 deletions(-) diff --git a/src/tile.test.js b/src/tile.test.js index e1dcf82..89a2ab4 100644 --- a/src/tile.test.js +++ b/src/tile.test.js @@ -28,12 +28,9 @@ describe('Tile', () => { buffer = jest.fn() defaults = { key: '0,0,0', - loader: jest.fn().mockImplementation((chunk, cb) => - cb( - null, // error - createMockChunk(chunk) - ) - ), + loader: jest + .fn() + .mockImplementation((chunk) => Promise.resolve(createMockChunk(chunk))), shape: [10, 1, 1], chunks: [5, 1, 1], dimensions: ['year', 'y', 'x'], @@ -148,14 +145,8 @@ describe('Tile', () => { ]) expect(defaults.loader).toHaveBeenCalledTimes(2) - expect(defaults.loader).toHaveBeenCalledWith( - [0, 0, 0], - expect.anything() - ) - expect(defaults.loader).toHaveBeenCalledWith( - [1, 0, 0], - expect.anything() - ) + expect(defaults.loader).toHaveBeenCalledWith([0, 0, 0]) + expect(defaults.loader).toHaveBeenCalledWith([1, 0, 0]) }) it('does not repeat loading for any chunks have been loaded', async () => { @@ -267,15 +258,11 @@ describe('Tile', () => { beforeEach(() => { resolvers = [] - const loader = jest.fn().mockImplementation((chunk, cb) => - new Promise((resolve) => { - resolvers.push(resolve) - }).then(() => { - cb( - null, // error - createMockChunk(chunk) - ) - }) + const loader = jest.fn().mockImplementation( + (chunk) => + new Promise((resolve) => { + resolvers.push(() => resolve(createMockChunk(chunk))) + }) ) tile = new Tile({ ...defaults, loader }) }) @@ -441,12 +428,11 @@ describe('Tile', () => { const selector = {} const tile = new Tile({ ...defaults, - loader: jest.fn().mockImplementation((chunk, cb) => - cb( - null, // error - ndarray([1, 2, 3, 4], [4, 1, 1]) - ) - ), + loader: jest + .fn() + .mockImplementation((chunk) => + Promise.resolve(ndarray([1, 2, 3, 4], [4, 1, 1])) + ), shape: [4, 1, 1], chunks: [4, 1, 1], dimensions: ['band', 'y', 'x'], @@ -491,12 +477,11 @@ describe('Tile', () => { const selector = {} const tile = new Tile({ ...defaults, - loader: jest.fn().mockImplementation((chunk, cb) => - cb( - null, // error - ndarray([1, 2, 3, 4], [2, 2]) - ) - ), + loader: jest + .fn() + .mockImplementation((chunk) => + Promise.resolve(ndarray([1, 2, 3, 4], [2, 2])) + ), shape: [2, 2], chunks: [2, 2], dimensions: ['y', 'x'], From 2494cd1008a62b3cbe9c8dab0f064d954c5b9c6f Mon Sep 17 00:00:00 2001 From: Shane Loeffler Date: Tue, 28 Oct 2025 16:43:27 -0500 Subject: [PATCH 05/14] make Raster respect prop changes, cache metadata --- src/initialize-store.js | 30 ++++++++++++++++++++++++------ src/raster.js | 21 ++++++++++++++++++++- src/tiles.js | 9 ++++++++- 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/src/initialize-store.js b/src/initialize-store.js index 32b9034..70ca20e 100644 --- a/src/initialize-store.js +++ b/src/initialize-store.js @@ -10,7 +10,13 @@ const wrapGet = (getFn) => { }) } -const initializeStore = async (source, version, variable, coordinateKeys) => { +const initializeStore = async ( + source, + version, + variable, + coordinateKeys, + metadataCache = {} +) => { let metadata let loaders let dimensions @@ -20,13 +26,20 @@ const initializeStore = async (source, version, variable, coordinateKeys) => { let dtype let levels, maxZoom, tileSize, crs const coordinates = {} + const cacheKey = `${source}-${version}` + switch (version) { case 'v2': try { - // Fetch consolidated metadata directly - const zmetadata = await fetch(`${source}/.zmetadata`).then((res) => - res.json() - ) + let zmetadata + if (metadataCache[cacheKey]) { + zmetadata = metadataCache[cacheKey] + } else { + zmetadata = await fetch(`${source}/.zmetadata`).then((res) => + res.json() + ) + metadataCache[cacheKey] = zmetadata + } metadata = { metadata: zmetadata.metadata } const rootAttrs = zmetadata.metadata['.zattrs'] ;({ levels, maxZoom, tileSize, crs } = getPyramidMetadata( @@ -125,7 +138,12 @@ const initializeStore = async (source, version, variable, coordinateKeys) => { break case 'v3': - metadata = await fetch(`${source}/zarr.json`).then((res) => res.json()) + if (metadataCache[cacheKey]) { + metadata = metadataCache[cacheKey] + } else { + metadata = await fetch(`${source}/zarr.json`).then((res) => res.json()) + metadataCache[cacheKey] = metadata + } ;({ levels, maxZoom, tileSize, crs } = getPyramidMetadata( metadata.attributes.multiscales )) diff --git a/src/raster.js b/src/raster.js index 0b5ef8e..9c7dec0 100644 --- a/src/raster.js +++ b/src/raster.js @@ -29,6 +29,7 @@ const Raster = (props) => { const tiles = useRef() const camera = useRef() const lastQueried = useRef() + const metadataCache = useRef({}) camera.current = { center: center, zoom: zoom } @@ -51,6 +52,7 @@ const Raster = (props) => { ...props, setLoading, clearLoading, + metadataCache: metadataCache.current, invalidate: () => { map.triggerRepaint() }, @@ -58,7 +60,24 @@ const Raster = (props) => { setRegionDataInvalidated(new Date().getTime()) }, }) - }, []) + + return () => { + if (tiles.current) { + tiles.current.active = {} + tiles.current.loaders = {} + } + } + }, [ + props.source, + props.variable, + props.mode, + props.version, + props.projection, + props.frag, + props.fillValue, + props.order, + props.maxCachedTiles, + ]) useEffect(() => { if (props.setLoading) { diff --git a/src/tiles.js b/src/tiles.js index 206e9bd..25dac45 100644 --- a/src/tiles.js +++ b/src/tiles.js @@ -48,6 +48,7 @@ export const createTiles = (regl, opts) => { version = 'v2', projection, maxCachedTiles = 500, + metadataCache = {}, }) { this.tiles = {} this.loaders = {} @@ -113,7 +114,13 @@ export const createTiles = (regl, opts) => { }) this.initialized = new Promise((resolve) => { const loadingID = this.setLoading('metadata') - initializeStore(source, version, variable, Object.keys(selector)).then( + initializeStore( + source, + version, + variable, + Object.keys(selector), + metadataCache + ).then( ({ metadata, loaders, From dedf81d22f8c034c050dad311131e820a3f32307 Mon Sep 17 00:00:00 2001 From: Shane Loeffler Date: Tue, 28 Oct 2025 16:45:56 -0500 Subject: [PATCH 06/14] single loader per store --- src/initialize-store.js | 56 ++++++++++++++++++++--------------------- src/raster.js | 2 +- src/tiles.js | 21 ++++++++-------- 3 files changed, 38 insertions(+), 41 deletions(-) diff --git a/src/initialize-store.js b/src/initialize-store.js index 70ca20e..9c6e6c1 100644 --- a/src/initialize-store.js +++ b/src/initialize-store.js @@ -2,12 +2,10 @@ import zarr from 'zarr-js' import { getPyramidMetadata } from './utils' -// return promises for all for consistency -const wrapGet = (getFn) => { - return (chunkIndices) => - new Promise((resolve, reject) => { - getFn(chunkIndices, (err, out) => (err ? reject(err) : resolve(out))) - }) +const createLoader = ({ callGet, variable }) => { + return { + load: ({ level, chunk }) => callGet(`${level}/${variable}`, chunk), + } } const initializeStore = async ( @@ -18,13 +16,13 @@ const initializeStore = async ( metadataCache = {} ) => { let metadata - let loaders let dimensions let shape let chunks let fill_value let dtype let levels, maxZoom, tileSize, crs + let loader const coordinates = {} const cacheKey = `${source}-${version}` @@ -85,15 +83,9 @@ const initializeStore = async ( }) ) - loaders = {} - levels.forEach((level) => { - const key = `${level}/${variable}` - loaders[key] = (chunkIndices) => callGet(key, chunkIndices) - }) - - coordinateKeys.forEach((key) => { - const coordKey = `${levels[0]}/${key}` - loaders[coordKey] = (chunkIndices) => callGet(coordKey, chunkIndices) + loader = createLoader({ + callGet, + variable, }) } catch (e) { // Fallback to openGroup @@ -130,9 +122,15 @@ const initializeStore = async ( }) ) - loaders = {} - Object.keys(rawLoaders).forEach((key) => { - loaders[key] = wrapGet(rawLoaders[key]) + loader = createLoader({ + callGet: (key, chunkIndices) => + new Promise((resolve, reject) => { + rawLoaders[key](chunkIndices, (err, out) => { + if (err) return reject(err) + resolve(out) + }) + }), + variable, }) } @@ -191,16 +189,16 @@ const initializeStore = async ( }) ) - loaders = {} - levels.forEach((level) => { - const key = `${level}/${variable}` - const meta = level === 0 ? arrayMetadata : null - loaders[key] = (chunkIndices) => callGet(key, chunkIndices, meta) - }) + const getChunk = (key, chunkIndices) => { + const meta = key.startsWith(`${levels[0]}/${variable}`) + ? arrayMetadata + : null + return callGet(key, chunkIndices, meta) + } - coordinateKeys.forEach((key) => { - const coordKey = `${levels[0]}/${key}` - loaders[coordKey] = (chunkIndices) => callGet(coordKey, chunkIndices) + loader = createLoader({ + callGet: getChunk, + variable, }) break default: @@ -211,7 +209,7 @@ const initializeStore = async ( return { metadata, - loaders, + loader, dimensions, shape, chunks, diff --git a/src/raster.js b/src/raster.js index 9c7dec0..68997d0 100644 --- a/src/raster.js +++ b/src/raster.js @@ -64,7 +64,7 @@ const Raster = (props) => { return () => { if (tiles.current) { tiles.current.active = {} - tiles.current.loaders = {} + tiles.current.loader = null } } }, [ diff --git a/src/tiles.js b/src/tiles.js index 25dac45..1b12c77 100644 --- a/src/tiles.js +++ b/src/tiles.js @@ -51,7 +51,6 @@ export const createTiles = (regl, opts) => { metadataCache = {}, }) { this.tiles = {} - this.loaders = {} this.active = {} this.display = display this.clim = clim @@ -112,6 +111,8 @@ export const createTiles = (regl, opts) => { } } }) + this.loader = null + this.availableLevels = new Set() this.initialized = new Promise((resolve) => { const loadingID = this.setLoading('metadata') initializeStore( @@ -123,7 +124,7 @@ export const createTiles = (regl, opts) => { ).then( ({ metadata, - loaders, + loader, dimensions, shape, chunks, @@ -181,11 +182,8 @@ export const createTiles = (regl, opts) => { this.ndim = this.dimensions.length this.coordinates = coordinates - - levels.forEach((z) => { - const loader = loaders[z + '/' + variable] - this.loaders[z] = loader - }) + this.loader = loader + this.availableLevels = new Set(levels) resolve(true) this.clearLoading(loadingID) @@ -329,12 +327,13 @@ export const createTiles = (regl, opts) => { this._initializeTile = (key, level) => { if (!this.tiles[key]) { - const loader = this.loaders[level] - if (!loader) return + if (!this.loader || !this.availableLevels.has(level)) return this._removeOldestTile() + const loadChunk = (chunk) => this.loader.load({ level, chunk }) + this.tiles[key] = new Tile({ key, - loader, + loader: loadChunk, shape: this.shape, chunks: this.chunks, dimensions: this.dimensions, @@ -388,7 +387,7 @@ export const createTiles = (regl, opts) => { Object.keys(this.active).map( (key) => new Promise((resolve) => { - if (this.loaders[level]) { + if (this.loader && this.availableLevels.has(level)) { const tileIndex = keyToTile(key) const tile = this._initializeTile(key, level) From 9e69236ff9dc93995d059df09ed236175558ec07 Mon Sep 17 00:00:00 2001 From: Shane Loeffler Date: Wed, 29 Oct 2025 17:33:01 -0500 Subject: [PATCH 07/14] refactor initializeStore to class --- src/initialize-store.js | 226 ---------------------------------------- src/raster.js | 35 ++++--- src/tiles.js | 144 ++++++++++++------------- src/zarr-store.js | 190 +++++++++++++++++++++++++++++++++ 4 files changed, 280 insertions(+), 315 deletions(-) delete mode 100644 src/initialize-store.js create mode 100644 src/zarr-store.js diff --git a/src/initialize-store.js b/src/initialize-store.js deleted file mode 100644 index 9c6e6c1..0000000 --- a/src/initialize-store.js +++ /dev/null @@ -1,226 +0,0 @@ -import zarr from 'zarr-js' - -import { getPyramidMetadata } from './utils' - -const createLoader = ({ callGet, variable }) => { - return { - load: ({ level, chunk }) => callGet(`${level}/${variable}`, chunk), - } -} - -const initializeStore = async ( - source, - version, - variable, - coordinateKeys, - metadataCache = {} -) => { - let metadata - let dimensions - let shape - let chunks - let fill_value - let dtype - let levels, maxZoom, tileSize, crs - let loader - const coordinates = {} - const cacheKey = `${source}-${version}` - - switch (version) { - case 'v2': - try { - let zmetadata - if (metadataCache[cacheKey]) { - zmetadata = metadataCache[cacheKey] - } else { - zmetadata = await fetch(`${source}/.zmetadata`).then((res) => - res.json() - ) - metadataCache[cacheKey] = zmetadata - } - metadata = { metadata: zmetadata.metadata } - const rootAttrs = zmetadata.metadata['.zattrs'] - ;({ levels, maxZoom, tileSize, crs } = getPyramidMetadata( - rootAttrs.multiscales - )) - - const zattrs = metadata.metadata[`${levels[0]}/${variable}/.zattrs`] - const zarray = metadata.metadata[`${levels[0]}/${variable}/.zarray`] - dimensions = zattrs['_ARRAY_DIMENSIONS'] - shape = zarray.shape - chunks = zarray.chunks - fill_value = zarray.fill_value - dtype = zarray.dtype - - const getCache = new Map() - - const callGet = async (key, chunkIndices) => { - let getPromise = getCache.get(key) - - if (!getPromise) { - const arrayMeta = metadata.metadata[`${key}/.zarray`] - getPromise = new Promise((resolve, reject) => { - zarr(window.fetch, version).open( - `${source}/${key}`, - (err, get) => (err ? reject(err) : resolve(get)), - arrayMeta - ) - }) - getCache.set(key, getPromise) - } - - const get = await getPromise - return new Promise((resolve, reject) => { - get(chunkIndices, (err, out) => (err ? reject(err) : resolve(out))) - }) - } - - await Promise.all( - coordinateKeys.map(async (key) => { - const coordKey = `${levels[0]}/${key}` - const chunk = await callGet(coordKey, [0]) - coordinates[key] = Array.from(chunk.data) - }) - ) - - loader = createLoader({ - callGet, - variable, - }) - } catch (e) { - // Fallback to openGroup - let rawLoaders - await new Promise((resolve) => - zarr(window.fetch, version).openGroup(source, (err, l, m) => { - rawLoaders = l - metadata = m - resolve() - }) - ) - ;({ levels, maxZoom, tileSize, crs } = getPyramidMetadata( - metadata.metadata['.zattrs'].multiscales - )) - - const zattrs = metadata.metadata[`${levels[0]}/${variable}/.zattrs`] - const zarray = metadata.metadata[`${levels[0]}/${variable}/.zarray`] - dimensions = zattrs['_ARRAY_DIMENSIONS'] - shape = zarray.shape - chunks = zarray.chunks - fill_value = zarray.fill_value - dtype = zarray.dtype - - await Promise.all( - coordinateKeys.map((key) => { - const coordKey = `${levels[0]}/${key}` - return new Promise((resolve, reject) => { - rawLoaders[coordKey]([0], (err, chunk) => { - if (err) return reject(err) - coordinates[key] = Array.from(chunk.data) - resolve() - }) - }) - }) - ) - - loader = createLoader({ - callGet: (key, chunkIndices) => - new Promise((resolve, reject) => { - rawLoaders[key](chunkIndices, (err, out) => { - if (err) return reject(err) - resolve(out) - }) - }), - variable, - }) - } - - break - case 'v3': - if (metadataCache[cacheKey]) { - metadata = metadataCache[cacheKey] - } else { - metadata = await fetch(`${source}/zarr.json`).then((res) => res.json()) - metadataCache[cacheKey] = metadata - } - ;({ levels, maxZoom, tileSize, crs } = getPyramidMetadata( - metadata.attributes.multiscales - )) - - const arrayMetadata = await fetch( - `${source}/${levels[0]}/${variable}/zarr.json` - ).then((res) => res.json()) - - dimensions = arrayMetadata.attributes['_ARRAY_DIMENSIONS'] - shape = arrayMetadata.shape - const isSharded = arrayMetadata.codecs[0].name == 'sharding_indexed' - chunks = isSharded - ? arrayMetadata.codecs[0].configuration.chunk_shape - : arrayMetadata.chunk_grid.configuration.chunk_shape - fill_value = arrayMetadata.fill_value - // dtype = arrayMetadata.data_type - - const getCache = new Map() - - const callGet = async (key, chunkIndices, meta = null) => { - let getPromise = getCache.get(key) - - if (!getPromise) { - getPromise = new Promise((resolve, reject) => { - zarr(window.fetch, version).open( - `${source}/${key}`, - (err, get) => (err ? reject(err) : resolve(get)), - meta - ) - }) - getCache.set(key, getPromise) - } - - const get = await getPromise - return new Promise((resolve, reject) => { - get(chunkIndices, (err, out) => (err ? reject(err) : resolve(out))) - }) - } - - await Promise.all( - coordinateKeys.map(async (key) => { - const coordKey = `${levels[0]}/${key}` - const chunk = await callGet(coordKey, [0]) - coordinates[key] = Array.from(chunk.data) - }) - ) - - const getChunk = (key, chunkIndices) => { - const meta = key.startsWith(`${levels[0]}/${variable}`) - ? arrayMetadata - : null - return callGet(key, chunkIndices, meta) - } - - loader = createLoader({ - callGet: getChunk, - variable, - }) - break - default: - throw new Error( - `Unexpected Zarr version: ${version}. Must be one of 'v2', 'v3'.` - ) - } - - return { - metadata, - loader, - dimensions, - shape, - chunks, - fill_value, - dtype, - coordinates, - levels, - maxZoom, - tileSize, - crs, - } -} - -export default initializeStore diff --git a/src/raster.js b/src/raster.js index 68997d0..4a7f71f 100644 --- a/src/raster.js +++ b/src/raster.js @@ -1,10 +1,11 @@ -import React, { useRef, useEffect, useState } from 'react' +import React, { useRef, useEffect, useState, useMemo } from 'react' import { useRegl } from './regl' import { useMap } from './map-provider' import { useControls } from './use-controls' import { createTiles } from './tiles' import { useRegion } from './region/context' import { useSetLoading } from './loading' +import ZarrStore from './zarr-store' const Raster = (props) => { const { @@ -29,10 +30,20 @@ const Raster = (props) => { const tiles = useRef() const camera = useRef() const lastQueried = useRef() - const metadataCache = useRef({}) camera.current = { center: center, zoom: zoom } + const store = useMemo( + () => + new ZarrStore({ + source: props.source, + version: props.version, + variable: props.variable, + coordinateKeys: Object.keys(selector), + }), + [props.source, props.version, props.variable] + ) + const queryRegion = async (r, s) => { const queryStart = new Date().getTime() lastQueried.current = queryStart @@ -47,12 +58,18 @@ const Raster = (props) => { } } + useEffect(() => { + return () => { + store.cleanup() + } + }, [store]) + useEffect(() => { tiles.current = createTiles(regl, { ...props, setLoading, clearLoading, - metadataCache: metadataCache.current, + store, invalidate: () => { map.triggerRepaint() }, @@ -67,17 +84,7 @@ const Raster = (props) => { tiles.current.loader = null } } - }, [ - props.source, - props.variable, - props.mode, - props.version, - props.projection, - props.frag, - props.fillValue, - props.order, - props.maxCachedTiles, - ]) + }, [store]) useEffect(() => { if (props.setLoading) { diff --git a/src/tiles.js b/src/tiles.js index 1b12c77..b323247 100644 --- a/src/tiles.js +++ b/src/tiles.js @@ -22,13 +22,11 @@ import { } from './utils' import { DEFAULT_FILL_VALUES } from './constants' import Tile from './tile' -import initializeStore from './initialize-store' export const createTiles = (regl, opts) => { return new Tiles(opts) function Tiles({ - source, colormap, clim, opacity, @@ -45,10 +43,9 @@ export const createTiles = (regl, opts) => { invalidateRegion, setMetadata, order, - version = 'v2', projection, maxCachedTiles = 500, - metadataCache = {}, + store, }) { this.tiles = {} this.active = {} @@ -113,84 +110,81 @@ export const createTiles = (regl, opts) => { }) this.loader = null this.availableLevels = new Set() - this.initialized = new Promise((resolve) => { + if (!store) { + throw new Error('Tiles requires a ZarrStore instance.') + } + this.store = store + + this.initialized = (async () => { const loadingID = this.setLoading('metadata') - initializeStore( - source, - version, - variable, - Object.keys(selector), - metadataCache - ).then( - ({ - metadata, - loader, - dimensions, - shape, - chunks, - fill_value, - dtype, - coordinates, - levels, - maxZoom, - tileSize, - crs, - }) => { - if (setMetadata) setMetadata(metadata) - this.maxZoom = maxZoom - this.level = zoomToLevel(this.zoom, maxZoom) - const position = getPositions(tileSize, mode) - this.position = regl.buffer(position) - this.size = tileSize - // Respect `projection` prop when provided, otherwise rely on `crs` value from metadata - this.projectionIndex = projection - ? ['mercator', 'equirectangular'].indexOf(projection) - : ['EPSG:3857', 'EPSG:4326'].indexOf(crs) - this.projection = ['mercator', 'equirectangular'][ - this.projectionIndex - ] - - if (!this.projection) { - this.projection = null - throw new Error( - projection - ? `Unexpected \`projection\` prop provided: '${projection}'. Must be one of 'mercator', 'equirectangular'.` - : `Unexpected \`crs\` found in metadata: '${crs}'. Must be one of 'EPSG:3857', 'EPSG:4326'.` - ) - } - if (mode === 'grid' || mode === 'dotgrid') { - this.count = position.length - } - if (mode === 'texture') { - this.count = 6 - } + await this.store.initialized + const { + metadata, + loader, + dimensions, + shape, + chunks, + fill_value, + dtype, + coordinates, + levels, + maxZoom, + tileSize, + crs, + } = this.store.describe() + + if (setMetadata) setMetadata(metadata) + this.maxZoom = maxZoom + this.level = zoomToLevel(this.zoom, maxZoom) + const position = getPositions(tileSize, mode) + this.position = regl.buffer(position) + this.size = tileSize + // Respect `projection` prop when provided, otherwise rely on `crs` value from metadata + this.projectionIndex = projection + ? ['mercator', 'equirectangular'].indexOf(projection) + : ['EPSG:3857', 'EPSG:4326'].indexOf(crs) + this.projection = ['mercator', 'equirectangular'][this.projectionIndex] - this.dimensions = dimensions - this.shape = shape - this.chunks = chunks - this.fillValue = fillValue ?? fill_value ?? DEFAULT_FILL_VALUES[dtype] + if (!this.projection) { + this.projection = null + throw new Error( + projection + ? `Unexpected \`projection\` prop provided: '${projection}'. Must be one of 'mercator', 'equirectangular'.` + : `Unexpected \`crs\` found in metadata: '${crs}'. Must be one of 'EPSG:3857', 'EPSG:4326'.` + ) + } - if (mode === 'texture') { - const emptyTexture = ndarray( - new Float32Array(Array(1).fill(this.fillValue)), - [1, 1] - ) - initialize = () => regl.texture(emptyTexture) - } + if (mode === 'grid' || mode === 'dotgrid') { + this.count = position.length + } + if (mode === 'texture') { + this.count = 6 + } - this.ndim = this.dimensions.length + this.dimensions = dimensions + this.shape = shape + this.chunks = chunks + this.fillValue = fillValue ?? fill_value ?? DEFAULT_FILL_VALUES[dtype] - this.coordinates = coordinates - this.loader = loader - this.availableLevels = new Set(levels) + if (mode === 'texture') { + const emptyTexture = ndarray( + new Float32Array(Array(1).fill(this.fillValue)), + [1, 1] + ) + initialize = () => regl.texture(emptyTexture) + } - resolve(true) - this.clearLoading(loadingID) - this.invalidate() - } - ) - }) + this.ndim = this.dimensions.length + + this.coordinates = coordinates + this.loader = loader + this.availableLevels = new Set(levels) + + this.clearLoading(loadingID) + this.invalidate() + return true + })() this.drawTiles = regl({ vert: vert(mode, this.bands), diff --git a/src/zarr-store.js b/src/zarr-store.js new file mode 100644 index 0000000..c7e9b74 --- /dev/null +++ b/src/zarr-store.js @@ -0,0 +1,190 @@ +import zarr from 'zarr-js' +import { getPyramidMetadata } from './utils' + +class ZarrStore { + constructor({ source, version = 'v2', variable, coordinateKeys = [] }) { + if (!source) { + throw new Error('source is a required parameter') + } + if (!variable) { + throw new Error('variable is a required parameter') + } + this.source = source + this.version = version + this.variable = variable + this.coordinateKeys = coordinateKeys + this.metadata = null + this.arrayMetadata = null + this.dimensions = null + this.shape = null + this.chunks = null + this.fill_value = null + this.dtype = null + this.levels = [] + this.maxZoom = null + this.tileSize = null + this.crs = null + this.coordinates = {} + this.loader = null + this._chunkHandles = new Map() + + this.initialized = this._initialize() + } + + async _initialize() { + if (this.version === 'v2') { + await this._loadV2() + } else if (this.version === 'v3') { + await this._loadV3() + } else { + throw new Error( + `Unexpected Zarr version: ${this.version}. Must be one of 'v2', 'v3'.` + ) + } + + this.loader = { + load: ({ level, chunk }) => + this.getChunk(`${level}/${this.variable}`, chunk), + } + + if (this.coordinateKeys.length) { + await Promise.all( + this.coordinateKeys.map((key) => + this.getChunk(`${this.levels[0]}/${key}`, [0]).then((chunk) => { + this.coordinates[key] = Array.from(chunk.data) + }) + ) + ) + } + + return this + } + + cleanup() { + this._chunkHandles.clear() + this.loader = null + this.coordinates = {} + } + + describe() { + return { + metadata: this.metadata, + loader: this.loader, + dimensions: this.dimensions, + shape: this.shape, + chunks: this.chunks, + fill_value: this.fill_value, + dtype: this.dtype, + coordinates: this.coordinates, + levels: this.levels, + maxZoom: this.maxZoom, + tileSize: this.tileSize, + crs: this.crs, + } + } + + async getChunk(key, chunkIndices) { + let handle = this._chunkHandles.get(key) + + if (!handle) { + let meta = null + if (this.version === 'v2') { + meta = this.metadata?.metadata?.[`${key}/.zarray`] || null + } else if (this.version === 'v3') { + meta = key.startsWith(`${this.levels[0]}/${this.variable}`) + ? this.arrayMetadata + : null + } + + handle = new Promise((resolve, reject) => { + zarr(window.fetch, this.version).open( + `${this.source}/${key}`, + (err, get) => (err ? reject(err) : resolve(get)), + meta + ) + }).catch((err) => { + this._chunkHandles.delete(key) + throw err + }) + this._chunkHandles.set(key, handle) + } + + const get = await handle + return new Promise((resolve, reject) => { + get(chunkIndices, (err, out) => (err ? reject(err) : resolve(out))) + }) + } + + async _loadV2() { + const cacheKey = `${this.version}:${this.source}` + let zmetadata = ZarrStore._cache.get(cacheKey) + if (!zmetadata) { + zmetadata = await fetch(`${this.source}/.zmetadata`).then((res) => + res.json() + ) + ZarrStore._cache.set(cacheKey, zmetadata) + } + + this.metadata = { metadata: zmetadata.metadata } + + const pyramid = getPyramidMetadata( + zmetadata.metadata['.zattrs'].multiscales + ) + this.levels = pyramid.levels + this.maxZoom = pyramid.maxZoom + this.tileSize = pyramid.tileSize + this.crs = pyramid.crs + + const basePath = `${this.levels[0]}/${this.variable}` + const zattrs = this.metadata.metadata[`${basePath}/.zattrs`] + const zarray = this.metadata.metadata[`${basePath}/.zarray`] + + this.dimensions = zattrs['_ARRAY_DIMENSIONS'] + this.shape = zarray.shape + this.chunks = zarray.chunks + this.fill_value = zarray.fill_value + this.dtype = zarray.dtype + } + + async _loadV3() { + const metadataCacheKey = `${this.version}:${this.source}` + let metadata = ZarrStore._cache.get(metadataCacheKey) + if (!metadata) { + metadata = await fetch(`${this.source}/zarr.json`).then((res) => + res.json() + ) + ZarrStore._cache.set(metadataCacheKey, metadata) + } + this.metadata = metadata + + const pyramid = getPyramidMetadata(this.metadata.attributes.multiscales) + this.levels = pyramid.levels + this.maxZoom = pyramid.maxZoom + this.tileSize = pyramid.tileSize + this.crs = pyramid.crs + + const arrayKey = `${this.levels[0]}/${this.variable}` + const arrayCacheKey = `${this.version}:${this.source}/${arrayKey}` + let arrayMetadata = ZarrStore._cache.get(arrayCacheKey) + if (!arrayMetadata) { + arrayMetadata = await fetch(`${this.source}/${arrayKey}/zarr.json`).then( + (res) => res.json() + ) + ZarrStore._cache.set(arrayCacheKey, arrayMetadata) + } + this.arrayMetadata = arrayMetadata + + this.dimensions = this.arrayMetadata.attributes['_ARRAY_DIMENSIONS'] + this.shape = this.arrayMetadata.shape + const isSharded = this.arrayMetadata.codecs[0].name === 'sharding_indexed' + this.chunks = isSharded + ? this.arrayMetadata.codecs[0].configuration.chunk_shape + : this.arrayMetadata.chunk_grid.configuration.chunk_shape + this.fill_value = this.arrayMetadata.fill_value + this.dtype = this.arrayMetadata.data_type || null + } +} + +ZarrStore._cache = new Map() + +export default ZarrStore From 90ae4087b62f6849a5987fd012c72a67282d649e Mon Sep 17 00:00:00 2001 From: Shane Loeffler Date: Thu, 30 Oct 2025 09:41:48 -0500 Subject: [PATCH 08/14] simplify chunk getting --- src/tiles.js | 10 ++++------ src/zarr-store.js | 18 +++++------------- 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/src/tiles.js b/src/tiles.js index b323247..e8df084 100644 --- a/src/tiles.js +++ b/src/tiles.js @@ -108,7 +108,6 @@ export const createTiles = (regl, opts) => { } } }) - this.loader = null this.availableLevels = new Set() if (!store) { throw new Error('Tiles requires a ZarrStore instance.') @@ -121,7 +120,6 @@ export const createTiles = (regl, opts) => { await this.store.initialized const { metadata, - loader, dimensions, shape, chunks, @@ -178,7 +176,6 @@ export const createTiles = (regl, opts) => { this.ndim = this.dimensions.length this.coordinates = coordinates - this.loader = loader this.availableLevels = new Set(levels) this.clearLoading(loadingID) @@ -321,9 +318,10 @@ export const createTiles = (regl, opts) => { this._initializeTile = (key, level) => { if (!this.tiles[key]) { - if (!this.loader || !this.availableLevels.has(level)) return + if (!this.availableLevels.has(level)) return this._removeOldestTile() - const loadChunk = (chunk) => this.loader.load({ level, chunk }) + const loadChunk = (chunk) => + this.store.getChunk(`${level}/${this.variable}`, chunk) this.tiles[key] = new Tile({ key, @@ -381,7 +379,7 @@ export const createTiles = (regl, opts) => { Object.keys(this.active).map( (key) => new Promise((resolve) => { - if (this.loader && this.availableLevels.has(level)) { + if (this.availableLevels.has(level)) { const tileIndex = keyToTile(key) const tile = this._initializeTile(key, level) diff --git a/src/zarr-store.js b/src/zarr-store.js index c7e9b74..cffff97 100644 --- a/src/zarr-store.js +++ b/src/zarr-store.js @@ -25,8 +25,7 @@ class ZarrStore { this.tileSize = null this.crs = null this.coordinates = {} - this.loader = null - this._chunkHandles = new Map() + this._getterCache = new Map() this.initialized = this._initialize() } @@ -42,11 +41,6 @@ class ZarrStore { ) } - this.loader = { - load: ({ level, chunk }) => - this.getChunk(`${level}/${this.variable}`, chunk), - } - if (this.coordinateKeys.length) { await Promise.all( this.coordinateKeys.map((key) => @@ -61,15 +55,13 @@ class ZarrStore { } cleanup() { - this._chunkHandles.clear() - this.loader = null + this._getterCache.clear() this.coordinates = {} } describe() { return { metadata: this.metadata, - loader: this.loader, dimensions: this.dimensions, shape: this.shape, chunks: this.chunks, @@ -84,7 +76,7 @@ class ZarrStore { } async getChunk(key, chunkIndices) { - let handle = this._chunkHandles.get(key) + let handle = this._getterCache.get(key) if (!handle) { let meta = null @@ -103,10 +95,10 @@ class ZarrStore { meta ) }).catch((err) => { - this._chunkHandles.delete(key) + this._getterCache.delete(key) throw err }) - this._chunkHandles.set(key, handle) + this._getterCache.set(key, handle) } const get = await handle From 82bda58cb7b6c49b75144d4e735f0fce04c5ebf0 Mon Sep 17 00:00:00 2001 From: Shane Loeffler Date: Thu, 30 Oct 2025 11:37:57 -0500 Subject: [PATCH 09/14] remove old cleanup --- src/raster.js | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/raster.js b/src/raster.js index 4a7f71f..28449cc 100644 --- a/src/raster.js +++ b/src/raster.js @@ -77,13 +77,6 @@ const Raster = (props) => { setRegionDataInvalidated(new Date().getTime()) }, }) - - return () => { - if (tiles.current) { - tiles.current.active = {} - tiles.current.loader = null - } - } }, [store]) useEffect(() => { From bc47b841e718a76509b1b474f67fdaaa32e43fd5 Mon Sep 17 00:00:00 2001 From: Shane Loeffler Date: Thu, 30 Oct 2025 16:30:32 -0500 Subject: [PATCH 10/14] encapsulate initialization state --- src/tiles.js | 2 +- src/zarr-store.js | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/tiles.js b/src/tiles.js index e8df084..b86fec4 100644 --- a/src/tiles.js +++ b/src/tiles.js @@ -117,7 +117,7 @@ export const createTiles = (regl, opts) => { this.initialized = (async () => { const loadingID = this.setLoading('metadata') - await this.store.initialized + await this.store.initialized() const { metadata, dimensions, diff --git a/src/zarr-store.js b/src/zarr-store.js index cffff97..a28ff5d 100644 --- a/src/zarr-store.js +++ b/src/zarr-store.js @@ -27,7 +27,12 @@ class ZarrStore { this.coordinates = {} this._getterCache = new Map() - this.initialized = this._initialize() + this._initialized = this._initialize() + } + + async initialized() { + await this._initialized + return true } async _initialize() { From 28b33378c5af7f443f74017a7a3e4104a475302b Mon Sep 17 00:00:00 2001 From: Shane Loeffler Date: Thu, 30 Oct 2025 16:33:57 -0500 Subject: [PATCH 11/14] simplify loadChunks promises --- src/tile.js | 47 +++++++++++++++++++++-------------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/src/tile.js b/src/tile.js index c30d09e..4f8c776 100644 --- a/src/tile.js +++ b/src/tile.js @@ -46,32 +46,27 @@ class Tile { async loadChunks(chunks) { const updated = await Promise.all( - chunks.map( - (chunk) => - new Promise((resolve) => { - const key = chunk.join('.') - if (this.chunkedData[key]) { - resolve(false) - } else { - this._loading[key] = true - this._ready[key] = new Promise((innerResolve) => { - this._loader(chunk) - .then((data) => { - this.chunkedData[key] = data - this._loading[key] = false - innerResolve(true) - resolve(true) - }) - .catch((err) => { - console.error('Error loading chunk', key, err) - this._loading[key] = false - innerResolve(false) - resolve(false) - }) - }) - } - }) - ) + chunks.map(async (chunk) => { + const key = chunk.join('.') + if (this.chunkedData[key]) { + return false + } else { + this._loading[key] = true + this._ready[key] = this._loader(chunk) + .then((data) => { + this.chunkedData[key] = data + this._loading[key] = false + return true + }) + .catch((err) => { + console.error('Error loading chunk', key, err) + this._loading[key] = false + return false + }) + + return this._ready[key] + } + }) ) return updated.some(Boolean) From 099a66412d9a20ef3decc1052a3cc5813e0e8634 Mon Sep 17 00:00:00 2001 From: Shane Loeffler Date: Thu, 30 Oct 2025 17:36:19 -0500 Subject: [PATCH 12/14] zarr store testing --- src/zarr-store.test.js | 279 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 279 insertions(+) create mode 100644 src/zarr-store.test.js diff --git a/src/zarr-store.test.js b/src/zarr-store.test.js new file mode 100644 index 0000000..5b4675e --- /dev/null +++ b/src/zarr-store.test.js @@ -0,0 +1,279 @@ +jest.mock('zarr-js', () => { + const handlers = new Map() + const openMock = jest.fn((url, callback) => { + const handler = handlers.get(url) + if (!handler) { + callback(new Error(`Missing handler for URL: ${url}`)) + return + } + callback(null, handler) + }) + + const factory = jest.fn(() => ({ open: openMock })) + + factory.__set = (url, handler) => handlers.set(url, handler) + factory.__clear = () => { + handlers.clear() + openMock.mockClear() + factory.mockClear() + } + factory.__openMock = openMock + + return factory +}) + +import zarr from 'zarr-js' +import ZarrStore from './zarr-store' + +const mockJsonResponse = (value) => + Promise.resolve({ + json: () => Promise.resolve(value), + }) + +const createV2Metadata = () => ({ + metadata: { + '.zattrs': { + multiscales: [ + { + datasets: [ + { path: '0', pixels_per_tile: 256, crs: 'EPSG:3857' }, + { path: '1', pixels_per_tile: 256, crs: 'EPSG:3857' }, + ], + }, + ], + }, + '0/temp/.zattrs': { + _ARRAY_DIMENSIONS: ['time', 'y', 'x'], + }, + '0/temp/.zarray': { + shape: [2, 256, 256], + chunks: [1, 256, 256], + fill_value: -9999, + dtype: ' ({ + attributes: { + multiscales: [ + { + datasets: [ + { path: '0', pixels_per_tile: 256, crs: 'EPSG:3857' }, + { path: '1', pixels_per_tile: 256, crs: 'EPSG:3857' }, + ], + }, + ], + }, +}) + +const createV3ArrayMetadata = () => ({ + attributes: { + _ARRAY_DIMENSIONS: ['time', 'y', 'x'], + }, + shape: [2, 256, 256], + chunk_grid: { + configuration: { + chunk_shape: [1, 256, 256], + }, + }, + codecs: [{ name: 'gzip' }], + fill_value: 0, + data_type: ' { + const handler = jest.fn((chunkIndices, callback) => { + callback(null, { data: new Float32Array(values) }) + }) + zarr.__set(url, handler) + return handler +} + +describe('ZarrStore', () => { + const originalGlobalFetch = global.fetch + const hasWindow = typeof window !== 'undefined' + const originalWindowFetch = hasWindow ? window.fetch : undefined + + let fetchMock + let createdWindow + + beforeEach(() => { + createdWindow = false + fetchMock = jest.fn() + + global.fetch = fetchMock + + if (!hasWindow) { + global.window = {} + createdWindow = true + } + window.fetch = fetchMock + + zarr.__clear() + ZarrStore._cache.clear() + }) + + afterEach(() => { + if (originalGlobalFetch !== undefined) { + global.fetch = originalGlobalFetch + } else { + delete global.fetch + } + + if (createdWindow) { + delete global.window + } else if (hasWindow) { + if (originalWindowFetch !== undefined) { + window.fetch = originalWindowFetch + } else { + delete window.fetch + } + } + }) + + it('loads v2 metadata and coordinates while reusing cached metadata', async () => { + const source = 'https://example.com/v2' + const coordinateValues = [2000, 2001] + + const zmetadata = createV2Metadata() + registerChunk(`${source}/0/time`, coordinateValues) + + fetchMock.mockResolvedValueOnce(mockJsonResponse(zmetadata)) + + const store = new ZarrStore({ + source, + version: 'v2', + variable: 'temp', + coordinateKeys: ['time'], + }) + + await expect(store.initialized()).resolves.toBe(true) + + expect(fetchMock).toHaveBeenCalledTimes(1) + expect(fetchMock).toHaveBeenCalledWith(`${source}/.zmetadata`) + + expect(store.describe()).toEqual({ + metadata: { metadata: zmetadata.metadata }, + dimensions: ['time', 'y', 'x'], + shape: [2, 256, 256], + chunks: [1, 256, 256], + fill_value: -9999, + dtype: ' { + const source = 'https://example.com/v3' + const metadata = createV3Metadata() + const arrayMetadata = createV3ArrayMetadata() + + fetchMock + .mockResolvedValueOnce(mockJsonResponse(metadata)) + .mockResolvedValueOnce(mockJsonResponse(arrayMetadata)) + + const store = new ZarrStore({ + source, + version: 'v3', + variable: 'temp', + }) + + await expect(store.initialized()).resolves.toBe(true) + + expect(fetchMock).toHaveBeenNthCalledWith(1, `${source}/zarr.json`) + expect(fetchMock).toHaveBeenNthCalledWith(2, `${source}/0/temp/zarr.json`) + + expect(store.describe()).toEqual({ + metadata, + dimensions: ['time', 'y', 'x'], + shape: [2, 256, 256], + chunks: [1, 256, 256], + fill_value: 0, + dtype: ' { + expect(() => new ZarrStore({ variable: 'temp' })).toThrow( + 'source is a required parameter' + ) + expect(() => new ZarrStore({ source: 'https://example.com' })).toThrow( + 'variable is a required parameter' + ) + }) + + it('reuses cached getter functions when requesting the same chunk key', async () => { + const source = 'https://example.com/v2' + const zmetadata = createV2Metadata() + const chunkValues = [1, 2, 3, 4] + + fetchMock.mockResolvedValueOnce(mockJsonResponse(zmetadata)) + const chunkHandler = registerChunk(`${source}/0/temp`, chunkValues) + + const store = new ZarrStore({ + source, + version: 'v2', + variable: 'temp', + }) + + await expect(store.initialized()).resolves.toBe(true) + + const firstChunk = await store.getChunk('0/temp', [0, 0]) + expect(firstChunk.data).toEqual(new Float32Array(chunkValues)) + + const secondChunk = await store.getChunk('0/temp', [0, 1]) + expect(secondChunk.data).toEqual(new Float32Array(chunkValues)) + + expect(zarr).toHaveBeenCalledTimes(1) + expect(zarr.__openMock).toHaveBeenCalledTimes(1) + expect(zarr.__openMock).toHaveBeenCalledWith( + `${source}/0/temp`, + expect.any(Function), + expect.anything() + ) + expect(chunkHandler).toHaveBeenCalledTimes(2) + expect(chunkHandler).toHaveBeenNthCalledWith( + 1, + [0, 0], + expect.any(Function) + ) + expect(chunkHandler).toHaveBeenNthCalledWith( + 2, + [0, 1], + expect.any(Function) + ) + }) +}) From 16d9eb7d2b7041bf28b73c468aad91b8851ab26a Mon Sep 17 00:00:00 2001 From: Shane Loeffler Date: Fri, 31 Oct 2025 12:51:28 -0500 Subject: [PATCH 13/14] Update src/tile.js Co-authored-by: Kata Martin --- src/tile.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tile.js b/src/tile.js index 4f8c776..3cf1cfc 100644 --- a/src/tile.js +++ b/src/tile.js @@ -46,7 +46,7 @@ class Tile { async loadChunks(chunks) { const updated = await Promise.all( - chunks.map(async (chunk) => { + chunks.map((chunk) => { const key = chunk.join('.') if (this.chunkedData[key]) { return false From c23d6e65d7b5cc74e14ee5994e171e2b43b32e9f Mon Sep 17 00:00:00 2001 From: Shane Loeffler Date: Fri, 31 Oct 2025 12:56:22 -0500 Subject: [PATCH 14/14] split up tests --- src/zarr-store.test.js | 328 ++++++++++++++++++++++++++--------------- 1 file changed, 208 insertions(+), 120 deletions(-) diff --git a/src/zarr-store.test.js b/src/zarr-store.test.js index 5b4675e..25df80c 100644 --- a/src/zarr-store.test.js +++ b/src/zarr-store.test.js @@ -132,148 +132,236 @@ describe('ZarrStore', () => { } }) - it('loads v2 metadata and coordinates while reusing cached metadata', async () => { - const source = 'https://example.com/v2' - const coordinateValues = [2000, 2001] - - const zmetadata = createV2Metadata() - registerChunk(`${source}/0/time`, coordinateValues) - - fetchMock.mockResolvedValueOnce(mockJsonResponse(zmetadata)) + it('requires source and variable options', () => { + expect(() => new ZarrStore({ variable: 'temp' })).toThrow( + 'source is a required parameter' + ) + expect(() => new ZarrStore({ source: 'https://example.com' })).toThrow( + 'variable is a required parameter' + ) + }) - const store = new ZarrStore({ - source, - version: 'v2', - variable: 'temp', - coordinateKeys: ['time'], + describe('v2', () => { + it('loads metadata and coordinates arrays on initialization', async () => { + const source = 'https://example.com/v2' + const coordinateValues = [2000, 2001] + + const zmetadata = createV2Metadata() + registerChunk(`${source}/0/time`, coordinateValues) + + fetchMock.mockResolvedValueOnce(mockJsonResponse(zmetadata)) + + const store = new ZarrStore({ + source, + version: 'v2', + variable: 'temp', + coordinateKeys: ['time'], + }) + + await expect(store.initialized()).resolves.toBe(true) + + expect(fetchMock).toHaveBeenCalledTimes(1) + expect(fetchMock).toHaveBeenCalledWith(`${source}/.zmetadata`) + + expect(store.describe()).toEqual({ + metadata: { metadata: zmetadata.metadata }, + dimensions: ['time', 'y', 'x'], + shape: [2, 256, 256], + chunks: [1, 256, 256], + fill_value: -9999, + dtype: ' { + const source = 'https://example.com/v2-cached' + const coordinateValues = [2000, 2001] - expect(fetchMock).toHaveBeenCalledTimes(1) - expect(fetchMock).toHaveBeenCalledWith(`${source}/.zmetadata`) + const zmetadata = createV2Metadata() + registerChunk(`${source}/0/time`, coordinateValues) - expect(store.describe()).toEqual({ - metadata: { metadata: zmetadata.metadata }, - dimensions: ['time', 'y', 'x'], - shape: [2, 256, 256], - chunks: [1, 256, 256], - fill_value: -9999, - dtype: ' { - const source = 'https://example.com/v3' - const metadata = createV3Metadata() - const arrayMetadata = createV3ArrayMetadata() + fetchMock.mockClear() - fetchMock - .mockResolvedValueOnce(mockJsonResponse(metadata)) - .mockResolvedValueOnce(mockJsonResponse(arrayMetadata)) + const secondStore = new ZarrStore({ + source, + version: 'v2', + variable: 'temp', + coordinateKeys: ['time'], + }) - const store = new ZarrStore({ - source, - version: 'v3', - variable: 'temp', + await expect(secondStore.initialized()).resolves.toBe(true) + expect(fetchMock).not.toHaveBeenCalled() }) - await expect(store.initialized()).resolves.toBe(true) - - expect(fetchMock).toHaveBeenNthCalledWith(1, `${source}/zarr.json`) - expect(fetchMock).toHaveBeenNthCalledWith(2, `${source}/0/temp/zarr.json`) - - expect(store.describe()).toEqual({ - metadata, - dimensions: ['time', 'y', 'x'], - shape: [2, 256, 256], - chunks: [1, 256, 256], - fill_value: 0, - dtype: ' { + const source = 'https://example.com/v2-chunks' + const zmetadata = createV2Metadata() + const chunkValues = [1, 2, 3, 4] + + fetchMock.mockResolvedValueOnce(mockJsonResponse(zmetadata)) + const chunkHandler = registerChunk(`${source}/0/temp`, chunkValues) + + const store = new ZarrStore({ + source, + version: 'v2', + variable: 'temp', + }) + + await expect(store.initialized()).resolves.toBe(true) + + const firstChunk = await store.getChunk('0/temp', [0, 0]) + expect(firstChunk.data).toEqual(new Float32Array(chunkValues)) + + const secondChunk = await store.getChunk('0/temp', [0, 1]) + expect(secondChunk.data).toEqual(new Float32Array(chunkValues)) + + expect(zarr).toHaveBeenCalledTimes(1) + expect(zarr.__openMock).toHaveBeenCalledTimes(1) + expect(zarr.__openMock).toHaveBeenCalledWith( + `${source}/0/temp`, + expect.any(Function), + expect.anything() + ) + expect(chunkHandler).toHaveBeenCalledTimes(2) + expect(chunkHandler).toHaveBeenNthCalledWith( + 1, + [0, 0], + expect.any(Function) + ) + expect(chunkHandler).toHaveBeenNthCalledWith( + 2, + [0, 1], + expect.any(Function) + ) }) + }) - fetchMock.mockClear() - - const secondStore = new ZarrStore({ - source, - version: 'v3', - variable: 'temp', + describe('v3', () => { + it('loads metadata and coordinates arrays on initialization', async () => { + const source = 'https://example.com/v3' + const metadata = createV3Metadata() + const arrayMetadata = createV3ArrayMetadata() + + fetchMock + .mockResolvedValueOnce(mockJsonResponse(metadata)) + .mockResolvedValueOnce(mockJsonResponse(arrayMetadata)) + + const store = new ZarrStore({ + source, + version: 'v3', + variable: 'temp', + }) + + await expect(store.initialized()).resolves.toBe(true) + + expect(fetchMock).toHaveBeenNthCalledWith(1, `${source}/zarr.json`) + expect(fetchMock).toHaveBeenNthCalledWith(2, `${source}/0/temp/zarr.json`) + + expect(store.describe()).toEqual({ + metadata, + dimensions: ['time', 'y', 'x'], + shape: [2, 256, 256], + chunks: [1, 256, 256], + fill_value: 0, + dtype: ' { - expect(() => new ZarrStore({ variable: 'temp' })).toThrow( - 'source is a required parameter' - ) - expect(() => new ZarrStore({ source: 'https://example.com' })).toThrow( - 'variable is a required parameter' - ) - }) + it('reuses cached metadata', async () => { + const source = 'https://example.com/v3-cached' + const metadata = createV3Metadata() + const arrayMetadata = createV3ArrayMetadata() - it('reuses cached getter functions when requesting the same chunk key', async () => { - const source = 'https://example.com/v2' - const zmetadata = createV2Metadata() - const chunkValues = [1, 2, 3, 4] + fetchMock + .mockResolvedValueOnce(mockJsonResponse(metadata)) + .mockResolvedValueOnce(mockJsonResponse(arrayMetadata)) - fetchMock.mockResolvedValueOnce(mockJsonResponse(zmetadata)) - const chunkHandler = registerChunk(`${source}/0/temp`, chunkValues) + const firstStore = new ZarrStore({ + source, + version: 'v3', + variable: 'temp', + }) - const store = new ZarrStore({ - source, - version: 'v2', - variable: 'temp', - }) + await expect(firstStore.initialized()).resolves.toBe(true) + expect(fetchMock).toHaveBeenCalledTimes(2) - await expect(store.initialized()).resolves.toBe(true) + fetchMock.mockClear() - const firstChunk = await store.getChunk('0/temp', [0, 0]) - expect(firstChunk.data).toEqual(new Float32Array(chunkValues)) + const secondStore = new ZarrStore({ + source, + version: 'v3', + variable: 'temp', + }) - const secondChunk = await store.getChunk('0/temp', [0, 1]) - expect(secondChunk.data).toEqual(new Float32Array(chunkValues)) + await expect(secondStore.initialized()).resolves.toBe(true) + expect(fetchMock).not.toHaveBeenCalled() + }) - expect(zarr).toHaveBeenCalledTimes(1) - expect(zarr.__openMock).toHaveBeenCalledTimes(1) - expect(zarr.__openMock).toHaveBeenCalledWith( - `${source}/0/temp`, - expect.any(Function), - expect.anything() - ) - expect(chunkHandler).toHaveBeenCalledTimes(2) - expect(chunkHandler).toHaveBeenNthCalledWith( - 1, - [0, 0], - expect.any(Function) - ) - expect(chunkHandler).toHaveBeenNthCalledWith( - 2, - [0, 1], - expect.any(Function) - ) + it('returns cached values when requesting the same chunk', async () => { + const source = 'https://example.com/v3-chunks' + const metadata = createV3Metadata() + const arrayMetadata = createV3ArrayMetadata() + const chunkValues = [5, 6, 7, 8] + + fetchMock + .mockResolvedValueOnce(mockJsonResponse(metadata)) + .mockResolvedValueOnce(mockJsonResponse(arrayMetadata)) + const chunkHandler = registerChunk(`${source}/0/temp`, chunkValues) + + const store = new ZarrStore({ + source, + version: 'v3', + variable: 'temp', + }) + + await expect(store.initialized()).resolves.toBe(true) + + const firstChunk = await store.getChunk('0/temp', [0, 0]) + expect(firstChunk.data).toEqual(new Float32Array(chunkValues)) + + const secondChunk = await store.getChunk('0/temp', [0, 1]) + expect(secondChunk.data).toEqual(new Float32Array(chunkValues)) + + expect(zarr).toHaveBeenCalledTimes(1) + expect(zarr.__openMock).toHaveBeenCalledTimes(1) + expect(zarr.__openMock).toHaveBeenCalledWith( + `${source}/0/temp`, + expect.any(Function), + expect.anything() + ) + expect(chunkHandler).toHaveBeenCalledTimes(2) + expect(chunkHandler).toHaveBeenNthCalledWith( + 1, + [0, 0], + expect.any(Function) + ) + expect(chunkHandler).toHaveBeenNthCalledWith( + 2, + [0, 1], + expect.any(Function) + ) + }) }) })