Add TIFF Key-Value Store Implementation (take 2)

tensorstore/driver/BUILD

@@ -23,6 +23,7 @@ DRIVERS = [
     "virtual_chunked",
     "zarr",
     "zarr3",
+    "tiff",
 ]
 
 DOCTEST_SOURCES = glob([

tensorstore/driver/index.rst

@@ -37,6 +37,7 @@ Chunked storage drivers
    zarr3/index
    n5/index
    neuroglancer_precomputed/index
+   tiff/index
 
 .. json:schema:: KeyValueStoreBackedChunkDriver
 

tensorstore/driver/tiff/BUILD

@@ -0,0 +1,219 @@
+load("//bazel:tensorstore.bzl", "tensorstore_cc_library", "tensorstore_cc_test")
+load("//docs:doctest.bzl", "doctest_test")
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+DOCTEST_SOURCES = glob([
+    "**/*.rst",
+    "**/*.yml",
+])
+
+doctest_test(
+    name = "doctest_test",
+    srcs = DOCTEST_SOURCES,
+)
+
+filegroup(
+    name = "doc_sources",
+    srcs = DOCTEST_SOURCES,
+)
+
+tensorstore_cc_library(
+    name = "tiff",
+    deps = [
+        ":driver",
+        ":zlib_compressor",
+        ":zstd_compressor",
+    ],
+)
+
+tensorstore_cc_library(
+    name = "driver",
+    srcs = ["driver.cc"],
+    deps = [
+        ":metadata",
+        "//tensorstore:array",
+        "//tensorstore:chunk_layout",
+        "//tensorstore:index",
+        "//tensorstore/driver",
+        "//tensorstore/driver:chunk_cache_driver",
+        "//tensorstore/driver:kvs_backed_chunk_driver",
+        "//tensorstore/internal/cache:async_cache",
+        "//tensorstore/internal/cache:kvs_backed_chunk_cache",
+        "//tensorstore/internal/json_binding:staleness_bound",
+        "//tensorstore/kvstore",
+        "//tensorstore/kvstore:generation",
+        "//tensorstore/kvstore/tiff:tiff_key_value_store",
+        "//tensorstore/util:result",
+        "//tensorstore/util:status",
+        "//tensorstore/util:str_cat",
+        "//tensorstore/util/execution",
+        "//tensorstore/util/execution:any_receiver",
+        "//tensorstore/util/garbage_collection",
+        "@com_github_nlohmann_json//:json",
+        "@com_google_absl//absl/log:absl_log",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings:cord",
+    ],
+    alwayslink = 1,
+)
+
+tensorstore_cc_library(
+    name = "metadata",
+    srcs = ["metadata.cc"],
+    hdrs = ["metadata.h"],
+    deps = [
+        ":compressor",
+        ":zlib_compressor",
+        ":zstd_compressor",
+        "//tensorstore:chunk_layout",
+        "//tensorstore:codec_spec",
+        "//tensorstore:data_type",
+        "//tensorstore:index",
+        "//tensorstore:schema",
+        "//tensorstore/index_space:dimension_units",
+        "//tensorstore/internal/log:verbose_flag",
+        "//tensorstore/internal/riegeli:array_endian_codec",
+        "//tensorstore/kvstore/tiff:tiff_details",
+        "//tensorstore/kvstore/tiff:tiff_dir_cache",
+        "//tensorstore/util/garbage_collection",
+        "@com_github_nlohmann_json//:json",
+        "@com_google_absl//absl/log:absl_log",
+        "@com_google_riegeli//riegeli/bytes:cord_reader",
+    ],
+    alwayslink = 1,
+)
+
+tensorstore_cc_test(
+    name = "metadata_test",
+    size = "small",
+    srcs = ["metadata_test.cc"],
+    deps = [
+        ":compressor",
+        ":metadata",
+        "//tensorstore:array",
+        "//tensorstore:codec_spec",
+        "//tensorstore:data_type",
+        "//tensorstore:index",
+        "//tensorstore/internal:json_gtest",
+        "//tensorstore/internal/json_binding:gtest",
+        "//tensorstore/internal/riegeli:array_endian_codec",
+        "//tensorstore/kvstore/tiff:tiff_details",
+        "//tensorstore/kvstore/tiff:tiff_dir_cache",
+        "//tensorstore/util:status_testutil",
+        "@com_github_nlohmann_json//:json",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings:cord",
+        "@com_google_googletest//:gtest_main",
+        "@com_google_riegeli//riegeli/bytes:cord_reader",
+        "@com_google_riegeli//riegeli/bytes:cord_writer",
+    ],
+)
+
+tensorstore_cc_library(
+    name = "compressor",
+    srcs = ["compressor.cc"],
+    hdrs = [
+        "compressor.h",
+        "compressor_registry.h",
+    ],
+    deps = [
+        "//tensorstore/internal:json_registry",
+        "//tensorstore/internal/compression:json_specified_compressor",
+        "//tensorstore/kvstore/tiff:tiff_details",
+        "@com_google_absl//absl/container:flat_hash_map",
+    ],
+)
+
+tensorstore_cc_test(
+    name = "driver_test",
+    size = "small",
+    srcs = ["driver_test.cc"],
+    deps = [
+        ":driver",
+        ":metadata",
+        "//tensorstore:array",
+        "//tensorstore:codec_spec",
+        "//tensorstore:data_type",
+        "//tensorstore:index",
+        "//tensorstore/driver:driver_testutil",
+        "//tensorstore/internal:global_initializer",
+        "//tensorstore/internal:json_gtest",
+        "//tensorstore/internal/json_binding:gtest",
+        "//tensorstore/internal/riegeli:array_endian_codec",
+        "//tensorstore/kvstore",
+        "//tensorstore/kvstore:test_matchers",
+        "//tensorstore/kvstore:test_util",
+        "//tensorstore/kvstore/memory",
+        "//tensorstore/kvstore/tiff:tiff_details",
+        "//tensorstore/kvstore/tiff:tiff_dir_cache",
+        "//tensorstore/kvstore/tiff:tiff_test_util",
+        "//tensorstore/util:status_testutil",
+        "@com_github_nlohmann_json//:json",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings:cord",
+        "@com_google_googletest//:gtest_main",
+        "@com_google_riegeli//riegeli/bytes:cord_reader",
+        "@com_google_riegeli//riegeli/bytes:cord_writer",
+    ],
+)
+
+tensorstore_cc_test(
+    name = "golden_file_test",
+    size = "small",
+    srcs = ["golden_file_test.cc"],
+    args = [
+        "--tensorstore_test_data_dir=" +
+        package_name() + "/testdata",
+    ],
+    data = [":testdata"],
+    deps = [
+        ":driver",
+        "//tensorstore",
+        "//tensorstore:array",
+        "//tensorstore:context",
+        "//tensorstore:index",
+        "//tensorstore:open",
+        "//tensorstore:open_mode",
+        "//tensorstore/internal:path",
+        "//tensorstore/kvstore/file",
+        "//tensorstore/util:status_testutil",
+        "@com_google_absl//absl/flags:flag",
+        "@com_google_absl//absl/log:absl_log",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
+filegroup(
+    name = "testdata",
+    srcs = glob(
+        include = [
+            "testdata/**",
+        ],
+        exclude = ["testdata/*.py"],
+    ),
+)
+
+tensorstore_cc_library(
+    name = "zstd_compressor",
+    srcs = ["zstd_compressor.cc"],
+    deps = [
+        ":compressor",
+        "//tensorstore/internal/compression:zstd_compressor",
+        "//tensorstore/internal/json_binding",
+    ],
+    alwayslink = 1,
+)
+
+tensorstore_cc_library(
+    name = "zlib_compressor",
+    srcs = ["zlib_compressor.cc"],
+    deps = [
+        ":compressor",
+        "//tensorstore/internal/compression:zlib_compressor",
+        "//tensorstore/internal/json_binding",
+    ],
+    alwayslink = 1,
+)

tensorstore/driver/tiff/compressor.cc

@@ -0,0 +1,72 @@
+// Copyright 2025 The TensorStore Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tensorstore/driver/tiff/compressor.h"  // For Compressor alias declaration
+
+#include <string>
+#include <string_view>
+#include <utility>
+
+#include "absl/base/no_destructor.h"
+#include "absl/container/flat_hash_map.h"
+#include "tensorstore/driver/tiff/compressor_registry.h"
+#include "tensorstore/internal/compression/json_specified_compressor.h"
+#include "tensorstore/internal/json_binding/bindable.h"
+#include "tensorstore/internal/json_binding/enum.h"
+#include "tensorstore/internal/json_binding/json_binding.h"
+#include "tensorstore/internal/json_registry.h"
+#include "tensorstore/kvstore/tiff/tiff_details.h"
+
+namespace tensorstore {
+namespace internal_tiff {
+
+namespace jb = tensorstore::internal_json_binding;
+using ::tensorstore::internal_tiff_kvstore::CompressionType;
+
+internal::JsonSpecifiedCompressor::Registry& GetTiffCompressorRegistry() {
+  static absl::NoDestructor<internal::JsonSpecifiedCompressor::Registry>
+      registry;
+  return *registry;
+}
+
+// Defines the mapping from TIFF numeric tag values to the string IDs used
+// for compressor registration and CodecSpec JSON representation.
+const static auto* const kCompressionTypeToStringIdMap =
+    new absl::flat_hash_map<CompressionType, std::string_view>{
+        {CompressionType::kNone, "raw"},         // No compression
+        {CompressionType::kZStd, "zstd"},        // Zstandard compression
+        {CompressionType::kDeflate, "zlib"},  // Deflate/Zlib compression.
+        // { CompressionType::kPackBits, "packbits" },
+    };
+
+const absl::flat_hash_map<CompressionType, std::string_view>&
+GetTiffCompressionMap() {
+  return *kCompressionTypeToStringIdMap;
+}
+
+TENSORSTORE_DEFINE_JSON_DEFAULT_BINDER(Compressor, [](auto is_loading,
+                                                      const auto& options,
+                                                      auto* obj, auto* j) {
+  auto& registry = GetTiffCompressorRegistry();
+  return jb::Object(
+      jb::Member("type",
+                 jb::MapValue(
+                     registry.KeyBinder(),
+                     // Map "raw" to a default-constructed Compressor (nullptr)
+                     std::make_pair(Compressor{}, std::string("raw")))),
+      registry.RegisteredObjectBinder())(is_loading, options, obj, j);
+})
+
+}  // namespace internal_tiff
+}  // namespace tensorstore

tensorstore/driver/tiff/compressor.h

@@ -0,0 +1,34 @@
+// Copyright 2025 The TensorStore Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TENSORSTORE_DRIVER_TIFF_COMPRESSOR_H_
+#define TENSORSTORE_DRIVER_TIFF_COMPRESSOR_H_
+
+// Include the base class required by the JsonSpecifiedCompressor registry
+#include "tensorstore/internal/compression/json_specified_compressor.h"
+#include "tensorstore/internal/json_binding/bindable.h"  // For binder macro
+
+namespace tensorstore {
+namespace internal_tiff {
+
+class Compressor : public internal::JsonSpecifiedCompressor::Ptr {
+ public:
+  TENSORSTORE_DECLARE_JSON_DEFAULT_BINDER(
+      Compressor, internal::JsonSpecifiedCompressor::FromJsonOptions,
+      internal::JsonSpecifiedCompressor::ToJsonOptions)
+};
+}  // namespace internal_tiff
+}  // namespace tensorstore
+
+#endif  // TENSORSTORE_DRIVER_TIFF_COMPRESSOR_H_

tensorstore/driver/tiff/compressor_registry.h

@@ -0,0 +1,46 @@
+// Copyright 2025 The TensorStore Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TENSORSTORE_DRIVER_TIFF_COMPRESSOR_REGISTRY_H_
+#define TENSORSTORE_DRIVER_TIFF_COMPRESSOR_REGISTRY_H_
+
+#include <string_view>
+
+#include "absl/container/flat_hash_map.h"
+#include "tensorstore/internal/compression/json_specified_compressor.h"
+#include "tensorstore/internal/json_registry.h"
+#include "tensorstore/kvstore/tiff/tiff_details.h"
+
+namespace tensorstore {
+namespace internal_tiff {
+
+// Returns the global registry instance for TIFF compressors.
+// This registry maps string IDs (like "lzw", "deflate") to factories/binders
+// capable of creating JsonSpecifiedCompressor instances.
+internal::JsonSpecifiedCompressor::Registry& GetTiffCompressorRegistry();
+
+// Returns the map from TIFF Compression tag enum to string ID.
+const absl::flat_hash_map<internal_tiff_kvstore::CompressionType,
+                          std::string_view>&
+GetTiffCompressionMap();
+
+template <typename T, typename Binder>
+void RegisterCompressor(std::string_view id, Binder binder) {
+  GetTiffCompressorRegistry().Register<T>(id, binder);
+}
+
+}  // namespace internal_tiff
+}  // namespace tensorstore
+
+#endif  // TENSORSTORE_DRIVER_TIFF_COMPRESSOR_REGISTRY_H_