refac: replaced Manifest defined by protobuf

.gitmodules

@@ -26,9 +26,6 @@
 	path = thirdparty/glog/glog-0.5.0
 	url = https://github.com/google/glog.git
 	ignore = all
-[submodule "thirdparty/protobuf/protobuf-3.21.12"]
-	path = thirdparty/protobuf/protobuf-3.21.12
-	url = https://github.com/protocolbuffers/protobuf.git
 [submodule "thirdparty/lz4/lz4-1.9.4"]
 	path = thirdparty/lz4/lz4-1.9.4
 	url = https://github.com/lz4/lz4.git

examples/c++/CMakeLists.txt

@@ -37,15 +37,13 @@ if(CMAKE_BUILD_TYPE STREQUAL "Debug")
     else ()
         set(GFLAGS_LIB gflags_nothreads_debug)
     endif ()
-    set(PROTOBUF_LIB protobufd)
 else()
     set(GLOG_LIB glog)
     if (WIN32)
         set(GFLAGS_LIB gflags_nothreads_static)
     else ()
         set(GFLAGS_LIB gflags_nothreads)
     endif ()
-    set(PROTOBUF_LIB protobuf)
 endif()
 
 # --- Dependency groups ---
@@ -77,12 +75,10 @@ if (NOT WIN32)
             antlr4-runtime
             ${GLOG_LIB}
             ${GFLAGS_LIB}
-            ${PROTOBUF_LIB}
             lz4
     )
 else ()
     # Windows static libraries use different naming conventions
-    set(PROTOBUF_LIB libprotobuf)
     set(zvec_ailego_deps
             arrow_static
             parquet_static
@@ -102,7 +98,6 @@ else ()
             antlr4-runtime-static
             ${GLOG_LIB}
             ${GFLAGS_LIB}
-            ${PROTOBUF_LIB}
             lz4
             rpcrt4
             shlwapi

examples/c/CMakeLists.txt

@@ -50,11 +50,9 @@ find_package(Threads REQUIRED)
 if(CMAKE_BUILD_TYPE STREQUAL "Debug")
     set(GLOG_LIB glogd)
     set(GFLAGS_LIB gflags_nothreads_debug)
-    set(PROTOBUF_LIB protobufd)
 else()
     set(GLOG_LIB glog)
     set(GFLAGS_LIB gflags_nothreads)
-    set(PROTOBUF_LIB protobuf)
 endif()
 
 # --- Dependency groups ---
@@ -71,14 +69,12 @@ if(NOT WIN32)
         antlr4-runtime
         ${GLOG_LIB}
         ${GFLAGS_LIB}
-        ${PROTOBUF_LIB}
         lz4
         ${CMAKE_THREAD_LIBS_INIT}
         ${CMAKE_DL_LIBS}
     )
 else()
     # Windows static libraries use different naming conventions
-    set(PROTOBUF_LIB libprotobuf)
     set(zvec_c_api_deps
         roaring
         rocksdb
@@ -91,7 +87,6 @@ else()
         antlr4-runtime-static
         ${GLOG_LIB}
         ${GFLAGS_LIB}
-        ${PROTOBUF_LIB}
         lz4
         ${CMAKE_THREAD_LIBS_INIT}
         rpcrt4

src/binding/c/CMakeLists.txt

@@ -136,7 +136,6 @@ elseif(APPLE)
             Arrow::arrow_acero
             rocksdb
             glog
-            libprotobuf
             antlr4
     )
 

src/db/CMakeLists.txt

@@ -1,12 +1,6 @@
 include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
 include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
 
-cc_proto_library(
-  NAME zvec_proto STATIC
-  SRCS proto/*.proto
-  PROTOROOT ./
-)
-
 cc_directory(common)
 cc_directory(index)
 cc_directory(sqlengine)
@@ -15,8 +9,8 @@ file(GLOB_RECURSE ALL_DB_SRCS *.cc *.c *.h)
 
 cc_library(
   NAME zvec_db STATIC STRICT SRCS_NO_GLOB PACKED
-  SRCS ${ALL_DB_SRCS} ${CMAKE_CURRENT_BINARY_DIR}/proto/zvec.pb.cc
-  INCS . ${CMAKE_CURRENT_BINARY_DIR}
+  SRCS ${ALL_DB_SRCS}
+  INCS .
   PUBINCS ${PROJECT_ROOT_DIR}/src/include
   LIBS
     zvec_ailego
@@ -25,11 +19,9 @@ cc_library(
     roaring
     rocksdb
     antlr4
-    libprotobuf
     Arrow::arrow_static
     Arrow::arrow_compute
     Arrow::arrow_dataset
     Arrow::arrow_acero
-  DEPS zvec_proto
   VERSION "${PROXIMA_ZVEC_VERSION}"
 )

src/db/common/binary_codec.h

@@ -0,0 +1,155 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#include <cstdint>
+#include <cstring>
+#include <string>
+#include <vector>
+
+namespace zvec {
+
+// CRC32 implementation (IEEE polynomial, same as zlib)
+class CRC32 {
+ public:
+  static uint32_t Compute(const void *data, size_t length) {
+    const uint8_t *bytes = static_cast<const uint8_t *>(data);
+    uint32_t crc = 0xFFFFFFFF;
+    for (size_t i = 0; i < length; ++i) {
+      crc ^= bytes[i];
+      for (int j = 0; j < 8; ++j) {
+        crc = (crc >> 1) ^ (0xEDB88320 & (-(crc & 1)));
+      }
+    }
+    return crc ^ 0xFFFFFFFF;
+  }
+};
+
+// Binary writer: appends data to an internal buffer
+class BinaryWriter {
+ public:
+  void PutUint8(uint8_t value) {
+    buffer_.push_back(value);
+  }
+
+  void PutUint32(uint32_t value) {
+    const size_t offset = buffer_.size();
+    buffer_.resize(offset + sizeof(uint32_t));
+    std::memcpy(buffer_.data() + offset, &value, sizeof(uint32_t));
+  }
+
+  void PutUint64(uint64_t value) {
+    const size_t offset = buffer_.size();
+    buffer_.resize(offset + sizeof(uint64_t));
+    std::memcpy(buffer_.data() + offset, &value, sizeof(uint64_t));
+  }
+
+  void PutInt32(int32_t value) {
+    PutUint32(static_cast<uint32_t>(value));
+  }
+
+  void PutBool(bool value) {
+    PutUint8(value ? 1 : 0);
+  }
+
+  void PutString(const std::string &value) {
+    PutUint32(static_cast<uint32_t>(value.size()));
+    if (!value.empty()) {
+      const size_t offset = buffer_.size();
+      buffer_.resize(offset + value.size());
+      std::memcpy(buffer_.data() + offset, value.data(), value.size());
+    }
+  }
+
+  const std::vector<uint8_t> &buffer() const {
+    return buffer_;
+  }
+
+  size_t size() const {
+    return buffer_.size();
+  }
+
+  const uint8_t *data() const {
+    return buffer_.data();
+  }
+
+ private:
+  std::vector<uint8_t> buffer_;
+};
+
+// Binary reader: reads data from a byte buffer
+class BinaryReader {
+ public:
+  BinaryReader(const uint8_t *data, size_t size)
+      : data_(data), size_(size), offset_(0) {}
+
+  bool GetUint8(uint8_t *value) {
+    if (offset_ + sizeof(uint8_t) > size_) return false;
+    *value = data_[offset_];
+    offset_ += sizeof(uint8_t);
+    return true;
+  }
+
+  bool GetUint32(uint32_t *value) {
+    if (offset_ + sizeof(uint32_t) > size_) return false;
+    std::memcpy(value, data_ + offset_, sizeof(uint32_t));
+    offset_ += sizeof(uint32_t);
+    return true;
+  }
+
+  bool GetUint64(uint64_t *value) {
+    if (offset_ + sizeof(uint64_t) > size_) return false;
+    std::memcpy(value, data_ + offset_, sizeof(uint64_t));
+    offset_ += sizeof(uint64_t);
+    return true;
+  }
+
+  bool GetInt32(int32_t *value) {
+    uint32_t raw;
+    if (!GetUint32(&raw)) return false;
+    *value = static_cast<int32_t>(raw);
+    return true;
+  }
+
+  bool GetBool(bool *value) {
+    uint8_t raw;
+    if (!GetUint8(&raw)) return false;
+    *value = (raw != 0);
+    return true;
+  }
+
+  bool GetString(std::string *value) {
+    uint32_t length;
+    if (!GetUint32(&length)) return false;
+    if (offset_ + length > size_) return false;
+    value->assign(reinterpret_cast<const char *>(data_ + offset_), length);
+    offset_ += length;
+    return true;
+  }
+
+  size_t offset() const {
+    return offset_;
+  }
+
+  size_t remaining() const {
+    return size_ - offset_;
+  }
+
+ private:
+  const uint8_t *data_;
+  size_t size_;
+  size_t offset_;
+};
+
+}  // namespace zvec

src/db/index/CMakeLists.txt

@@ -5,7 +5,6 @@ cc_library(
     NAME zvec_index STATIC STRICT
     SRCS *.cc segment/*.cc column/vector_column/*.cc column/inverted_column/*.cc storage/*.cc storage/wal/*.cc common/*.cc
     LIBS zvec_common
-         zvec_proto
          rocksdb
          core_interface
          Arrow::arrow_static