From 204f726170fe29934a8443d592afca833cc7151f Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sat, 15 Nov 2025 09:14:43 +0100 Subject: [PATCH 01/99] Update usage error for server to use remove data folder URL --- crates/modelardb_server/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/modelardb_server/src/main.rs b/crates/modelardb_server/src/main.rs index 784ada52..e3cd876f 100644 --- a/crates/modelardb_server/src/main.rs +++ b/crates/modelardb_server/src/main.rs @@ -68,7 +68,7 @@ async fn main() -> Result<()> { { cluster_mode_and_data_folders } else { - print_usage_and_exit_with_error("[server_mode] local_data_folder_url [manager_url]"); + print_usage_and_exit_with_error("[server_mode] local_data_folder_url [remote_data_folder_url]"); }; let context = Arc::new(Context::try_new(data_folders, cluster_mode.clone()).await?); From 6caa15fa94e5f1b154efcf1ff86b63b600c7f771 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sat, 15 Nov 2025 09:20:35 +0100 Subject: [PATCH 02/99] Re-order deployment options --- crates/modelardb_server/src/data_folders.rs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/crates/modelardb_server/src/data_folders.rs b/crates/modelardb_server/src/data_folders.rs index f495eccc..45044a8c 100644 --- a/crates/modelardb_server/src/data_folders.rs +++ b/crates/modelardb_server/src/data_folders.rs @@ -58,6 +58,7 @@ impl DataFolders { ) -> Result<(ClusterMode, Self)> { // Match the provided command line arguments to the supported inputs. match arguments { + // Single edge without a cluster. &["edge", local_data_folder_url] | &[local_data_folder_url] => { let local_data_folder = DataFolder::open_local_url(local_data_folder_url).await?; @@ -66,9 +67,11 @@ impl DataFolders { Self::new(local_data_folder.clone(), None, local_data_folder), )) } - &["cloud", local_data_folder_url, manager_url] => { + // Edge node in a cluster. + &["edge", local_data_folder_url, remote_data_folder_url] + | &[local_data_folder_url, remote_data_folder_url] => { let (manager, storage_configuration) = - Manager::register_node(manager_url, ServerMode::Cloud).await?; + Manager::register_node(remote_data_folder_url, ServerMode::Edge).await?; let local_data_folder = DataFolder::open_local_url(local_data_folder_url).await?; @@ -78,16 +81,16 @@ impl DataFolders { Ok(( ClusterMode::MultiNode(manager), Self::new( + local_data_folder.clone(), + Some(remote_data_folder), local_data_folder, - Some(remote_data_folder.clone()), - remote_data_folder, ), )) } - &["edge", local_data_folder_url, manager_url] - | &[local_data_folder_url, manager_url] => { + // Cloud node in a cluster. + &["cloud", local_data_folder_url, remote_data_folder_url] => { let (manager, storage_configuration) = - Manager::register_node(manager_url, ServerMode::Edge).await?; + Manager::register_node(remote_data_folder_url, ServerMode::Cloud).await?; let local_data_folder = DataFolder::open_local_url(local_data_folder_url).await?; @@ -97,9 +100,9 @@ impl DataFolders { Ok(( ClusterMode::MultiNode(manager), Self::new( - local_data_folder.clone(), - Some(remote_data_folder), local_data_folder, + Some(remote_data_folder.clone()), + remote_data_folder, ), )) } From 2b73043ba00758ff6d6a17878e8fb4402277104f Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sat, 15 Nov 2025 16:35:31 +0100 Subject: [PATCH 03/99] Add EnvironmentVar to storage errors --- crates/modelardb_storage/src/error.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/modelardb_storage/src/error.rs b/crates/modelardb_storage/src/error.rs index b5ebbbb0..cd39ca8e 100644 --- a/crates/modelardb_storage/src/error.rs +++ b/crates/modelardb_storage/src/error.rs @@ -15,6 +15,7 @@ //! The [`Error`] and [`Result`] types used throughout `modelardb_storage`. +use std::env::VarError; use std::error::Error; use std::fmt::{Display, Formatter}; use std::io::Error as IoError; @@ -42,6 +43,8 @@ pub enum ModelarDbStorageError { DataFusion(DataFusionError), /// Error returned by Delta Lake. DeltaLake(DeltaTableError), + /// Error returned by environment variables. + EnvironmentVar(VarError), /// Error returned when an invalid argument was passed. InvalidArgument(String), /// Error returned from IO operations. @@ -66,6 +69,7 @@ impl Display for ModelarDbStorageError { Self::Arrow(reason) => write!(f, "Arrow Error: {reason}"), Self::DataFusion(reason) => write!(f, "DataFusion Error: {reason}"), Self::DeltaLake(reason) => write!(f, "Delta Lake Error: {reason}"), + Self::EnvironmentVar(reason) => write!(f, "Environment Variable Error: {reason}"), Self::InvalidArgument(reason) => write!(f, "Invalid Argument Error: {reason}"), Self::Io(reason) => write!(f, "Io Error: {reason}"), Self::ObjectStore(reason) => write!(f, "Object Store Error: {reason}"), @@ -85,6 +89,7 @@ impl Error for ModelarDbStorageError { Self::Arrow(reason) => Some(reason), Self::DataFusion(reason) => Some(reason), Self::DeltaLake(reason) => Some(reason), + Self::EnvironmentVar(reason) => Some(reason), Self::InvalidArgument(_reason) => None, Self::Io(reason) => Some(reason), Self::ObjectStore(reason) => Some(reason), @@ -115,6 +120,12 @@ impl From for ModelarDbStorageError { } } +impl From for ModelarDbStorageError { + fn from(error: VarError) -> Self { + Self::EnvironmentVar(error) + } +} + impl From for ModelarDbStorageError { fn from(error: IoError) -> Self { Self::Io(error) From 7d979bd902b8ee704358ac210c479ae6791153ce Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sat, 15 Nov 2025 16:38:15 +0100 Subject: [PATCH 04/99] Update open_object_store to use remote_url instead of StorageConfiguration --- crates/modelardb_storage/src/data_folder.rs | 46 +++++++++++---------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/crates/modelardb_storage/src/data_folder.rs b/crates/modelardb_storage/src/data_folder.rs index 2ae4d5cc..b05429d9 100644 --- a/crates/modelardb_storage/src/data_folder.rs +++ b/crates/modelardb_storage/src/data_folder.rs @@ -16,9 +16,9 @@ //! Implementation of the type used to interact with local and remote storage through a Delta Lake. use std::collections::HashMap; -use std::fs; use std::path::Path as StdPath; use std::sync::Arc; +use std::{env, fs}; use arrow::array::{ ArrayRef, ArrowPrimitiveType, BinaryArray, BooleanArray, Float32Array, Int16Array, RecordBatch, @@ -45,7 +45,6 @@ use deltalake::operations::write::writer::{DeltaWriter, WriterConfig}; use deltalake::protocol::{DeltaOperation, SaveMode}; use deltalake::{DeltaOps, DeltaTable, DeltaTableError}; use futures::{StreamExt, TryStreamExt}; -use modelardb_types::flight::protocol; use modelardb_types::functions::{try_convert_bytes_to_schema, try_convert_schema_to_bytes}; use modelardb_types::schemas::{COMPRESSED_SCHEMA, FIELD_COLUMN}; use modelardb_types::types::{ @@ -151,13 +150,16 @@ impl DataFolder { } /// Create a new [`DataFolder`] that manages Delta tables in the remote object store given by - /// `storage_configuration`. Returns [`ModelarDbStorageError`] if a connection to the specified - /// object store could not be created. - pub async fn open_object_store( - storage_configuration: protocol::manager_metadata::StorageConfiguration, - ) -> Result { - match storage_configuration { - protocol::manager_metadata::StorageConfiguration::S3Configuration(s3_configuration) => { + /// `remote_url`. Note that only the object store type and bucket/container name are given in + /// the url. The remaining parameters are retrieved from environment variables. Returns + /// [`ModelarDbStorageError`] if a connection to the specified object store could not be created. + pub async fn open_object_store(remote_url: &str) -> Result { + match remote_url.split_once("://") { + Some(("s3", bucket_name)) => { + let endpoint = env::var("AWS_ENDPOINT")?; + let access_key_id = env::var("AWS_ACCESS_KEY_ID")?; + let secret_access_key = env::var("AWS_SECRET_ACCESS_KEY")?; + // Register the S3 storage handlers to allow the use of Amazon S3 object stores. This is // required at runtime to initialize the S3 storage implementation in the deltalake_aws // storage subcrate. It is safe to call this function multiple times as the handlers are @@ -165,23 +167,23 @@ impl DataFolder { deltalake::aws::register_handlers(None); Self::open_s3( - s3_configuration.endpoint, - s3_configuration.bucket_name, - s3_configuration.access_key_id, - s3_configuration.secret_access_key, + endpoint, + bucket_name.to_owned(), + access_key_id, + secret_access_key, ) .await } - protocol::manager_metadata::StorageConfiguration::AzureConfiguration( - azure_configuration, - ) => { - Self::open_azure( - azure_configuration.account_name, - azure_configuration.access_key, - azure_configuration.container_name, - ) - .await + Some(("azureblobstorage", container_name)) => { + let account_name = env::var("AZURE_STORAGE_ACCOUNT_NAME")?; + let access_key = env::var("AZURE_STORAGE_ACCESS_KEY")?; + + Self::open_azure(account_name, access_key, container_name.to_owned()).await } + _ => Err(ModelarDbStorageError::InvalidArgument( + "Remote data folder URL must be s3://bucket-name or azureblobstorage://container-name." + .to_owned(), + )), } } From ef80caaa3996df862734ff52e54768b7fc6968c1 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sat, 15 Nov 2025 16:41:12 +0100 Subject: [PATCH 05/99] Be specific with what objects stores are supported --- crates/modelardb_storage/src/data_folder.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/modelardb_storage/src/data_folder.rs b/crates/modelardb_storage/src/data_folder.rs index b05429d9..da534279 100644 --- a/crates/modelardb_storage/src/data_folder.rs +++ b/crates/modelardb_storage/src/data_folder.rs @@ -150,8 +150,9 @@ impl DataFolder { } /// Create a new [`DataFolder`] that manages Delta tables in the remote object store given by - /// `remote_url`. Note that only the object store type and bucket/container name are given in - /// the url. The remaining parameters are retrieved from environment variables. Returns + /// `remote_url`. If `remote_url` has the schema `s3`, the object store is an Amazon S3 bucket. + /// If `remote_url` has the schema `azureblobstorage`, the object store is an Azure Blob Storage + /// container. The remaining parameters are retrieved from environment variables. Returns /// [`ModelarDbStorageError`] if a connection to the specified object store could not be created. pub async fn open_object_store(remote_url: &str) -> Result { match remote_url.split_once("://") { From e41a649066c7c05d02dbdcf561dc024d2d86c9ec Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sat, 15 Nov 2025 16:42:24 +0100 Subject: [PATCH 06/99] Rename to open_remote_url for consistency with open_local_url --- crates/modelardb_manager/src/main.rs | 2 +- crates/modelardb_storage/src/data_folder.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/modelardb_manager/src/main.rs b/crates/modelardb_manager/src/main.rs index 8e0525f7..984a65e8 100644 --- a/crates/modelardb_manager/src/main.rs +++ b/crates/modelardb_manager/src/main.rs @@ -72,7 +72,7 @@ async fn main() -> Result<()> { let remote_storage_configuration = modelardb_types::flight::argument_to_storage_configuration(remote_data_folder_str)?; let remote_data_folder = - DataFolder::open_object_store(remote_storage_configuration.clone()).await?; + DataFolder::open_remote_url(remote_storage_configuration.clone()).await?; remote_data_folder .create_and_register_manager_metadata_data_folder_tables() diff --git a/crates/modelardb_storage/src/data_folder.rs b/crates/modelardb_storage/src/data_folder.rs index da534279..8f660fba 100644 --- a/crates/modelardb_storage/src/data_folder.rs +++ b/crates/modelardb_storage/src/data_folder.rs @@ -154,7 +154,7 @@ impl DataFolder { /// If `remote_url` has the schema `azureblobstorage`, the object store is an Azure Blob Storage /// container. The remaining parameters are retrieved from environment variables. Returns /// [`ModelarDbStorageError`] if a connection to the specified object store could not be created. - pub async fn open_object_store(remote_url: &str) -> Result { + pub async fn open_remote_url(remote_url: &str) -> Result { match remote_url.split_once("://") { Some(("s3", bucket_name)) => { let endpoint = env::var("AWS_ENDPOINT")?; From f2c57225e69d3abc4637556951cc68545d9c77a6 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sat, 15 Nov 2025 16:45:03 +0100 Subject: [PATCH 07/99] No longer use Manager::register_node --- crates/modelardb_server/src/data_folders.rs | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/crates/modelardb_server/src/data_folders.rs b/crates/modelardb_server/src/data_folders.rs index 45044a8c..bcb8a7e3 100644 --- a/crates/modelardb_server/src/data_folders.rs +++ b/crates/modelardb_server/src/data_folders.rs @@ -16,12 +16,10 @@ //! Implementation of a struct that provides access to the local and remote data storage components. use modelardb_storage::data_folder::DataFolder; -use modelardb_types::types::ServerMode; use crate::ClusterMode; use crate::Result; use crate::error::ModelarDbServerError; -use crate::manager::Manager; /// Folders for storing metadata and data in Apache Parquet files locally and remotely. #[derive(Clone)] @@ -70,16 +68,13 @@ impl DataFolders { // Edge node in a cluster. &["edge", local_data_folder_url, remote_data_folder_url] | &[local_data_folder_url, remote_data_folder_url] => { - let (manager, storage_configuration) = - Manager::register_node(remote_data_folder_url, ServerMode::Edge).await?; - let local_data_folder = DataFolder::open_local_url(local_data_folder_url).await?; let remote_data_folder = - DataFolder::open_object_store(storage_configuration).await?; + DataFolder::open_remote_url(remote_data_folder_url).await?; Ok(( - ClusterMode::MultiNode(manager), + ClusterMode::MultiNode, Self::new( local_data_folder.clone(), Some(remote_data_folder), @@ -89,16 +84,13 @@ impl DataFolders { } // Cloud node in a cluster. &["cloud", local_data_folder_url, remote_data_folder_url] => { - let (manager, storage_configuration) = - Manager::register_node(remote_data_folder_url, ServerMode::Cloud).await?; - let local_data_folder = DataFolder::open_local_url(local_data_folder_url).await?; let remote_data_folder = - DataFolder::open_object_store(storage_configuration).await?; + DataFolder::open_remote_url(remote_data_folder_url).await?; Ok(( - ClusterMode::MultiNode(manager), + ClusterMode::MultiNode, Self::new( local_data_folder, Some(remote_data_folder.clone()), From f9207784194d19437c58e1d09ed2c882517d775f Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sat, 15 Nov 2025 16:53:51 +0100 Subject: [PATCH 08/99] Remove ManagerMetadata and StorageConfiguration from protobuf --- .../modelardb_types/src/flight/protocol.proto | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/crates/modelardb_types/src/flight/protocol.proto b/crates/modelardb_types/src/flight/protocol.proto index 85242ac1..900b4a34 100644 --- a/crates/modelardb_types/src/flight/protocol.proto +++ b/crates/modelardb_types/src/flight/protocol.proto @@ -17,33 +17,6 @@ syntax = "proto3"; package modelardb.flight.protocol; -// Metadata for the ModelarDB cluster manager, including its unique key and storage configuration. -message ManagerMetadata { - // Key used to uniquely identify the cluster manager. - string key = 1; - - // Storage configuration used to connect to an S3 object store. - message S3Configuration { - string endpoint = 1; - string bucket_name = 2; - string access_key_id = 3; - string secret_access_key = 4; - } - - // Storage configuration used to connect to an Azure Blob Storage object store. - message AzureConfiguration { - string account_name = 1; - string access_key = 2; - string container_name = 3; - } - - // Storage configuration used by the cluster manager. - oneof storage_configuration { - S3Configuration s3_configuration = 2; - AzureConfiguration azure_configuration = 3; - } -} - // Metadata for a node in the ModelarDB cluster, including its URL and server mode. message NodeMetadata { enum ServerMode { From 118b2f99cac06750a3a40a48df091ece6305c4fd Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sat, 15 Nov 2025 16:59:19 +0100 Subject: [PATCH 09/99] Remove argument_to_storage_configuration since it is unused --- crates/modelardb_types/src/flight/mod.rs | 112 ----------------------- 1 file changed, 112 deletions(-) diff --git a/crates/modelardb_types/src/flight/mod.rs b/crates/modelardb_types/src/flight/mod.rs index 88325f74..a9d0d2b9 100644 --- a/crates/modelardb_types/src/flight/mod.rs +++ b/crates/modelardb_types/src/flight/mod.rs @@ -17,7 +17,6 @@ //! defined in `flight/protocol.proto`. The module also provides functions to serialize and //! deserialize encoded messages to and from bytes. -use std::env; use std::sync::Arc; use arrow::datatypes::Schema; @@ -35,49 +34,6 @@ pub mod protocol { include!(concat!(env!("OUT_DIR"), "/modelardb.flight.protocol.rs")); } -/// Parse `argument` and encode it into a [`StorageConfiguration`](protocol::manager_metadata::StorageConfiguration) -/// protobuf message. If `argument` is not a valid remote data folder, return [`ModelarDbTypesError`]. -pub fn argument_to_storage_configuration( - argument: &str, -) -> Result { - match argument.split_once("://") { - Some(("s3", bucket_name)) => { - let endpoint = env::var("AWS_ENDPOINT")?; - let access_key_id = env::var("AWS_ACCESS_KEY_ID")?; - let secret_access_key = env::var("AWS_SECRET_ACCESS_KEY")?; - - Ok( - protocol::manager_metadata::StorageConfiguration::S3Configuration( - protocol::manager_metadata::S3Configuration { - endpoint, - bucket_name: bucket_name.to_owned(), - access_key_id, - secret_access_key, - }, - ), - ) - } - Some(("azureblobstorage", container_name)) => { - let account_name = env::var("AZURE_STORAGE_ACCOUNT_NAME")?; - let access_key = env::var("AZURE_STORAGE_ACCESS_KEY")?; - - Ok( - protocol::manager_metadata::StorageConfiguration::AzureConfiguration( - protocol::manager_metadata::AzureConfiguration { - account_name, - access_key, - container_name: container_name.to_owned(), - }, - ), - ) - } - _ => Err(ModelarDbTypesError::InvalidArgument( - "Remote data folder must be s3://bucket-name or azureblobstorage://container-name." - .to_owned(), - )), - } -} - /// Encode `node` into a [`NodeMetadata`](protocol::NodeMetadata) protobuf message. pub fn encode_node(node: &Node) -> Result { let server_mode = match node.mode { @@ -293,79 +249,11 @@ fn decode_generated_column_expressions( mod test { use super::*; - use std::sync::{LazyLock, Mutex}; - use modelardb_test::table::{ self, NORMAL_TABLE_NAME, TIME_SERIES_TABLE_NAME, normal_table_schema, time_series_table_metadata, }; - /// Lock used for env::set_var() as it is not guaranteed to be thread-safe. - static SET_VAR_LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); - - // Tests for argument_to_storage_configuration(). - #[test] - fn test_s3_argument_to_storage_configuration() { - // env::set_var is safe to call in a single-threaded program. - unsafe { - let _mutex_guard = SET_VAR_LOCK.lock(); - env::set_var("AWS_ENDPOINT", "test_endpoint"); - env::set_var("AWS_ACCESS_KEY_ID", "test_access_key_id"); - env::set_var("AWS_SECRET_ACCESS_KEY", "test_secret_access_key"); - } - - let storage_configuration = - argument_to_storage_configuration("s3://test_bucket_name").unwrap(); - - match storage_configuration { - protocol::manager_metadata::StorageConfiguration::S3Configuration(s3_configuration) => { - assert_eq!(s3_configuration.endpoint, "test_endpoint"); - assert_eq!(s3_configuration.bucket_name, "test_bucket_name"); - assert_eq!(s3_configuration.access_key_id, "test_access_key_id"); - assert_eq!(s3_configuration.secret_access_key, "test_secret_access_key"); - } - _ => panic!("Expected S3 connection type."), - } - } - - #[test] - fn test_azureblobstorage_argument_to_storage_configuration() { - // env::set_var is safe to call in a single-threaded program. - unsafe { - let _mutex_guard = SET_VAR_LOCK.lock(); - env::set_var("AZURE_STORAGE_ACCOUNT_NAME", "test_storage_account_name"); - env::set_var("AZURE_STORAGE_ACCESS_KEY", "test_storage_access_key"); - } - - let storage_configuration = - argument_to_storage_configuration("azureblobstorage://test_container_name").unwrap(); - - match storage_configuration { - protocol::manager_metadata::StorageConfiguration::AzureConfiguration( - azure_configuration, - ) => { - assert_eq!( - azure_configuration.account_name, - "test_storage_account_name" - ); - assert_eq!(azure_configuration.access_key, "test_storage_access_key"); - assert_eq!(azure_configuration.container_name, "test_container_name"); - } - _ => panic!("Expected Azure connection type."), - } - } - - #[test] - fn test_invalid_argument_to_storage_configuration() { - let result = argument_to_storage_configuration("googlecloudstorage://test"); - - assert_eq!( - result.unwrap_err().to_string(), - "Invalid Argument Error: Remote data folder must be s3://bucket-name or azureblobstorage://container-name." - .to_owned() - ); - } - // Test for encode_node() and decode_node_metadata(). #[test] fn test_encode_and_decode_node_metadata() { From b29e51de82694a61f10a22369ce3efb3a8d4ef22 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sat, 15 Nov 2025 17:03:36 +0100 Subject: [PATCH 10/99] Remove encode_node and decode_node_metadata since they are unused. --- crates/modelardb_types/src/flight/mod.rs | 42 ------------------------ 1 file changed, 42 deletions(-) diff --git a/crates/modelardb_types/src/flight/mod.rs b/crates/modelardb_types/src/flight/mod.rs index a9d0d2b9..1dc504fa 100644 --- a/crates/modelardb_types/src/flight/mod.rs +++ b/crates/modelardb_types/src/flight/mod.rs @@ -34,36 +34,6 @@ pub mod protocol { include!(concat!(env!("OUT_DIR"), "/modelardb.flight.protocol.rs")); } -/// Encode `node` into a [`NodeMetadata`](protocol::NodeMetadata) protobuf message. -pub fn encode_node(node: &Node) -> Result { - let server_mode = match node.mode { - ServerMode::Edge => protocol::node_metadata::ServerMode::Edge, - ServerMode::Cloud => protocol::node_metadata::ServerMode::Cloud, - }; - - Ok(protocol::NodeMetadata { - url: node.url.clone(), - server_mode: server_mode as i32, - }) -} - -/// Decode a [`NodeMetadata`](protocol::NodeMetadata) protobuf message into a [`Node`]. -pub fn decode_node_metadata(node_metadata: &protocol::NodeMetadata) -> Result { - let server_mode = match protocol::node_metadata::ServerMode::try_from(node_metadata.server_mode) - { - Ok(protocol::node_metadata::ServerMode::Edge) => ServerMode::Edge, - Ok(protocol::node_metadata::ServerMode::Cloud) => ServerMode::Cloud, - _ => { - return Err(ModelarDbTypesError::InvalidArgument(format!( - "Unknown server mode: {}.", - node_metadata.server_mode - ))); - } - }; - - Ok(Node::new(node_metadata.url.clone(), server_mode)) -} - /// Encode the metadata for a normal table into a [`TableMetadata`](protocol::TableMetadata) /// protobuf message and serialize it. If the schema cannot be converted to bytes, return /// [`ModelarDbTypesError`]. @@ -254,18 +224,6 @@ mod test { time_series_table_metadata, }; - // Test for encode_node() and decode_node_metadata(). - #[test] - fn test_encode_and_decode_node_metadata() { - let node = Node::new("grpc://server:9999".to_string(), ServerMode::Edge); - - let encoded_node_metadata = encode_node(&node).unwrap(); - let decoded_node = decode_node_metadata(&encoded_node_metadata).unwrap(); - - assert_eq!(node.url, decoded_node.url); - assert_eq!(node.mode, decoded_node.mode); - } - // Tests for serializing and deserializing table metadata. #[test] fn test_serialize_and_deserialize_normal_table_metadata() { From ad616163257c4565cc8036999b8e6cbec9d047f7 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sat, 15 Nov 2025 17:14:24 +0100 Subject: [PATCH 11/99] Remove NodeMetadata from protocol buffers since it is no longer used --- crates/modelardb_types/src/flight/mod.rs | 2 +- crates/modelardb_types/src/flight/protocol.proto | 14 -------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/crates/modelardb_types/src/flight/mod.rs b/crates/modelardb_types/src/flight/mod.rs index 1dc504fa..aa077450 100644 --- a/crates/modelardb_types/src/flight/mod.rs +++ b/crates/modelardb_types/src/flight/mod.rs @@ -28,7 +28,7 @@ use prost::bytes::Bytes; use crate::error::{ModelarDbTypesError, Result}; use crate::functions::{try_convert_bytes_to_schema, try_convert_schema_to_bytes}; -use crate::types::{ErrorBound, GeneratedColumn, Node, ServerMode, Table, TimeSeriesTableMetadata}; +use crate::types::{ErrorBound, GeneratedColumn, Table, TimeSeriesTableMetadata}; pub mod protocol { include!(concat!(env!("OUT_DIR"), "/modelardb.flight.protocol.rs")); diff --git a/crates/modelardb_types/src/flight/protocol.proto b/crates/modelardb_types/src/flight/protocol.proto index 900b4a34..d26b19e2 100644 --- a/crates/modelardb_types/src/flight/protocol.proto +++ b/crates/modelardb_types/src/flight/protocol.proto @@ -17,20 +17,6 @@ syntax = "proto3"; package modelardb.flight.protocol; -// Metadata for a node in the ModelarDB cluster, including its URL and server mode. -message NodeMetadata { - enum ServerMode { - CLOUD = 0; - EDGE = 1; - } - - // gRPC URL of the node. - string url = 1; - - // Mode indicating whether the node is a cloud or edge server. - ServerMode server_mode = 2; -} - // Metadata for a normal table or a time series table. message TableMetadata { // Metadata for a normal table, including its name and schema. From 6b78c8e2e73defa82400d5624556a339cb248d3f Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sat, 15 Nov 2025 17:26:27 +0100 Subject: [PATCH 12/99] Update DataFolders tests to no longer mention manager --- crates/modelardb_server/src/data_folders.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/crates/modelardb_server/src/data_folders.rs b/crates/modelardb_server/src/data_folders.rs index bcb8a7e3..851a4c19 100644 --- a/crates/modelardb_server/src/data_folders.rs +++ b/crates/modelardb_server/src/data_folders.rs @@ -68,11 +68,11 @@ impl DataFolders { // Edge node in a cluster. &["edge", local_data_folder_url, remote_data_folder_url] | &[local_data_folder_url, remote_data_folder_url] => { - let local_data_folder = DataFolder::open_local_url(local_data_folder_url).await?; - let remote_data_folder = DataFolder::open_remote_url(remote_data_folder_url).await?; + let local_data_folder = DataFolder::open_local_url(local_data_folder_url).await?; + Ok(( ClusterMode::MultiNode, Self::new( @@ -84,11 +84,11 @@ impl DataFolders { } // Cloud node in a cluster. &["cloud", local_data_folder_url, remote_data_folder_url] => { - let local_data_folder = DataFolder::open_local_url(local_data_folder_url).await?; - let remote_data_folder = DataFolder::open_remote_url(remote_data_folder_url).await?; + let local_data_folder = DataFolder::open_local_url(local_data_folder_url).await?; + Ok(( ClusterMode::MultiNode, Self::new( @@ -121,7 +121,7 @@ mod tests { } #[tokio::test] - async fn test_try_from_edge_command_line_arguments_without_manager() { + async fn test_try_from_edge_command_line_arguments_without_remote_url() { let temp_dir = tempfile::tempdir().unwrap(); let temp_dir_str = temp_dir.path().to_str().unwrap(); @@ -129,7 +129,7 @@ mod tests { } #[tokio::test] - async fn test_try_from_edge_command_line_arguments_without_server_mode_and_manager() { + async fn test_try_from_edge_command_line_arguments_without_server_mode_and_remote_url() { let temp_dir = tempfile::tempdir().unwrap(); let temp_dir_str = temp_dir.path().to_str().unwrap(); @@ -154,9 +154,8 @@ mod tests { assert_eq!( result.err().unwrap().to_string(), - format!( - "Invalid Argument Error: Could not connect to manager at '{temp_dir_str}': transport error", - ) + "ModelarDB Storage Error: Invalid Argument Error: Remote data folder URL must be \ + s3://bucket-name or azureblobstorage://container-name.", ); } } From 6c35561c43dfd9b361355a6081841260338fe12c Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sat, 15 Nov 2025 18:17:41 +0100 Subject: [PATCH 13/99] Add file structure for cluster struct --- crates/modelardb_server/src/cluster/mod.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 crates/modelardb_server/src/cluster/mod.rs diff --git a/crates/modelardb_server/src/cluster/mod.rs b/crates/modelardb_server/src/cluster/mod.rs new file mode 100644 index 00000000..e726063f --- /dev/null +++ b/crates/modelardb_server/src/cluster/mod.rs @@ -0,0 +1,22 @@ +/* Copyright 2025 The ModelarDB Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +pub struct Cluster; + +impl Cluster { + pub fn new() -> Self { + Self {} + } +} From c94e16a39f9f1fcc195937808b67253a58234cf3 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sat, 15 Nov 2025 18:32:07 +0100 Subject: [PATCH 14/99] Move data_folder.rs into module folder --- crates/modelardb_server/src/cluster/mod.rs | 22 ------------------- .../{data_folder.rs => data_folder/mod.rs} | 0 2 files changed, 22 deletions(-) delete mode 100644 crates/modelardb_server/src/cluster/mod.rs rename crates/modelardb_storage/src/{data_folder.rs => data_folder/mod.rs} (100%) diff --git a/crates/modelardb_server/src/cluster/mod.rs b/crates/modelardb_server/src/cluster/mod.rs deleted file mode 100644 index e726063f..00000000 --- a/crates/modelardb_server/src/cluster/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -/* Copyright 2025 The ModelarDB Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -pub struct Cluster; - -impl Cluster { - pub fn new() -> Self { - Self {} - } -} diff --git a/crates/modelardb_storage/src/data_folder.rs b/crates/modelardb_storage/src/data_folder/mod.rs similarity index 100% rename from crates/modelardb_storage/src/data_folder.rs rename to crates/modelardb_storage/src/data_folder/mod.rs From 4c6c4598d81bd7f0821abd60f6b8cd7c73f0f769 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sat, 15 Nov 2025 18:47:33 +0100 Subject: [PATCH 15/99] Move and rename ManagerMetadata trait to ClusterMetadata --- .../src/data_folder/cluster.rs | 306 ++++++++++++++++++ .../modelardb_storage/src/data_folder/mod.rs | 2 + 2 files changed, 308 insertions(+) create mode 100644 crates/modelardb_storage/src/data_folder/cluster.rs diff --git a/crates/modelardb_storage/src/data_folder/cluster.rs b/crates/modelardb_storage/src/data_folder/cluster.rs new file mode 100644 index 00000000..7c9d22d7 --- /dev/null +++ b/crates/modelardb_storage/src/data_folder/cluster.rs @@ -0,0 +1,306 @@ +/* Copyright 2025 The ModelarDB Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Management of the Delta Lake for the cluster. Metadata which is unique to the cluster, such as +//! the key and the nodes, are handled here. + +use std::str::FromStr; +use std::sync::Arc; + +use arrow::array::{Array, StringArray}; +use arrow::datatypes::{DataType, Field, Schema}; +use deltalake::DeltaTableError; +use deltalake::datafusion::logical_expr::{col, lit}; +use modelardb_types::types::{Node, ServerMode}; +use uuid::Uuid; + +use crate::data_folder::DataFolder; +use crate::error::Result; +use crate::{register_metadata_table, sql_and_concat}; + +/// Trait that extends [`DataFolder`] to provide management of the Delta Lake for the cluster. +pub trait ClusterMetadata { + async fn create_and_register_cluster_metadata_tables(&self) -> Result<()>; + async fn cluster_key(&self) -> Result; + async fn save_node(&self, node: Node) -> Result<()>; + async fn remove_node(&self, url: &str) -> Result<()>; + async fn nodes(&self) -> Result>; +} + +impl ClusterMetadata for DataFolder { + /// If they do not already exist, create the tables that are specific to the cluster and + /// register them with the Apache DataFusion session context. + /// * The `cluster_metadata` table contains metadata for the cluster itself. It is assumed that + /// this table will only have a single row since there can only be a single cluster. + /// * The `nodes` table contains metadata for each node that is in the cluster. + /// + /// If the tables exist or were created, return [`Ok`], otherwise return + /// [`ModelarDbStorageError`](crate::error::ModelarDbStorageError). + async fn create_and_register_cluster_metadata_tables(&self) -> Result<()> { + // Create and register the cluster_metadata table if it does not exist. + let delta_table = self + .create_metadata_table( + "cluster_metadata", + &Schema::new(vec![Field::new("key", DataType::Utf8, false)]), + ) + .await?; + + register_metadata_table(self.session_context(), "cluster_metadata", delta_table)?; + + // Create and register the nodes table if it does not exist. + let delta_table = self + .create_metadata_table( + "nodes", + &Schema::new(vec![ + Field::new("url", DataType::Utf8, false), + Field::new("mode", DataType::Utf8, false), + ]), + ) + .await?; + + register_metadata_table(self.session_context(), "nodes", delta_table)?; + + Ok(()) + } + + /// Retrieve the key for the cluster from the `cluster_metadata` table. If a key does not + /// already exist, create one and save it to the Delta Lake. If a key could not be retrieved + /// or created, return [`ModelarDbStorageError`](crate::error::ModelarDbStorageError). + async fn cluster_key(&self) -> Result { + let sql = "SELECT key FROM metadata.cluster_metadata"; + let batch = sql_and_concat(self.session_context(), sql).await?; + + let keys = modelardb_types::array!(batch, 0, StringArray); + if keys.is_empty() { + let cluster_key = Uuid::new_v4(); + + // Add a new row to the cluster_metadata table to persist the key. + self.write_columns_to_metadata_table( + "cluster_metadata", + vec![Arc::new(StringArray::from(vec![cluster_key.to_string()]))], + ) + .await?; + + Ok(cluster_key) + } else { + let cluster_key: String = keys.value(0).to_owned(); + + Ok(cluster_key + .parse() + .map_err(|error: uuid::Error| DeltaTableError::Generic(error.to_string()))?) + } + } + + /// Save the node to the Delta Lake and return [`Ok`]. If the node could not be saved, return + /// [`ModelarDbStorageError`](crate::error::ModelarDbStorageError). + async fn save_node(&self, node: Node) -> Result<()> { + self.write_columns_to_metadata_table( + "nodes", + vec![ + Arc::new(StringArray::from(vec![node.url])), + Arc::new(StringArray::from(vec![node.mode.to_string()])), + ], + ) + .await?; + + Ok(()) + } + + /// Remove the row in the `nodes` table that corresponds to the node with `url` and return + /// [`Ok`]. If the row could not be removed, return + /// [`ModelarDbStorageError`](crate::error::ModelarDbStorageError). + async fn remove_node(&self, url: &str) -> Result<()> { + let delta_ops = self.metadata_delta_ops("nodes").await?; + + delta_ops + .delete() + .with_predicate(col("url").eq(lit(url))) + .await?; + + Ok(()) + } + + /// Return the nodes currently in the cluster that have been persisted to the Delta Lake. If the + /// nodes could not be retrieved, [`ModelarDbStorageError`](crate::error::ModelarDbStorageError) + /// is returned. + async fn nodes(&self) -> Result> { + let mut nodes: Vec = vec![]; + + let sql = "SELECT url, mode FROM metadata.nodes"; + let batch = sql_and_concat(self.session_context(), sql).await?; + + let url_array = modelardb_types::array!(batch, 0, StringArray); + let mode_array = modelardb_types::array!(batch, 1, StringArray); + + for row_index in 0..batch.num_rows() { + let url = url_array.value(row_index).to_owned(); + let mode = mode_array.value(row_index).to_owned(); + + let server_mode = ServerMode::from_str(&mode) + .map_err(|error| DeltaTableError::Generic(error.to_string()))?; + + nodes.push(Node::new(url, server_mode)); + } + + Ok(nodes) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use tempfile::TempDir; + + // Tests for ClusterMetadata. + #[tokio::test] + async fn test_create_cluster_metadata_tables() { + let (_temp_dir, data_folder) = create_data_folder().await; + + // Verify that the tables were created, registered, and has the expected columns. + assert!( + data_folder + .session_context() + .sql("SELECT key FROM metadata.cluster_metadata") + .await + .is_ok() + ); + + assert!( + data_folder + .session_context() + .sql("SELECT url, mode FROM metadata.nodes") + .await + .is_ok() + ); + } + + #[tokio::test] + async fn test_new_cluster_key() { + let (_temp_dir, data_folder) = create_data_folder().await; + + // Verify that the cluster key is created and saved correctly. + let cluster_key = data_folder.cluster_key().await.unwrap(); + + let sql = "SELECT key FROM metadata.cluster_metadata"; + let batch = sql_and_concat(data_folder.session_context(), sql) + .await + .unwrap(); + + assert_eq!( + **batch.column(0), + StringArray::from(vec![cluster_key.to_string()]) + ); + } + + #[tokio::test] + async fn test_existing_cluster_key() { + let (_temp_dir, data_folder) = create_data_folder().await; + + // Verify that only a single key is created and saved when retrieving multiple times. + let cluster_key_1 = data_folder.cluster_key().await.unwrap(); + let cluster_key_2 = data_folder.cluster_key().await.unwrap(); + + let sql = "SELECT key FROM metadata.cluster_metadata"; + let batch = sql_and_concat(data_folder.session_context(), sql) + .await + .unwrap(); + + assert_eq!(cluster_key_1, cluster_key_2); + assert_eq!(batch.column(0).len(), 1); + } + + #[tokio::test] + async fn test_save_node() { + let (_temp_dir, data_folder) = create_data_folder().await; + + let node_1 = Node::new("url_1".to_string(), ServerMode::Edge); + data_folder.save_node(node_1.clone()).await.unwrap(); + + let node_2 = Node::new("url_2".to_string(), ServerMode::Edge); + data_folder.save_node(node_2.clone()).await.unwrap(); + + // Verify that the nodes are saved correctly. + let sql = "SELECT url, mode FROM metadata.nodes"; + let batch = sql_and_concat(data_folder.session_context(), sql) + .await + .unwrap(); + + assert_eq!( + **batch.column(0), + StringArray::from(vec![node_2.url.clone(), node_1.url.clone()]) + ); + assert_eq!( + **batch.column(1), + StringArray::from(vec![node_2.mode.to_string(), node_1.mode.to_string()]) + ); + } + + #[tokio::test] + async fn test_remove_node() { + let (_temp_dir, data_folder) = create_data_folder().await; + + let node_1 = Node::new("url_1".to_string(), ServerMode::Edge); + data_folder.save_node(node_1.clone()).await.unwrap(); + + let node_2 = Node::new("url_2".to_string(), ServerMode::Edge); + data_folder.save_node(node_2.clone()).await.unwrap(); + + data_folder.remove_node(&node_1.url).await.unwrap(); + + // Verify that node_1 is removed correctly. + let sql = "SELECT url, mode FROM metadata.nodes"; + let batch = sql_and_concat(data_folder.session_context(), sql) + .await + .unwrap(); + + assert_eq!( + **batch.column(0), + StringArray::from(vec![node_2.url.clone()]) + ); + assert_eq!( + **batch.column(1), + StringArray::from(vec![node_2.mode.to_string()]) + ); + } + + #[tokio::test] + async fn test_nodes() { + let (_temp_dir, data_folder) = create_data_folder().await; + + let node_1 = Node::new("url_1".to_string(), ServerMode::Edge); + data_folder.save_node(node_1.clone()).await.unwrap(); + + let node_2 = Node::new("url_2".to_string(), ServerMode::Edge); + data_folder.save_node(node_2.clone()).await.unwrap(); + + let nodes = data_folder.nodes().await.unwrap(); + + assert_eq!(nodes, vec![node_2, node_1]); + } + + async fn create_data_folder() -> (TempDir, DataFolder) { + let temp_dir = tempfile::tempdir().unwrap(); + + let data_folder = DataFolder::open_local(temp_dir.path()).await.unwrap(); + + data_folder + .create_and_register_cluster_metadata_tables() + .await + .unwrap(); + + (temp_dir, data_folder) + } +} diff --git a/crates/modelardb_storage/src/data_folder/mod.rs b/crates/modelardb_storage/src/data_folder/mod.rs index 8f660fba..c4982e7b 100644 --- a/crates/modelardb_storage/src/data_folder/mod.rs +++ b/crates/modelardb_storage/src/data_folder/mod.rs @@ -15,6 +15,8 @@ //! Implementation of the type used to interact with local and remote storage through a Delta Lake. +pub mod cluster; + use std::collections::HashMap; use std::path::Path as StdPath; use std::sync::Arc; From ea362f4c2939c90e9e06445d103ef2709a69b810 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sat, 15 Nov 2025 21:59:13 +0100 Subject: [PATCH 16/99] Added basic structure for new Cluster struct --- crates/modelardb_server/src/cluster.rs | 47 ++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 crates/modelardb_server/src/cluster.rs diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs new file mode 100644 index 00000000..a3a5c1bc --- /dev/null +++ b/crates/modelardb_server/src/cluster.rs @@ -0,0 +1,47 @@ +/* Copyright 2025 The ModelarDB Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Functionality to perform operations on every node in the cluster. + +use modelardb_storage::data_folder::DataFolder; +use modelardb_storage::data_folder::cluster::ClusterMetadata; + +use crate::error::Result; + +/// Stores the currently managed nodes in the cluster and allows for performing operations that need +/// to be applied to every single node in the cluster. +pub struct Cluster { + /// Key identifying the cluster. The key is used to validate communication within the cluster + /// between nodes. + key: String, + /// The remote data folder that each node in the cluster should be synchronized with. + /// When a table is created, dropped, vacuumed, or truncated, it is done in the + /// remote data folder first. + remote_data_folder: DataFolder, +} + +impl Cluster { + /// Try to retrieve the cluster key from the remote data folder and create a new cluster + /// instance. If the cluster key could not be retrieved from the remote data folder, return + /// [`ModelarDbServerError`](crate::error::ModelarDbServerError). + pub async fn try_new(remote_data_folder: DataFolder) -> Result { + let key = remote_data_folder.cluster_key().await?.to_string(); + + Ok(Self { + key, + remote_data_folder, + }) + } +} From dcc17d29d29875f641e0c8f9ba220ef7e8a921be Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 08:57:22 +0100 Subject: [PATCH 17/99] Add method to get the most capable query node --- Cargo.lock | 1 + crates/modelardb_server/Cargo.toml | 1 + crates/modelardb_server/src/cluster.rs | 26 ++++++++++++++++++++++++-- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3e9bbcb2..97ab260c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3338,6 +3338,7 @@ dependencies = [ "object_store", "proptest", "prost", + "rand 0.9.2", "snmalloc-rs", "tempfile", "tokio", diff --git a/crates/modelardb_server/Cargo.toml b/crates/modelardb_server/Cargo.toml index 4e8e8de7..a56f44d6 100644 --- a/crates/modelardb_server/Cargo.toml +++ b/crates/modelardb_server/Cargo.toml @@ -38,6 +38,7 @@ modelardb_storage = { path = "../modelardb_storage" } modelardb_types = { path = "../modelardb_types" } object_store = { workspace = true, features = ["aws", "azure"] } prost.workspace = true +rand.workspace = true snmalloc-rs = { workspace = true, features = ["build_cc"] } tokio = { workspace = true, features = ["rt-multi-thread", "signal"] } tokio-stream.workspace = true diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index a3a5c1bc..cd04f8c0 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -15,10 +15,14 @@ //! Functionality to perform operations on every node in the cluster. +use rand::rng; +use rand::seq::IteratorRandom; + use modelardb_storage::data_folder::DataFolder; use modelardb_storage::data_folder::cluster::ClusterMetadata; +use modelardb_types::types::Node; -use crate::error::Result; +use crate::error::{ModelarDbServerError, Result}; /// Stores the currently managed nodes in the cluster and allows for performing operations that need /// to be applied to every single node in the cluster. @@ -35,7 +39,7 @@ pub struct Cluster { impl Cluster { /// Try to retrieve the cluster key from the remote data folder and create a new cluster /// instance. If the cluster key could not be retrieved from the remote data folder, return - /// [`ModelarDbServerError`](crate::error::ModelarDbServerError). + /// [`ModelarDbServerError`]. pub async fn try_new(remote_data_folder: DataFolder) -> Result { let key = remote_data_folder.cluster_key().await?.to_string(); @@ -44,4 +48,22 @@ impl Cluster { remote_data_folder, }) } + + /// Return the cloud node in the cluster that is currently most capable of running a query. + /// Note that the most capable node is currently selected at random. If there are no cloud nodes + /// in the cluster, return [`ModelarDbServerError`]. + pub async fn query_node(&mut self) -> Result { + let nodes = self.remote_data_folder.nodes().await?; + + let cloud_nodes = nodes + .iter() + .filter(|n| n.mode == modelardb_types::types::ServerMode::Cloud); + + let mut rng = rng(); + cloud_nodes.choose(&mut rng).cloned().ok_or_else(|| { + ModelarDbServerError::InvalidState( + "There are no cloud nodes to execute the query in the cluster.".to_owned(), + ) + }) + } } From da316614720aac6a62fc5ecbc69e5eab2003f378 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 09:09:42 +0100 Subject: [PATCH 18/99] Add cluster_do_get to new cluster struct --- crates/modelardb_server/src/cluster.rs | 55 ++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index cd04f8c0..eb6c9ebe 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -15,12 +15,20 @@ //! Functionality to perform operations on every node in the cluster. -use rand::rng; -use rand::seq::IteratorRandom; +use std::str::FromStr; +use arrow_flight::Ticket; +use arrow_flight::flight_service_client::FlightServiceClient; +use futures::StreamExt; +use futures::stream::FuturesUnordered; +use log::info; use modelardb_storage::data_folder::DataFolder; use modelardb_storage::data_folder::cluster::ClusterMetadata; use modelardb_types::types::Node; +use rand::rng; +use rand::seq::IteratorRandom; +use tonic::Request; +use tonic::metadata::{Ascii, MetadataValue}; use crate::error::{ModelarDbServerError, Result}; @@ -29,7 +37,7 @@ use crate::error::{ModelarDbServerError, Result}; pub struct Cluster { /// Key identifying the cluster. The key is used to validate communication within the cluster /// between nodes. - key: String, + key: MetadataValue, /// The remote data folder that each node in the cluster should be synchronized with. /// When a table is created, dropped, vacuumed, or truncated, it is done in the /// remote data folder first. @@ -43,6 +51,9 @@ impl Cluster { pub async fn try_new(remote_data_folder: DataFolder) -> Result { let key = remote_data_folder.cluster_key().await?.to_string(); + // Convert the key to a MetadataValue since it is used in tonic requests. + let key = MetadataValue::from_str(&key).expect("UUID Version 4 should be valid ASCII."); + Ok(Self { key, remote_data_folder, @@ -66,4 +77,42 @@ impl Cluster { ) }) } + + /// For each node in the cluster, execute the given `sql` statement with the cluster key as + /// metadata. If the statement was successfully executed for each node, return [`Ok`], otherwise + /// return [`ModelarDbServerError`]. + pub async fn cluster_do_get(&self, sql: &str) -> Result<()> { + let nodes = self.remote_data_folder.nodes().await?; + + let mut do_get_futures: FuturesUnordered<_> = nodes + .iter() + .map(|node| self.connect_and_do_get(&node.url, sql)) + .collect(); + + // TODO: Fix issue where we return immediately if we encounter an error. If it is a + // connection error, we either need to retry later or remove the node. + // Run the futures concurrently and log when the statement has been executed on each node. + while let Some(result) = do_get_futures.next().await { + info!("Executed statement `{sql}` on node with url '{}'.", result?); + } + + Ok(()) + } + + /// Connect to the Apache Arrow Flight server given by `url` and execute the given `sql` + /// statement with the key as metadata. If the statement was successfully executed, return the + /// url of the node to simplify logging, otherwise return [`ModelarDbServerError`]. + async fn connect_and_do_get(&self, url: &str, sql: &str) -> Result { + let mut flight_client = FlightServiceClient::connect(url.to_owned()).await?; + + // Add the key to the request metadata to indicate that the request is a cluster operation. + let mut request = Request::new(Ticket::new(sql.to_owned())); + request + .metadata_mut() + .insert("x-cluster-key", self.key.clone()); + + flight_client.do_get(request).await?; + + Ok(url.to_owned()) + } } From 132179f70019d543458c869d16f095a505bcaccc Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 09:15:57 +0100 Subject: [PATCH 19/99] Add cluster_do_action to new cluster struct --- crates/modelardb_server/src/cluster.rs | 45 ++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index eb6c9ebe..ebf27e0c 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -17,8 +17,8 @@ use std::str::FromStr; -use arrow_flight::Ticket; use arrow_flight::flight_service_client::FlightServiceClient; +use arrow_flight::{Action, Ticket}; use futures::StreamExt; use futures::stream::FuturesUnordered; use log::info; @@ -100,8 +100,8 @@ impl Cluster { } /// Connect to the Apache Arrow Flight server given by `url` and execute the given `sql` - /// statement with the key as metadata. If the statement was successfully executed, return the - /// url of the node to simplify logging, otherwise return [`ModelarDbServerError`]. + /// statement with the cluster key as metadata. If the statement was successfully executed, + /// return the url of the node to simplify logging, otherwise return [`ModelarDbServerError`]. async fn connect_and_do_get(&self, url: &str, sql: &str) -> Result { let mut flight_client = FlightServiceClient::connect(url.to_owned()).await?; @@ -115,4 +115,43 @@ impl Cluster { Ok(url.to_owned()) } + + /// For each node in the cluster, execute the given `action` with the cluster key as metadata. + /// If the action was successfully executed for each node, return [`Ok`], otherwise return + /// [`ModelarDbServerError`]. + pub async fn cluster_do_action(&self, action: Action) -> Result<()> { + let nodes = self.remote_data_folder.nodes().await?; + + let mut action_futures: FuturesUnordered<_> = nodes + .iter() + .map(|node| self.connect_and_do_action(&node.url, action.clone())) + .collect(); + + // Run the futures concurrently and log when the action has been executed on each node. + while let Some(result) = action_futures.next().await { + info!( + "Executed action `{}` on node with url '{}'.", + action.r#type, result? + ); + } + + Ok(()) + } + + /// Connect to the Apache Arrow Flight server given by `url` and make a request to do `action` + /// with the cluster key as metadata. If the action was successfully executed, return the url + /// of the node to simplify logging, otherwise return [`ModelarDbServerError`]. + async fn connect_and_do_action(&self, url: &str, action: Action) -> Result { + let mut flight_client = FlightServiceClient::connect(url.to_owned()).await?; + + // Add the key to the request metadata to indicate that the request is a cluster operation. + let mut request = Request::new(action); + request + .metadata_mut() + .insert("x-cluster-key", self.key.clone()); + + flight_client.do_action(request).await?; + + Ok(url.to_owned()) + } } From ca085f017d84b34f35d85175a63a83e4ad0624cd Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 09:52:43 +0100 Subject: [PATCH 20/99] Create and register cluster tables when cluster is created --- crates/modelardb_server/src/cluster.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index ebf27e0c..ee762ddd 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -45,10 +45,14 @@ pub struct Cluster { } impl Cluster { - /// Try to retrieve the cluster key from the remote data folder and create a new cluster - /// instance. If the cluster key could not be retrieved from the remote data folder, return - /// [`ModelarDbServerError`]. + /// Try to retrieve the cluster key from the remote data folder and create a new cluster instance. + /// If the cluster key could not be retrieved from the remote data folder or the cluster metadata + /// tables do not exist and could not be created, return [`ModelarDbServerError`]. pub async fn try_new(remote_data_folder: DataFolder) -> Result { + remote_data_folder + .create_and_register_cluster_metadata_tables() + .await?; + let key = remote_data_folder.cluster_key().await?.to_string(); // Convert the key to a MetadataValue since it is used in tonic requests. From 595d6a7a3db3bc971e30c5799a8b5ef162077fdd Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 10:04:20 +0100 Subject: [PATCH 21/99] Add unit tests for new cluster struct --- crates/modelardb_server/src/cluster.rs | 49 ++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index ee762ddd..0e08608c 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -159,3 +159,52 @@ impl Cluster { Ok(url.to_owned()) } } + +#[cfg(test)] +mod test { + use super::*; + + use modelardb_types::types::ServerMode; + use tempfile::TempDir; + + // Tests for Cluster. + #[tokio::test] + async fn test_query_node() { + let (_temp_dir, mut cluster) = create_cluster_with_edge().await; + + let cloud_node = Node::new("cloud".to_owned(), ServerMode::Cloud); + cluster + .remote_data_folder + .save_node(cloud_node.clone()) + .await + .unwrap(); + + assert_eq!(cluster.query_node().await.unwrap(), cloud_node); + } + + #[tokio::test] + async fn test_query_node_no_cloud_nodes() { + let (_temp_dir, mut cluster) = create_cluster_with_edge().await; + let result = cluster.query_node().await; + + assert_eq!( + result.unwrap_err().to_string(), + "Invalid State Error: There are no cloud nodes to execute the query in the cluster." + ); + } + + async fn create_cluster_with_edge() -> (TempDir, Cluster) { + let temp_dir = tempfile::tempdir().unwrap(); + let temp_dir_url = temp_dir.path().to_str().unwrap(); + let local_data_folder = DataFolder::open_local_url(temp_dir_url).await.unwrap(); + + let cluster = Cluster::try_new(local_data_folder.clone()).await.unwrap(); + + local_data_folder + .save_node(Node::new("edge".to_owned(), ServerMode::Edge)) + .await + .unwrap(); + + (temp_dir, cluster) + } +} From f13fffcf9b5867f6d5444eec6d99ebda49486459 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 10:15:12 +0100 Subject: [PATCH 22/99] Save node automatically when creating cluster object --- crates/modelardb_server/src/cluster.rs | 34 ++++++++++++-------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 0e08608c..7596e825 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -45,14 +45,17 @@ pub struct Cluster { } impl Cluster { - /// Try to retrieve the cluster key from the remote data folder and create a new cluster instance. - /// If the cluster key could not be retrieved from the remote data folder or the cluster metadata - /// tables do not exist and could not be created, return [`ModelarDbServerError`]. - pub async fn try_new(remote_data_folder: DataFolder) -> Result { + /// Create and register the cluster metadata tables and save the given `node` in the cluster. + /// It is assumed that `node` corresponds to the local system running `modelardbd`. If the + /// cluster metadata tables do not exist and could not be created or the node could not be + /// saved, return [`ModelarDbServerError`]. + pub async fn try_new(node: Node, remote_data_folder: DataFolder) -> Result { remote_data_folder .create_and_register_cluster_metadata_tables() .await?; + remote_data_folder.save_node(node).await?; + let key = remote_data_folder.cluster_key().await?.to_string(); // Convert the key to a MetadataValue since it is used in tonic requests. @@ -170,21 +173,19 @@ mod test { // Tests for Cluster. #[tokio::test] async fn test_query_node() { - let (_temp_dir, mut cluster) = create_cluster_with_edge().await; - let cloud_node = Node::new("cloud".to_owned(), ServerMode::Cloud); - cluster - .remote_data_folder - .save_node(cloud_node.clone()) - .await - .unwrap(); + let (_temp_dir, mut cluster) = create_cluster_with_node(cloud_node.clone()).await; - assert_eq!(cluster.query_node().await.unwrap(), cloud_node); + let query_node = cluster.query_node().await.unwrap(); + + assert_eq!(query_node, cloud_node); } #[tokio::test] async fn test_query_node_no_cloud_nodes() { - let (_temp_dir, mut cluster) = create_cluster_with_edge().await; + let edge_node = Node::new("edge".to_owned(), ServerMode::Edge); + let (_temp_dir, mut cluster) = create_cluster_with_node(edge_node).await; + let result = cluster.query_node().await; assert_eq!( @@ -193,15 +194,12 @@ mod test { ); } - async fn create_cluster_with_edge() -> (TempDir, Cluster) { + async fn create_cluster_with_node(node: Node) -> (TempDir, Cluster) { let temp_dir = tempfile::tempdir().unwrap(); let temp_dir_url = temp_dir.path().to_str().unwrap(); let local_data_folder = DataFolder::open_local_url(temp_dir_url).await.unwrap(); - let cluster = Cluster::try_new(local_data_folder.clone()).await.unwrap(); - - local_data_folder - .save_node(Node::new("edge".to_owned(), ServerMode::Edge)) + let cluster = Cluster::try_new(node, local_data_folder.clone()) .await .unwrap(); From dc04a65bc31337a6097d013c85575d35d8d5d9a2 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 10:45:27 +0100 Subject: [PATCH 23/99] Put new Cluster struct in ClusterMode::MultiNode --- crates/modelardb_server/src/cluster.rs | 1 + crates/modelardb_server/src/configuration.rs | 13 +++++++----- crates/modelardb_server/src/data_folders.rs | 21 ++++++++++++++++---- crates/modelardb_server/src/main.rs | 10 +++++----- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 7596e825..0dbfa040 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -34,6 +34,7 @@ use crate::error::{ModelarDbServerError, Result}; /// Stores the currently managed nodes in the cluster and allows for performing operations that need /// to be applied to every single node in the cluster. +#[derive(Clone)] pub struct Cluster { /// Key identifying the cluster. The key is used to validate communication within the cluster /// between nodes. diff --git a/crates/modelardb_server/src/configuration.rs b/crates/modelardb_server/src/configuration.rs index 90ff04a9..777153f4 100644 --- a/crates/modelardb_server/src/configuration.rs +++ b/crates/modelardb_server/src/configuration.rs @@ -242,12 +242,12 @@ mod tests { use std::sync::Arc; use modelardb_storage::data_folder::DataFolder; + use modelardb_types::types::{Node, ServerMode}; use tempfile::TempDir; use tokio::sync::RwLock; - use uuid::Uuid; + use crate::cluster::Cluster; use crate::data_folders::DataFolders; - use crate::manager::Manager; use crate::storage::StorageEngine; // Tests for ConfigurationManager. @@ -416,14 +416,17 @@ mod tests { let data_folders = DataFolders::new( local_data_folder.clone(), - Some(remote_data_folder), + Some(remote_data_folder.clone()), local_data_folder, ); - let manager = Manager::new(Uuid::new_v4().to_string()); + let edge_node = Node::new("edge".to_owned(), ServerMode::Edge); + let cluster = Cluster::try_new(edge_node, remote_data_folder) + .await + .unwrap(); let configuration_manager = Arc::new(RwLock::new(ConfigurationManager::new( - ClusterMode::MultiNode(manager), + ClusterMode::MultiNode(cluster), ))); let storage_engine = Arc::new(RwLock::new( diff --git a/crates/modelardb_server/src/data_folders.rs b/crates/modelardb_server/src/data_folders.rs index 851a4c19..85b9c2be 100644 --- a/crates/modelardb_server/src/data_folders.rs +++ b/crates/modelardb_server/src/data_folders.rs @@ -15,11 +15,15 @@ //! Implementation of a struct that provides access to the local and remote data storage components. +use std::env; + use modelardb_storage::data_folder::DataFolder; +use modelardb_types::types::{Node, ServerMode}; -use crate::ClusterMode; use crate::Result; +use crate::cluster::Cluster; use crate::error::ModelarDbServerError; +use crate::{ClusterMode, PORT}; /// Folders for storing metadata and data in Apache Parquet files locally and remotely. #[derive(Clone)] @@ -54,6 +58,9 @@ impl DataFolders { pub async fn try_from_command_line_arguments( arguments: &[&str], ) -> Result<(ClusterMode, Self)> { + let ip_address = env::var("MODELARDBD_IP_ADDRESS").unwrap_or("127.0.0.1".to_string()); + let url_with_port = format!("grpc://{ip_address}:{}", &PORT.to_string()); + // Match the provided command line arguments to the supported inputs. match arguments { // Single edge without a cluster. @@ -73,8 +80,11 @@ impl DataFolders { let local_data_folder = DataFolder::open_local_url(local_data_folder_url).await?; + let node = Node::new(url_with_port, ServerMode::Edge); + let cluster = Cluster::try_new(node, remote_data_folder.clone()).await?; + Ok(( - ClusterMode::MultiNode, + ClusterMode::MultiNode(cluster), Self::new( local_data_folder.clone(), Some(remote_data_folder), @@ -89,8 +99,11 @@ impl DataFolders { let local_data_folder = DataFolder::open_local_url(local_data_folder_url).await?; + let node = Node::new(url_with_port, ServerMode::Cloud); + let cluster = Cluster::try_new(node, remote_data_folder.clone()).await?; + Ok(( - ClusterMode::MultiNode, + ClusterMode::MultiNode(cluster), Self::new( local_data_folder, Some(remote_data_folder.clone()), @@ -141,7 +154,7 @@ mod tests { .await .unwrap(); - assert_eq!(cluster_mode, ClusterMode::SingleNode); + assert!(matches!(cluster_mode, ClusterMode::SingleNode)); assert!(data_folders.maybe_remote_data_folder.is_none()); } diff --git a/crates/modelardb_server/src/main.rs b/crates/modelardb_server/src/main.rs index e3cd876f..c57f7c03 100644 --- a/crates/modelardb_server/src/main.rs +++ b/crates/modelardb_server/src/main.rs @@ -22,16 +22,17 @@ mod error; mod manager; mod remote; mod storage; +mod cluster; use std::sync::{Arc, LazyLock}; use std::{env, process}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; +use crate::cluster::Cluster; use crate::context::Context; use crate::data_folders::DataFolders; use crate::error::Result; -use crate::manager::Manager; #[global_allocator] static ALLOC: snmalloc_rs::SnMalloc = snmalloc_rs::SnMalloc; @@ -42,10 +43,10 @@ pub static PORT: LazyLock = /// The different possible modes that a ModelarDB server can be deployed in, assigned when the /// server is started. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone)] pub enum ClusterMode { SingleNode, - MultiNode(Manager), + MultiNode(Cluster), } /// Setup tracing that prints to stdout, parse the command line arguments to extract @@ -77,8 +78,7 @@ async fn main() -> Result<()> { context.register_normal_tables().await?; context.register_time_series_tables().await?; - if let ClusterMode::MultiNode(manager) = &cluster_mode { - manager.retrieve_and_create_tables(&context).await?; + if let ClusterMode::MultiNode(_cluster) = &cluster_mode { } // Setup CTRL+C handler. From f0622019a54f06d6ad4e4b80a5fc6244440b33b7 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 15:57:12 +0100 Subject: [PATCH 24/99] Move validate_local_tables_exist_remotely to new Cluster struct --- crates/modelardb_server/src/cluster.rs | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 0dbfa040..19a2986c 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -16,6 +16,7 @@ //! Functionality to perform operations on every node in the cluster. use std::str::FromStr; +use std::sync::Arc; use arrow_flight::flight_service_client::FlightServiceClient; use arrow_flight::{Action, Ticket}; @@ -30,6 +31,7 @@ use rand::seq::IteratorRandom; use tonic::Request; use tonic::metadata::{Ascii, MetadataValue}; +use crate::context::Context; use crate::error::{ModelarDbServerError, Result}; /// Stores the currently managed nodes in the cluster and allows for performing operations that need @@ -164,6 +166,31 @@ impl Cluster { } } +/// Validate that all tables in the local data folder exist in the remote data folder. If any table +/// does not exist in the remote data folder, return [`ModelarDbServerError`]. +async fn validate_local_tables_exist_remotely( + local_data_folder: &DataFolder, + remote_data_folder: &DataFolder, +) -> Result<()> { + let local_table_names = local_data_folder.table_names().await?; + let remote_table_names = remote_data_folder.table_names().await?; + + let invalid_tables: Vec = local_table_names + .iter() + .filter(|table| !remote_table_names.contains(table)) + .cloned() + .collect(); + + if !invalid_tables.is_empty() { + return Err(ModelarDbServerError::InvalidState(format!( + "The following tables do not exist in the remote data folder: {}.", + invalid_tables.join(", ") + ))); + } + + Ok(()) +} + #[cfg(test)] mod test { use super::*; From 42015d421f136ea45266829015d68589edc5e532 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 15:59:43 +0100 Subject: [PATCH 25/99] Move validate_normal_tables to new Cluster struct --- crates/modelardb_server/src/cluster.rs | 39 ++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 19a2986c..37dbdf6d 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -20,6 +20,8 @@ use std::sync::Arc; use arrow_flight::flight_service_client::FlightServiceClient; use arrow_flight::{Action, Ticket}; +use datafusion::arrow::datatypes::Schema; +use datafusion::catalog::TableProvider; use futures::StreamExt; use futures::stream::FuturesUnordered; use log::info; @@ -191,6 +193,43 @@ async fn validate_local_tables_exist_remotely( Ok(()) } +/// For each normal table in the remote data folder, if the table also exists in the local data +/// folder, validate that the schemas are identical. If the schemas are not identical, return +/// [`ModelarDbServerError`]. Return a vector containing the name and schema of each normal table +/// that is in the remote data folder but not in the local data folder. +async fn validate_normal_tables( + local_data_folder: &DataFolder, + remote_data_folder: &DataFolder, +) -> Result)>> { + let mut missing_normal_tables = vec![]; + + let remote_normal_tables = remote_data_folder.normal_table_names().await?; + + for table_name in remote_normal_tables { + let remote_schema = normal_table_schema(remote_data_folder, &table_name).await?; + + if let Ok(local_schema) = normal_table_schema(local_data_folder, &table_name).await { + if remote_schema != local_schema { + return Err(ModelarDbServerError::InvalidState(format!( + "The normal table '{table_name}' has a different schema in the local data \ + folder compared to the remote data folder.", + ))); + } + } else { + missing_normal_tables.push((table_name, remote_schema)); + } + } + + Ok(missing_normal_tables) +} + +/// Retrieve the schema of a normal table from the Delta Lake in the data folder. If the table does +/// not exist, or the schema could not be retrieved, return [`ModelarDbServerError`]. +async fn normal_table_schema(data_folder: &DataFolder, table_name: &str) -> Result> { + let delta_table = data_folder.delta_table(table_name).await?; + Ok(TableProvider::schema(&delta_table)) +} + #[cfg(test)] mod test { use super::*; From 544fea90d853bea7374b2aea699dfebc0432eb0b Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 16:01:13 +0100 Subject: [PATCH 26/99] Move validate_time_series_tables to new Cluster struct --- crates/modelardb_server/src/cluster.rs | 35 ++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 37dbdf6d..b489c016 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -230,6 +230,41 @@ async fn normal_table_schema(data_folder: &DataFolder, table_name: &str) -> Resu Ok(TableProvider::schema(&delta_table)) } +/// For each time series table in the remote data folder, if the table also exists in the local +/// data folder, validate that the metadata is identical. If the metadata is not identical, return +/// [`ModelarDbServerError`]. Return a vector containing the metadata of each time series table +/// that is in the remote data folder but not in the local data folder. +async fn validate_time_series_tables( + local_data_folder: &DataFolder, + remote_data_folder: &DataFolder, +) -> Result> { + let mut missing_time_series_tables = vec![]; + + let remote_time_series_tables = remote_data_folder.time_series_table_names().await?; + + for table_name in remote_time_series_tables { + let remote_metadata = remote_data_folder + .time_series_table_metadata_for_time_series_table(&table_name) + .await?; + + if let Ok(local_metadata) = local_data_folder + .time_series_table_metadata_for_time_series_table(&table_name) + .await + { + if remote_metadata != local_metadata { + return Err(ModelarDbServerError::InvalidState(format!( + "The time series table '{table_name}' has different metadata in the local data \ + folder compared to the remote data folder.", + ))); + } + } else { + missing_time_series_tables.push(remote_metadata); + } + } + + Ok(missing_time_series_tables) +} + #[cfg(test)] mod test { use super::*; From eb6e7b9d579ff6971346668ec985113cd80c3212 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 16:03:31 +0100 Subject: [PATCH 27/99] Move retrieve_and_create_tables to new Cluster struct --- crates/modelardb_server/src/cluster.rs | 32 +++++++++++++++++++++++++- crates/modelardb_server/src/main.rs | 9 +++++--- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index b489c016..b0b888ad 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -27,7 +27,7 @@ use futures::stream::FuturesUnordered; use log::info; use modelardb_storage::data_folder::DataFolder; use modelardb_storage::data_folder::cluster::ClusterMetadata; -use modelardb_types::types::Node; +use modelardb_types::types::{Node, TimeSeriesTableMetadata}; use rand::rng; use rand::seq::IteratorRandom; use tonic::Request; @@ -72,6 +72,36 @@ impl Cluster { }) } + /// Initialize the local database schema with the normal tables and time series tables from the + /// cluster's database schema using the remote data folder. If the tables to create could not be + /// retrieved from the remote data folder, or the tables could not be created, + /// return [`ModelarDbServerError`]. + pub(crate) async fn retrieve_and_create_tables(&self, context: &Arc) -> Result<()> { + let local_data_folder = &context.data_folders.local_data_folder; + + validate_local_tables_exist_remotely(local_data_folder, &self.remote_data_folder).await?; + + // Validate that all tables that are in both the local and remote data folder are identical. + let missing_normal_tables = + validate_normal_tables(local_data_folder, &self.remote_data_folder).await?; + + let missing_time_series_tables = + validate_time_series_tables(local_data_folder, &self.remote_data_folder).await?; + + // For each table that does not already exist locally, create the table. + for (table_name, schema) in missing_normal_tables { + context.create_normal_table(&table_name, &schema).await?; + } + + for time_series_table_metadata in missing_time_series_tables { + context + .create_time_series_table(&time_series_table_metadata) + .await?; + } + + Ok(()) + } + /// Return the cloud node in the cluster that is currently most capable of running a query. /// Note that the most capable node is currently selected at random. If there are no cloud nodes /// in the cluster, return [`ModelarDbServerError`]. diff --git a/crates/modelardb_server/src/main.rs b/crates/modelardb_server/src/main.rs index c57f7c03..39599641 100644 --- a/crates/modelardb_server/src/main.rs +++ b/crates/modelardb_server/src/main.rs @@ -15,6 +15,7 @@ //! Implementation of ModelarDB's main function. +mod cluster; mod configuration; mod context; mod data_folders; @@ -22,7 +23,6 @@ mod error; mod manager; mod remote; mod storage; -mod cluster; use std::sync::{Arc, LazyLock}; use std::{env, process}; @@ -69,7 +69,9 @@ async fn main() -> Result<()> { { cluster_mode_and_data_folders } else { - print_usage_and_exit_with_error("[server_mode] local_data_folder_url [remote_data_folder_url]"); + print_usage_and_exit_with_error( + "[server_mode] local_data_folder_url [remote_data_folder_url]", + ); }; let context = Arc::new(Context::try_new(data_folders, cluster_mode.clone()).await?); @@ -78,7 +80,8 @@ async fn main() -> Result<()> { context.register_normal_tables().await?; context.register_time_series_tables().await?; - if let ClusterMode::MultiNode(_cluster) = &cluster_mode { + if let ClusterMode::MultiNode(cluster) = &cluster_mode { + cluster.retrieve_and_create_tables(&context).await?; } // Setup CTRL+C handler. From 518e3677f16bd61b59f36545140b5c01f2016cc5 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 16:37:11 +0100 Subject: [PATCH 28/99] Move temp_dir out of test util function --- crates/modelardb_server/src/cluster.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index b0b888ad..26cbf9fe 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -305,8 +305,10 @@ mod test { // Tests for Cluster. #[tokio::test] async fn test_query_node() { + let temp_dir = tempfile::tempdir().unwrap(); let cloud_node = Node::new("cloud".to_owned(), ServerMode::Cloud); - let (_temp_dir, mut cluster) = create_cluster_with_node(cloud_node.clone()).await; + + let mut cluster = create_cluster_with_node(temp_dir, cloud_node.clone()).await; let query_node = cluster.query_node().await.unwrap(); @@ -315,8 +317,10 @@ mod test { #[tokio::test] async fn test_query_node_no_cloud_nodes() { + let temp_dir = tempfile::tempdir().unwrap(); let edge_node = Node::new("edge".to_owned(), ServerMode::Edge); - let (_temp_dir, mut cluster) = create_cluster_with_node(edge_node).await; + + let mut cluster = create_cluster_with_node(temp_dir, edge_node).await; let result = cluster.query_node().await; @@ -326,15 +330,12 @@ mod test { ); } - async fn create_cluster_with_node(node: Node) -> (TempDir, Cluster) { - let temp_dir = tempfile::tempdir().unwrap(); + async fn create_cluster_with_node(temp_dir: TempDir, node: Node) -> Cluster { let temp_dir_url = temp_dir.path().to_str().unwrap(); let local_data_folder = DataFolder::open_local_url(temp_dir_url).await.unwrap(); - let cluster = Cluster::try_new(node, local_data_folder.clone()) + Cluster::try_new(node, local_data_folder.clone()) .await - .unwrap(); - - (temp_dir, cluster) + .unwrap() } } From 87886c24b946f6cd4556fb069c38294c0b12b097 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 16:51:28 +0100 Subject: [PATCH 29/99] Add test util method to create context --- crates/modelardb_server/src/cluster.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 26cbf9fe..90c92441 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -302,7 +302,32 @@ mod test { use modelardb_types::types::ServerMode; use tempfile::TempDir; + use crate::ClusterMode; + use crate::data_folders::DataFolders; + // Tests for Cluster. + + /// Create a [`Context`] for an edge node within a cluster. Note that both the local and remote + /// data folder in the context uses a local [`DataFolder`]. + async fn create_context(local_temp_dir: TempDir, remote_temp_dir: TempDir) -> Context { + let temp_dir_url = local_temp_dir.path().to_str().unwrap(); + let local_data_folder = DataFolder::open_local_url(temp_dir_url).await.unwrap(); + + let edge_node = Node::new("edge".to_owned(), ServerMode::Edge); + let cluster = create_cluster_with_node(remote_temp_dir, edge_node).await; + + Context::try_new( + DataFolders::new( + local_data_folder.clone(), + Some(cluster.remote_data_folder.clone()), + local_data_folder, + ), + ClusterMode::MultiNode(cluster), + ) + .await + .unwrap() + } + #[tokio::test] async fn test_query_node() { let temp_dir = tempfile::tempdir().unwrap(); @@ -330,6 +355,7 @@ mod test { ); } + /// Create a [`Cluster`] that uses a local [`DataFolder`] for the remote data folder. async fn create_cluster_with_node(temp_dir: TempDir, node: Node) -> Cluster { let temp_dir_url = temp_dir.path().to_str().unwrap(); let local_data_folder = DataFolder::open_local_url(temp_dir_url).await.unwrap(); From 8ccbd6d7bd24f901f3e30265f22bc4ab9d76105e Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 17:18:27 +0100 Subject: [PATCH 30/99] Use a reference for the TempDir to avoid dropping it --- crates/modelardb_server/src/cluster.rs | 28 ++++++++++++++------------ 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 90c92441..866bedb4 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -309,23 +309,25 @@ mod test { /// Create a [`Context`] for an edge node within a cluster. Note that both the local and remote /// data folder in the context uses a local [`DataFolder`]. - async fn create_context(local_temp_dir: TempDir, remote_temp_dir: TempDir) -> Context { + async fn create_context(local_temp_dir: &TempDir, remote_temp_dir: &TempDir) -> Arc { let temp_dir_url = local_temp_dir.path().to_str().unwrap(); let local_data_folder = DataFolder::open_local_url(temp_dir_url).await.unwrap(); let edge_node = Node::new("edge".to_owned(), ServerMode::Edge); let cluster = create_cluster_with_node(remote_temp_dir, edge_node).await; - Context::try_new( - DataFolders::new( - local_data_folder.clone(), - Some(cluster.remote_data_folder.clone()), - local_data_folder, - ), - ClusterMode::MultiNode(cluster), + Arc::new( + Context::try_new( + DataFolders::new( + local_data_folder.clone(), + Some(cluster.remote_data_folder.clone()), + local_data_folder, + ), + ClusterMode::MultiNode(cluster), + ) + .await + .unwrap(), ) - .await - .unwrap() } #[tokio::test] @@ -333,7 +335,7 @@ mod test { let temp_dir = tempfile::tempdir().unwrap(); let cloud_node = Node::new("cloud".to_owned(), ServerMode::Cloud); - let mut cluster = create_cluster_with_node(temp_dir, cloud_node.clone()).await; + let mut cluster = create_cluster_with_node(&temp_dir, cloud_node.clone()).await; let query_node = cluster.query_node().await.unwrap(); @@ -345,7 +347,7 @@ mod test { let temp_dir = tempfile::tempdir().unwrap(); let edge_node = Node::new("edge".to_owned(), ServerMode::Edge); - let mut cluster = create_cluster_with_node(temp_dir, edge_node).await; + let mut cluster = create_cluster_with_node(&temp_dir, edge_node).await; let result = cluster.query_node().await; @@ -356,7 +358,7 @@ mod test { } /// Create a [`Cluster`] that uses a local [`DataFolder`] for the remote data folder. - async fn create_cluster_with_node(temp_dir: TempDir, node: Node) -> Cluster { + async fn create_cluster_with_node(temp_dir: &TempDir, node: Node) -> Cluster { let temp_dir_url = temp_dir.path().to_str().unwrap(); let local_data_folder = DataFolder::open_local_url(temp_dir_url).await.unwrap(); From f00f511c80968739c176a2a4459dbb79c852cbec Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 17:24:28 +0100 Subject: [PATCH 31/99] Add test for retrieving and creating missing local normal table --- crates/modelardb_server/src/cluster.rs | 55 +++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 866bedb4..d7d4717f 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -299,13 +299,66 @@ async fn validate_time_series_tables( mod test { use super::*; - use modelardb_types::types::ServerMode; + use datafusion::arrow::datatypes::{ArrowPrimitiveType, Field}; + use modelardb_test::table::NORMAL_TABLE_NAME; + use modelardb_types::types::{ArrowValue, ServerMode}; use tempfile::TempDir; use crate::ClusterMode; use crate::data_folders::DataFolders; // Tests for Cluster. + #[tokio::test] + async fn test_retrieve_and_create_missing_local_normal_table() { + let local_temp_dir = tempfile::tempdir().unwrap(); + let remote_temp_dir = tempfile::tempdir().unwrap(); + + let context = create_context(&local_temp_dir, &remote_temp_dir).await; + let data_folders = context.data_folders.clone(); + + // Create a normal table in the remote data folder that should be retrieved and created + // in the local data folder. + create_normal_table( + "column", + data_folders.maybe_remote_data_folder.clone().unwrap(), + ) + .await; + + retrieve_and_create_tables(&context).await; + + assert_eq!( + vec![NORMAL_TABLE_NAME], + data_folders.local_data_folder.table_names().await.unwrap() + ); + } + + #[tokio::test] + async fn test_retrieve_and_create_invalid_local_normal_table() {} + + async fn create_normal_table(column_name: &str, data_folder: DataFolder) { + let schema = Schema::new(vec![Field::new(column_name, ArrowValue::DATA_TYPE, false)]); + + data_folder + .create_normal_table(NORMAL_TABLE_NAME, &schema) + .await + .unwrap(); + + data_folder + .save_normal_table_metadata(NORMAL_TABLE_NAME) + .await + .unwrap(); + } + + + async fn retrieve_and_create_tables(context: &Arc) { + if let ClusterMode::MultiNode(cluster) = + &context.configuration_manager.read().await.cluster_mode + { + cluster.retrieve_and_create_tables(context).await.unwrap(); + } else { + panic!("Cluster should be a MultiNode cluster.") + } + } /// Create a [`Context`] for an edge node within a cluster. Note that both the local and remote /// data folder in the context uses a local [`DataFolder`]. From 78bc426b46cc850f15b580ffb7b273de1c4b21c5 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 17:31:15 +0100 Subject: [PATCH 32/99] Add test for trying to retrieve and create tables when a table exists but with a different schema --- crates/modelardb_server/src/cluster.rs | 34 +++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index d7d4717f..23445eda 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -324,7 +324,7 @@ mod test { ) .await; - retrieve_and_create_tables(&context).await; + retrieve_and_create_tables(&context).await.unwrap(); assert_eq!( vec![NORMAL_TABLE_NAME], @@ -333,7 +333,33 @@ mod test { } #[tokio::test] - async fn test_retrieve_and_create_invalid_local_normal_table() {} + async fn test_retrieve_and_create_invalid_local_normal_table() { + let local_temp_dir = tempfile::tempdir().unwrap(); + let remote_temp_dir = tempfile::tempdir().unwrap(); + + let context = create_context(&local_temp_dir, &remote_temp_dir).await; + let data_folders = context.data_folders.clone(); + + // Create a normal table in the local data folder with the same name as a normal table in + // the remote data folder, but with a different schema. + create_normal_table("local", data_folders.local_data_folder.clone()).await; + + create_normal_table( + "remote", + data_folders.maybe_remote_data_folder.clone().unwrap(), + ) + .await; + + let result = retrieve_and_create_tables(&context).await; + + assert_eq!( + result.unwrap_err().to_string(), + format!( + "Invalid State Error: The normal table '{NORMAL_TABLE_NAME}' has a different schema \ + in the local data folder compared to the remote data folder." + ) + ); + } async fn create_normal_table(column_name: &str, data_folder: DataFolder) { let schema = Schema::new(vec![Field::new(column_name, ArrowValue::DATA_TYPE, false)]); @@ -350,11 +376,11 @@ mod test { } - async fn retrieve_and_create_tables(context: &Arc) { + async fn retrieve_and_create_tables(context: &Arc) -> Result<()> { if let ClusterMode::MultiNode(cluster) = &context.configuration_manager.read().await.cluster_mode { - cluster.retrieve_and_create_tables(context).await.unwrap(); + cluster.retrieve_and_create_tables(context).await } else { panic!("Cluster should be a MultiNode cluster.") } From 39d3b8560b29c7168388030739a3703b3d61364b Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 17:37:39 +0100 Subject: [PATCH 33/99] Now also testing that we accept tables with same name and schema --- crates/modelardb_server/src/cluster.rs | 33 +++++++++++++++++++++----- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 23445eda..81e9ca25 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -317,8 +317,23 @@ mod test { let data_folders = context.data_folders.clone(); // Create a normal table in the remote data folder that should be retrieved and created - // in the local data folder. + // in the local data folder and one that already exists. create_normal_table( + "normal_table_1", + "column", + data_folders.local_data_folder.clone(), + ) + .await; + + create_normal_table( + "normal_table_1", + "column", + data_folders.maybe_remote_data_folder.clone().unwrap(), + ) + .await; + + create_normal_table( + "normal_table_2", "column", data_folders.maybe_remote_data_folder.clone().unwrap(), ) @@ -327,7 +342,7 @@ mod test { retrieve_and_create_tables(&context).await.unwrap(); assert_eq!( - vec![NORMAL_TABLE_NAME], + vec!["normal_table_2", "normal_table_1"], data_folders.local_data_folder.table_names().await.unwrap() ); } @@ -342,9 +357,15 @@ mod test { // Create a normal table in the local data folder with the same name as a normal table in // the remote data folder, but with a different schema. - create_normal_table("local", data_folders.local_data_folder.clone()).await; + create_normal_table( + "normal_table", + "local", + data_folders.local_data_folder.clone(), + ) + .await; create_normal_table( + "normal_table", "remote", data_folders.maybe_remote_data_folder.clone().unwrap(), ) @@ -361,16 +382,16 @@ mod test { ); } - async fn create_normal_table(column_name: &str, data_folder: DataFolder) { + async fn create_normal_table(table_name: &str, column_name: &str, data_folder: DataFolder) { let schema = Schema::new(vec![Field::new(column_name, ArrowValue::DATA_TYPE, false)]); data_folder - .create_normal_table(NORMAL_TABLE_NAME, &schema) + .create_normal_table(table_name, &schema) .await .unwrap(); data_folder - .save_normal_table_metadata(NORMAL_TABLE_NAME) + .save_normal_table_metadata(table_name) .await .unwrap(); } From 8a4d8371c53b7d184eb5671fb790a4012c2fbdc4 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 17:46:17 +0100 Subject: [PATCH 34/99] Add test for retrieving and creating missing local time series table --- crates/modelardb_server/src/cluster.rs | 70 +++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 81e9ca25..aa235f2c 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -301,7 +301,7 @@ mod test { use datafusion::arrow::datatypes::{ArrowPrimitiveType, Field}; use modelardb_test::table::NORMAL_TABLE_NAME; - use modelardb_types::types::{ArrowValue, ServerMode}; + use modelardb_types::types::{ArrowTimestamp, ArrowValue, ErrorBound, ServerMode}; use tempfile::TempDir; use crate::ClusterMode; @@ -396,6 +396,74 @@ mod test { .unwrap(); } + #[tokio::test] + async fn test_retrieve_and_create_missing_local_time_series_table() { + let local_temp_dir = tempfile::tempdir().unwrap(); + let remote_temp_dir = tempfile::tempdir().unwrap(); + + let context = create_context(&local_temp_dir, &remote_temp_dir).await; + let data_folders = context.data_folders.clone(); + + // Create a time series table in the remote data folder that should be retrieved and created + // in the local data folder and one that already exists. + create_time_series_table( + "time_series_table_1", + "field", + data_folders.local_data_folder.clone(), + ) + .await; + + create_time_series_table( + "time_series_table_1", + "field", + data_folders.maybe_remote_data_folder.clone().unwrap(), + ) + .await; + + create_time_series_table( + "time_series_table_2", + "field", + data_folders.maybe_remote_data_folder.clone().unwrap(), + ) + .await; + + retrieve_and_create_tables(&context).await.unwrap(); + + assert_eq!( + vec!["time_series_table_2", "time_series_table_1"], + data_folders.local_data_folder.table_names().await.unwrap() + ); + } + + + async fn create_time_series_table( + table_name: &str, + column_name: &str, + data_folder: DataFolder, + ) { + let query_schema = Arc::new(Schema::new(vec![ + Field::new("timestamp", ArrowTimestamp::DATA_TYPE, false), + Field::new(column_name, ArrowValue::DATA_TYPE, false), + ])); + + let time_series_table_metadata = TimeSeriesTableMetadata::try_new( + table_name.to_owned(), + query_schema, + vec![ErrorBound::Lossless, ErrorBound::Lossless], + vec![None, None], + ) + .unwrap(); + + data_folder + .create_time_series_table(&time_series_table_metadata) + .await + .unwrap(); + + data_folder + .save_time_series_table_metadata(&time_series_table_metadata) + .await + .unwrap(); + } async fn retrieve_and_create_tables(context: &Arc) -> Result<()> { if let ClusterMode::MultiNode(cluster) = From 9b5b7aba05eef7e7a9554b95b32d6d3798f429fb Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 17:50:28 +0100 Subject: [PATCH 35/99] Add test for trying to retrieve and create a time series table with the same name but with different metadata --- crates/modelardb_server/src/cluster.rs | 40 ++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index aa235f2c..3e1cbcae 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -300,7 +300,6 @@ mod test { use super::*; use datafusion::arrow::datatypes::{ArrowPrimitiveType, Field}; - use modelardb_test::table::NORMAL_TABLE_NAME; use modelardb_types::types::{ArrowTimestamp, ArrowValue, ErrorBound, ServerMode}; use tempfile::TempDir; @@ -375,10 +374,8 @@ mod test { assert_eq!( result.unwrap_err().to_string(), - format!( - "Invalid State Error: The normal table '{NORMAL_TABLE_NAME}' has a different schema \ - in the local data folder compared to the remote data folder." - ) + "Invalid State Error: The normal table 'normal_table' has a different schema in the \ + local data folder compared to the remote data folder." ); } @@ -435,6 +432,39 @@ mod test { ); } + #[tokio::test] + async fn test_retrieve_and_create_invalid_local_time_series_table() { + let local_temp_dir = tempfile::tempdir().unwrap(); + let remote_temp_dir = tempfile::tempdir().unwrap(); + + let context = create_context(&local_temp_dir, &remote_temp_dir).await; + let data_folders = context.data_folders.clone(); + + // Create a time series table in the local data folder with the same name as a time series + // table in the remote data folder, but with a different schema. + create_time_series_table( + "time_series_table", + "local", + data_folders.local_data_folder.clone(), + ) + .await; + + create_time_series_table( + "time_series_table", + "remote", + data_folders.maybe_remote_data_folder.clone().unwrap(), + ) + .await; + + let result = retrieve_and_create_tables(&context).await; + + assert_eq!( + result.unwrap_err().to_string(), + "Invalid State Error: The time series table 'time_series_table' has different metadata \ + in the local data folder compared to the remote data folder." + ); + } + async fn create_time_series_table( table_name: &str, From d3f5eed0f4cd7ee4f0bae8e0e5046dabee19152a Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 17:54:29 +0100 Subject: [PATCH 36/99] Add test for trying to retrieve and create tables with tables that are not in the remote data folder --- crates/modelardb_server/src/cluster.rs | 59 ++++++++++++++++++++------ 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 3e1cbcae..a7c12acd 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -379,20 +379,6 @@ mod test { ); } - async fn create_normal_table(table_name: &str, column_name: &str, data_folder: DataFolder) { - let schema = Schema::new(vec![Field::new(column_name, ArrowValue::DATA_TYPE, false)]); - - data_folder - .create_normal_table(table_name, &schema) - .await - .unwrap(); - - data_folder - .save_normal_table_metadata(table_name) - .await - .unwrap(); - } - #[tokio::test] async fn test_retrieve_and_create_missing_local_time_series_table() { let local_temp_dir = tempfile::tempdir().unwrap(); @@ -465,6 +451,51 @@ mod test { ); } + #[tokio::test] + async fn test_retrieve_and_create_missing_remote_table() { + let local_temp_dir = tempfile::tempdir().unwrap(); + let remote_temp_dir = tempfile::tempdir().unwrap(); + + let context = create_context(&local_temp_dir, &remote_temp_dir).await; + let data_folders = context.data_folders.clone(); + + // Create tables in the local data folder that are not in the remote data folder. + create_normal_table( + "normal_table", + "local", + data_folders.local_data_folder.clone(), + ) + .await; + + create_time_series_table( + "time_series_table", + "local", + data_folders.local_data_folder.clone(), + ) + .await; + + let result = retrieve_and_create_tables(&context).await; + + assert_eq!( + result.unwrap_err().to_string(), + "Invalid State Error: The following tables do not exist in the remote data folder: \ + normal_table, time_series_table." + ); + } + + async fn create_normal_table(table_name: &str, column_name: &str, data_folder: DataFolder) { + let schema = Schema::new(vec![Field::new(column_name, ArrowValue::DATA_TYPE, false)]); + + data_folder + .create_normal_table(table_name, &schema) + .await + .unwrap(); + + data_folder + .save_normal_table_metadata(table_name) + .await + .unwrap(); + } async fn create_time_series_table( table_name: &str, From aee496a433d0146ec963aaaf6786c406d2ffadec Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 17:57:42 +0100 Subject: [PATCH 37/99] Add documentation to test util functions --- crates/modelardb_server/src/cluster.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index a7c12acd..602cf1c9 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -483,6 +483,8 @@ mod test { ); } + /// Create a normal table named `table_name` with a single column named `column_name` in + /// `data_folder`. async fn create_normal_table(table_name: &str, column_name: &str, data_folder: DataFolder) { let schema = Schema::new(vec![Field::new(column_name, ArrowValue::DATA_TYPE, false)]); @@ -497,6 +499,8 @@ mod test { .unwrap(); } + /// Create a time series table named `table_name` with a field column named `column_name` in + /// `data_folder`. async fn create_time_series_table( table_name: &str, column_name: &str, @@ -526,6 +530,8 @@ mod test { .unwrap(); } + /// Call [`Cluster::retrieve_and_create_tables`] if the [`ClusterMode`] of `context` is + /// [`ClusterMode::MultiNode`] and panic if not. async fn retrieve_and_create_tables(context: &Arc) -> Result<()> { if let ClusterMode::MultiNode(cluster) = &context.configuration_manager.read().await.cluster_mode From c04516fccf8fe778eb483f5734b6373f623691c4 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 18:01:41 +0100 Subject: [PATCH 38/99] Use constants for table names where possible --- crates/modelardb_server/src/cluster.rs | 31 ++++++++++++++++---------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 602cf1c9..e708a54e 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -300,6 +300,7 @@ mod test { use super::*; use datafusion::arrow::datatypes::{ArrowPrimitiveType, Field}; + use modelardb_test::table::{NORMAL_TABLE_NAME, TIME_SERIES_TABLE_NAME}; use modelardb_types::types::{ArrowTimestamp, ArrowValue, ErrorBound, ServerMode}; use tempfile::TempDir; @@ -357,14 +358,14 @@ mod test { // Create a normal table in the local data folder with the same name as a normal table in // the remote data folder, but with a different schema. create_normal_table( - "normal_table", + NORMAL_TABLE_NAME, "local", data_folders.local_data_folder.clone(), ) .await; create_normal_table( - "normal_table", + NORMAL_TABLE_NAME, "remote", data_folders.maybe_remote_data_folder.clone().unwrap(), ) @@ -374,8 +375,10 @@ mod test { assert_eq!( result.unwrap_err().to_string(), - "Invalid State Error: The normal table 'normal_table' has a different schema in the \ - local data folder compared to the remote data folder." + format!( + "Invalid State Error: The normal table '{NORMAL_TABLE_NAME}' has a different schema \ + in the local data folder compared to the remote data folder." + ) ); } @@ -429,14 +432,14 @@ mod test { // Create a time series table in the local data folder with the same name as a time series // table in the remote data folder, but with a different schema. create_time_series_table( - "time_series_table", + TIME_SERIES_TABLE_NAME, "local", data_folders.local_data_folder.clone(), ) .await; create_time_series_table( - "time_series_table", + TIME_SERIES_TABLE_NAME, "remote", data_folders.maybe_remote_data_folder.clone().unwrap(), ) @@ -446,8 +449,10 @@ mod test { assert_eq!( result.unwrap_err().to_string(), - "Invalid State Error: The time series table 'time_series_table' has different metadata \ - in the local data folder compared to the remote data folder." + format!( + "Invalid State Error: The time series table '{TIME_SERIES_TABLE_NAME}' has different \ + metadata in the local data folder compared to the remote data folder." + ) ); } @@ -461,14 +466,14 @@ mod test { // Create tables in the local data folder that are not in the remote data folder. create_normal_table( - "normal_table", + NORMAL_TABLE_NAME, "local", data_folders.local_data_folder.clone(), ) .await; create_time_series_table( - "time_series_table", + TIME_SERIES_TABLE_NAME, "local", data_folders.local_data_folder.clone(), ) @@ -478,8 +483,10 @@ mod test { assert_eq!( result.unwrap_err().to_string(), - "Invalid State Error: The following tables do not exist in the remote data folder: \ - normal_table, time_series_table." + format!( + "Invalid State Error: The following tables do not exist in the remote data folder: \ + {NORMAL_TABLE_NAME}, {TIME_SERIES_TABLE_NAME}." + ) ); } From c4dc0b8881a95db66b57d6866404b5196cd34514 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 18:39:24 +0100 Subject: [PATCH 39/99] Add method to remove node and add unit test --- crates/modelardb_server/src/cluster.rs | 31 +++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index e708a54e..0a411798 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -40,6 +40,8 @@ use crate::error::{ModelarDbServerError, Result}; /// to be applied to every single node in the cluster. #[derive(Clone)] pub struct Cluster { + /// Node that represents the local system running `modelardbd`. + node: Node, /// Key identifying the cluster. The key is used to validate communication within the cluster /// between nodes. key: MetadataValue, @@ -59,7 +61,7 @@ impl Cluster { .create_and_register_cluster_metadata_tables() .await?; - remote_data_folder.save_node(node).await?; + remote_data_folder.save_node(node.clone()).await?; let key = remote_data_folder.cluster_key().await?.to_string(); @@ -67,6 +69,7 @@ impl Cluster { let key = MetadataValue::from_str(&key).expect("UUID Version 4 should be valid ASCII."); Ok(Self { + node, key, remote_data_folder, }) @@ -196,6 +199,16 @@ impl Cluster { Ok(url.to_owned()) } + + /// Remove the node that was saved when the [`Cluster`] was created from the remote data folder. + /// If the node could not be removed, return [`ModelarDbServerError`]. Note that this method + /// should only be called when the process running `modelardbd` is stopped. + pub async fn remove_node(&self) -> Result<()> { + self.remote_data_folder + .remove_node(&self.node.url) + .await + .map_err(|error| error.into()) + } } /// Validate that all tables in the local data folder exist in the remote data folder. If any table @@ -599,6 +612,22 @@ mod test { ); } + #[tokio::test] + async fn test_remove_node() { + let temp_dir = tempfile::tempdir().unwrap(); + let edge_node = Node::new("edge".to_owned(), ServerMode::Edge); + + let cluster = create_cluster_with_node(&temp_dir, edge_node.clone()).await; + + let nodes = cluster.remote_data_folder.nodes().await.unwrap(); + assert_eq!(vec![edge_node], nodes); + + cluster.remove_node().await.unwrap(); + + let nodes = cluster.remote_data_folder.nodes().await.unwrap(); + assert!(nodes.is_empty()); + } + /// Create a [`Cluster`] that uses a local [`DataFolder`] for the remote data folder. async fn create_cluster_with_node(temp_dir: &TempDir, node: Node) -> Cluster { let temp_dir_url = temp_dir.path().to_str().unwrap(); From 9bd3735f208f6302e020cfa644f814f155e925d1 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 18:45:15 +0100 Subject: [PATCH 40/99] Use method to remove node when process is stopped --- crates/modelardb_server/src/main.rs | 10 ++++++++-- crates/modelardb_server/src/remote.rs | 6 +++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/crates/modelardb_server/src/main.rs b/crates/modelardb_server/src/main.rs index 39599641..128ab743 100644 --- a/crates/modelardb_server/src/main.rs +++ b/crates/modelardb_server/src/main.rs @@ -125,8 +125,8 @@ pub fn print_usage_and_exit_with_error(parameters: &str) -> ! { } /// Register a handler to execute when CTRL+C is pressed. The handler takes an exclusive lock for -/// the storage engine, flushes the data the storage engine currently buffers, and terminates the -/// system without releasing the lock. +/// the storage engine, flushes the data the storage engine currently buffers, removes the node +/// from the cluster if necessary, and terminates the system without releasing the lock. fn setup_ctrl_c_handler(context: &Arc) { let ctrl_c_context = context.clone(); tokio::spawn(async move { @@ -137,6 +137,12 @@ fn setup_ctrl_c_handler(context: &Arc) { // Stop the threads in the storage engine and close it. ctrl_c_context.storage_engine.write().await.close().unwrap(); + // If running in a cluster, remove the node from the remote data folder. + let configuration_manager = ctrl_c_context.configuration_manager.read().await; + if let ClusterMode::MultiNode(cluster) = &configuration_manager.cluster_mode { + cluster.remove_node().await.unwrap(); + } + std::process::exit(0) }); } diff --git a/crates/modelardb_server/src/remote.rs b/crates/modelardb_server/src/remote.rs index 8df16abe..0a6d6600 100644 --- a/crates/modelardb_server/src/remote.rs +++ b/crates/modelardb_server/src/remote.rs @@ -633,9 +633,9 @@ impl FlightService for FlightServiceHandler { /// object store. Note that data is only transferred to the remote object store if one was /// provided when starting the server. /// * `KillNode`: An extension of the `FlushNode` action that first flushes all data to disk, - /// then flushes all compressed data to the remote object store, and finally kills the process - /// that is running the server. Note that since the process is killed, a conventional response - /// cannot be returned. + /// then flushes all compressed data to the remote object store, then removes the node + /// from the cluster if necessary, and finally kills the process that is running the server. + /// Note that since the process is killed, a conventional response cannot be returned. /// * `GetConfiguration`: Get the current server configuration. The value of each setting in the /// configuration is returned in a [`Configuration`](protocol::Configuration) protobuf message. /// * `UpdateConfiguration`: Update a single setting in the configuration. The setting to update From b95ff679f51b125d5587ea787fc2e42610e6fe04 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 18:46:04 +0100 Subject: [PATCH 41/99] Use method to remove node when KillNode is called --- crates/modelardb_server/src/remote.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/modelardb_server/src/remote.rs b/crates/modelardb_server/src/remote.rs index 0a6d6600..1ccd8c69 100644 --- a/crates/modelardb_server/src/remote.rs +++ b/crates/modelardb_server/src/remote.rs @@ -711,6 +711,15 @@ impl FlightService for FlightServiceHandler { .await .map_err(error_to_status_internal)?; + // If running in a cluster, remove the node from the remote data folder. + let configuration_manager = self.context.configuration_manager.read().await; + if let ClusterMode::MultiNode(cluster) = &configuration_manager.cluster_mode { + cluster + .remove_node() + .await + .map_err(error_to_status_internal)?; + } + // Since the process is killed, a conventional response cannot be given. If the action // returns a "Stream removed" message, the edge was successfully flushed and killed. std::process::exit(0); From 7bed18396a39bbfde366e5cf0afa61027e46012f Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 21:33:45 +0100 Subject: [PATCH 42/99] query_node() no longer uses &mut self --- crates/modelardb_server/src/cluster.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 0a411798..a8a55c5f 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -108,7 +108,7 @@ impl Cluster { /// Return the cloud node in the cluster that is currently most capable of running a query. /// Note that the most capable node is currently selected at random. If there are no cloud nodes /// in the cluster, return [`ModelarDbServerError`]. - pub async fn query_node(&mut self) -> Result { + pub async fn query_node(&self) -> Result { let nodes = self.remote_data_folder.nodes().await?; let cloud_nodes = nodes @@ -590,8 +590,7 @@ mod test { let temp_dir = tempfile::tempdir().unwrap(); let cloud_node = Node::new("cloud".to_owned(), ServerMode::Cloud); - let mut cluster = create_cluster_with_node(&temp_dir, cloud_node.clone()).await; - + let cluster = create_cluster_with_node(&temp_dir, cloud_node.clone()).await; let query_node = cluster.query_node().await.unwrap(); assert_eq!(query_node, cloud_node); @@ -602,8 +601,7 @@ mod test { let temp_dir = tempfile::tempdir().unwrap(); let edge_node = Node::new("edge".to_owned(), ServerMode::Edge); - let mut cluster = create_cluster_with_node(&temp_dir, edge_node).await; - + let cluster = create_cluster_with_node(&temp_dir, edge_node).await; let result = cluster.query_node().await; assert_eq!( From 30ce062485dcb2f8535e0b56ac9f4a9e2a730852 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 21:35:04 +0100 Subject: [PATCH 43/99] Move get_flight_info to server remote --- crates/modelardb_server/src/remote.rs | 45 ++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/crates/modelardb_server/src/remote.rs b/crates/modelardb_server/src/remote.rs index 1ccd8c69..1d603aae 100644 --- a/crates/modelardb_server/src/remote.rs +++ b/crates/modelardb_server/src/remote.rs @@ -28,7 +28,7 @@ use std::sync::Arc; use arrow_flight::flight_service_client::FlightServiceClient; use arrow_flight::flight_service_server::{FlightService, FlightServiceServer}; use arrow_flight::{ - Action, ActionType, Criteria, Empty, FlightData, FlightDescriptor, FlightInfo, + Action, ActionType, Criteria, Empty, FlightData, FlightDescriptor, FlightEndpoint, FlightInfo, HandshakeRequest, HandshakeResponse, PollInfo, PutResult, Result as FlightResult, SchemaAsIpc, SchemaResult, Ticket, utils, }; @@ -394,12 +394,49 @@ impl FlightService for FlightServiceHandler { Ok(Response::new(Box::pin(output))) } - /// Not implemented. + /// Given a query, return [`FlightInfo`] containing [`FlightEndpoints`](FlightEndpoint) + /// describing which cloud nodes should be used to execute the query. The query must be + /// provided in `FlightDescriptor.cmd`. async fn get_flight_info( &self, - _request: Request, + request: Request, ) -> StdResult, Status> { - Err(Status::unimplemented("Not implemented.")) + let flight_descriptor = request.into_inner(); + + // Extract the query. + let query = str::from_utf8(&flight_descriptor.cmd) + .map_err(error_to_status_invalid_argument)? + .to_owned(); + + // Retrieve the cloud node that should execute the given query. + let configuration_manager = self.context.configuration_manager.read().await; + if let ClusterMode::MultiNode(cluster) = &configuration_manager.cluster_mode { + let cloud_node = cluster + .query_node() + .await + .map_err(error_to_status_internal)?; + + info!( + "Assigning query '{query}' to cloud node with url '{}'.", + cloud_node.url + ); + + // All data in the query result should be retrieved using a single endpoint. + let endpoint = FlightEndpoint::new() + .with_ticket(Ticket::new(query)) + .with_location(cloud_node.url); + + // schema is empty and total_records and total_bytes are -1 since we do not know + // anything about the result of the query at this point. + let flight_info = FlightInfo::new() + .with_descriptor(flight_descriptor) + .with_endpoint(endpoint) + .with_ordered(true); + + Ok(Response::new(flight_info)) + } else { + Err(Status::internal("The node is not running in a cluster.")) + } } /// Not implemented. From cc85ab1ef8afdb3c45951845b3e2b4bd9c546e08 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 16 Nov 2025 21:40:08 +0100 Subject: [PATCH 44/99] Reorder get_flight_info to extract cloud node first --- crates/modelardb_server/src/remote.rs | 51 +++++++++++++-------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/crates/modelardb_server/src/remote.rs b/crates/modelardb_server/src/remote.rs index 1d603aae..54e6887d 100644 --- a/crates/modelardb_server/src/remote.rs +++ b/crates/modelardb_server/src/remote.rs @@ -401,42 +401,39 @@ impl FlightService for FlightServiceHandler { &self, request: Request, ) -> StdResult, Status> { - let flight_descriptor = request.into_inner(); + // Retrieve the cloud node that should execute the given query. + let configuration_manager = self.context.configuration_manager.read().await; + let cloud_node = + if let ClusterMode::MultiNode(cluster) = &configuration_manager.cluster_mode { + cluster.query_node().await.map_err(error_to_status_internal) + } else { + Err(Status::internal("The node is not running in a cluster.")) + }?; // Extract the query. + let flight_descriptor = request.into_inner(); let query = str::from_utf8(&flight_descriptor.cmd) .map_err(error_to_status_invalid_argument)? .to_owned(); - // Retrieve the cloud node that should execute the given query. - let configuration_manager = self.context.configuration_manager.read().await; - if let ClusterMode::MultiNode(cluster) = &configuration_manager.cluster_mode { - let cloud_node = cluster - .query_node() - .await - .map_err(error_to_status_internal)?; - - info!( - "Assigning query '{query}' to cloud node with url '{}'.", - cloud_node.url - ); + info!( + "Assigning query '{query}' to cloud node with url '{}'.", + cloud_node.url + ); - // All data in the query result should be retrieved using a single endpoint. - let endpoint = FlightEndpoint::new() - .with_ticket(Ticket::new(query)) - .with_location(cloud_node.url); + // All data in the query result should be retrieved using a single endpoint. + let endpoint = FlightEndpoint::new() + .with_ticket(Ticket::new(query)) + .with_location(cloud_node.url); - // schema is empty and total_records and total_bytes are -1 since we do not know - // anything about the result of the query at this point. - let flight_info = FlightInfo::new() - .with_descriptor(flight_descriptor) - .with_endpoint(endpoint) - .with_ordered(true); + // schema is empty and total_records and total_bytes are -1 since we do not know + // anything about the result of the query at this point. + let flight_info = FlightInfo::new() + .with_descriptor(flight_descriptor) + .with_endpoint(endpoint) + .with_ordered(true); - Ok(Response::new(flight_info)) - } else { - Err(Status::internal("The node is not running in a cluster.")) - } + Ok(Response::new(flight_info)) } /// Not implemented. From 59e9a49162967404f1c9d4a08634794a71a5b495 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 17 Nov 2025 08:12:29 +0100 Subject: [PATCH 45/99] Fix bug where cluster operations was executed on all nodes except only on peers --- crates/modelardb_server/src/cluster.rs | 52 ++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index a8a55c5f..5b0a7e28 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -123,11 +123,11 @@ impl Cluster { }) } - /// For each node in the cluster, execute the given `sql` statement with the cluster key as - /// metadata. If the statement was successfully executed for each node, return [`Ok`], otherwise - /// return [`ModelarDbServerError`]. + /// For each peer node in the cluster, execute the given `sql` statement with the cluster key + /// as metadata. If the statement was successfully executed for each node, return [`Ok`], + /// otherwise return [`ModelarDbServerError`]. pub async fn cluster_do_get(&self, sql: &str) -> Result<()> { - let nodes = self.remote_data_folder.nodes().await?; + let nodes = self.peer_nodes().await?; let mut do_get_futures: FuturesUnordered<_> = nodes .iter() @@ -161,11 +161,11 @@ impl Cluster { Ok(url.to_owned()) } - /// For each node in the cluster, execute the given `action` with the cluster key as metadata. - /// If the action was successfully executed for each node, return [`Ok`], otherwise return - /// [`ModelarDbServerError`]. + /// For each peer node in the cluster, execute the given `action` with the cluster key as + /// metadata. If the action was successfully executed for each node, return [`Ok`], otherwise + /// return [`ModelarDbServerError`]. pub async fn cluster_do_action(&self, action: Action) -> Result<()> { - let nodes = self.remote_data_folder.nodes().await?; + let nodes = self.peer_nodes().await?; let mut action_futures: FuturesUnordered<_> = nodes .iter() @@ -200,6 +200,17 @@ impl Cluster { Ok(url.to_owned()) } + /// Return all nodes in the cluster except the node that was saved when the [`Cluster`] was + /// created. If the nodes could not be retrieved, return [`ModelarDbServerError`]. + async fn peer_nodes(&self) -> Result> { + let nodes = self.remote_data_folder.nodes().await?; + + Ok(nodes + .into_iter() + .filter(|node| node.url != self.node.url) + .collect()) + } + /// Remove the node that was saved when the [`Cluster`] was created from the remote data folder. /// If the node could not be removed, return [`ModelarDbServerError`]. Note that this method /// should only be called when the process running `modelardbd` is stopped. @@ -610,6 +621,31 @@ mod test { ); } + #[tokio::test] + async fn test_peer_nodes() { + let temp_dir = tempfile::tempdir().unwrap(); + + let cloud_node = Node::new("cloud".to_owned(), ServerMode::Cloud); + let cluster = create_cluster_with_node(&temp_dir, cloud_node).await; + + let peer_1 = Node::new("peer_1".to_owned(), ServerMode::Edge); + cluster + .remote_data_folder + .save_node(peer_1.clone()) + .await + .unwrap(); + + let peer_2 = Node::new("peer_2".to_owned(), ServerMode::Edge); + cluster + .remote_data_folder + .save_node(peer_2.clone()) + .await + .unwrap(); + + let peer_nodes = cluster.peer_nodes().await.unwrap(); + assert_eq!(vec![peer_2, peer_1], peer_nodes); + } + #[tokio::test] async fn test_remove_node() { let temp_dir = tempfile::tempdir().unwrap(); From 00478ea2a69e791b80f22c8fe002590f004a1050 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 17 Nov 2025 09:11:22 +0100 Subject: [PATCH 46/99] Add method to check if cluster key is in request metadata --- crates/modelardb_server/src/cluster.rs | 19 ++++++++++++------- crates/modelardb_server/src/remote.rs | 26 +++++++++++++++----------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 5b0a7e28..79a2b437 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -31,7 +31,7 @@ use modelardb_types::types::{Node, TimeSeriesTableMetadata}; use rand::rng; use rand::seq::IteratorRandom; use tonic::Request; -use tonic::metadata::{Ascii, MetadataValue}; +use tonic::metadata::{Ascii, MetadataMap, MetadataValue}; use crate::context::Context; use crate::error::{ModelarDbServerError, Result}; @@ -39,7 +39,7 @@ use crate::error::{ModelarDbServerError, Result}; /// Stores the currently managed nodes in the cluster and allows for performing operations that need /// to be applied to every single node in the cluster. #[derive(Clone)] -pub struct Cluster { +pub(crate) struct Cluster { /// Node that represents the local system running `modelardbd`. node: Node, /// Key identifying the cluster. The key is used to validate communication within the cluster @@ -56,7 +56,7 @@ impl Cluster { /// It is assumed that `node` corresponds to the local system running `modelardbd`. If the /// cluster metadata tables do not exist and could not be created or the node could not be /// saved, return [`ModelarDbServerError`]. - pub async fn try_new(node: Node, remote_data_folder: DataFolder) -> Result { + pub(crate) async fn try_new(node: Node, remote_data_folder: DataFolder) -> Result { remote_data_folder .create_and_register_cluster_metadata_tables() .await?; @@ -75,6 +75,11 @@ impl Cluster { }) } + /// Return the key identifying the cluster. + pub(crate) fn key(&self) -> &MetadataValue { + &self.key + } + /// Initialize the local database schema with the normal tables and time series tables from the /// cluster's database schema using the remote data folder. If the tables to create could not be /// retrieved from the remote data folder, or the tables could not be created, @@ -108,7 +113,7 @@ impl Cluster { /// Return the cloud node in the cluster that is currently most capable of running a query. /// Note that the most capable node is currently selected at random. If there are no cloud nodes /// in the cluster, return [`ModelarDbServerError`]. - pub async fn query_node(&self) -> Result { + pub(crate) async fn query_node(&self) -> Result { let nodes = self.remote_data_folder.nodes().await?; let cloud_nodes = nodes @@ -126,7 +131,7 @@ impl Cluster { /// For each peer node in the cluster, execute the given `sql` statement with the cluster key /// as metadata. If the statement was successfully executed for each node, return [`Ok`], /// otherwise return [`ModelarDbServerError`]. - pub async fn cluster_do_get(&self, sql: &str) -> Result<()> { + pub(crate) async fn cluster_do_get(&self, sql: &str) -> Result<()> { let nodes = self.peer_nodes().await?; let mut do_get_futures: FuturesUnordered<_> = nodes @@ -164,7 +169,7 @@ impl Cluster { /// For each peer node in the cluster, execute the given `action` with the cluster key as /// metadata. If the action was successfully executed for each node, return [`Ok`], otherwise /// return [`ModelarDbServerError`]. - pub async fn cluster_do_action(&self, action: Action) -> Result<()> { + pub(crate) async fn cluster_do_action(&self, action: Action) -> Result<()> { let nodes = self.peer_nodes().await?; let mut action_futures: FuturesUnordered<_> = nodes @@ -214,7 +219,7 @@ impl Cluster { /// Remove the node that was saved when the [`Cluster`] was created from the remote data folder. /// If the node could not be removed, return [`ModelarDbServerError`]. Note that this method /// should only be called when the process running `modelardbd` is stopped. - pub async fn remove_node(&self) -> Result<()> { + pub(crate) async fn remove_node(&self) -> Result<()> { self.remote_data_folder .remove_node(&self.node.url) .await diff --git a/crates/modelardb_server/src/remote.rs b/crates/modelardb_server/src/remote.rs index 54e6887d..7f94eb81 100644 --- a/crates/modelardb_server/src/remote.rs +++ b/crates/modelardb_server/src/remote.rs @@ -343,18 +343,22 @@ impl FlightServiceHandler { Ok(()) } - /// If the server was started with a manager, validate the request by checking that the key in - /// the request metadata matches the key of the manager. If the request is invalid, return a - /// [`Status`] with the code [`tonic::Code::Unauthenticated`]. - async fn validate_request(&self, request_metadata: &MetadataMap) -> StdResult<(), Status> { - let configuration_manager = self.context.configuration_manager.read().await; - - if let ClusterMode::MultiNode(manager) = &configuration_manager.cluster_mode { - manager - .validate_request(request_metadata) - .map_err(|error| Status::unauthenticated(error.to_string())) + /// Return `true` if the request contains the cluster key and `false` if not. If the request + /// contains a key that does not match the cluster key, return [`Status`]. + async fn cluster_key_in_request( + cluster: &Cluster, + request_metadata: &MetadataMap, + ) -> StdResult { + if let Some(request_key) = request_metadata.get("x-cluster-key") { + if cluster.key() == request_key { + Ok(true) + } else { + Err(Status::invalid_argument( + "The cluster key in the request does not match the cluster key in the configuration.", + )) + } } else { - Ok(()) + Ok(false) } } } From f8c36813ab831cdc64343f84ba90799a74191db2 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 17 Nov 2025 09:28:48 +0100 Subject: [PATCH 47/99] Use new methods to handle potentially creating tables in entire cluster --- crates/modelardb_server/src/remote.rs | 130 +++++++++++++++++--------- 1 file changed, 87 insertions(+), 43 deletions(-) diff --git a/crates/modelardb_server/src/remote.rs b/crates/modelardb_server/src/remote.rs index 7f94eb81..39101dac 100644 --- a/crates/modelardb_server/src/remote.rs +++ b/crates/modelardb_server/src/remote.rs @@ -55,6 +55,7 @@ use tonic::{Request, Response, Status, Streaming}; use tracing::{debug, error, info}; use crate::ClusterMode; +use crate::cluster::Cluster; use crate::context::Context; use crate::error::{ModelarDbServerError, Result}; @@ -244,6 +245,25 @@ pub fn table_name_from_flight_descriptor( .ok_or_else(|| Status::invalid_argument("No table name in FlightDescriptor.path.")) } +/// Return `true` if the request contains the cluster key and `false` if not. If the request +/// contains a key that does not match the cluster key, return [`Status`]. +fn cluster_key_in_request( + cluster: &Cluster, + request_metadata: &MetadataMap, +) -> StdResult { + if let Some(request_key) = request_metadata.get("x-cluster-key") { + if cluster.key() == request_key { + Ok(true) + } else { + Err(Status::invalid_argument( + "The cluster key in the request does not match the cluster key in the configuration.", + )) + } + } else { + Ok(false) + } +} + /// Return an empty stream of [`RecordBatches`](datafusion::arrow::record_batch::RecordBatch) that /// can be returned when a SQL command has been successfully executed but did not produce any rows /// to return. @@ -288,6 +308,65 @@ impl FlightServiceHandler { } } + /// Create a normal table with the given `name` and `schema`. If the node is running in a + /// cluster, the table is created in the remote data folder and locally in each node in the + /// cluster. If not, the table is only created locally. + async fn create_normal_table( + &self, + name: &str, + schema: &Schema, + request_metadata: &MetadataMap, + ) -> StdResult<(), Status> { + let configuration_manager = self.context.configuration_manager.read().await; + + // If the cluster key is in the request, the request is from a peer node, which means the + // table has already been created in the remote data folder and propagated to all nodes. + if let ClusterMode::MultiNode(cluster) = &configuration_manager.cluster_mode + && !cluster_key_in_request(cluster, request_metadata)? + { + cluster + .create_cluster_normal_table(name, schema) + .await + .map_err(error_to_status_invalid_argument)?; + } + + self.context + .create_normal_table(name, schema) + .await + .map_err(error_to_status_invalid_argument)?; + + Ok(()) + } + + /// Create a time series table with the given `time_series_table_metadata`. If the node is + /// running in a cluster, the table is created in the remote data folder and locally in each + /// node in the cluster. If not, the table is only created locally. + async fn create_time_series_table( + &self, + time_series_table_metadata: &TimeSeriesTableMetadata, + request_metadata: &MetadataMap, + ) -> StdResult<(), Status> { + let configuration_manager = self.context.configuration_manager.read().await; + + // If the cluster key is in the request, the request is from a peer node, which means the + // table has already been created in the remote data folder and propagated to all nodes. + if let ClusterMode::MultiNode(cluster) = &configuration_manager.cluster_mode + && !cluster_key_in_request(cluster, request_metadata)? + { + cluster + .create_cluster_time_series_table(time_series_table_metadata) + .await + .map_err(error_to_status_invalid_argument)?; + } + + self.context + .create_time_series_table(time_series_table_metadata) + .await + .map_err(error_to_status_invalid_argument)?; + + Ok(()) + } + /// While there is still more data to receive, ingest the data into the normal table. async fn ingest_into_normal_table( &self, @@ -342,25 +421,6 @@ impl FlightServiceHandler { Ok(()) } - - /// Return `true` if the request contains the cluster key and `false` if not. If the request - /// contains a key that does not match the cluster key, return [`Status`]. - async fn cluster_key_in_request( - cluster: &Cluster, - request_metadata: &MetadataMap, - ) -> StdResult { - if let Some(request_key) = request_metadata.get("x-cluster-key") { - if cluster.key() == request_key { - Ok(true) - } else { - Err(Status::invalid_argument( - "The cluster key in the request does not match the cluster key in the configuration.", - )) - } - } else { - Ok(false) - } - } } #[tonic::async_trait] @@ -493,22 +553,14 @@ impl FlightService for FlightServiceHandler { let sendable_record_batch_stream = match modelardb_statement { ModelarDbStatement::CreateNormalTable { name, schema } => { - self.validate_request(request.metadata()).await?; - - self.context - .create_normal_table(&name, &schema) - .await - .map_err(error_to_status_invalid_argument)?; + self.create_normal_table(&name, &schema, request.metadata()) + .await?; Ok(empty_record_batch_stream()) } ModelarDbStatement::CreateTimeSeriesTable(time_series_table_metadata) => { - self.validate_request(request.metadata()).await?; - - self.context - .create_time_series_table(&time_series_table_metadata) - .await - .map_err(error_to_status_invalid_argument)?; + self.create_time_series_table(&time_series_table_metadata, request.metadata()) + .await?; Ok(empty_record_batch_stream()) } @@ -689,24 +741,18 @@ impl FlightService for FlightServiceHandler { info!("Received request to perform action '{}'.", action.r#type); if action.r#type == "CreateTable" { - self.validate_request(request.metadata()).await?; - let table_metadata = modelardb_types::flight::deserialize_and_extract_table_metadata(&action.body) .map_err(error_to_status_invalid_argument)?; match table_metadata { Table::NormalTable(table_name, schema) => { - self.context - .create_normal_table(&table_name, &schema) - .await - .map_err(error_to_status_invalid_argument)?; + self.create_normal_table(&table_name, &schema, request.metadata()) + .await?; } Table::TimeSeriesTable(metadata) => { - self.context - .create_time_series_table(&metadata) - .await - .map_err(error_to_status_invalid_argument)?; + self.create_time_series_table(&metadata, request.metadata()) + .await?; } } @@ -737,8 +783,6 @@ impl FlightService for FlightServiceHandler { // Confirm the data was flushed. Ok(Response::new(Box::pin(stream::empty()))) } else if action.r#type == "KillNode" { - self.validate_request(request.metadata()).await?; - let mut storage_engine = self.context.storage_engine.write().await; storage_engine .flush() From 8d70670547a09fb0d5c0d7a204f7f9c135687353 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 17 Nov 2025 09:55:52 +0100 Subject: [PATCH 48/99] Add cluster methods to create table in remote data folder and in peer nodes --- crates/modelardb_server/src/cluster.rs | 64 ++++++++++++++++++++++++++ crates/modelardb_server/src/remote.rs | 8 ++-- 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 79a2b437..358ae94e 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -128,6 +128,70 @@ impl Cluster { }) } + /// Create a normal table with the given `table_name` and `schema` in the remote data folder and + /// in each peer node. If the normal table already exists or could not be created, return + /// [`ModelarDbServerError`]. + pub(crate) async fn create_cluster_normal_table( + &self, + table_name: &str, + schema: &Schema, + ) -> Result<()> { + // Create the normal table in the remote data folder. + self.remote_data_folder + .create_normal_table(table_name, schema) + .await?; + + self.remote_data_folder + .save_normal_table_metadata(table_name) + .await?; + + // Create the normal table in each peer node. + let protobuf_bytes = modelardb_types::flight::encode_and_serialize_normal_table_metadata( + table_name, schema, + )?; + + let action = Action { + r#type: "CreateTable".to_owned(), + body: protobuf_bytes.into(), + }; + + self.cluster_do_action(action).await?; + + Ok(()) + } + + /// Create a time series table with the given `time_series_table_metadata` in the remote data + /// folder and in each peer node. If the time series table already exists or could not be + /// created, return [`ModelarDbServerError`]. + pub(crate) async fn create_cluster_time_series_table( + &self, + time_series_table_metadata: &TimeSeriesTableMetadata, + ) -> Result<()> { + // Create the time series table in the remote data folder. + self.remote_data_folder + .create_time_series_table(time_series_table_metadata) + .await?; + + self.remote_data_folder + .save_time_series_table_metadata(time_series_table_metadata) + .await?; + + // Create the time series table in each peer node. + let protobuf_bytes = + modelardb_types::flight::encode_and_serialize_time_series_table_metadata( + time_series_table_metadata, + )?; + + let action = Action { + r#type: "CreateTable".to_owned(), + body: protobuf_bytes.into(), + }; + + self.cluster_do_action(action).await?; + + Ok(()) + } + /// For each peer node in the cluster, execute the given `sql` statement with the cluster key /// as metadata. If the statement was successfully executed for each node, return [`Ok`], /// otherwise return [`ModelarDbServerError`]. diff --git a/crates/modelardb_server/src/remote.rs b/crates/modelardb_server/src/remote.rs index 39101dac..ae0a60cb 100644 --- a/crates/modelardb_server/src/remote.rs +++ b/crates/modelardb_server/src/remote.rs @@ -308,12 +308,12 @@ impl FlightServiceHandler { } } - /// Create a normal table with the given `name` and `schema`. If the node is running in a + /// Create a normal table with the given `table_name` and `schema`. If the node is running in a /// cluster, the table is created in the remote data folder and locally in each node in the /// cluster. If not, the table is only created locally. async fn create_normal_table( &self, - name: &str, + table_name: &str, schema: &Schema, request_metadata: &MetadataMap, ) -> StdResult<(), Status> { @@ -325,13 +325,13 @@ impl FlightServiceHandler { && !cluster_key_in_request(cluster, request_metadata)? { cluster - .create_cluster_normal_table(name, schema) + .create_cluster_normal_table(table_name, schema) .await .map_err(error_to_status_invalid_argument)?; } self.context - .create_normal_table(name, schema) + .create_normal_table(table_name, schema) .await .map_err(error_to_status_invalid_argument)?; From adae82546e41ef00aace04916a0e00f7acc8d3e4 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 17 Nov 2025 10:02:42 +0100 Subject: [PATCH 49/99] Add method to server remote to handle potentially dropping tables in entire cluster --- crates/modelardb_server/src/remote.rs | 36 ++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/crates/modelardb_server/src/remote.rs b/crates/modelardb_server/src/remote.rs index ae0a60cb..48cc2fb5 100644 --- a/crates/modelardb_server/src/remote.rs +++ b/crates/modelardb_server/src/remote.rs @@ -367,6 +367,35 @@ impl FlightServiceHandler { Ok(()) } + /// Drop the table with the given `table_name`. If the node is running in a cluster, the table + /// is dropped in the remote data folder and locally in each node in the cluster. If not, the + /// table is only dropped locally. + async fn drop_table( + &self, + table_name: &str, + request_metadata: &MetadataMap, + ) -> StdResult<(), Status> { + let configuration_manager = self.context.configuration_manager.read().await; + + // If the cluster key is in the request, the request is from a peer node, which means the + // table has already been dropped in the remote data folder and propagated to all nodes. + if let ClusterMode::MultiNode(cluster) = &configuration_manager.cluster_mode + && !cluster_key_in_request(cluster, request_metadata)? + { + cluster + .drop_cluster_table(table_name) + .await + .map_err(error_to_status_invalid_argument)?; + } + + self.context + .drop_table(table_name) + .await + .map_err(error_to_status_invalid_argument)?; + + Ok(()) + } + /// While there is still more data to receive, ingest the data into the normal table. async fn ingest_into_normal_table( &self, @@ -603,13 +632,8 @@ impl FlightService for FlightServiceHandler { Ok(empty_record_batch_stream()) } ModelarDbStatement::DropTable(table_names) => { - self.validate_request(request.metadata()).await?; - for table_name in table_names { - self.context - .drop_table(&table_name) - .await - .map_err(error_to_status_invalid_argument)?; + self.drop_table(&table_name, request.metadata()).await?; } Ok(empty_record_batch_stream()) From 524fdde064f970eba6b44f52e29adc707c88b721 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 17 Nov 2025 10:06:34 +0100 Subject: [PATCH 50/99] Add method to drop table in remote data folder and in each peer node --- crates/modelardb_server/src/cluster.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 358ae94e..d11b354d 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -192,6 +192,23 @@ impl Cluster { Ok(()) } + /// Drop the normal table with the given `table_name` in the remote data folder and in each peer + /// node. If the normal table could not be dropped, return [`ModelarDbServerError`]. + pub(crate) async fn drop_cluster_table(&self, table_name: &str) -> Result<()> { + // Drop the table from the remote data folder. + self.remote_data_folder + .drop_table_metadata(table_name) + .await?; + + self.remote_data_folder.drop_table(table_name).await?; + + // Drop the table from each peer node. + self.cluster_do_get(&format!("DROP TABLE {table_name}")) + .await?; + + Ok(()) + } + /// For each peer node in the cluster, execute the given `sql` statement with the cluster key /// as metadata. If the statement was successfully executed for each node, return [`Ok`], /// otherwise return [`ModelarDbServerError`]. From 9ec8d09166b3b014c20e794712307f1ddada1075 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 17 Nov 2025 18:57:05 +0100 Subject: [PATCH 51/99] Remove unused startup functions in manager --- crates/modelardb_manager/src/main.rs | 59 +++------------------------- 1 file changed, 5 insertions(+), 54 deletions(-) diff --git a/crates/modelardb_manager/src/main.rs b/crates/modelardb_manager/src/main.rs index 984a65e8..a808228b 100644 --- a/crates/modelardb_manager/src/main.rs +++ b/crates/modelardb_manager/src/main.rs @@ -17,22 +17,20 @@ mod cluster; mod error; -mod metadata; mod remote; +use std::env; use std::sync::{Arc, LazyLock}; -use std::{env, process}; use modelardb_storage::data_folder::DataFolder; -use modelardb_types::flight::protocol; use tokio::sync::RwLock; use tonic::metadata::errors::InvalidMetadataValue; use tonic::metadata::{Ascii, MetadataValue}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; +use uuid::Uuid; use crate::cluster::Cluster; use crate::error::{ModelarDbManagerError, Result}; -use crate::metadata::ManagerMetadata; use crate::remote::start_apache_arrow_flight_server; /// The port of the Apache Arrow Flight Server. If the environment variable is not set, 9998 is used. @@ -43,9 +41,6 @@ pub static PORT: LazyLock = pub struct Context { /// [`DataFolder`] for storing metadata and data in Apache Parquet files. pub remote_data_folder: DataFolder, - /// Storage configuration encoded as a [`StorageConfiguration`](protocol::manager_metadata::StorageConfiguration) - /// protobuf message to make it possible to transfer the configuration using Apache Arrow Flight. - pub remote_storage_configuration: protocol::manager_metadata::StorageConfiguration, /// Cluster of nodes currently controlled by the manager. pub cluster: RwLock, /// Key used to identify requests coming from the manager. @@ -62,32 +57,12 @@ async fn main() -> Result<()> { let stdout_log = tracing_subscriber::fmt::layer(); tracing_subscriber::registry().with(stdout_log).init(); - let user_arguments = collect_command_line_arguments(3); - let user_arguments: Vec<&str> = user_arguments.iter().map(|arg| arg.as_str()).collect(); - let remote_data_folder_str = match user_arguments.as_slice() { - &[remote_data_folder_str] => remote_data_folder_str, - _ => print_usage_and_exit_with_error("remote_data_folder"), - }; + let remote_data_folder = DataFolder::open_remote_url("s3://test").await?; - let remote_storage_configuration = - modelardb_types::flight::argument_to_storage_configuration(remote_data_folder_str)?; - let remote_data_folder = - DataFolder::open_remote_url(remote_storage_configuration.clone()).await?; - - remote_data_folder - .create_and_register_manager_metadata_data_folder_tables() - .await?; - - let mut cluster = Cluster::new(); - let nodes = remote_data_folder.nodes().await?; - for node in nodes { - cluster.register_node(node)?; - } + let cluster = Cluster::new(); // Retrieve and parse the key to a tonic metadata value since it is used in tonic requests. - let key = remote_data_folder - .manager_key() - .await? + let key = Uuid::new_v4() .to_string() .parse() .map_err(|error: InvalidMetadataValue| { @@ -97,7 +72,6 @@ async fn main() -> Result<()> { // Create the Context. let context = Arc::new(Context { remote_data_folder, - remote_storage_configuration, cluster: RwLock::new(cluster), key, }); @@ -106,26 +80,3 @@ async fn main() -> Result<()> { Ok(()) } - -/// Collect the command line arguments that this program was started with. -pub fn collect_command_line_arguments(maximum_arguments: usize) -> Vec { - let mut args = std::env::args(); - args.next(); // Skip the executable. - - // Collect at most the maximum number of command line arguments plus one. The plus one argument - // is collected to trigger the default pattern when parsing the command line arguments with - // pattern matching, making it possible to handle errors caused by too many arguments. - args.by_ref().take(maximum_arguments + 1).collect() -} - -/// Prints a usage message with `parameters` appended to the name of the binary executing this -/// function to stderr and exits with status code one to indicate that an error has occurred. -pub fn print_usage_and_exit_with_error(parameters: &str) -> ! { - // The errors are consciously ignored as the program is terminating. - let binary_path = std::env::current_exe().unwrap(); - let binary_name = binary_path.file_name().unwrap().to_str().unwrap(); - - // Punctuation at the end does not seem to be common in the usage message of Unix tools. - eprintln!("Usage: {binary_name} {parameters}"); - process::exit(1); -} From fb42b50deb024c2d20772f436f3107f2f9fac978 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 17 Nov 2025 18:57:30 +0100 Subject: [PATCH 52/99] Remove unused metadata management in manager --- crates/modelardb_manager/src/metadata.rs | 307 ----------------------- 1 file changed, 307 deletions(-) delete mode 100644 crates/modelardb_manager/src/metadata.rs diff --git a/crates/modelardb_manager/src/metadata.rs b/crates/modelardb_manager/src/metadata.rs deleted file mode 100644 index 00fdda03..00000000 --- a/crates/modelardb_manager/src/metadata.rs +++ /dev/null @@ -1,307 +0,0 @@ -/* Copyright 2023 The ModelarDB Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -//! Management of the Delta Lake for the manager. Metadata which is unique to the manager, such as -//! metadata about registered edges, is handled here. - -use std::str::FromStr; -use std::sync::Arc; - -use arrow::array::{Array, StringArray}; -use arrow::datatypes::{DataType, Field, Schema}; -use deltalake::DeltaTableError; -use deltalake::datafusion::logical_expr::{col, lit}; -use modelardb_storage::data_folder::DataFolder; -use modelardb_storage::{register_metadata_table, sql_and_concat}; -use modelardb_types::types::{Node, ServerMode}; -use uuid::Uuid; - -use crate::error::Result; - -/// Stores the metadata required for reading from and writing to the normal tables and time series -/// tables and persisting edges. The data that needs to be persisted is stored in the Delta Lake. -pub trait ManagerMetadata { - async fn create_and_register_manager_metadata_data_folder_tables(&self) -> Result<()>; - async fn manager_key(&self) -> Result; - async fn save_node(&self, node: Node) -> Result<()>; - async fn remove_node(&self, url: &str) -> Result<()>; - async fn nodes(&self) -> Result>; -} - -impl ManagerMetadata for DataFolder { - /// If they do not already exist, create the tables that are specific to the manager metadata - /// Delta Lake and register them with the Apache DataFusion session context. - /// * The `manager_metadata` table contains metadata for the manager itself. It is assumed that - /// this table will only have a single row since there can only be a single manager. - /// * The `nodes` table contains metadata for each node that is controlled by the manager. - /// - /// If the tables exist or were created, return [`Ok`], otherwise return - /// [`ModelarDbManagerError`](crate::error::ModelarDbManagerError). - async fn create_and_register_manager_metadata_data_folder_tables(&self) -> Result<()> { - // Create and register the manager_metadata table if it does not exist. - let delta_table = self - .create_metadata_table( - "manager_metadata", - &Schema::new(vec![Field::new("key", DataType::Utf8, false)]), - ) - .await?; - - register_metadata_table(self.session_context(), "manager_metadata", delta_table)?; - - // Create and register the nodes table if it does not exist. - let delta_table = self - .create_metadata_table( - "nodes", - &Schema::new(vec![ - Field::new("url", DataType::Utf8, false), - Field::new("mode", DataType::Utf8, false), - ]), - ) - .await?; - - register_metadata_table(self.session_context(), "nodes", delta_table)?; - - Ok(()) - } - - /// Retrieve the key for the manager from the `manager_metadata` table. If a key does not - /// already exist, create one and save it to the Delta Lake. If a key could not be retrieved - /// or created, return [`ModelarDbManagerError`](crate::error::ModelarDbManagerError). - async fn manager_key(&self) -> Result { - let sql = "SELECT key FROM metadata.manager_metadata"; - let batch = sql_and_concat(self.session_context(), sql).await?; - - let keys = modelardb_types::array!(batch, 0, StringArray); - if keys.is_empty() { - let manager_key = Uuid::new_v4(); - - // Add a new row to the manager_metadata table to persist the key. - self.write_columns_to_metadata_table( - "manager_metadata", - vec![Arc::new(StringArray::from(vec![manager_key.to_string()]))], - ) - .await?; - - Ok(manager_key) - } else { - let manager_key: String = keys.value(0).to_owned(); - - Ok(manager_key - .parse() - .map_err(|error: uuid::Error| DeltaTableError::Generic(error.to_string()))?) - } - } - - /// Save the node to the Delta Lake and return [`Ok`]. If the node could not be saved, return - /// [`ModelarDbManagerError`](crate::error::ModelarDbManagerError). - async fn save_node(&self, node: Node) -> Result<()> { - self.write_columns_to_metadata_table( - "nodes", - vec![ - Arc::new(StringArray::from(vec![node.url])), - Arc::new(StringArray::from(vec![node.mode.to_string()])), - ], - ) - .await?; - - Ok(()) - } - - /// Remove the row in the `nodes` table that corresponds to the node with `url` and return - /// [`Ok`]. If the row could not be removed, return - /// [`ModelarDbManagerError`](crate::error::ModelarDbManagerError). - async fn remove_node(&self, url: &str) -> Result<()> { - let delta_ops = self.metadata_delta_ops("nodes").await?; - - delta_ops - .delete() - .with_predicate(col("url").eq(lit(url))) - .await?; - - Ok(()) - } - - /// Return the nodes currently controlled by the manager that have been persisted to the Delta - /// Lake. If the nodes could not be retrieved, - /// [`ModelarDbManagerError`](crate::error::ModelarDbManagerError) is returned. - async fn nodes(&self) -> Result> { - let mut nodes: Vec = vec![]; - - let sql = "SELECT url, mode FROM metadata.nodes"; - let batch = sql_and_concat(self.session_context(), sql).await?; - - let url_array = modelardb_types::array!(batch, 0, StringArray); - let mode_array = modelardb_types::array!(batch, 1, StringArray); - - for row_index in 0..batch.num_rows() { - let url = url_array.value(row_index).to_owned(); - let mode = mode_array.value(row_index).to_owned(); - - let server_mode = ServerMode::from_str(&mode) - .map_err(|error| DeltaTableError::Generic(error.to_string()))?; - - nodes.push(Node::new(url, server_mode)); - } - - Ok(nodes) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - use tempfile::TempDir; - - // Tests for MetadataManager. - #[tokio::test] - async fn test_create_manager_metadata_data_folder_tables() { - let (_temp_dir, data_folder) = create_data_folder().await; - - // Verify that the tables were created, registered, and has the expected columns. - assert!( - data_folder - .session_context() - .sql("SELECT key FROM metadata.manager_metadata") - .await - .is_ok() - ); - - assert!( - data_folder - .session_context() - .sql("SELECT url, mode FROM metadata.nodes") - .await - .is_ok() - ); - } - - #[tokio::test] - async fn test_new_manager_key() { - let (_temp_dir, data_folder) = create_data_folder().await; - - // Verify that the manager key is created and saved correctly. - let manager_key = data_folder.manager_key().await.unwrap(); - - let sql = "SELECT key FROM metadata.manager_metadata"; - let batch = sql_and_concat(data_folder.session_context(), sql) - .await - .unwrap(); - - assert_eq!( - **batch.column(0), - StringArray::from(vec![manager_key.to_string()]) - ); - } - - #[tokio::test] - async fn test_existing_manager_key() { - let (_temp_dir, data_folder) = create_data_folder().await; - - // Verify that only a single key is created and saved when retrieving multiple times. - let manager_key_1 = data_folder.manager_key().await.unwrap(); - let manager_key_2 = data_folder.manager_key().await.unwrap(); - - let sql = "SELECT key FROM metadata.manager_metadata"; - let batch = sql_and_concat(data_folder.session_context(), sql) - .await - .unwrap(); - - assert_eq!(manager_key_1, manager_key_2); - assert_eq!(batch.column(0).len(), 1); - } - - #[tokio::test] - async fn test_save_node() { - let (_temp_dir, data_folder) = create_data_folder().await; - - let node_1 = Node::new("url_1".to_string(), ServerMode::Edge); - data_folder.save_node(node_1.clone()).await.unwrap(); - - let node_2 = Node::new("url_2".to_string(), ServerMode::Edge); - data_folder.save_node(node_2.clone()).await.unwrap(); - - // Verify that the nodes are saved correctly. - let sql = "SELECT url, mode FROM metadata.nodes"; - let batch = sql_and_concat(data_folder.session_context(), sql) - .await - .unwrap(); - - assert_eq!( - **batch.column(0), - StringArray::from(vec![node_2.url.clone(), node_1.url.clone()]) - ); - assert_eq!( - **batch.column(1), - StringArray::from(vec![node_2.mode.to_string(), node_1.mode.to_string()]) - ); - } - - #[tokio::test] - async fn test_remove_node() { - let (_temp_dir, data_folder) = create_data_folder().await; - - let node_1 = Node::new("url_1".to_string(), ServerMode::Edge); - data_folder.save_node(node_1.clone()).await.unwrap(); - - let node_2 = Node::new("url_2".to_string(), ServerMode::Edge); - data_folder.save_node(node_2.clone()).await.unwrap(); - - data_folder.remove_node(&node_1.url).await.unwrap(); - - // Verify that node_1 is removed correctly. - let sql = "SELECT url, mode FROM metadata.nodes"; - let batch = sql_and_concat(data_folder.session_context(), sql) - .await - .unwrap(); - - assert_eq!( - **batch.column(0), - StringArray::from(vec![node_2.url.clone()]) - ); - assert_eq!( - **batch.column(1), - StringArray::from(vec![node_2.mode.to_string()]) - ); - } - - #[tokio::test] - async fn test_nodes() { - let (_temp_dir, data_folder) = create_data_folder().await; - - let node_1 = Node::new("url_1".to_string(), ServerMode::Edge); - data_folder.save_node(node_1.clone()).await.unwrap(); - - let node_2 = Node::new("url_2".to_string(), ServerMode::Edge); - data_folder.save_node(node_2.clone()).await.unwrap(); - - let nodes = data_folder.nodes().await.unwrap(); - - assert_eq!(nodes, vec![node_2, node_1]); - } - - async fn create_data_folder() -> (TempDir, DataFolder) { - let temp_dir = tempfile::tempdir().unwrap(); - - let data_folder = DataFolder::open_local(temp_dir.path()).await.unwrap(); - - data_folder - .create_and_register_manager_metadata_data_folder_tables() - .await - .unwrap(); - - (temp_dir, data_folder) - } -} From e87e1cce73ff6b498e80303d71fe1788367c419c Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 17 Nov 2025 18:58:40 +0100 Subject: [PATCH 53/99] Remove duplicated methods for table validation from old Manager struct --- crates/modelardb_server/src/manager.rs | 217 ------------------------- 1 file changed, 217 deletions(-) diff --git a/crates/modelardb_server/src/manager.rs b/crates/modelardb_server/src/manager.rs index 2e00e39a..14b27a48 100644 --- a/crates/modelardb_server/src/manager.rs +++ b/crates/modelardb_server/src/manager.rs @@ -16,24 +16,8 @@ //! Interface to connect to and interact with the manager, used if the server is started with a //! manager and needs to interact with it to initialize the Delta Lake. -use std::sync::Arc; -use std::{env, str}; - -use arrow_flight::flight_service_client::FlightServiceClient; -use arrow_flight::{Action, Result as FlightResult}; -use datafusion::arrow::datatypes::Schema; -use datafusion::catalog::TableProvider; -use modelardb_storage::data_folder::DataFolder; -use modelardb_types::flight::protocol; -use modelardb_types::types::{Node, ServerMode, TimeSeriesTableMetadata}; -use prost::Message; -use tokio::sync::RwLock; -use tonic::Request; use tonic::metadata::MetadataMap; -use tonic::transport::Channel; -use crate::PORT; -use crate::context::Context; use crate::error::{ModelarDbServerError, Result}; /// Manages metadata related to the manager and provides functionality for interacting with the manager. @@ -49,86 +33,6 @@ impl Manager { Self { key } } - /// Register the server as a node in the cluster and retrieve the key and connection information - /// from the manager. If the key and connection information could not be retrieved, - /// [`ModelarDbServerError`] is returned. - pub(crate) async fn register_node( - manager_url: &str, - server_mode: ServerMode, - ) -> Result<(Self, protocol::manager_metadata::StorageConfiguration)> { - let flight_client = Arc::new(RwLock::new( - FlightServiceClient::connect(manager_url.to_owned()) - .await - .map_err(|error| { - ModelarDbServerError::InvalidArgument(format!( - "Could not connect to manager at '{manager_url}': {error}", - )) - })?, - )); - - let ip_address = env::var("MODELARDBD_IP_ADDRESS").unwrap_or("127.0.0.1".to_string()); - let url_with_port = format!("grpc://{ip_address}:{}", &PORT.to_string()); - - // Add the url and mode of the server to the action request. - let node = Node::new(url_with_port, server_mode); - let node_metadata = modelardb_types::flight::encode_node(&node)?; - - let action = Action { - r#type: "RegisterNode".to_owned(), - body: node_metadata.encode_to_vec().into(), - }; - - let message = do_action_and_extract_result(&flight_client, action).await?; - - // Extract the key and the storage configuration for the remote object store from the response. - let manager_metadata = protocol::ManagerMetadata::decode(message.body)?; - - Ok(( - Manager::new(manager_metadata.key), - manager_metadata - .storage_configuration - .expect("The manager should have a remote storage configuration."), - )) - } - - /// Initialize the local database schema with the normal tables and time series tables from the - /// manager's database schema using the remote data folder. If the tables to create could not be - /// retrieved from the remote data folder, or the tables could not be created, - /// return [`ModelarDbServerError`]. - pub(crate) async fn retrieve_and_create_tables(&self, context: &Arc) -> Result<()> { - let local_data_folder = &context.data_folders.local_data_folder; - - let remote_data_folder = &context - .data_folders - .maybe_remote_data_folder - .clone() - .ok_or(ModelarDbServerError::InvalidState( - "Remote data folder is missing.".to_owned(), - ))?; - - validate_local_tables_exist_remotely(local_data_folder, remote_data_folder).await?; - - // Validate that all tables that are in both the local and remote data folder are identical. - let missing_normal_tables = - validate_normal_tables(local_data_folder, remote_data_folder).await?; - - let missing_time_series_tables = - validate_time_series_tables(local_data_folder, remote_data_folder).await?; - - // For each table that does not already exist locally, create the table. - for (table_name, schema) in missing_normal_tables { - context.create_normal_table(&table_name, &schema).await?; - } - - for time_series_table_metadata in missing_time_series_tables { - context - .create_time_series_table(&time_series_table_metadata) - .await?; - } - - Ok(()) - } - /// Validate the request by checking that the key in the request metadata matches the key of the /// manager. If the request is valid, return [`Ok`], otherwise return [`ModelarDbServerError`]. pub fn validate_request(&self, request_metadata: &MetadataMap) -> Result<()> { @@ -149,127 +53,6 @@ impl Manager { } } -/// Execute `action` using `flight_client` and extract the message inside the response. If `action` -/// could not be executed or the response is invalid or empty, return [`ModelarDbServerError`]. -async fn do_action_and_extract_result( - flight_client: &RwLock>, - action: Action, -) -> Result { - let response = flight_client - .write() - .await - .do_action(Request::new(action.clone())) - .await?; - - // Extract the message from the response. - let maybe_message = response.into_inner().message().await?; - - // Handle that the response is potentially empty. - maybe_message.ok_or_else(|| { - ModelarDbServerError::InvalidArgument(format!( - "Response for action request '{}' is empty.", - action.r#type - )) - }) -} - -/// Validate that all tables in the local data folder exist in the remote data folder. If any table -/// does not exist in the remote data folder, return [`ModelarDbServerError`]. -async fn validate_local_tables_exist_remotely( - local_data_folder: &DataFolder, - remote_data_folder: &DataFolder, -) -> Result<()> { - let local_table_names = local_data_folder.table_names().await?; - let remote_table_names = remote_data_folder.table_names().await?; - - let invalid_tables: Vec = local_table_names - .iter() - .filter(|table| !remote_table_names.contains(table)) - .cloned() - .collect(); - - if !invalid_tables.is_empty() { - return Err(ModelarDbServerError::InvalidState(format!( - "The following tables do not exist in the remote data folder: {}.", - invalid_tables.join(", ") - ))); - } - - Ok(()) -} - -/// For each normal table in the remote data folder, if the table also exists in the local data -/// folder, validate that the schemas are identical. If the schemas are not identical, return -/// [`ModelarDbServerError`]. Return a vector containing the name and schema of each normal table -/// that is in the remote data folder but not in the local data folder. -async fn validate_normal_tables( - local_data_folder: &DataFolder, - remote_data_folder: &DataFolder, -) -> Result)>> { - let mut missing_normal_tables = vec![]; - - let remote_normal_tables = remote_data_folder.normal_table_names().await?; - - for table_name in remote_normal_tables { - let remote_schema = normal_table_schema(remote_data_folder, &table_name).await?; - - if let Ok(local_schema) = normal_table_schema(local_data_folder, &table_name).await { - if remote_schema != local_schema { - return Err(ModelarDbServerError::InvalidState(format!( - "The normal table '{table_name}' has a different schema in the local data \ - folder compared to the remote data folder.", - ))); - } - } else { - missing_normal_tables.push((table_name, remote_schema)); - } - } - - Ok(missing_normal_tables) -} - -/// Retrieve the schema of a normal table from the Delta Lake in the data folder. If the table does -/// not exist, or the schema could not be retrieved, return [`ModelarDbServerError`]. -async fn normal_table_schema(data_folder: &DataFolder, table_name: &str) -> Result> { - let delta_table = data_folder.delta_table(table_name).await?; - Ok(TableProvider::schema(&delta_table)) -} - -/// For each time series table in the remote data folder, if the table also exists in the local -/// data folder, validate that the metadata is identical. If the metadata is not identical, return -/// [`ModelarDbServerError`]. Return a vector containing the metadata of each time series table -/// that is in the remote data folder but not in the local data folder. -async fn validate_time_series_tables( - local_data_folder: &DataFolder, - remote_data_folder: &DataFolder, -) -> Result> { - let mut missing_time_series_tables = vec![]; - - let remote_time_series_tables = remote_data_folder.time_series_table_names().await?; - - for table_name in remote_time_series_tables { - let remote_metadata = remote_data_folder - .time_series_table_metadata_for_time_series_table(&table_name) - .await?; - - if let Ok(local_metadata) = local_data_folder - .time_series_table_metadata_for_time_series_table(&table_name) - .await - { - if remote_metadata != local_metadata { - return Err(ModelarDbServerError::InvalidState(format!( - "The time series table '{table_name}' has different metadata in the local data \ - folder compared to the remote data folder.", - ))); - } - } else { - missing_time_series_tables.push(remote_metadata); - } - } - - Ok(missing_time_series_tables) -} - #[cfg(test)] mod tests { use super::*; From 547d2381064a0a4c7a9c24427d71201548e86f65 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 17 Nov 2025 18:59:09 +0100 Subject: [PATCH 54/99] Remove unused actions to register and remove nodes --- crates/modelardb_manager/src/remote.rs | 90 -------------------------- 1 file changed, 90 deletions(-) diff --git a/crates/modelardb_manager/src/remote.rs b/crates/modelardb_manager/src/remote.rs index 6567731a..ae1eb348 100644 --- a/crates/modelardb_manager/src/remote.rs +++ b/crates/modelardb_manager/src/remote.rs @@ -35,16 +35,13 @@ use arrow_flight::{ use futures::{Stream, stream}; use modelardb_storage::parser; use modelardb_storage::parser::ModelarDbStatement; -use modelardb_types::flight::protocol; use modelardb_types::types::{Table, TimeSeriesTableMetadata}; -use prost::Message; use tonic::transport::Server; use tonic::{Request, Response, Status, Streaming}; use tracing::info; use crate::Context; use crate::error::{ModelarDbManagerError, Result}; -use crate::metadata::ManagerMetadata; /// Start an Apache Arrow Flight server on 0.0.0.0:`port`. pub async fn start_apache_arrow_flight_server(context: Arc, port: u16) -> Result<()> { @@ -558,14 +555,6 @@ impl FlightService for FlightServiceHandler { /// * `CreateTable`: Create the table given in the [`TableMetadata`](protocol::TableMetadata) /// protobuf message in the action body. The table is created for each node in the cluster of /// nodes controlled by the manager. - /// * `RegisterNode`: Register either an edge or cloud node with the manager. The url and mode - /// of the node must be provided in the action body as a [`NodeMetadata`](protocol::NodeMetadata) - /// protobuf message. The node is added to the cluster of nodes controlled by the manager and - /// the key and object store used in the cluster is returned as a - /// [`ManagerMetadata`](protocol::ManagerMetadata) protobuf message. - /// * `RemoveNode`: Remove the node given in the [`NodeMetadata`](protocol::NodeMetadata) - /// protobuf message in the action body. The node is removed from the cluster of nodes - /// controlled by the manager and the process running on the node is killed. /// * `NodeType`: Get the type of the node. The type is always `manager`. The type of the node /// is returned as a string. async fn do_action( @@ -596,72 +585,6 @@ impl FlightService for FlightServiceHandler { // Confirm the tables were created. Ok(Response::new(Box::pin(stream::empty()))) - } else if action.r#type == "RegisterNode" { - // Extract the node from the action body. - let node_metadata = protocol::NodeMetadata::decode(action.body) - .map_err(error_to_status_invalid_argument)?; - let node = modelardb_types::flight::decode_node_metadata(&node_metadata) - .map_err(error_to_status_invalid_argument)?; - - // Use the cluster to register the node in memory. This returns an error if the node is - // already registered. - self.context - .cluster - .write() - .await - .register_node(node.clone()) - .map_err(error_to_status_internal)?; - - // Use the metadata manager to persist the node to the Delta Lake. Note that if this - // fails, the Delta Lake and the cluster will be out of sync until the manager is - // restarted. - self.context - .remote_data_folder - .save_node(node) - .await - .map_err(error_to_status_internal)?; - - let manager_metadata = protocol::ManagerMetadata { - key: self - .context - .key - .to_str() - .expect("key should not contain invalid characters.") - .to_owned(), - storage_configuration: Some(self.context.remote_storage_configuration.clone()), - }; - - let protobuf_bytes = manager_metadata.encode_to_vec(); - - // Return the key for the manager and the storage configuration for the remote object store. - Ok(Response::new(Box::pin(stream::once(async { - Ok(FlightResult { - body: protobuf_bytes.into(), - }) - })))) - } else if action.r#type == "RemoveNode" { - let node_metadata = protocol::NodeMetadata::decode(action.body) - .map_err(error_to_status_invalid_argument)?; - - // Remove the node with the given url from the Delta Lake. - self.context - .remote_data_folder - .remove_node(&node_metadata.url) - .await - .map_err(error_to_status_internal)?; - - // Remove the node with the given url from the cluster and kill it. Note that if this - // fails, the cluster and Delta Lake will be out of sync until the manager is restarted. - self.context - .cluster - .write() - .await - .remove_node(&node_metadata.url, &self.context.key) - .await - .map_err(error_to_status_internal)?; - - // Confirm the node was removed. - Ok(Response::new(Box::pin(stream::empty()))) } else if action.r#type == "NodeType" { let flight_result = FlightResult { body: "manager".bytes().collect(), @@ -686,17 +609,6 @@ impl FlightService for FlightServiceHandler { .to_owned(), }; - let register_node_action = ActionType { - r#type: "RegisterNode".to_owned(), - description: "Register either an edge or cloud node with the manager.".to_owned(), - }; - - let remove_node_action = ActionType { - r#type: "RemoveNode".to_owned(), - description: "Remove a node from the manager and kill the process running on the node." - .to_owned(), - }; - let node_type_action = ActionType { r#type: "NodeType".to_owned(), description: "Get the type of the node.".to_owned(), @@ -704,8 +616,6 @@ impl FlightService for FlightServiceHandler { let output = stream::iter(vec![ Ok(create_tables_action), - Ok(register_node_action), - Ok(remove_node_action), Ok(node_type_action), ]); From a96945c7d8b3d97602719a7066e0510808d1c1c2 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 17 Nov 2025 19:07:21 +0100 Subject: [PATCH 55/99] Remove unused Manager struct from server --- crates/modelardb_server/src/cluster.rs | 2 +- crates/modelardb_server/src/main.rs | 3 +- crates/modelardb_server/src/manager.rs | 104 ------------------------- 3 files changed, 2 insertions(+), 107 deletions(-) delete mode 100644 crates/modelardb_server/src/manager.rs diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index d11b354d..27ed4edc 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -31,7 +31,7 @@ use modelardb_types::types::{Node, TimeSeriesTableMetadata}; use rand::rng; use rand::seq::IteratorRandom; use tonic::Request; -use tonic::metadata::{Ascii, MetadataMap, MetadataValue}; +use tonic::metadata::{Ascii, MetadataValue}; use crate::context::Context; use crate::error::{ModelarDbServerError, Result}; diff --git a/crates/modelardb_server/src/main.rs b/crates/modelardb_server/src/main.rs index 128ab743..b93fb876 100644 --- a/crates/modelardb_server/src/main.rs +++ b/crates/modelardb_server/src/main.rs @@ -20,7 +20,6 @@ mod configuration; mod context; mod data_folders; mod error; -mod manager; mod remote; mod storage; @@ -44,7 +43,7 @@ pub static PORT: LazyLock = /// The different possible modes that a ModelarDB server can be deployed in, assigned when the /// server is started. #[derive(Clone)] -pub enum ClusterMode { +pub(crate) enum ClusterMode { SingleNode, MultiNode(Cluster), } diff --git a/crates/modelardb_server/src/manager.rs b/crates/modelardb_server/src/manager.rs deleted file mode 100644 index 14b27a48..00000000 --- a/crates/modelardb_server/src/manager.rs +++ /dev/null @@ -1,104 +0,0 @@ -/* Copyright 2023 The ModelarDB Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -//! Interface to connect to and interact with the manager, used if the server is started with a -//! manager and needs to interact with it to initialize the Delta Lake. - -use tonic::metadata::MetadataMap; - -use crate::error::{ModelarDbServerError, Result}; - -/// Manages metadata related to the manager and provides functionality for interacting with the manager. -#[derive(Clone, Debug, PartialEq)] -pub struct Manager { - /// Key received from the manager when registering, used to validate future requests that are - /// only allowed to come from the manager. - key: String, -} - -impl Manager { - pub fn new(key: String) -> Self { - Self { key } - } - - /// Validate the request by checking that the key in the request metadata matches the key of the - /// manager. If the request is valid, return [`Ok`], otherwise return [`ModelarDbServerError`]. - pub fn validate_request(&self, request_metadata: &MetadataMap) -> Result<()> { - let request_key = - request_metadata - .get("x-manager-key") - .ok_or(ModelarDbServerError::InvalidState( - "Missing manager key.".to_owned(), - ))?; - - if &self.key != request_key { - Err(ModelarDbServerError::InvalidState(format!( - "Manager key '{request_key:?}' is invalid.", - ))) - } else { - Ok(()) - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - use uuid::Uuid; - - // Tests for validate_request(). - #[tokio::test] - async fn test_validate_request() { - let manager = create_manager(); - let mut request_metadata = MetadataMap::new(); - request_metadata.append("x-manager-key", manager.key.parse().unwrap()); - - assert!(manager.validate_request(&request_metadata).is_ok()); - } - - #[tokio::test] - async fn test_validate_request_without_key() { - let manager = create_manager(); - let request_metadata = MetadataMap::new(); - - let result = manager.validate_request(&request_metadata); - - assert_eq!( - result.unwrap_err().to_string(), - "Invalid State Error: Missing manager key." - ); - } - - #[tokio::test] - async fn test_validate_request_with_invalid_key() { - let manager = create_manager(); - let mut request_metadata = MetadataMap::new(); - - let key = Uuid::new_v4().to_string(); - request_metadata.append("x-manager-key", key.parse().unwrap()); - - let result = manager.validate_request(&request_metadata); - - assert_eq!( - result.unwrap_err().to_string(), - format!("Invalid State Error: Manager key '\"{key}\"' is invalid.") - ); - } - - fn create_manager() -> Manager { - Manager::new(Uuid::new_v4().to_string()) - } -} From 2ebf2b396ef3fcb1546eb3866d5e9090b484379e Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 17 Nov 2025 19:12:34 +0100 Subject: [PATCH 56/99] Allow async fn in trait by disabling warning --- crates/modelardb_storage/src/data_folder/cluster.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/modelardb_storage/src/data_folder/cluster.rs b/crates/modelardb_storage/src/data_folder/cluster.rs index 7c9d22d7..c4dd5eb5 100644 --- a/crates/modelardb_storage/src/data_folder/cluster.rs +++ b/crates/modelardb_storage/src/data_folder/cluster.rs @@ -31,6 +31,7 @@ use crate::error::Result; use crate::{register_metadata_table, sql_and_concat}; /// Trait that extends [`DataFolder`] to provide management of the Delta Lake for the cluster. +#[allow(async_fn_in_trait)] pub trait ClusterMetadata { async fn create_and_register_cluster_metadata_tables(&self) -> Result<()>; async fn cluster_key(&self) -> Result; From 6db84d9be31af2b3b3bb098a1c1ad8daab98def5 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 17 Nov 2025 19:19:27 +0100 Subject: [PATCH 57/99] Use allow to remote clippy warnings --- crates/modelardb_manager/src/remote.rs | 5 +---- crates/modelardb_server/src/main.rs | 1 + crates/modelardb_server/src/remote.rs | 1 + 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/modelardb_manager/src/remote.rs b/crates/modelardb_manager/src/remote.rs index ae1eb348..9517b59c 100644 --- a/crates/modelardb_manager/src/remote.rs +++ b/crates/modelardb_manager/src/remote.rs @@ -614,10 +614,7 @@ impl FlightService for FlightServiceHandler { description: "Get the type of the node.".to_owned(), }; - let output = stream::iter(vec![ - Ok(create_tables_action), - Ok(node_type_action), - ]); + let output = stream::iter(vec![Ok(create_tables_action), Ok(node_type_action)]); Ok(Response::new(Box::pin(output))) } diff --git a/crates/modelardb_server/src/main.rs b/crates/modelardb_server/src/main.rs index b93fb876..9ffe41be 100644 --- a/crates/modelardb_server/src/main.rs +++ b/crates/modelardb_server/src/main.rs @@ -42,6 +42,7 @@ pub static PORT: LazyLock = /// The different possible modes that a ModelarDB server can be deployed in, assigned when the /// server is started. +#[allow(clippy::large_enum_variant)] #[derive(Clone)] pub(crate) enum ClusterMode { SingleNode, diff --git a/crates/modelardb_server/src/remote.rs b/crates/modelardb_server/src/remote.rs index 48cc2fb5..e45159b0 100644 --- a/crates/modelardb_server/src/remote.rs +++ b/crates/modelardb_server/src/remote.rs @@ -247,6 +247,7 @@ pub fn table_name_from_flight_descriptor( /// Return `true` if the request contains the cluster key and `false` if not. If the request /// contains a key that does not match the cluster key, return [`Status`]. +#[allow(clippy::result_large_err)] fn cluster_key_in_request( cluster: &Cluster, request_metadata: &MetadataMap, From 828c8a258da64bd5862c6f2bcb908e2077b18ea5 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 17 Nov 2025 20:19:05 +0100 Subject: [PATCH 58/99] Use Statement::DropSecret instead of Statement::Notify for Vacuum --- crates/modelardb_storage/src/parser.rs | 31 ++++++++++++++------------ 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/crates/modelardb_storage/src/parser.rs b/crates/modelardb_storage/src/parser.rs index cb9d3fdc..1b59b9f5 100644 --- a/crates/modelardb_storage/src/parser.rs +++ b/crates/modelardb_storage/src/parser.rs @@ -132,11 +132,11 @@ pub fn tokenize_and_parse_sql_statement(sql_statement: &str) -> Result Ok(ModelarDbStatement::Vacuum( - channel.value.split_terminator(';').map(|s| s.to_owned()).collect(), - payload.and_then(|p| p.parse::().ok()), + Statement::DropSecret { name, storage_specifier, .. } => Ok(ModelarDbStatement::Vacuum( + name.value.split_terminator(';').map(|s| s.to_owned()).collect(), + storage_specifier.and_then(|p| p.value.parse::().ok()), )), Statement::Explain { .. } => Ok(ModelarDbStatement::Statement(statement)), Statement::Query(ref boxed_query) => { @@ -493,12 +493,12 @@ impl ModelarDbDialect { } } - /// Parse VACUUM \[table_name\[, table_name\]+\] \[RETAIN num_seconds\] to a [`Statement::NOTIFY`] - /// with the table names in the `channel` field and the optional retention period in the `payload` - /// field. Note that [`Statement::NOTIFY`] is used since [`Statement`] does not have a `Vacuum` - /// variant. A [`ParserError`] is returned if VACUUM is not the first word, the table names - /// cannot be extracted, or the retention period is not a valid positive integer that is at - /// most [`MAX_RETENTION_PERIOD_IN_SECONDS`] seconds. + /// Parse VACUUM \[table_name\[, table_name\]+\] \[RETAIN num_seconds\] to a + /// [`Statement::DropSecret`] with the table names in the `name` field and the optional + /// retention period in the `storage_specifier` field. Note that [`Statement::DropSecret`] is + /// used since [`Statement`] does not have a `Vacuum` variant. A [`ParserError`] is returned if + /// VACUUM is not the first word, the table names cannot be extracted, or the retention period + /// is not a valid positive integer that is at most [`MAX_RETENTION_PERIOD_IN_SECONDS`] seconds. fn parse_vacuum(&self, parser: &mut Parser) -> StdResult { // VACUUM. parser.expect_keyword(Keyword::VACUUM)?; @@ -543,10 +543,13 @@ impl ModelarDbDialect { None }; - // Return Statement::NOTIFY as a substitute for Vacuum. - Ok(Statement::NOTIFY { - channel: Ident::new(table_names.join(";")), - payload: maybe_retention_period_in_seconds.map(|period| period.to_string()), + // Return Statement::DropSecret as a substitute for Vacuum. + Ok(Statement::DropSecret { + if_exists: false, + temporary: None, + name: Ident::new(table_names.join(";")), + storage_specifier: maybe_retention_period_in_seconds + .map(|period| Ident::new(period.to_string())), }) } From 2eb6cfd65617fba3741ca3927c1e93930ceb4ada Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 17 Nov 2025 22:29:44 +0100 Subject: [PATCH 59/99] Add support for parsing VACUUM CLUSTER statements --- crates/modelardb_storage/src/parser.rs | 39 +++++++++++++++++--------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/crates/modelardb_storage/src/parser.rs b/crates/modelardb_storage/src/parser.rs index 1b59b9f5..4cfe7d7d 100644 --- a/crates/modelardb_storage/src/parser.rs +++ b/crates/modelardb_storage/src/parser.rs @@ -69,7 +69,7 @@ pub enum ModelarDbStatement { /// TRUNCATE TABLE. TruncateTable(Vec), /// VACUUM. - Vacuum(Vec, Option), + Vacuum(Vec, Option, bool), } /// Tokenizes and parses the SQL statement in `sql` and returns its parsed representation in the form @@ -134,9 +134,10 @@ pub fn tokenize_and_parse_sql_statement(sql_statement: &str) -> Result Ok(ModelarDbStatement::Vacuum( + Statement::DropSecret { name, storage_specifier, if_exists, .. } => Ok(ModelarDbStatement::Vacuum( name.value.split_terminator(';').map(|s| s.to_owned()).collect(), storage_specifier.and_then(|p| p.value.parse::().ok()), + if_exists, )), Statement::Explain { .. } => Ok(ModelarDbStatement::Statement(statement)), Statement::Query(ref boxed_query) => { @@ -175,7 +176,7 @@ pub fn tokenize_and_parse_sql_expression( /// SQL dialect that extends `sqlparsers's` [`GenericDialect`] with support for parsing CREATE TIME /// SERIES TABLE table_name DDL statements, INCLUDE 'address'\[, 'address'\]+ DQL statements, and -/// VACUUM \[table_name\[, table_name\]+\] \[RETAIN num_seconds\] statements. +/// VACUUM \[CLUSTER\] \[table_name\[, table_name\]+\] \[RETAIN num_seconds\] statements. #[derive(Debug)] struct ModelarDbDialect { /// Dialect to use for identifying identifiers. @@ -493,16 +494,27 @@ impl ModelarDbDialect { } } - /// Parse VACUUM \[table_name\[, table_name\]+\] \[RETAIN num_seconds\] to a - /// [`Statement::DropSecret`] with the table names in the `name` field and the optional - /// retention period in the `storage_specifier` field. Note that [`Statement::DropSecret`] is - /// used since [`Statement`] does not have a `Vacuum` variant. A [`ParserError`] is returned if - /// VACUUM is not the first word, the table names cannot be extracted, or the retention period - /// is not a valid positive integer that is at most [`MAX_RETENTION_PERIOD_IN_SECONDS`] seconds. + /// Parse VACUUM \[CLUSTER\] \[table_name\[, table_name\]+\] \[RETAIN num_seconds\] to a + /// [`Statement::DropSecret`] with the table names in the `name` field, the cluster flag in + /// the `if_exists` field, and the optional retention period in the `storage_specifier` field. + /// Note that [`Statement::DropSecret`] is used since [`Statement`] does not have a `Vacuum` + /// variant. A [`ParserError`] is returned if VACUUM is not the first word, the table names + /// cannot be extracted, or the retention period is not a valid positive integer that is at + /// most [`MAX_RETENTION_PERIOD_IN_SECONDS`] seconds. fn parse_vacuum(&self, parser: &mut Parser) -> StdResult { // VACUUM. parser.expect_keyword(Keyword::VACUUM)?; + // If the next token is CLUSTER, consume it and set the flag to vacuum the entire cluster. + let vacuum_cluster = if let Token::Word(word) = parser.peek_nth_token(0).token + && word.keyword == Keyword::CLUSTER + { + parser.expect_keyword(Keyword::CLUSTER)?; + true + } else { + false + }; + let mut table_names = vec![]; // If the next token is a word that is not RETAIN, attempt to parse table names. @@ -545,7 +557,7 @@ impl ModelarDbDialect { // Return Statement::DropSecret as a substitute for Vacuum. Ok(Statement::DropSecret { - if_exists: false, + if_exists: vacuum_cluster, temporary: None, name: Ident::new(table_names.join(";")), storage_specifier: maybe_retention_period_in_seconds @@ -602,9 +614,10 @@ impl Dialect for ModelarDbDialect { /// as a CREATE TIME SERIES TABLE DDL statement. If not, check if the next token is INCLUDE, if so, /// attempt to parse the token stream as an INCLUDE 'address'\[, 'address'\]+ DQL statement. /// If not, check if the next token is VACUUM, if so, attempt to parse the token stream as a - /// VACUUM \[table_name\[, table_name\]+\] \[RETAIN num_seconds\] statement. If all checks fail, - /// [`None`] is returned so [`sqlparser`] uses its parsing methods for all other statements. - /// If parsing succeeds, a [`Statement`] is returned, and if not, a [`ParserError`] is returned. + /// VACUUM \[CLUSTER\] \[table_name\[, table_name\]+\] \[RETAIN num_seconds\] statement. If all + /// checks fail, [`None`] is returned so [`sqlparser`] uses its parsing methods for all other + /// statements. If parsing succeeds, a [`Statement`] is returned, and if not, a [`ParserError`] + /// is returned. fn parse_statement(&self, parser: &mut Parser) -> Option> { if self.next_tokens_are_create_time_series_table(parser) { Some(self.parse_create_time_series_table(parser)) From d5f9ebdba2fa4cc73f4d5992eb7357de06c9d8a2 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 17 Nov 2025 22:31:59 +0100 Subject: [PATCH 60/99] Add more unit tests for testing correct VACUUM CLUSTER statements --- crates/modelardb_storage/src/parser.rs | 71 +++++++++++++++++++++++--- 1 file changed, 63 insertions(+), 8 deletions(-) diff --git a/crates/modelardb_storage/src/parser.rs b/crates/modelardb_storage/src/parser.rs index 4cfe7d7d..cf6eebb2 100644 --- a/crates/modelardb_storage/src/parser.rs +++ b/crates/modelardb_storage/src/parser.rs @@ -1799,25 +1799,27 @@ mod tests { #[test] fn test_tokenize_and_parse_vacuum_all_tables() { - let (table_names, maybe_retention_period_in_seconds) = + let (table_names, maybe_retention_period_in_seconds, cluster) = parse_vacuum_and_extract_table_names("VACUUM"); assert!(table_names.is_empty()); assert!(maybe_retention_period_in_seconds.is_none()); + assert!(!cluster) } #[test] fn test_tokenize_and_parse_vacuum_single_table() { - let (table_names, maybe_retention_period_in_seconds) = + let (table_names, maybe_retention_period_in_seconds, cluster) = parse_vacuum_and_extract_table_names("VACUUM table_name"); assert_eq!(table_names, vec!["table_name".to_owned()]); assert!(maybe_retention_period_in_seconds.is_none()); + assert!(!cluster) } #[test] fn test_tokenize_and_parse_vacuum_multiple_tables() { - let (table_names, maybe_retention_period_in_seconds) = + let (table_names, maybe_retention_period_in_seconds, cluster) = parse_vacuum_and_extract_table_names("VACUUM table_name_1, table_name_2"); assert_eq!( @@ -1825,20 +1827,22 @@ mod tests { vec!["table_name_1".to_owned(), "table_name_2".to_owned()] ); assert!(maybe_retention_period_in_seconds.is_none()); + assert!(!cluster) } #[test] fn test_tokenize_and_parse_vacuum_with_retention_period() { - let (table_names, maybe_retention_period_in_seconds) = + let (table_names, maybe_retention_period_in_seconds, cluster) = parse_vacuum_and_extract_table_names("VACUUM RETAIN 30"); assert!(table_names.is_empty()); assert_eq!(maybe_retention_period_in_seconds, Some(30)); + assert!(!cluster) } #[test] fn test_tokenize_and_parse_vacuum_multiple_tables_with_retention_period() { - let (table_names, maybe_retention_period_in_seconds) = + let (table_names, maybe_retention_period_in_seconds, cluster) = parse_vacuum_and_extract_table_names("VACUUM table_name_1, table_name_2 RETAIN 30"); assert_eq!( @@ -1846,14 +1850,65 @@ mod tests { vec!["table_name_1".to_owned(), "table_name_2".to_owned()] ); assert_eq!(maybe_retention_period_in_seconds, Some(30)); + assert!(!cluster) } - fn parse_vacuum_and_extract_table_names(sql_statement: &str) -> (Vec, Option) { + #[test] + fn test_tokenize_and_parse_vacuum_cluster() { + let (table_names, maybe_retention_period_in_seconds, cluster) = + parse_vacuum_and_extract_table_names("VACUUM CLUSTER"); + + assert!(table_names.is_empty()); + assert!(maybe_retention_period_in_seconds.is_none()); + assert!(cluster) + } + + #[test] + fn test_tokenize_and_parse_vacuum_cluster_with_multiple_tables() { + let (table_names, maybe_retention_period_in_seconds, cluster) = + parse_vacuum_and_extract_table_names("VACUUM CLUSTER table_name_1, table_name_2"); + + assert_eq!( + table_names, + vec!["table_name_1".to_owned(), "table_name_2".to_owned()] + ); + assert!(maybe_retention_period_in_seconds.is_none()); + assert!(cluster) + } + + #[test] + fn test_tokenize_and_parse_vacuum_cluster_with_retention_period() { + let (table_names, maybe_retention_period_in_seconds, cluster) = + parse_vacuum_and_extract_table_names("VACUUM CLUSTER RETAIN 30"); + + assert!(table_names.is_empty()); + assert_eq!(maybe_retention_period_in_seconds, Some(30)); + assert!(cluster) + } + + #[test] + fn test_tokenize_and_parse_vacuum_cluster_with_multiple_tables_and_retention_period() { + let (table_names, maybe_retention_period_in_seconds, cluster) = + parse_vacuum_and_extract_table_names( + "VACUUM CLUSTER table_name_1, table_name_2 RETAIN 30", + ); + + assert_eq!( + table_names, + vec!["table_name_1".to_owned(), "table_name_2".to_owned()] + ); + assert_eq!(maybe_retention_period_in_seconds, Some(30)); + assert!(cluster) + } + + fn parse_vacuum_and_extract_table_names( + sql_statement: &str, + ) -> (Vec, Option, bool) { let modelardb_statement = tokenize_and_parse_sql_statement(sql_statement).unwrap(); match modelardb_statement { - ModelarDbStatement::Vacuum(table_names, maybe_retention_period_in_seconds) => { - (table_names, maybe_retention_period_in_seconds) + ModelarDbStatement::Vacuum(table_names, maybe_retention_period_in_seconds, cluster) => { + (table_names, maybe_retention_period_in_seconds, cluster) } _ => panic!("Expected ModelarDbStatement::Vacuum."), } From 1d0af6a969e1d92369ba76cdf9e05489336a0bf1 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 17 Nov 2025 22:33:21 +0100 Subject: [PATCH 61/99] Add more unit tests for testing incorrect VACUUM CLUSTER statements --- crates/modelardb_storage/src/parser.rs | 40 ++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/crates/modelardb_storage/src/parser.rs b/crates/modelardb_storage/src/parser.rs index cf6eebb2..55188ca7 100644 --- a/crates/modelardb_storage/src/parser.rs +++ b/crates/modelardb_storage/src/parser.rs @@ -2058,4 +2058,44 @@ mod tests { "Parser Error: sql parser error: Expected: end of statement, found: table_1 at Line: 1, Column: 18" ); } + + #[test] + fn test_tokenize_and_parse_vacuum_cluster_last() { + let result = tokenize_and_parse_sql_statement("VACUUM table_1, table_2 RETAIN 30 CLUSTER"); + + assert_eq!( + result.unwrap_err().to_string(), + "Parser Error: sql parser error: Expected: end of statement, found: CLUSTER at Line: 1, Column: 35" + ); + } + + #[test] + fn test_tokenize_and_parse_vacuum_cluster_after_tables_before_retain() { + let result = tokenize_and_parse_sql_statement("VACUUM table_1, table_2 CLUSTER RETAIN 30"); + + assert_eq!( + result.unwrap_err().to_string(), + "Parser Error: sql parser error: Expected: end of statement, found: CLUSTER at Line: 1, Column: 25" + ); + } + + #[test] + fn test_tokenize_and_parse_vacuum_cluster_trailing_comma() { + let result = tokenize_and_parse_sql_statement("VACUUM CLUSTER, table_1"); + + assert_eq!( + result.unwrap_err().to_string(), + "Parser Error: sql parser error: Expected: end of statement, found: , at Line: 1, Column: 15" + ); + } + + #[test] + fn test_tokenize_and_parse_vacuum_retain_and_cluster_mixed() { + let result = tokenize_and_parse_sql_statement("VACUUM RETAIN CLUSTER 30"); + + assert_eq!( + result.unwrap_err().to_string(), + "Parser Error: sql parser error: Expected: literal integer, found: CLUSTER at Line: 1, Column: 15" + ); + } } From 0c8c9ca038dc5b637e9dd1ca37399775f1aa40b7 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 18 Nov 2025 15:47:03 +0100 Subject: [PATCH 62/99] Add util method to parse table names --- crates/modelardb_storage/src/parser.rs | 44 ++++++++++++++++---------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/crates/modelardb_storage/src/parser.rs b/crates/modelardb_storage/src/parser.rs index 55188ca7..137117db 100644 --- a/crates/modelardb_storage/src/parser.rs +++ b/crates/modelardb_storage/src/parser.rs @@ -515,26 +515,14 @@ impl ModelarDbDialect { false }; - let mut table_names = vec![]; - // If the next token is a word that is not RETAIN, attempt to parse table names. - if let Token::Word(word) = parser.peek_nth_token(0).token + let table_names = if let Token::Word(word) = parser.peek_nth_token(0).token && word.keyword != Keyword::RETAIN { - loop { - match self.parse_word_value(parser) { - Ok(table_name) => { - table_names.push(table_name); - if Token::Comma == parser.peek_nth_token(0).token { - parser.next_token(); - } else { - break; - }; - } - Err(error) => return Err(error), - } - } - } + self.parse_table_names(parser)? + } else { + vec![] + }; // If the next token is RETAIN, attempt to parse the retention period in seconds. let maybe_retention_period_in_seconds = if let Token::Word(word) = @@ -578,6 +566,28 @@ impl ModelarDbDialect { _ => parser.expected("literal integer", token_with_location), } } + /// Return a list of table names parsed from the token stream. It is assumed that the table + /// names are separated by commas. A [`ParserError`] is returned if the table names cannot be + /// extracted. + fn parse_table_names(&self, parser: &mut Parser) -> StdResult, ParserError> { + let mut table_names = vec![]; + + loop { + match self.parse_word_value(parser) { + Ok(table_name) => { + table_names.push(table_name); + if Token::Comma == parser.peek_nth_token(0).token { + parser.next_token(); + } else { + break; + }; + } + Err(error) => return Err(error), + } + } + + Ok(table_names) + } } /// Create a [`Setting`] that is repurposed for storing `address` as ClickHouse's SETTINGS is not From 92cd70d461c6a6d1a21eb7cae2e7d606d6d8021d Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 18 Nov 2025 15:50:12 +0100 Subject: [PATCH 63/99] Update doc comments to add new TRUNCATE statements --- crates/modelardb_storage/src/parser.rs | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/crates/modelardb_storage/src/parser.rs b/crates/modelardb_storage/src/parser.rs index 137117db..9bcd6d8c 100644 --- a/crates/modelardb_storage/src/parser.rs +++ b/crates/modelardb_storage/src/parser.rs @@ -175,8 +175,9 @@ pub fn tokenize_and_parse_sql_expression( } /// SQL dialect that extends `sqlparsers's` [`GenericDialect`] with support for parsing CREATE TIME -/// SERIES TABLE table_name DDL statements, INCLUDE 'address'\[, 'address'\]+ DQL statements, and -/// VACUUM \[CLUSTER\] \[table_name\[, table_name\]+\] \[RETAIN num_seconds\] statements. +/// SERIES TABLE table_name DDL statements, INCLUDE 'address'\[, 'address'\]+ DQL statements, +/// VACUUM \[CLUSTER\] \[table_name\[, table_name\]+\] \[RETAIN num_seconds\] statements, and +/// TRUNCATE \[CLUSTER\] table_name\[, table_name\]+ statements. #[derive(Debug)] struct ModelarDbDialect { /// Dialect to use for identifying identifiers. @@ -624,10 +625,11 @@ impl Dialect for ModelarDbDialect { /// as a CREATE TIME SERIES TABLE DDL statement. If not, check if the next token is INCLUDE, if so, /// attempt to parse the token stream as an INCLUDE 'address'\[, 'address'\]+ DQL statement. /// If not, check if the next token is VACUUM, if so, attempt to parse the token stream as a - /// VACUUM \[CLUSTER\] \[table_name\[, table_name\]+\] \[RETAIN num_seconds\] statement. If all - /// checks fail, [`None`] is returned so [`sqlparser`] uses its parsing methods for all other - /// statements. If parsing succeeds, a [`Statement`] is returned, and if not, a [`ParserError`] - /// is returned. + /// VACUUM \[CLUSTER\] \[table_name\[, table_name\]+\] \[RETAIN num_seconds\] statement. If not, + /// check if the next token is TRUNCATE, if so, attempt to parse the token stream as a + /// TRUNCATE \[CLUSTER\] table_name\[, table_name\]+ statement. If all checks fail, [`None`] is + /// returned so [`sqlparser`] uses its parsing methods for all other statements. If parsing + /// succeeds, a [`Statement`] is returned, and if not, a [`ParserError`] is returned. fn parse_statement(&self, parser: &mut Parser) -> Option> { if self.next_tokens_are_create_time_series_table(parser) { Some(self.parse_create_time_series_table(parser)) @@ -1215,12 +1217,6 @@ fn semantic_checks_for_truncate( identity: Option, cascade: Option, on_cluster: Option, -) -> StdResult, ParserError> { - if partitions.is_some() - || !table - || identity.is_some() - || cascade.is_some() - || on_cluster.is_some() { Err(ParserError::ParserError( "Only TRUNCATE TABLE is supported.".to_owned(), From 71366a370dc596031e8abbcd55484d57f80f486b Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 18 Nov 2025 15:56:42 +0100 Subject: [PATCH 64/99] Add custom parsing for TRUNCATE statements to support TRUNCATE CLUSTER --- crates/modelardb_storage/src/parser.rs | 71 +++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 6 deletions(-) diff --git a/crates/modelardb_storage/src/parser.rs b/crates/modelardb_storage/src/parser.rs index 9bcd6d8c..1a1047d8 100644 --- a/crates/modelardb_storage/src/parser.rs +++ b/crates/modelardb_storage/src/parser.rs @@ -67,7 +67,7 @@ pub enum ModelarDbStatement { /// DROP TABLE. DropTable(Vec), /// TRUNCATE TABLE. - TruncateTable(Vec), + TruncateTable(Vec, bool), /// VACUUM. Vacuum(Vec, Option, bool), } @@ -122,7 +122,7 @@ pub fn tokenize_and_parse_sql_statement(sql_statement: &str) -> Result { - let table_names = semantic_checks_for_truncate( + let (table_names, cluster) = semantic_checks_for_truncate( table_names, partitions, table, @@ -130,7 +130,8 @@ pub fn tokenize_and_parse_sql_statement(sql_statement: &str) -> Result parser.expected("literal integer", token_with_location), } } + + /// Return [`true`] if the token stream starts with TRUNCATE, otherwise [`false`] is returned. + /// The method does not consume tokens. + fn next_token_is_truncate(&self, parser: &Parser) -> bool { + // TRUNCATE. + if let Token::Word(word) = parser.peek_nth_token(0).token { + word.keyword == Keyword::TRUNCATE + } else { + false + } + } + + /// Parse TRUNCATE \[CLUSTER\] table_name\[, table_name\]+ to a [`Statement::Truncate`] with the + /// table names in the `table_names` field and the cluster flag in the `on_cluster` field. + /// A [`ParserError`] is returned if TRUNCATE is not the first word or the table names cannot be + /// extracted. + fn parse_truncate(&self, parser: &mut Parser) -> StdResult { + // TRUNCATE. + parser.expect_keyword(Keyword::TRUNCATE)?; + + // If the next token is CLUSTER, consume it and set the flag to truncate the entire cluster. + let truncate_cluster = if let Token::Word(word) = parser.peek_nth_token(0).token + && word.keyword == Keyword::CLUSTER + { + parser.expect_keyword(Keyword::CLUSTER)?; + + // A boolean is not used since we reuse the `on_cluster` field in Statement::Truncate. + // Some() with any value signifies that the flag is set. + Some(Ident::new("cluster")) + } else { + None + }; + + // Parse the table names. At least one table name is required. + let table_names = self.parse_table_names(parser)?; + + // Convert the table names to the type required by Statement::Truncate. + let truncate_table_targets = table_names + .iter() + .map(|table_name| TruncateTableTarget { + name: ObjectName(vec![ObjectNamePart::Identifier(Ident::new(table_name))]), + only: false, + }) + .collect(); + + Ok(Statement::Truncate { + table_names: truncate_table_targets, + partitions: None, + table: true, + identity: None, + cascade: None, + on_cluster: truncate_cluster, + }) + } + /// Return a list of table names parsed from the token stream. It is assumed that the table /// names are separated by commas. A [`ParserError`] is returned if the table names cannot be /// extracted. @@ -637,6 +693,8 @@ impl Dialect for ModelarDbDialect { Some(self.parse_include_query(parser)) } else if self.next_token_is_vacuum(parser) { Some(self.parse_vacuum(parser)) + } else if self.next_token_is_truncate(parser) { + Some(self.parse_truncate(parser)) } else { None } @@ -1217,9 +1275,10 @@ fn semantic_checks_for_truncate( identity: Option, cascade: Option, on_cluster: Option, - { +) -> StdResult<(Vec, bool), ParserError> { + if partitions.is_some() || !table || identity.is_some() || cascade.is_some() { Err(ParserError::ParserError( - "Only TRUNCATE TABLE is supported.".to_owned(), + "Only TRUNCATE [CLUSTER] table_name[, table_name]+ is supported.".to_owned(), )) } else { let mut table_names = Vec::with_capacity(names.len()); @@ -1235,7 +1294,7 @@ fn semantic_checks_for_truncate( table_names.push(table_name); } - Ok(table_names) + Ok((table_names, on_cluster.is_some())) } } From 6461c941dd6ee04003a0beb276d78f2bd7852f12 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 18 Nov 2025 15:57:39 +0100 Subject: [PATCH 65/99] Add unit tests that ensure the correct TRUNCATE CLUSTER syntax works --- crates/modelardb_storage/src/parser.rs | 41 ++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/crates/modelardb_storage/src/parser.rs b/crates/modelardb_storage/src/parser.rs index 1a1047d8..0286c636 100644 --- a/crates/modelardb_storage/src/parser.rs +++ b/crates/modelardb_storage/src/parser.rs @@ -2163,4 +2163,45 @@ mod tests { "Parser Error: sql parser error: Expected: literal integer, found: CLUSTER at Line: 1, Column: 15" ); } + + #[test] + fn test_tokenize_and_parse_truncate_single_table() { + let (table_names, cluster) = parse_truncate_and_extract_table_names("TRUNCATE table_name"); + + assert_eq!(table_names, vec!["table_name".to_owned()]); + assert!(!cluster) + } + + #[test] + fn test_tokenize_and_parse_truncate_multiple_tables() { + let (table_names, cluster) = + parse_truncate_and_extract_table_names("TRUNCATE table_name_1, table_name_2"); + + assert_eq!( + table_names, + vec!["table_name_1".to_owned(), "table_name_2".to_owned()] + ); + assert!(!cluster) + } + + #[test] + fn test_tokenize_and_parse_truncate_cluster_multiple_tables() { + let (table_names, cluster) = + parse_truncate_and_extract_table_names("TRUNCATE CLUSTER table_name_1, table_name_2"); + + assert_eq!( + table_names, + vec!["table_name_1".to_owned(), "table_name_2".to_owned()] + ); + assert!(cluster) + } + + fn parse_truncate_and_extract_table_names(sql_statement: &str) -> (Vec, bool) { + let modelardb_statement = tokenize_and_parse_sql_statement(sql_statement).unwrap(); + + match modelardb_statement { + ModelarDbStatement::TruncateTable(table_names, cluster) => (table_names, cluster), + _ => panic!("Expected ModelarDbStatement::TruncateTable."), + } + } } From 80a3ab1fed3018cad2c44a36afdc256d1d4eee13 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 18 Nov 2025 15:59:24 +0100 Subject: [PATCH 66/99] Add unit tests that ensure the wrong TRUNCATE CLUSTER syntax fails --- crates/modelardb_storage/src/parser.rs | 80 ++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/crates/modelardb_storage/src/parser.rs b/crates/modelardb_storage/src/parser.rs index 0286c636..3d4005bf 100644 --- a/crates/modelardb_storage/src/parser.rs +++ b/crates/modelardb_storage/src/parser.rs @@ -2204,4 +2204,84 @@ mod tests { _ => panic!("Expected ModelarDbStatement::TruncateTable."), } } + + #[test] + fn test_tokenize_and_parse_truncate_no_tables() { + let result = tokenize_and_parse_sql_statement("TRUNCATE"); + + assert_eq!( + result.unwrap_err().to_string(), + "Parser Error: sql parser error: Expected: word, found: EOF" + ); + } + + #[test] + fn test_tokenize_and_parse_truncate_trailing_comma() { + let result = tokenize_and_parse_sql_statement("TRUNCATE table_name,"); + + assert_eq!( + result.unwrap_err().to_string(), + "Parser Error: sql parser error: Expected: word, found: EOF" + ); + } + + #[test] + fn test_tokenize_and_parse_truncate_leading_comma() { + let result = tokenize_and_parse_sql_statement("TRUNCATE ,table_name"); + + assert_eq!( + result.unwrap_err().to_string(), + "Parser Error: sql parser error: Expected: word, found: , at Line: 1, Column: 10" + ); + } + + #[test] + fn test_tokenize_and_parse_truncate_only_comma() { + let result = tokenize_and_parse_sql_statement("TRUNCATE,"); + + assert_eq!( + result.unwrap_err().to_string(), + "Parser Error: sql parser error: Expected: word, found: , at Line: 1, Column: 9" + ); + } + + #[test] + fn test_tokenize_and_parse_truncate_quoted_table_name() { + let result = tokenize_and_parse_sql_statement("TRUNCATE 'table_name'"); + + assert_eq!( + result.unwrap_err().to_string(), + "Parser Error: sql parser error: Expected: word, found: 'table_name' at Line: 1, Column: 10" + ); + } + + #[test] + fn test_tokenize_and_parse_truncate_only_cluster() { + let result = tokenize_and_parse_sql_statement("TRUNCATE CLUSTER"); + + assert_eq!( + result.unwrap_err().to_string(), + "Parser Error: sql parser error: Expected: word, found: EOF" + ); + } + + #[test] + fn test_tokenize_and_parse_truncate_cluster_after_tables() { + let result = tokenize_and_parse_sql_statement("TRUNCATE table_1, table_2 CLUSTER"); + + assert_eq!( + result.unwrap_err().to_string(), + "Parser Error: sql parser error: Expected: end of statement, found: CLUSTER at Line: 1, Column: 27" + ); + } + + #[test] + fn test_tokenize_and_parse_truncate_cluster_trailing_comma() { + let result = tokenize_and_parse_sql_statement("TRUNCATE CLUSTER, table_name"); + + assert_eq!( + result.unwrap_err().to_string(), + "Parser Error: sql parser error: Expected: word, found: , at Line: 1, Column: 17" + ); + } } From 468ea9d69d24ce77f71dbac1159c2b81a728fe23 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 18 Nov 2025 16:19:29 +0100 Subject: [PATCH 67/99] Add method to truncate entire cluster if CLUSTER is included in statement --- crates/modelardb_server/src/remote.rs | 36 +++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/crates/modelardb_server/src/remote.rs b/crates/modelardb_server/src/remote.rs index e45159b0..7905a6e6 100644 --- a/crates/modelardb_server/src/remote.rs +++ b/crates/modelardb_server/src/remote.rs @@ -397,6 +397,35 @@ impl FlightServiceHandler { Ok(()) } + /// Truncate the table with the given `table_name`. If the node is running in a cluster and + /// `truncate_cluster` is `true`, the table is truncated in the remote data folder and locally + /// in each node in the cluster. If not, the table is only truncated locally. + async fn truncate_table( + &self, + table_name: &str, + truncate_cluster: bool, + ) -> StdResult<(), Status> { + let configuration_manager = self.context.configuration_manager.read().await; + + if truncate_cluster { + if let ClusterMode::MultiNode(cluster) = &configuration_manager.cluster_mode { + cluster + .truncate_cluster_table(table_name) + .await + .map_err(error_to_status_invalid_argument)?; + } else { + return Err(Status::internal("The node is not running in a cluster.")); + } + } + + self.context + .truncate_table(table_name) + .await + .map_err(error_to_status_invalid_argument)?; + + Ok(()) + } + /// While there is still more data to receive, ingest the data into the normal table. async fn ingest_into_normal_table( &self, @@ -622,12 +651,9 @@ impl FlightService for FlightServiceHandler { ) .await } - ModelarDbStatement::TruncateTable(table_names) => { + ModelarDbStatement::TruncateTable(table_names, cluster) => { for table_name in table_names { - self.context - .truncate_table(&table_name) - .await - .map_err(error_to_status_invalid_argument)?; + self.truncate_table(&table_name, cluster).await?; } Ok(empty_record_batch_stream()) From 0fcd72fcbdefcbe9d875860a915f5d9e4059abb9 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 18 Nov 2025 16:19:58 +0100 Subject: [PATCH 68/99] Add method to vacuum entire cluster if CLUSTER is included in statement --- crates/modelardb_server/src/remote.rs | 42 +++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/crates/modelardb_server/src/remote.rs b/crates/modelardb_server/src/remote.rs index 7905a6e6..d5e1ac9b 100644 --- a/crates/modelardb_server/src/remote.rs +++ b/crates/modelardb_server/src/remote.rs @@ -426,6 +426,36 @@ impl FlightServiceHandler { Ok(()) } + /// Vacuum the table with the given `table_name`. If the node is running in a cluster and + /// `vacuum_cluster` is `true`, the table is vacuumed in the remote data folder and locally in + /// each node in the cluster. If not, the table is only vacuumed locally. + async fn vacuum_table( + &self, + table_name: &str, + maybe_retention_period_in_seconds: Option, + vacuum_cluster: bool, + ) -> StdResult<(), Status> { + let configuration_manager = self.context.configuration_manager.read().await; + + if vacuum_cluster { + if let ClusterMode::MultiNode(cluster) = &configuration_manager.cluster_mode { + cluster + .vacuum_cluster_table(table_name, maybe_retention_period_in_seconds) + .await + .map_err(error_to_status_invalid_argument)?; + } else { + return Err(Status::internal("The node is not running in a cluster.")); + } + } + + self.context + .vacuum_table(table_name, maybe_retention_period_in_seconds) + .await + .map_err(error_to_status_invalid_argument)?; + + Ok(()) + } + /// While there is still more data to receive, ingest the data into the normal table. async fn ingest_into_normal_table( &self, @@ -665,7 +695,11 @@ impl FlightService for FlightServiceHandler { Ok(empty_record_batch_stream()) } - ModelarDbStatement::Vacuum(mut table_names, maybe_retention_period_in_seconds) => { + ModelarDbStatement::Vacuum( + mut table_names, + maybe_retention_period_in_seconds, + cluster, + ) => { // Vacuum all tables if no table names are provided. if table_names.is_empty() { table_names = self @@ -676,10 +710,8 @@ impl FlightService for FlightServiceHandler { }; for table_name in table_names { - self.context - .vacuum_table(&table_name, maybe_retention_period_in_seconds) - .await - .map_err(error_to_status_invalid_argument)?; + self.vacuum_table(&table_name, maybe_retention_period_in_seconds, cluster) + .await?; } Ok(empty_record_batch_stream()) From 1a5779c4ddf2551bcbea54d489a22f7817555ba9 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 18 Nov 2025 16:31:13 +0100 Subject: [PATCH 69/99] Implement cluster methods to vacuum and truncate cluster table --- crates/modelardb_server/src/cluster.rs | 41 ++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 27ed4edc..b5ffc58e 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -192,8 +192,8 @@ impl Cluster { Ok(()) } - /// Drop the normal table with the given `table_name` in the remote data folder and in each peer - /// node. If the normal table could not be dropped, return [`ModelarDbServerError`]. + /// Drop the table with the given `table_name` in the remote data folder and in each peer + /// node. If the table could not be dropped, return [`ModelarDbServerError`]. pub(crate) async fn drop_cluster_table(&self, table_name: &str) -> Result<()> { // Drop the table from the remote data folder. self.remote_data_folder @@ -209,6 +209,43 @@ impl Cluster { Ok(()) } + /// Truncate the table with the given `table_name` in the remote data folder and in each + /// peer node. If the table could not be truncated, return [`ModelarDbServerError`]. + pub(crate) async fn truncate_cluster_table(&self, table_name: &str) -> Result<()> { + // Truncate the table in the remote data folder. + self.remote_data_folder.truncate_table(table_name).await?; + + // Truncate the table in each peer node. + self.cluster_do_get(&format!("TRUNCATE {table_name}")) + .await?; + + Ok(()) + } + + /// Vacuum the table with the given `table_name` in the remote data folder and in each peer + /// node. If the table could not be vacuumed, return [`ModelarDbServerError`]. + pub(crate) async fn vacuum_cluster_table( + &self, + table_name: &str, + maybe_retention_period_in_seconds: Option, + ) -> Result<()> { + // Vacuum the table in the remote data folder. + self.remote_data_folder + .vacuum_table(table_name, maybe_retention_period_in_seconds) + .await?; + + // Vacuum the table in each peer node. + let vacuum_sql = if let Some(retention_period) = maybe_retention_period_in_seconds { + format!("VACUUM {table_name} RETAIN {retention_period}") + } else { + format!("VACUUM {table_name}") + }; + + self.cluster_do_get(&vacuum_sql).await?; + + Ok(()) + } + /// For each peer node in the cluster, execute the given `sql` statement with the cluster key /// as metadata. If the statement was successfully executed for each node, return [`Ok`], /// otherwise return [`ModelarDbServerError`]. From 8080524ca312945d7ffdd2fa9458063078c4c20d Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 18 Nov 2025 16:32:04 +0100 Subject: [PATCH 70/99] Remove TABLE from TRUNCATE TABLE --- crates/modelardb_embedded/src/operations/client.rs | 2 +- crates/modelardb_manager/src/remote.rs | 10 +++++----- crates/modelardb_server/src/remote.rs | 2 +- crates/modelardb_server/tests/integration_test.rs | 2 +- crates/modelardb_storage/src/parser.rs | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/modelardb_embedded/src/operations/client.rs b/crates/modelardb_embedded/src/operations/client.rs index fabc6be5..a20e36fe 100644 --- a/crates/modelardb_embedded/src/operations/client.rs +++ b/crates/modelardb_embedded/src/operations/client.rs @@ -341,7 +341,7 @@ impl Operations for Client { /// Truncate the table with the name in `table_name`. If the table could not be truncated, /// [`ModelarDbEmbeddedError`] is returned. async fn truncate(&mut self, table_name: &str) -> Result<()> { - let ticket = Ticket::new(format!("TRUNCATE TABLE {table_name}")); + let ticket = Ticket::new(format!("TRUNCATE {table_name}")); self.flight_client.do_get(ticket).await?; Ok(()) diff --git a/crates/modelardb_manager/src/remote.rs b/crates/modelardb_manager/src/remote.rs index 9517b59c..2c35d6fc 100644 --- a/crates/modelardb_manager/src/remote.rs +++ b/crates/modelardb_manager/src/remote.rs @@ -300,7 +300,7 @@ impl FlightServiceHandler { .cluster .read() .await - .cluster_do_get(&format!("TRUNCATE TABLE {table_name}"), &self.context.key) + .cluster_do_get(&format!("TRUNCATE {table_name}"), &self.context.key) .await .map_err(error_to_status_internal)?; @@ -457,7 +457,7 @@ impl FlightService for FlightServiceHandler { } /// Execute a SQL statement provided in UTF-8 and return the schema of the result followed by - /// the result itself. Currently, CREATE TABLE, CREATE TIME SERIES TABLE, TRUNCATE TABLE, + /// the result itself. Currently, CREATE TABLE, CREATE TIME SERIES TABLE, TRUNCATE, /// DROP TABLE, and VACUUM are supported. async fn do_get( &self, @@ -491,7 +491,7 @@ impl FlightService for FlightServiceHandler { self.save_and_create_cluster_time_series_table(time_series_table_metadata) .await?; } - ModelarDbStatement::TruncateTable(table_names) => { + ModelarDbStatement::TruncateTable(table_names, ..) => { for table_name in table_names { self.truncate_cluster_table(&table_name).await?; } @@ -501,7 +501,7 @@ impl FlightService for FlightServiceHandler { self.drop_cluster_table(&table_name).await?; } } - ModelarDbStatement::Vacuum(mut table_names, maybe_retention_period_in_seconds) => { + ModelarDbStatement::Vacuum(mut table_names, maybe_retention_period_in_seconds, ..) => { // Vacuum all tables if no table names are provided. if table_names.is_empty() { table_names = self @@ -520,7 +520,7 @@ impl FlightService for FlightServiceHandler { // .. is not used so a compile error is raised if a new ModelarDbStatement is added. ModelarDbStatement::Statement(_) | ModelarDbStatement::IncludeSelect(..) => { return Err(Status::invalid_argument( - "Expected CREATE TABLE, CREATE TIME SERIES TABLE, TRUNCATE TABLE, or DROP TABLE.", + "Expected CREATE TABLE, CREATE TIME SERIES TABLE, TRUNCATE, or DROP TABLE.", )); } }; diff --git a/crates/modelardb_server/src/remote.rs b/crates/modelardb_server/src/remote.rs index d5e1ac9b..ff828af6 100644 --- a/crates/modelardb_server/src/remote.rs +++ b/crates/modelardb_server/src/remote.rs @@ -619,7 +619,7 @@ impl FlightService for FlightServiceHandler { /// Execute a SQL statement provided in UTF-8 and return the schema of the result followed by /// the result itself. Currently, CREATE TABLE, CREATE TIME SERIES TABLE, EXPLAIN, INCLUDE, - /// SELECT, INSERT, TRUNCATE TABLE, DROP TABLE, and VACUUM are supported. + /// SELECT, INSERT, TRUNCATE, DROP TABLE, and VACUUM are supported. async fn do_get( &self, request: Request, diff --git a/crates/modelardb_server/tests/integration_test.rs b/crates/modelardb_server/tests/integration_test.rs index 9a563471..c615d464 100644 --- a/crates/modelardb_server/tests/integration_test.rs +++ b/crates/modelardb_server/tests/integration_test.rs @@ -250,7 +250,7 @@ impl TestContext { &mut self, table_name: &str, ) -> Result>, Status> { - let ticket = Ticket::new(format!("TRUNCATE TABLE {table_name}")); + let ticket = Ticket::new(format!("TRUNCATE {table_name}")); self.client.do_get(ticket).await } diff --git a/crates/modelardb_storage/src/parser.rs b/crates/modelardb_storage/src/parser.rs index 3d4005bf..9a5379e6 100644 --- a/crates/modelardb_storage/src/parser.rs +++ b/crates/modelardb_storage/src/parser.rs @@ -66,7 +66,7 @@ pub enum ModelarDbStatement { IncludeSelect(Statement, Vec), /// DROP TABLE. DropTable(Vec), - /// TRUNCATE TABLE. + /// TRUNCATE. TruncateTable(Vec, bool), /// VACUUM. Vacuum(Vec, Option, bool), @@ -75,7 +75,7 @@ pub enum ModelarDbStatement { /// Tokenizes and parses the SQL statement in `sql` and returns its parsed representation in the form /// of a [`ModelarDbStatement`]. Returns a [`ModelarDbStorageError`] if `sql` is empty, contains /// multiple statements, or the statement is unsupported. Currently, CREATE TABLE, CREATE TIME SERIES -/// TABLE, INSERT, EXPLAIN, INCLUDE, SELECT, TRUNCATE TABLE, DROP TABLE, and VACUUM are supported. +/// TABLE, INSERT, EXPLAIN, INCLUDE, SELECT, TRUNCATE, DROP TABLE, and VACUUM are supported. pub fn tokenize_and_parse_sql_statement(sql_statement: &str) -> Result { let mut statements = Parser::parse_sql(&ModelarDbDialect::new(), sql_statement)?; From 398a4bcdb6bc7d8a49d3462f1738119b10c3f1ba Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 18 Nov 2025 16:47:35 +0100 Subject: [PATCH 71/99] Move for loop down to avoid sending a request for each table when cluster truncating --- crates/modelardb_server/src/cluster.rs | 21 ++++++++++++--------- crates/modelardb_server/src/remote.rs | 26 +++++++++++++------------- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index b5ffc58e..a18cb971 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -209,15 +209,18 @@ impl Cluster { Ok(()) } - /// Truncate the table with the given `table_name` in the remote data folder and in each - /// peer node. If the table could not be truncated, return [`ModelarDbServerError`]. - pub(crate) async fn truncate_cluster_table(&self, table_name: &str) -> Result<()> { - // Truncate the table in the remote data folder. - self.remote_data_folder.truncate_table(table_name).await?; - - // Truncate the table in each peer node. - self.cluster_do_get(&format!("TRUNCATE {table_name}")) - .await?; + /// Truncate the tables in `table_names` in the remote data folder and in each peer node. + /// If the tables could not be truncated, return [`ModelarDbServerError`]. + pub(crate) async fn truncate_cluster_tables(&self, table_names: &[String]) -> Result<()> { + let truncate_sql = format!("TRUNCATE {}", table_names.join(", ")); + + // Truncate the tables in the remote data folder. + for table_name in table_names { + self.remote_data_folder.truncate_table(table_name).await?; + } + + // Truncate the tables in each peer node. + self.cluster_do_get(&truncate_sql).await?; Ok(()) } diff --git a/crates/modelardb_server/src/remote.rs b/crates/modelardb_server/src/remote.rs index ff828af6..07a3fee5 100644 --- a/crates/modelardb_server/src/remote.rs +++ b/crates/modelardb_server/src/remote.rs @@ -397,12 +397,12 @@ impl FlightServiceHandler { Ok(()) } - /// Truncate the table with the given `table_name`. If the node is running in a cluster and - /// `truncate_cluster` is `true`, the table is truncated in the remote data folder and locally - /// in each node in the cluster. If not, the table is only truncated locally. - async fn truncate_table( + /// Truncate the tables in `table_names`. If the node is running in a cluster and + /// `truncate_cluster` is `true`, the tables are truncated in the remote data folder and + /// locally in each node in the cluster. If not, the tables are only truncated locally. + async fn truncate_tables( &self, - table_name: &str, + table_names: &[String], truncate_cluster: bool, ) -> StdResult<(), Status> { let configuration_manager = self.context.configuration_manager.read().await; @@ -410,7 +410,7 @@ impl FlightServiceHandler { if truncate_cluster { if let ClusterMode::MultiNode(cluster) = &configuration_manager.cluster_mode { cluster - .truncate_cluster_table(table_name) + .truncate_cluster_tables(table_names) .await .map_err(error_to_status_invalid_argument)?; } else { @@ -418,10 +418,12 @@ impl FlightServiceHandler { } } - self.context - .truncate_table(table_name) - .await - .map_err(error_to_status_invalid_argument)?; + for table_name in table_names { + self.context + .truncate_table(table_name) + .await + .map_err(error_to_status_invalid_argument)?; + } Ok(()) } @@ -682,9 +684,7 @@ impl FlightService for FlightServiceHandler { .await } ModelarDbStatement::TruncateTable(table_names, cluster) => { - for table_name in table_names { - self.truncate_table(&table_name, cluster).await?; - } + self.truncate_tables(&table_names, cluster).await?; Ok(empty_record_batch_stream()) } From 62de621311352a447a6e304281a2a124d9711c31 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 18 Nov 2025 16:55:10 +0100 Subject: [PATCH 72/99] Move for loop down to avoid sending a request for each table when cluster vacuuming --- crates/modelardb_server/src/cluster.rs | 30 +++++++++++++++----------- crates/modelardb_server/src/remote.rs | 28 ++++++++++++------------ 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index a18cb971..6d8b79c5 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -212,36 +212,40 @@ impl Cluster { /// Truncate the tables in `table_names` in the remote data folder and in each peer node. /// If the tables could not be truncated, return [`ModelarDbServerError`]. pub(crate) async fn truncate_cluster_tables(&self, table_names: &[String]) -> Result<()> { - let truncate_sql = format!("TRUNCATE {}", table_names.join(", ")); - // Truncate the tables in the remote data folder. for table_name in table_names { self.remote_data_folder.truncate_table(table_name).await?; } // Truncate the tables in each peer node. + let truncate_sql = format!("TRUNCATE {}", table_names.join(", ")); self.cluster_do_get(&truncate_sql).await?; Ok(()) } - /// Vacuum the table with the given `table_name` in the remote data folder and in each peer - /// node. If the table could not be vacuumed, return [`ModelarDbServerError`]. - pub(crate) async fn vacuum_cluster_table( + /// Vacuum the tables in `table_names` in the remote data folder and in each peer node. If the + /// tables could not be vacuumed, return [`ModelarDbServerError`]. + pub(crate) async fn vacuum_cluster_tables( &self, - table_name: &str, + table_names: &[String], maybe_retention_period_in_seconds: Option, ) -> Result<()> { - // Vacuum the table in the remote data folder. - self.remote_data_folder - .vacuum_table(table_name, maybe_retention_period_in_seconds) - .await?; + // Vacuum the tables in the remote data folder. + for table_name in table_names { + self.remote_data_folder + .vacuum_table(table_name, maybe_retention_period_in_seconds) + .await?; + } - // Vacuum the table in each peer node. + // Vacuum the tables in each peer node. let vacuum_sql = if let Some(retention_period) = maybe_retention_period_in_seconds { - format!("VACUUM {table_name} RETAIN {retention_period}") + format!( + "VACUUM {} RETAIN {retention_period}", + table_names.join(", ") + ) } else { - format!("VACUUM {table_name}") + format!("VACUUM {}", table_names.join(", ")) }; self.cluster_do_get(&vacuum_sql).await?; diff --git a/crates/modelardb_server/src/remote.rs b/crates/modelardb_server/src/remote.rs index 07a3fee5..2db10db9 100644 --- a/crates/modelardb_server/src/remote.rs +++ b/crates/modelardb_server/src/remote.rs @@ -428,12 +428,12 @@ impl FlightServiceHandler { Ok(()) } - /// Vacuum the table with the given `table_name`. If the node is running in a cluster and - /// `vacuum_cluster` is `true`, the table is vacuumed in the remote data folder and locally in - /// each node in the cluster. If not, the table is only vacuumed locally. - async fn vacuum_table( + /// Vacuum the tables in `table_names`. If the node is running in a cluster and `vacuum_cluster` + /// is `true`, the tables are vacuumed in the remote data folder and locally in each node in the + /// cluster. If not, the tables are only vacuumed locally. + async fn vacuum_tables( &self, - table_name: &str, + table_names: &[String], maybe_retention_period_in_seconds: Option, vacuum_cluster: bool, ) -> StdResult<(), Status> { @@ -442,7 +442,7 @@ impl FlightServiceHandler { if vacuum_cluster { if let ClusterMode::MultiNode(cluster) = &configuration_manager.cluster_mode { cluster - .vacuum_cluster_table(table_name, maybe_retention_period_in_seconds) + .vacuum_cluster_tables(table_names, maybe_retention_period_in_seconds) .await .map_err(error_to_status_invalid_argument)?; } else { @@ -450,10 +450,12 @@ impl FlightServiceHandler { } } - self.context - .vacuum_table(table_name, maybe_retention_period_in_seconds) - .await - .map_err(error_to_status_invalid_argument)?; + for table_name in table_names { + self.context + .vacuum_table(table_name, maybe_retention_period_in_seconds) + .await + .map_err(error_to_status_invalid_argument)?; + } Ok(()) } @@ -709,10 +711,8 @@ impl FlightService for FlightServiceHandler { .table_names(); }; - for table_name in table_names { - self.vacuum_table(&table_name, maybe_retention_period_in_seconds, cluster) - .await?; - } + self.vacuum_tables(&table_names, maybe_retention_period_in_seconds, cluster) + .await?; Ok(empty_record_batch_stream()) } From 134ba38a9a23587abee04fac85b95e52f36cc54c Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 18 Nov 2025 19:41:41 +0100 Subject: [PATCH 73/99] Update the NodeType action to return SingleEdge, ClusterEdge, or ClusterCloud --- crates/modelardb_server/src/cluster.rs | 5 +++++ crates/modelardb_server/src/remote.rs | 18 ++++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 6d8b79c5..71397052 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -75,6 +75,11 @@ impl Cluster { }) } + /// Return the node that represents the local system running `modelardbd`. + pub(crate) fn node(&self) -> &Node { + &self.node + } + /// Return the key identifying the cluster. pub(crate) fn key(&self) -> &MetadataValue { &self.key diff --git a/crates/modelardb_server/src/remote.rs b/crates/modelardb_server/src/remote.rs index 2db10db9..bc191a1e 100644 --- a/crates/modelardb_server/src/remote.rs +++ b/crates/modelardb_server/src/remote.rs @@ -44,7 +44,7 @@ use futures::stream::{self, BoxStream, SelectAll}; use modelardb_storage::parser::{self, ModelarDbStatement}; use modelardb_types::flight::protocol; use modelardb_types::functions; -use modelardb_types::types::{Table, TimeSeriesTableMetadata}; +use modelardb_types::types::{ServerMode, Table, TimeSeriesTableMetadata}; use prost::Message; use tokio::sync::mpsc::{self, Sender}; use tokio::task; @@ -814,8 +814,8 @@ impl FlightService for FlightServiceHandler { /// * `UpdateConfiguration`: Update a single setting in the configuration. The setting to update /// and the new value are provided in the [`UpdateConfiguration`](protocol::UpdateConfiguration) /// protobuf message in the action body. - /// * `NodeType`: Get the type of the node. The type is always `server`. The type of the node - /// is returned as a string. + /// * `NodeType`: Get the type of the node. The type is `SingleEdge`, `ClusterEdge`, or + /// `ClusterCloud`. The type of the node is returned as a string. async fn do_action( &self, request: Request, @@ -958,8 +958,18 @@ impl FlightService for FlightServiceHandler { // Confirm the configuration was updated. Ok(Response::new(Box::pin(stream::empty()))) } else if action.r#type == "NodeType" { + let configuration_manager = self.context.configuration_manager.read().await; + + let node_type = match &configuration_manager.cluster_mode { + ClusterMode::SingleNode => "SingleEdge", + ClusterMode::MultiNode(cluster) => match cluster.node().mode { + ServerMode::Edge => "ClusterEdge", + ServerMode::Cloud => "ClusterCloud", + }, + }; + let flight_result = FlightResult { - body: "server".bytes().collect(), + body: node_type.bytes().collect(), }; Ok(Response::new(Box::pin(stream::once(async { From 865a9244cdc55b755dcc4bfe7b6cb62ab3076a61 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 18 Nov 2025 19:55:48 +0100 Subject: [PATCH 74/99] Remove Node enum and instead just use URL when connecting to Client --- .../src/operations/client.rs | 104 ++---------------- 1 file changed, 7 insertions(+), 97 deletions(-) diff --git a/crates/modelardb_embedded/src/operations/client.rs b/crates/modelardb_embedded/src/operations/client.rs index a20e36fe..cf3f75d9 100644 --- a/crates/modelardb_embedded/src/operations/client.rs +++ b/crates/modelardb_embedded/src/operations/client.rs @@ -42,106 +42,20 @@ use crate::operations::{ }; use crate::{Aggregate, TableType}; -/// Types of nodes that can be connected to by [`Client`]. -#[derive(Clone)] -pub enum Node { - /// The Apache Arrow Flight server URL of a ModelarDB server node. - Server(String), - /// The Apache Arrow Flight server URL of a ModelarDB manager node. - Manager(String), -} - -impl Node { - /// Returns the URL of the node. - pub fn url(&self) -> &str { - match self { - Node::Server(url) => url, - Node::Manager(url) => url, - } - } - - /// Returns the type of the node. - pub fn node_type(&self) -> &str { - match self { - Node::Server(_) => "server", - Node::Manager(_) => "manager", - } - } -} - /// Client for connecting to ModelarDB Apache Arrow Flight servers. #[derive(Clone)] pub struct Client { - /// The node that the client is connected to. - pub(crate) node: Node, /// Apache Arrow Flight client connected to the Apache Arrow Flight server of the ModelarDB node. pub(crate) flight_client: FlightServiceClient, } impl Client { - /// Create a new [`Client`] that is connected to the node with the URL in `node`. If a - /// connection to the node could not be established or if the actual type of the node does not - /// match `node`, [`ModelarDbEmbeddedError`] is returned. - pub async fn connect(node: Node) -> Result { - // Retrieve the actual type of the node to ensure it matches the expected type. - let mut flight_client = FlightServiceClient::connect(node.url().to_owned()).await?; - - let action = Action { - r#type: "NodeType".to_owned(), - body: vec![].into(), - }; + /// Create a new [`Client`] that is connected to the node with `url`. If a connection + /// to the node could not be established, [`ModelarDbEmbeddedError`] is returned. + pub async fn connect(url: &str) -> Result { + let flight_client = FlightServiceClient::connect(url.to_owned()).await?; - let response = flight_client.do_action(action).await?; - - if let Some(response_message) = response.into_inner().message().await? { - let actual_node_type = str::from_utf8(&response_message.body)?; - let expected_node_type = node.node_type(); - - if actual_node_type == expected_node_type { - Ok(Client { - node, - flight_client, - }) - } else { - Err(ModelarDbEmbeddedError::InvalidArgument(format!( - "The actual node type '{actual_node_type}' does not match the expected node type '{expected_node_type}'." - ))) - } - } else { - Err(ModelarDbEmbeddedError::from(Status::internal( - "Could not retrieve the node type from the node.", - ))) - } - } - - /// Returns the client that should be used to execute the given command. If the client is - /// connected to a server, this will be the same client. If the client is connected to a - /// manager, this will be a cloud node in the cluster. If the cluster does not have at least one - /// cloud node or if a cloud node could not be retrieved, [`ModelarDbEmbeddedError`] is - /// returned. - async fn client_for_command(&mut self, command: &str) -> Result> { - match self.node { - Node::Server(_) => Ok(self.flight_client.clone()), - Node::Manager(_) => { - let request = FlightDescriptor::new_cmd(command.to_owned()); - let flight_info = self.flight_client.get_flight_info(request).await?; - - // Retrieve the location in the endpoint from the returned flight info. - if let [endpoint] = flight_info.into_inner().endpoint.as_slice() { - if let [location] = endpoint.location.as_slice() { - Ok(FlightServiceClient::connect(location.uri.clone()).await?) - } else { - Err(ModelarDbEmbeddedError::from(Status::internal( - "Endpoint did not contain exactly one location.", - ))) - } - } else { - Err(ModelarDbEmbeddedError::from(Status::internal( - "Flight info did not contain exactly one endpoint.", - ))) - } - } - } + Ok(Client { flight_client }) } } @@ -226,8 +140,6 @@ impl Operations for Client { /// the schema of `uncompressed_data` does not match the schema of the table, or the data could /// not be written to the table, [`ModelarDbEmbeddedError`] is returned. async fn write(&mut self, table_name: &str, uncompressed_data: RecordBatch) -> Result<()> { - let mut write_client = self.client_for_command("WRITE").await?; - // Include the table name in the flight descriptor. let flight_descriptor = FlightDescriptor::new_path(vec![table_name.to_owned()]); @@ -239,7 +151,7 @@ impl Operations for Client { maybe_flight_data.expect("Flight data should be validated by FlightDataEncoderBuilder.") }); - write_client.do_put(flight_data_stream).await?; + self.flight_client.do_put(flight_data_stream).await?; Ok(()) } @@ -247,10 +159,8 @@ impl Operations for Client { /// Executes the SQL in `sql` and returns the result as a [`RecordBatchStream`]. If the SQL /// could not be executed, [`ModelarDbEmbeddedError`] is returned. async fn read(&mut self, sql: &str) -> Result>> { - let mut sql_client = self.client_for_command(sql).await?; - let ticket = Ticket::new(sql.to_owned()); - let stream = sql_client.do_get(ticket).await?.into_inner(); + let stream = self.flight_client.do_get(ticket).await?.into_inner(); let record_batch_stream = FlightRecordBatchStream::new_from_flight_data( // Convert tonic::Status to FlightError. From afa2b5d0ecc6f0748749e19f33870a9dacc766d2 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 18 Nov 2025 20:09:58 +0100 Subject: [PATCH 75/99] Use URL instead of Node in C-API --- crates/modelardb_embedded/src/capi.rs | 25 ++++++------------- .../src/operations/data_folder.rs | 3 +-- 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/crates/modelardb_embedded/src/capi.rs b/crates/modelardb_embedded/src/capi.rs index 911bc24e..209666a5 100644 --- a/crates/modelardb_embedded/src/capi.rs +++ b/crates/modelardb_embedded/src/capi.rs @@ -45,7 +45,7 @@ use tokio::runtime::Runtime; use crate::error::{ModelarDbEmbeddedError, Result}; use crate::operations::Operations; -use crate::operations::client::{Client, Node}; +use crate::operations::client::Client; use crate::operations::data_folder::DataFolderDataSink; use crate::record_batch_stream_to_record_batch; use crate::{Aggregate, TableType}; @@ -197,29 +197,20 @@ unsafe fn open_azure( Ok(data_folder) } -/// Creates a [`Client`] that is connected to the Apache Arrow Flight server URL in `node_url_ptr` +/// Creates a [`Client`] that is connected to the Apache Arrow Flight server URL in `url_ptr` /// and returns a pointer to the [`Client`] or a zero-initialized pointer if an error occurs. -/// Assumes `node_url_ptr` points to a valid C string. +/// Assumes `url_ptr` points to a valid C string. #[unsafe(no_mangle)] -pub unsafe extern "C" fn modelardb_embedded_connect( - node_url_ptr: *const c_char, - is_server_node: bool, -) -> *const c_void { - let maybe_client = unsafe { connect(node_url_ptr, is_server_node) }; +pub unsafe extern "C" fn modelardb_embedded_connect(url_ptr: *const c_char) -> *const c_void { + let maybe_client = unsafe { connect(url_ptr) }; set_error_and_return_value_ptr(maybe_client) } /// See documentation for [`modelardb_embedded_connect()`]. -unsafe fn connect(node_url_ptr: *const c_char, is_server_node: bool) -> Result { - let node_url_str = unsafe { c_char_ptr_to_str(node_url_ptr)? }; - - let node = if is_server_node { - Node::Server(node_url_str.to_owned()) - } else { - Node::Manager(node_url_str.to_owned()) - }; +unsafe fn connect(url_ptr: *const c_char) -> Result { + let url_str = unsafe { c_char_ptr_to_str(url_ptr)? }; - TOKIO_RUNTIME.block_on(Client::connect(node)) + TOKIO_RUNTIME.block_on(Client::connect(url_str)) } /// Moves the value in `maybe_value` to a [`Box`] and returns a pointer to it if `maybe_value` is diff --git a/crates/modelardb_embedded/src/operations/data_folder.rs b/crates/modelardb_embedded/src/operations/data_folder.rs index 7350a2cc..2de24a3c 100644 --- a/crates/modelardb_embedded/src/operations/data_folder.rs +++ b/crates/modelardb_embedded/src/operations/data_folder.rs @@ -593,7 +593,7 @@ mod tests { use tempfile::TempDir; use tonic::transport::Channel; - use crate::operations::client::{Client, Node}; + use crate::operations::client::Client; use crate::record_batch_stream_to_record_batch; const NORMAL_TABLE_NAME: &str = "normal_table"; @@ -2730,7 +2730,6 @@ mod tests { fn lazy_modelardb_client() -> Client { Client { - node: Node::Server("localhost".to_owned()), flight_client: FlightServiceClient::new( Channel::from_static("localhost").connect_lazy(), ), From 2f0d6826d24eec755256def98301591dc82d678b Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 18 Nov 2025 20:23:17 +0100 Subject: [PATCH 76/99] Use str url instead of Node class when connecting to Client --- .../bindings/python/docs/api/node.rst | 21 ---------- .../bindings/python/modelardb/__init__.py | 1 - .../bindings/python/modelardb/node.py | 39 ------------------- .../bindings/python/modelardb/operations.py | 25 +++++------- 4 files changed, 10 insertions(+), 76 deletions(-) delete mode 100644 crates/modelardb_embedded/bindings/python/docs/api/node.rst delete mode 100644 crates/modelardb_embedded/bindings/python/modelardb/node.py diff --git a/crates/modelardb_embedded/bindings/python/docs/api/node.rst b/crates/modelardb_embedded/bindings/python/docs/api/node.rst deleted file mode 100644 index 8c741706..00000000 --- a/crates/modelardb_embedded/bindings/python/docs/api/node.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. Copyright 2025 The ModelarDB Contributors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -node -==== - -.. automodule:: modelardb.node - :show-inheritance: - :undoc-members: - :members: diff --git a/crates/modelardb_embedded/bindings/python/modelardb/__init__.py b/crates/modelardb_embedded/bindings/python/modelardb/__init__.py index 7799d8e2..a586ae32 100644 --- a/crates/modelardb_embedded/bindings/python/modelardb/__init__.py +++ b/crates/modelardb_embedded/bindings/python/modelardb/__init__.py @@ -16,7 +16,6 @@ from .error_bound import AbsoluteErrorBound, RelativeErrorBound from .ffi_array import FFIArray -from .node import Server, Manager from .operations import ( Aggregate, Operations, diff --git a/crates/modelardb_embedded/bindings/python/modelardb/node.py b/crates/modelardb_embedded/bindings/python/modelardb/node.py deleted file mode 100644 index b22c0c69..00000000 --- a/crates/modelardb_embedded/bindings/python/modelardb/node.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2025 The ModelarDB Contributors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from dataclasses import dataclass - - -@dataclass -class Server: - """A ModelarDB edge or cloud server node. - - :param url: The URL of the ModelarDB server node. - :type url: str - """ - - def __init__(self, url: str): - self.url: str = url - - -@dataclass -class Manager: - """A ModelarDB manager node. - - :param url: The URL of the ModelarDB manager node. - :type url: str - """ - - def __init__(self, url: str): - self.url: str = url diff --git a/crates/modelardb_embedded/bindings/python/modelardb/operations.py b/crates/modelardb_embedded/bindings/python/modelardb/operations.py index be3738fe..b979e9c9 100644 --- a/crates/modelardb_embedded/bindings/python/modelardb/operations.py +++ b/crates/modelardb_embedded/bindings/python/modelardb/operations.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import sys import platform import warnings @@ -26,7 +25,6 @@ from pyarrow import MapArray, RecordBatch, Schema, StringArray from pyarrow.cffi import ffi -from .node import Server, Manager from .error_bound import AbsoluteErrorBound from .table import NormalTable, TimeSeriesTable from .ffi_array import FFIArray @@ -120,8 +118,7 @@ def __find_library(build: str) -> str: char* access_key_ptr, char* container_name_ptr); - void* modelardb_embedded_connect(char* node_url_ptr, - bool is_server_node); + void* modelardb_embedded_connect(char* url_ptr); int modelardb_embedded_close(void* maybe_operations_ptr, bool is_data_folder); @@ -322,19 +319,17 @@ def open_azure(cls, account_name: str, access_key: str, container_name: str): return self @classmethod - def connect(cls, node: Server | Manager): + def connect(cls, url: str): """Create a connection to an :obj:`Operations` node. - :param node: The ModelarDB node to connect to. - :type node: Server | Manager + :param url: The URL of the ModelarDB node to connect to. + :type url: str """ self: Operations = cls() - node_url_ptr = ffi.new("char[]", bytes(node.url, "UTF-8")) + url_ptr = ffi.new("char[]", bytes(url, "UTF-8")) - self.__operations_ptr = self.__library.modelardb_embedded_connect( - node_url_ptr, isinstance(node, Server) - ) + self.__operations_ptr = self.__library.modelardb_embedded_connect(url_ptr) self.__is_data_folder = False if self.__operations_ptr == ffi.NULL: @@ -847,10 +842,10 @@ def open_azure(account_name: str, access_key: str, container_name: str) -> Opera return Operations.open_azure(account_name, access_key, container_name) -def connect(node: Server | Manager) -> Operations: +def connect(url: str) -> Operations: """Create a connection to an :obj:`Operations` node. - :param node: The ModelarDB node to connect to. - :type node: Server | Manager + :param url: The URL of the ModelarDB node to connect to. + :type url: str """ - return Operations.connect(node) + return Operations.connect(url) From 1fdfcf73f6bfea2d0244896d6ec9f9a1a18d5ad7 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 18 Nov 2025 20:59:18 +0100 Subject: [PATCH 77/99] Add modelardb_type method to Operations API --- .../src/operations/client.rs | 23 +++++++++++++- .../src/operations/data_folder.rs | 8 ++++- .../modelardb_embedded/src/operations/mod.rs | 30 ++++++++++++++++++- .../tests/integration_test.rs | 2 +- 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/crates/modelardb_embedded/src/operations/client.rs b/crates/modelardb_embedded/src/operations/client.rs index cf3f75d9..8f0e1352 100644 --- a/crates/modelardb_embedded/src/operations/client.rs +++ b/crates/modelardb_embedded/src/operations/client.rs @@ -19,6 +19,7 @@ use std::any::Any; use std::collections::HashMap; use std::pin::Pin; use std::str; +use std::str::FromStr; use std::sync::Arc; use arrow::datatypes::Schema; @@ -38,7 +39,8 @@ use tonic::{Request, Status}; use crate::error::{ModelarDbEmbeddedError, Result}; use crate::operations::{ - Operations, generate_read_time_series_table_sql, try_new_time_series_table_metadata, + ModelarDBType, Operations, generate_read_time_series_table_sql, + try_new_time_series_table_metadata, }; use crate::{Aggregate, TableType}; @@ -66,6 +68,25 @@ impl Operations for Client { self } + /// Returns the type of the ModelarDB node that the client is connected to. + async fn modelardb_type(&mut self) -> Result { + // Retrieve the node type from the ModelarDB node. + let action = Action { + r#type: "NodeType".to_owned(), + body: vec![].into(), + }; + + let response = self.flight_client.do_action(Request::new(action)).await?; + + let message = response + .into_inner() + .message() + .await? + .expect("Flight message should exist."); + + ModelarDBType::from_str(str::from_utf8(&message.body)?) + } + /// Creates a table with the name in `table_name` and the information in `table_type`. If the /// table already exists or if the table could not be created, [`ModelarDbEmbeddedError`] is /// returned. diff --git a/crates/modelardb_embedded/src/operations/data_folder.rs b/crates/modelardb_embedded/src/operations/data_folder.rs index 2de24a3c..cea8a050 100644 --- a/crates/modelardb_embedded/src/operations/data_folder.rs +++ b/crates/modelardb_embedded/src/operations/data_folder.rs @@ -35,7 +35,8 @@ use modelardb_storage::data_folder::DataFolder; use crate::error::{ModelarDbEmbeddedError, Result}; use crate::operations::{ - Operations, generate_read_time_series_table_sql, try_new_time_series_table_metadata, + ModelarDBType, Operations, generate_read_time_series_table_sql, + try_new_time_series_table_metadata, }; use crate::{Aggregate, TableType}; @@ -112,6 +113,11 @@ impl Operations for DataFolder { self } + /// Returns that the ModelarDB instance is a data folder. + async fn modelardb_type(&mut self) -> Result { + Ok(ModelarDBType::DataFolder) + } + /// Creates a table with the name in `table_name` and the information in `table_type`. If the /// table could not be created or registered with Apache DataFusion, [`ModelarDbEmbeddedError`] /// is returned. diff --git a/crates/modelardb_embedded/src/operations/mod.rs b/crates/modelardb_embedded/src/operations/mod.rs index a5e97627..f434d475 100644 --- a/crates/modelardb_embedded/src/operations/mod.rs +++ b/crates/modelardb_embedded/src/operations/mod.rs @@ -21,6 +21,7 @@ pub mod data_folder; use std::any::Any; use std::collections::HashMap; use std::pin::Pin; +use std::str::FromStr; use std::sync::Arc; use arrow::datatypes::Schema; @@ -31,9 +32,33 @@ use datafusion::execution::RecordBatchStream; use modelardb_storage::parser::tokenize_and_parse_sql_expression; use modelardb_types::types::{ErrorBound, GeneratedColumn, TimeSeriesTableMetadata}; -use crate::error::Result; +use crate::error::{ModelarDbEmbeddedError, Result}; use crate::{Aggregate, TableType}; +/// Different types of ModelarDB instances that the Operations API can interact with. +pub enum ModelarDBType { + SingleEdge, + ClusterEdge, + ClusterCloud, + DataFolder, +} + +impl FromStr for ModelarDBType { + type Err = ModelarDbEmbeddedError; + + fn from_str(value: &str) -> Result { + match value { + "SingleEdge" => Ok(ModelarDBType::SingleEdge), + "ClusterEdge" => Ok(ModelarDBType::ClusterEdge), + "ClusterCloud" => Ok(ModelarDBType::ClusterCloud), + "DataFolder" => Ok(ModelarDBType::DataFolder), + _ => Err(ModelarDbEmbeddedError::InvalidArgument(format!( + "'{value}' is not a valid value for ModelarDBType." + ))), + } + } +} + /// Trait for interacting with ModelarDB, either through an Apache Arrow Flight server or a data /// folder. #[async_trait] @@ -42,6 +67,9 @@ pub trait Operations: Sync + Send { /// implementation. fn as_any(&self) -> &dyn Any; + /// Returns the type of the ModelarDB instance that this [`Operations`] instance interacts with. + async fn modelardb_type(&mut self) -> Result; + /// Creates a table with the name in `table_name` and the information in `table_type`. async fn create(&mut self, table_name: &str, table_type: TableType) -> Result<()>; diff --git a/crates/modelardb_server/tests/integration_test.rs b/crates/modelardb_server/tests/integration_test.rs index c615d464..d2f77789 100644 --- a/crates/modelardb_server/tests/integration_test.rs +++ b/crates/modelardb_server/tests/integration_test.rs @@ -1396,7 +1396,7 @@ async fn test_can_get_node_type() { let mut test_context = TestContext::new().await; let response_bytes = test_context.retrieve_action_bytes("NodeType").await; - assert_eq!(str::from_utf8(&response_bytes).unwrap(), "server"); + assert_eq!(str::from_utf8(&response_bytes).unwrap(), "SingleEdge"); } #[tokio::test] From 07f25a0c8b53bb84104e637f41bcf36ff616d665 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 18 Nov 2025 22:04:56 +0100 Subject: [PATCH 78/99] Implement C-API method to get ModelarDB type of Operations instance --- crates/modelardb_embedded/src/capi.rs | 46 +++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/crates/modelardb_embedded/src/capi.rs b/crates/modelardb_embedded/src/capi.rs index 209666a5..7a124fc5 100644 --- a/crates/modelardb_embedded/src/capi.rs +++ b/crates/modelardb_embedded/src/capi.rs @@ -257,6 +257,52 @@ pub unsafe extern "C" fn modelardb_embedded_close( } } +/// Returns the ModelarDB type of the [`DataFolder`] or [`Client`] in `maybe_operations_ptr`. Assumes +/// `maybe_operations_ptr` points to a [`DataFolder`] or [`Client`]; `modelardb_type_array_ptr` is +/// a valid pointer to enough memory for an Apache Arrow C Data Interface Array; and +/// `modelardb_type_array_schema_ptr` is a valid pointer to enough memory for an Apache Arrow C +/// Data Interface Schema. Note that only a single value is written to the C Data Interface Array. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn modelardb_embedded_modelardb_type( + maybe_operations_ptr: *mut c_void, + is_data_folder: bool, + modelardb_type_array_ptr: *mut FFI_ArrowArray, + modelardb_type_array_schema_ptr: *mut FFI_ArrowSchema, +) -> c_int { + let maybe_unit = unsafe { + modelardb_type( + maybe_operations_ptr, + is_data_folder, + modelardb_type_array_ptr, + modelardb_type_array_schema_ptr, + ) + }; + + set_error_and_return_code(maybe_unit) +} + +/// See documentation for [`modelardb_embedded_modelardb_type()`]. +unsafe fn modelardb_type( + maybe_operations_ptr: *mut c_void, + is_data_folder: bool, + modelardb_type_array_ptr: *mut FFI_ArrowArray, + modelardb_type_array_schema_ptr: *mut FFI_ArrowSchema, +) -> Result<()> { + let modelardb = unsafe { c_void_to_operations(maybe_operations_ptr, is_data_folder)? }; + + let modelardb_type = TOKIO_RUNTIME.block_on(modelardb.modelardb_type())?; + let modelardb_type_str = format!("{:?}", modelardb_type); + + let modelardb_type_array = StringArray::from(vec![modelardb_type_str]); + let modelardb_type_array_data = modelardb_type_array.into_data(); + let (out_array, out_schema) = ffi::to_ffi(&modelardb_type_array_data)?; + + unsafe { modelardb_type_array_ptr.write(out_array) }; + unsafe { modelardb_type_array_schema_ptr.write(out_schema) }; + + Ok(()) +} + /// Creates a table with the name in `table_name_ptr`, the schema in `schema_ptr`, and the error /// bounds in `error_bounds_ptr` in the [`DataFolder`] or [`Client`] in `maybe_operations_ptr`. /// Assumes `maybe_operations_ptr` points to a [`DataFolder`] or [`Client`]; `table_name_ptr` points From 7f663b1c84df8ff5b8a309e6ed2af070107b7b28 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 18 Nov 2025 22:08:15 +0100 Subject: [PATCH 79/99] Implement method on Python interface to get ModelarDB type --- .../bindings/python/modelardb/operations.py | 23 +++++++++++++++++++ .../bindings/python/tests/test_operations.py | 5 ++++ .../modelardb_embedded/src/operations/mod.rs | 1 + 3 files changed, 29 insertions(+) diff --git a/crates/modelardb_embedded/bindings/python/modelardb/operations.py b/crates/modelardb_embedded/bindings/python/modelardb/operations.py index b979e9c9..88e5e676 100644 --- a/crates/modelardb_embedded/bindings/python/modelardb/operations.py +++ b/crates/modelardb_embedded/bindings/python/modelardb/operations.py @@ -123,6 +123,11 @@ def __find_library(build: str) -> str: int modelardb_embedded_close(void* maybe_operations_ptr, bool is_data_folder); + int modelardb_embedded_modelardb_type(void* maybe_operations_ptr, + bool is_data_folder, + struct ArrowArray* modelardb_type_array_ptr, + struct ArrowSchema* modelardb_type_array_schema_ptr); + int modelardb_embedded_create(void* maybe_operations_ptr, bool is_data_folder, char* table_name_ptr, @@ -344,6 +349,24 @@ def __del__(self): ) self.__check_return_code_and_raise_error(return_code) + def modelardb_type(self) -> str: + """Returns the type of the ModelarDB instance that :obj:`Operations` is connected to. + + :return: The type of the ModelarDB instance. + :rtype: str + """ + modelardb_type_ffi = FFIArray.from_type(StringArray) + + return_code = self.__library.modelardb_embedded_modelardb_type( + self.__operations_ptr, + self.__is_data_folder, + modelardb_type_ffi.array_ptr, + modelardb_type_ffi.schema_ptr, + ) + self.__check_return_code_and_raise_error(return_code) + + return modelardb_type_ffi.array().to_pylist()[0] + def create(self, table_name: str, table_type: NormalTable | TimeSeriesTable): """Creates a table with `table_name`, `schema`, and `error_bounds`. diff --git a/crates/modelardb_embedded/bindings/python/tests/test_operations.py b/crates/modelardb_embedded/bindings/python/tests/test_operations.py index 4e62c8cd..d0cb8651 100644 --- a/crates/modelardb_embedded/bindings/python/tests/test_operations.py +++ b/crates/modelardb_embedded/bindings/python/tests/test_operations.py @@ -42,6 +42,11 @@ def test_data_folder_init(self): new_data_folder = Operations.open_local(temp_dir) self.assertEqual(data_folder.tables(), new_data_folder.tables()) + def test_data_folder_modelardb_type(self): + with TemporaryDirectory() as temp_dir: + data_folder = Operations.open_local(temp_dir) + self.assertEqual(data_folder.modelardb_type(), "DataFolder") + def test_data_folder_create_normal_table(self): with TemporaryDirectory() as temp_dir: data_folder = Operations.open_local(temp_dir) diff --git a/crates/modelardb_embedded/src/operations/mod.rs b/crates/modelardb_embedded/src/operations/mod.rs index f434d475..c4e4f467 100644 --- a/crates/modelardb_embedded/src/operations/mod.rs +++ b/crates/modelardb_embedded/src/operations/mod.rs @@ -36,6 +36,7 @@ use crate::error::{ModelarDbEmbeddedError, Result}; use crate::{Aggregate, TableType}; /// Different types of ModelarDB instances that the Operations API can interact with. +#[derive(Debug)] pub enum ModelarDBType { SingleEdge, ClusterEdge, From cb3be715b367a4fc535873aefd9c4794faee8035 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 18 Nov 2025 22:17:39 +0100 Subject: [PATCH 80/99] Use ModelarDBType enum on Python side for return value of modelardb_type() --- .../bindings/python/modelardb/__init__.py | 1 + .../bindings/python/modelardb/operations.py | 16 +++++++++++++--- .../bindings/python/tests/test_operations.py | 4 +++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/crates/modelardb_embedded/bindings/python/modelardb/__init__.py b/crates/modelardb_embedded/bindings/python/modelardb/__init__.py index a586ae32..cd6ed2fa 100644 --- a/crates/modelardb_embedded/bindings/python/modelardb/__init__.py +++ b/crates/modelardb_embedded/bindings/python/modelardb/__init__.py @@ -18,6 +18,7 @@ from .ffi_array import FFIArray from .operations import ( Aggregate, + ModelarDBType, Operations, open_memory, open_local, diff --git a/crates/modelardb_embedded/bindings/python/modelardb/operations.py b/crates/modelardb_embedded/bindings/python/modelardb/operations.py index 88e5e676..a05e67c9 100644 --- a/crates/modelardb_embedded/bindings/python/modelardb/operations.py +++ b/crates/modelardb_embedded/bindings/python/modelardb/operations.py @@ -30,6 +30,15 @@ from .ffi_array import FFIArray +class ModelarDBType(Enum): + """Different types of ModelarDB instances that the Operations API can interact with.""" + + SingleEdge = 0 + ClusterEdge = 1 + ClusterCloud = 2 + DataFolder = 3 + + class Aggregate(Enum): """Aggregate operations supported by :meth:`Operations.read`.""" @@ -349,11 +358,11 @@ def __del__(self): ) self.__check_return_code_and_raise_error(return_code) - def modelardb_type(self) -> str: + def modelardb_type(self) -> ModelarDBType: """Returns the type of the ModelarDB instance that :obj:`Operations` is connected to. :return: The type of the ModelarDB instance. - :rtype: str + :rtype: ModelarDBType """ modelardb_type_ffi = FFIArray.from_type(StringArray) @@ -365,7 +374,8 @@ def modelardb_type(self) -> str: ) self.__check_return_code_and_raise_error(return_code) - return modelardb_type_ffi.array().to_pylist()[0] + modelardb_type_str = modelardb_type_ffi.array().to_pylist()[0] + return ModelarDBType[modelardb_type_str] def create(self, table_name: str, table_type: NormalTable | TimeSeriesTable): """Creates a table with `table_name`, `schema`, and `error_bounds`. diff --git a/crates/modelardb_embedded/bindings/python/tests/test_operations.py b/crates/modelardb_embedded/bindings/python/tests/test_operations.py index d0cb8651..2d5f0aac 100644 --- a/crates/modelardb_embedded/bindings/python/tests/test_operations.py +++ b/crates/modelardb_embedded/bindings/python/tests/test_operations.py @@ -20,6 +20,7 @@ import pyarrow from modelardb import ( Aggregate, + ModelarDBType, AbsoluteErrorBound, TimeSeriesTable, NormalTable, @@ -45,7 +46,8 @@ def test_data_folder_init(self): def test_data_folder_modelardb_type(self): with TemporaryDirectory() as temp_dir: data_folder = Operations.open_local(temp_dir) - self.assertEqual(data_folder.modelardb_type(), "DataFolder") + + self.assertEqual(data_folder.modelardb_type(), ModelarDBType.DataFolder) def test_data_folder_create_normal_table(self): with TemporaryDirectory() as temp_dir: From 8e015b4e07829a3a23d81b4c70ba074593f1319e Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Thu, 20 Nov 2025 09:19:45 +0100 Subject: [PATCH 81/99] Remove manager from cluster Docker Compose setup --- docker-compose-cluster.yml | 48 +++++++++++++------------------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/docker-compose-cluster.yml b/docker-compose-cluster.yml index e1647b45..150bc4fc 100644 --- a/docker-compose-cluster.yml +++ b/docker-compose-cluster.yml @@ -12,7 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -version: "3.8" +x-aws-variables: &aws-variables + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + AWS_DEFAULT_REGION: eu-central-1 + AWS_ENDPOINT: http://minio-server:9000 + AWS_ALLOW_HTTP: true services: # Object store services. @@ -41,19 +46,17 @@ services: " # ModelarDB services. - modelardb-manager: + modelardb-edge: image: modelardb build: . - container_name: modelardb-manager - command: ["target/debug/modelardbm", "s3://modelardb"] + container_name: modelardb-edge + command: [ "target/debug/modelardbd", "edge", "data/edge", "s3://modelardb" ] ports: - - "9998:9998" + - "9999:9999" environment: - AWS_ACCESS_KEY_ID: minioadmin - AWS_SECRET_ACCESS_KEY: minioadmin - AWS_DEFAULT_REGION: eu-central-1 - AWS_ENDPOINT: http://minio-server:9000 - AWS_ALLOW_HTTP: true + <<: *aws-variables + MODELARDBD_IP_ADDRESS: modelardb-edge + MODELARDBD_PORT: 9999 depends_on: create-bucket: condition: service_completed_successfully @@ -61,35 +64,16 @@ services: test: [ "CMD", "echo", "ready" ] interval: 2s - modelardb-edge: - image: modelardb - container_name: modelardb-edge - command: ["target/debug/modelardbd", "edge", "data/edge", "grpc://modelardb-manager:9998"] - ports: - - "9999:9999" - environment: - MODELARDBD_UNCOMPRESSED_DATA_BUFFER_CAPACITY: 640 - MODELARDBD_COMPRESSED_RESERVED_MEMORY_IN_BYTES: 10000 - MODELARDBD_TRANSFER_BATCH_SIZE_IN_BYTES: 10000 - MODELARDBD_IP_ADDRESS: host.docker.internal - AWS_ALLOW_HTTP: true - depends_on: - modelardb-manager: - condition: service_healthy - healthcheck: - test: [ "CMD", "echo", "ready" ] - interval: 1s - modelardb-cloud: image: modelardb container_name: modelardb-cloud - command: [ "target/debug/modelardbd", "cloud", "data/cloud", "grpc://modelardb-manager:9998"] + command: [ "target/debug/modelardbd", "cloud", "data/cloud", "s3://modelardb" ] ports: - "9997:9997" environment: + <<: *aws-variables MODELARDBD_PORT: 9997 - MODELARDBD_IP_ADDRESS: host.docker.internal - AWS_ALLOW_HTTP: true + MODELARDBD_IP_ADDRESS: modelardb-cloud depends_on: modelardb-edge: condition: service_healthy From 3419b4c6908946ae7937d0087b4ad57305efc281 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Thu, 20 Nov 2025 09:37:23 +0100 Subject: [PATCH 82/99] Add one more edge and cloud node to lcluster setup --- docker-compose-cluster.yml | 43 +++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/docker-compose-cluster.yml b/docker-compose-cluster.yml index 150bc4fc..a279fb13 100644 --- a/docker-compose-cluster.yml +++ b/docker-compose-cluster.yml @@ -46,16 +46,16 @@ services: " # ModelarDB services. - modelardb-edge: + modelardb-edge-1: &modelardb-base image: modelardb build: . - container_name: modelardb-edge + container_name: modelardb-edge-1 command: [ "target/debug/modelardbd", "edge", "data/edge", "s3://modelardb" ] ports: - "9999:9999" environment: <<: *aws-variables - MODELARDBD_IP_ADDRESS: modelardb-edge + MODELARDBD_IP_ADDRESS: modelardb-edge-1 MODELARDBD_PORT: 9999 depends_on: create-bucket: @@ -64,16 +64,43 @@ services: test: [ "CMD", "echo", "ready" ] interval: 2s - modelardb-cloud: - image: modelardb - container_name: modelardb-cloud + modelardb-edge-2: + <<: *modelardb-base + container_name: modelardb-edge-2 + ports: + - "9998:9998" + environment: + <<: *aws-variables + MODELARDBD_PORT: 9998 + MODELARDBD_IP_ADDRESS: modelardb-edge-2 + depends_on: + modelardb-edge-1: + condition: service_healthy + + modelardb-cloud-1: + <<: *modelardb-base + container_name: modelardb-cloud-1 command: [ "target/debug/modelardbd", "cloud", "data/cloud", "s3://modelardb" ] ports: - "9997:9997" environment: <<: *aws-variables MODELARDBD_PORT: 9997 - MODELARDBD_IP_ADDRESS: modelardb-cloud + MODELARDBD_IP_ADDRESS: modelardb-cloud-1 + depends_on: + modelardb-edge-2: + condition: service_healthy + + modelardb-cloud-2: + <<: *modelardb-base + container_name: modelardb-cloud-2 + command: [ "target/debug/modelardbd", "cloud", "data/cloud", "s3://modelardb" ] + ports: + - "9996:9996" + environment: + <<: *aws-variables + MODELARDBD_PORT: 9996 + MODELARDBD_IP_ADDRESS: modelardb-cloud-2 depends_on: - modelardb-edge: + modelardb-cloud-1: condition: service_healthy From 6926178f1e719ccf6e278f8ed7c8cf2281912ad9 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Thu, 20 Nov 2025 10:24:36 +0100 Subject: [PATCH 83/99] No longer use base service to avoid multiple builds --- docker-compose-cluster.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/docker-compose-cluster.yml b/docker-compose-cluster.yml index a279fb13..175557e5 100644 --- a/docker-compose-cluster.yml +++ b/docker-compose-cluster.yml @@ -46,9 +46,9 @@ services: " # ModelarDB services. - modelardb-edge-1: &modelardb-base - image: modelardb + modelardb-edge-1: build: . + image: modelardb container_name: modelardb-edge-1 command: [ "target/debug/modelardbd", "edge", "data/edge", "s3://modelardb" ] ports: @@ -65,8 +65,9 @@ services: interval: 2s modelardb-edge-2: - <<: *modelardb-base + image: modelardb container_name: modelardb-edge-2 + command: [ "target/debug/modelardbd", "edge", "data/edge", "s3://modelardb" ] ports: - "9998:9998" environment: @@ -76,9 +77,12 @@ services: depends_on: modelardb-edge-1: condition: service_healthy + healthcheck: + test: [ "CMD", "echo", "ready" ] + interval: 2s modelardb-cloud-1: - <<: *modelardb-base + image: modelardb container_name: modelardb-cloud-1 command: [ "target/debug/modelardbd", "cloud", "data/cloud", "s3://modelardb" ] ports: @@ -90,9 +94,12 @@ services: depends_on: modelardb-edge-2: condition: service_healthy + healthcheck: + test: [ "CMD", "echo", "ready" ] + interval: 2s modelardb-cloud-2: - <<: *modelardb-base + image: modelardb container_name: modelardb-cloud-2 command: [ "target/debug/modelardbd", "cloud", "data/cloud", "s3://modelardb" ] ports: From e312937c9c422cc807fa029d429bbfa9083607ca Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Thu, 20 Nov 2025 13:55:14 +0100 Subject: [PATCH 84/99] Update Docker section in docs to no longer mention manager --- docs/user/README.md | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/docs/user/README.md b/docs/user/README.md index b6f3c3ab..04c634e4 100644 --- a/docs/user/README.md +++ b/docs/user/README.md @@ -407,14 +407,13 @@ Two different [Docker](https://docs.docker.com/) environments are included to ma different use cases of ModelarDB. The first environment sets up a single instance of `modelardbd` that only uses local storage. Data can be ingested into this instance, compressed, and saved to local storage. The compressed data in local storage can then be queried. The second environment covers the more complex use case of ModelarDB where multiple -instances of `modelardbd` are deployed in a cluster with a manager that is responsible for managing the cluster. A -single edge node and a single cloud node is set up in the cluster. Data can be ingested into the edge or cloud node, -compressed, and transferred to a remote object store. The compressed data in the remote object store can then be queried -through the cloud node or by directing the query through the manager node. +instances of `modelardbd` are deployed in a cluster. Two edge nodes and two cloud nodes are set up in the cluster. Data +can be ingested into the edge or cloud nodes, compressed, and transferred to a remote object store. The compressed data +in the remote object store can then be queried through the cloud nodes. Note that since [Rust](https://www.rust-lang.org/) is a compiled language and a more dynamic ModelarDB configuration might be needed, it is not recommended to use the [Docker](https://docs.docker.com/) environments during active -development of ModelarDB. They are however ideal to use for experimenting with ModelarDB or when developing +development of ModelarDB. They are, however, ideal to use for experimenting with ModelarDB or when developing software that utilizes ModelarDB. Downloading [Docker Desktop](https://docs.docker.com/desktop/) is recommended to make maintenance of the created containers easier. @@ -459,8 +458,7 @@ The cluster deployment sets up a MinIO object store and a MinIO client is used t the object store. MinIO can be administered through its [web interface](http://127.0.0.1:9001). The default username and password, `minioadmin`, can be used to log in. -The cluster itself consists of a manager node, an edge node, and a cloud node. The manager node can be accessed using -the URL `grpc://127.0.0.1:9998`, the edge node using the URL `grpc://127.0.0.1:9999`, and the cloud node using the URL -`grpc://127.0.0.1:9997`. Tables can be created through the manager node and data can be ingested, compressed, and -transferred to the object store through the edge node or the cloud node. The compressed data in the MinIO object store -can then be queried through the cloud node or by directing the query through the manager node. +The cluster itself consists of two edge nodes and two cloud nodes. The edge nodes can be accessed using +the URLs `grpc://127.0.0.1:9999` and `grpc://127.0.0.1:9998`, and the cloud nodes using the URLs `grpc://127.0.0.1:9997` +and `grpc://127.0.0.1:9996`. Tables can be created and data can be ingested, compressed, and transferred to the object +store through all four nodes. The compressed data in the MinIO object store can then be queried through the cloud nodes. From 5d407fc0a3b17d1ce36eec3d354f5302ef33694b Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Thu, 20 Nov 2025 14:11:01 +0100 Subject: [PATCH 85/99] Update dev docs to match the new module structure --- docs/dev/README.md | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/docs/dev/README.md b/docs/dev/README.md index 515758db..1131609e 100644 --- a/docs/dev/README.md +++ b/docs/dev/README.md @@ -37,16 +37,11 @@ data folders from programming languages. - **Error** - Error type used throughout the crate, a single error type is used for simplicity. - **C-API** - A C-API for using modelardb_embedded from other programming languages through a C-FFI. - **ModelarDB** - Module providing functionality for reading from and writing to ModelarDB instances and data folders. -- [modelardb_manager](/crates/modelardb_manager) - ModelarDB's manager in the form of the binary `modelardbm`. - - **Cluster** - Manages edge and cloud nodes currently controlled by the ModelarDB manager and provides functionality - for balancing query workloads across multiple cloud nodes. - - **Error** - Error type used throughout the crate, a single error type is used for simplicity. - - **Metadata** - Manages metadata stored in Delta Lake, e.g., information about the manager itself, the nodes - controlled by the manager, and the database schema and compressed data in the cluster. - - **Remote** - A public interface for interacting with the ModelarDB manager using Apache Arrow Flight. - [modelardb_server](/crates/modelardb_server) - ModelarDB's DBMS server in the form of the binary `modelardbd`. - **Storage** - Manages uncompressed data, compresses uncompressed data, manages compressed data, and writes compressed data to Delta Lake. + - **Cluster** - Manages edge and cloud nodes in the cluster and provides functionality for performing operations on + all peer nodes and for balancing query workloads across multiple cloud nodes. - **Configuration** - Manages the configuration of the ModelarDB DBMS server and provides functionality for updating the configuration. - **Context** - A type that contains all of the components in the ModelarDB DBMS server and makes it easy to share and @@ -54,22 +49,19 @@ data folders from programming languages. - **Data Folders** - A type for managing data and metadata in a local data folder, an Amazon S3 bucket, or an Microsoft Azure Blob Storage container. - **Error** - Error type used throughout the crate, a single error type is used for simplicity. - - **Manager** - Manages metadata related to the ModelarDB manager and provides functionality for interacting with the - ModelarDB manager. - **Remote** - A public interface to interact with the ModelarDB DBMS server using Apache Arrow Flight. -- [modelardb_storage](/crates/modelardb_storage) - Library providing functionality for reading from and writing to -storage. - - **Metadata** - Manages metadata stored in Delta Lake, e.g., information about the tables' schema and compressed - data. +- [modelardb_storage](/crates/modelardb_storage) - Library providing functionality for reading from and writing to storage. + - **Data Folder** - Module providing functionality for interacting with local and remote storage through a Delta Lake. - **Optimizer** - Rules for rewriting Apache DataFusion's physical plans for time series tables so aggregates are computed from compressed segments instead of from reconstructed data points. - **Query** - Types that implement traits provided by Apache DataFusion so SQL queries can be executed for ModelarDB tables. - - **Delta Lake** - Module providing functionality for reading from and writing to a delta lake. - **Error** - Error type used throughout the crate, a single error type is used for simplicity. - **Parser** - Extensions to Apache DataFusion's SQL parser. The first extension adds support for creating time series tables with a timestamp, one or more fields, and zero or more tags. The second adds support for adding a `INCLUDE - address[, address+]` clause before `SELECT`. + address[, address+]` clause before `SELECT`. The third adds support for + `VACUUM [CLUSTER] [table_name[, table_name]+] [RETAIN num_seconds]` statements. The final extension adds support for + `TRUNCATE [CLUSTER] table_name[, table_name]+` statements. - [modelardb_test](/crates/modelardb_test) - Library providing functionality for testing ModelarDB. - **Data Generation** - Functionality for generating data with a specific structure for use in tests. - **Table** - Constants and functionality for testing normal tables and time series tables. From 5c30230ef2521ff15c1c82d6d467e218204671ac Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Thu, 20 Nov 2025 15:31:38 +0100 Subject: [PATCH 86/99] Update user docs to no longer mention the manager --- docs/user/README.md | 128 +++++++++++++++++++------------------------- 1 file changed, 55 insertions(+), 73 deletions(-) diff --git a/docs/user/README.md b/docs/user/README.md index 04c634e4..4ce57977 100644 --- a/docs/user/README.md +++ b/docs/user/README.md @@ -38,36 +38,32 @@ The following commands are for Ubuntu Server. However, equivalent commands shoul - Run Tests: `cargo test` - Run DBMS Server: `cargo run --bin modelardbd path_to_local_data_folder` - Run Client: `cargo run --bin modelardb [server_address] [query_file]` -6. Move `modelardbd`, `modelardbm`,`modelardb`, and `modelardbb` from the `target` directory to any - directory. +6. Move `modelardbd`, `modelardb`, and `modelardbb` from the `target` directory to any directory. 7. Install and test the Python bindings for `modelardb_embedded` using Python: * Install: `python3 -m pip install .` * Run Tests: `python3 -m unittest` ## Usage -ModelarDB consists of four binaries and a library with bindings: `modelardbd` is a DBMS server that manages data and -executes SQL queries, `modelardbm` is a manager for one or more `modelardbd` instances deployed on the *edge* or in the -*cloud*, `modelardb` is a command-line client for connecting to a `modelardbd` instance and executing commands and SQL -queries, `modelardbb` is a command-line bulk loader that operates without `modelardbd` as it reads from and writes to -`modelardbd`'s data folder directly, and `modelardb_embedded` is an embeddable library for executing queries against and -writing to `modelardbd` or its data folder directly. `modelardbd` uses local storage on the edge and an Amazon -S3-compatible or Azure Blob Storage object store in the cloud. `modelardbd` can be deployed alone or together with -`modelardbm` depending on the use cases. - -`modelardbd` can be deployed on a single node to manage data in a local folder. In this configuration `modelardbm` is -not needed. `modelardbd` can also be deployed in a distributed configuration across edge and cloud. In this -configuration, `modelardbm` must first be deployed in the cloud to manage the `modelardbd` instances on the edge and in -the cloud. Specifically, `modelardbm` is responsible for keeping the database schema consistent across all `modelardbd` -instances in the cluster. After deploying `modelardbm`, instances of `modelardbd` can be deployed on the edge and in -cloud. While the `modelardbd` instances on the edge and in cloud provides the same functionality, the primary purpose of -the instances on the edge is to collect data and transfer it to an object store in the cloud while the primary purpose -of the instances in the cloud is to execute queries on the data in the object store. +ModelarDB consists of three binaries and a library with bindings: `modelardbd` is a DBMS server that manages data and +executes SQL queries, `modelardb` is a command-line client for connecting to a `modelardbd` instance and executing +commands and SQL queries, `modelardbb` is a command-line bulk loader that operates without `modelardbd` as it reads +from and writes to `modelardbd`'s data folder directly, and `modelardb_embedded` is an embeddable library for executing +queries against and writing to `modelardbd` or its data folder directly. `modelardbd` uses local storage on the edge +and an Amazon S3-compatible or Azure Blob Storage object store in the cloud. `modelardbd` can be deployed alone or +together with other instances of `modelardbd` in a cluster architecture, depending on the use case. + +`modelardbd` can be deployed on a single node to manage data in a local folder. `modelardbd` can also be deployed in a +distributed configuration across edge and cloud. In this configuration, the shared remote object store is responsible +for keeping the database schema consistent across all `modelardbd` instances in the cluster. While the `modelardbd` +instances on the edge and in cloud provides the same functionality, the primary purpose of the instances on the edge is +to collect data and transfer it to an object store in the cloud while the primary purpose of the instances in the cloud +is to execute queries on the data in the object store. Thus, when `modelardbd` is deployed in edge mode, it executes queries against local storage for low latency queries on the latest data and when it is deployed in cloud mode, it executes queries against the object store for efficient -analytics on all the data transferred to the cloud. `modelardbm` implements the [Apache Arrow Flight +analytics on all the data transferred to the cloud. `modelardbd` implements the [Apache Arrow Flight protocol](https://arrow.apache.org/docs/format/Flight.html#downloading-data) for looking up the `modelardbd` instance in -the cloud to use for executing each query, thus providing a single, workload-balanced, interface for querying the data +the cloud to use for executing each query, thus providing a workload-balanced interface for querying the data in the object store using the `modelardbd` instances in the cloud. ### Start Server @@ -87,17 +83,15 @@ modelardbd path_to_local_data_folder modelardbd edge path_to_local_data_folder ``` -2. Start `modelardbd` in edge mode with a manager - A running instance of `modelardbm` and an Amazon S3-compatible or -Azure Blob Storage object store are required to start `modelardbd` in edge mode with a manager. This configuration -supports ingesting data to a local folder on the edge, querying the data in the local folder on the edge, and -transferring data in the local data folder to the object store. -3. Start `modelardbd` in cloud mode with a manager - As above, a running instance of `modelardbm` and an Amazon -S3-compatible or Azure Blob Storage object store are required to start `modelardbd` in cloud mode with a manager. This -configuration supports ingesting data to a local folder in the cloud, transferring data in the local data folder to the -object store, and querying the data in the object store in the cloud. +2. Start `modelardbd` in edge mode in a cluster - An Amazon S3-compatible or Azure Blob Storage object store is required +to start `modelardbd` in a cluster. This configuration supports ingesting data to a local folder on the edge, querying +the data in the local folder on the edge, and transferring data in the local data folder to the object store. +3. Start `modelardbd` in cloud mode in a cluster - As above, an Amazon S3-compatible or Azure Blob Storage object store +is required. This configuration supports ingesting data to a local folder in the cloud, transferring data in the local +data folder to the object store, and querying the data in the object store in the cloud. The following environment variables must be set to appropriate values if an Amazon S3-compatible object store is used so -`modelardbm` and `modelardbd` know how to connect to it: +`modelardbd` know how to connect to it: ```shell AWS_ENDPOINT @@ -116,13 +110,14 @@ AWS_SECRET_ACCESS_KEY="sp7TDyD2ZruJ0VdFHmkacT1Y90PVPF7p" ``` Then, assuming a bucket named `wind-turbine` has been created through MinIO's command line tool or web interface, -`modelardbm` can be started with the following command: +`modelardbd` can be started in edge mode with transfer of the ingested data to the object store using the +following command: ```shell -modelardbm s3://wind-turbine +modelardbd edge path_to_local_data_folder s3://wind-turbine ``` -`modelardbm` also supports using [Azure Blob Storage](https://azure.microsoft.com/en-us/products/storage/blobs/) +`modelardbd` also supports using [Azure Blob Storage](https://azure.microsoft.com/en-us/products/storage/blobs/) for the remote object store. To use Azure Blob Storage, set the following environment variables: ```shell @@ -130,34 +125,24 @@ AZURE_STORAGE_ACCOUNT_NAME AZURE_STORAGE_ACCESS_KEY ``` -Then, assuming a container named `wind-turbine` has been created, `modelardbm` can be started with the following +Then, assuming a container named `wind-turbine` has been created, `modelardbd` can be started with the following command: ```shell -modelardbm azureblobstorage://wind-turbine +modelardbd edge path_to_local_data_folder azureblobstorage://wind-turbine ``` -When a manager is running, `modelardbd` can be started in edge mode with transfer of the ingested data to the object -store given to `modelardbm` using the following command: - -```shell -modelardbd edge path_to_local_data_folder manager_url -```` - To run `modelardbd` in cloud mode simply replace `edge` with `cloud` as shown below. Be aware that when running in cloud -mode the `modelardbd` instance will execute queries against the object store given to `modelardbm` and not against local -storage. +mode the `modelardbd` instance will execute queries against the object store and not against local storage. ```shell -modelardbd cloud path_to_local_data_folder manager_url +modelardbd cloud path_to_local_data_folder s3://wind-turbine ``` -In both cases, access to the object store is automatically provided by the manager when `modelardbd` instances are -connected to it. Note that the manager uses `9998` as the default port and that the DBMS server uses `9999` as the -default port. The ports can be changed by specifying different ports with the following environment variables: +Note that the DBMS server uses `9999` as the default port. The port can be changed by specifying a different port with +the following environment variable: ```shell -MODELARDBM_PORT=8888 # By default modelardbm uses port 9998. MODELARDBD_PORT=8889 # By default modelardbd uses port 9999. ``` @@ -200,12 +185,10 @@ result = flight_client.do_get(ticket) print(list(result)) ``` -When running a cluster of `modelardbd` instances, it is required to use `modelardbm` to create tables. This is a -necessity as `modelardbm` is responsible for keeping the database schema consistent across all `modelardbd` instances in -the cluster. The process for creating a table on the manager is the same as when creating the table directly on a -`modelardbd` instance, as shown above. The only difference is that the gRPC URL should be changed to connect to the -manager instead of the DBMS server. When the table is created through `modelardbm`, the table is automatically created -in all `modelardbd` instances managed by `modelardbm`. +When running a cluster of `modelardbd` instances, any instance in the cluster can be used to create tables. The +process for creating a table in a cluster is the same as when creating the table directly on a single `modelardbd` +instance, as shown above. When the table is created through `modelardbd` in a cluster, the table is automatically +created in all peer `modelardbd` instances. After creating a table, data can be ingested into `modelardbd` with `INSERT` in `modelardb`. Be aware that `INSERT` statements currently must contain values for all columns but that the values for generated columns will be dropped by @@ -248,8 +231,8 @@ sources](https://www.influxdata.com/time-series-platform/telegraf/telegraf-input [MQTT](https://mqtt.org/) and [OPC-UA](https://opcfoundation.org/about/opc-technologies/opc-ua/). It should be noted that the ingested data is only transferred to the remote object store when `modelardbd` is deployed -in edge mode with a manager or in cloud mode with a manager. When `modelardbd` is deployed in edge mode without a -manager, the ingested data is only stored in local storage. +in edge mode in a cluster or in cloud mode in a cluster. When `modelardbd` is deployed in edge mode without a cluster, +the ingested data is only stored in local storage. ### Execute Queries ModelarDB includes a command-line client in the form of `modelardb`. To interactively execute SQL statements against a @@ -332,23 +315,23 @@ for flight_stream_chunk in flight_stream_reader: print(pandas_data_frame) ``` -When running a cluster of `modelardbd` instances, it is recommended to use `modelardbm` to query the data in the remote -object store. As mentioned above, `modelardbm` implements the [Apache Arrow Flight -protocol](https://arrow.apache.org/docs/format/Flight.html#downloading-data) for querying data. This means that a -request is made to the manager first to determine which `modelardbd` cloud instance in the cluster should be queried. -The workload of the queries sent to the manager is balanced across all `modelardbd` cloud instances and at least one -`modelardbd` cloud instance must be running for the manager to accept queries. The following Python example shows how to -execute a simple SQL query using a `modelardbm` instance and how to process the resulting stream of data points using +When running a cluster of `modelardbd` instances, it is recommended to use the +[Apache Arrow Flight protocol](https://arrow.apache.org/docs/format/Flight.html#downloading-data) to query the data in +the remote object store. This means that a request is made to the `get_flight_info()` endpoint first to determine which +`modelardbd` cloud instance in the cluster should be queried. The workload of the queries sent to this endpoint is +balanced across all `modelardbd` cloud instances and at least one `modelardbd` cloud instance must be running for the +endpoint to accept queries. The following Python example shows how to execute a simple workload-balanced SQL query +using a `modelardbd` instance and how to process the resulting stream of data points using [`pyarrow`](https://pypi.org/project/pyarrow/) and [`pandas`](https://pypi.org/project/pandas/). It should be noted that -the manager is only responsible for workload balancing and that the query is sent directly to the `modelardbd` cloud -instance chosen by the manager which then sends the result set directly back to the client. +the endpoint is only responsible for workload balancing and that the query is sent directly to the `modelardbd` cloud +instance chosen by the endpoint which then sends the result set directly back to the client. ```python from pyarrow import flight -manager_client = flight.FlightClient("grpc://127.0.0.1:9998") +modelardbd_client = flight.FlightClient("grpc://127.0.0.1:9999") query_descriptor = flight.FlightDescriptor.for_command("SELECT * FROM wind_turbine LIMIT 10") -flight_info = manager_client.get_flight_info(query_descriptor) +flight_info = modelardbd_client.get_flight_info(query_descriptor) endpoint = flight_info.endpoints[0] cloud_node_url = endpoint.locations[0] @@ -364,10 +347,10 @@ for flight_stream_chunk in flight_stream_reader: ### Embed Library ModelarDB includes an embeddable library in the form of `modelardb_embedded`. It allows programming languages to execute -queries against or write to `modelardbd`, `modelardbm`, or a data folder directly. A C-API allows other programming +queries against or write to `modelardbd` or a data folder directly. A C-API allows other programming languages than Rust to also use `modelardb_embedded`. The location where queries and writes are executed is specified using the set of `open_*()` methods and the `connect()` method. The other methods in `modelardb_embedded` work for both -local disk, object store, `modelardbd`, and `modelardbm` with a few exceptions. Thus, a program can be developed against +local disk, object store, and `modelardbd` with a few exceptions. Thus, a program can be developed against a small data set in a local data folder and then scaled by switching to `modelardbd`. ```python @@ -382,8 +365,8 @@ remote_data_folder = modelardb.open_s3(endpoint, bucket_name, access_key_id, sec # Execute queries and write to a data folder in Microsoft Azure Blob Storage. remote_data_folder = modelardb.open_azure(account_name, access_key, container_name) -# Execute queries and write to a `modelardbd` or `modelardbm` instance. -modelardb_node = modelardb.connect(node) +# Execute queries and write to a `modelardbd` instance. +modelardb_node = modelardb.connect(url) ``` ## ModelarDB configuration @@ -392,7 +375,6 @@ variables is provided here. If an environment variable is not set, the specified | **Variable** | **Default** | **Description** | |--------------------------------------------------|-------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| MODELARDBM_PORT | 9998 | The port of the manager Apache Arrow Flight Server. | | MODELARDBD_PORT | 9999 | The port of the server Apache Arrow Flight Server. | | MODELARDBD_IP_ADDRESS | 127.0.0.1 | The IP address of the Apache Arrow Flight Server. | | MODELARDBD_MULTIVARIATE_RESERVED_MEMORY_IN_BYTES | 512 MB | The amount of memory to reserve for storing multivariate time series. | From 51b6db06faaf9f257273983265029703589d8a6e Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Thu, 20 Nov 2025 15:35:51 +0100 Subject: [PATCH 87/99] Remove modelardb_manager crate --- Cargo.lock | 20 - crates/modelardb_manager/Cargo.toml | 44 -- crates/modelardb_manager/src/cluster.rs | 296 ----------- crates/modelardb_manager/src/error.rs | 118 ----- crates/modelardb_manager/src/main.rs | 82 ---- crates/modelardb_manager/src/remote.rs | 621 ------------------------ 6 files changed, 1181 deletions(-) delete mode 100644 crates/modelardb_manager/Cargo.toml delete mode 100644 crates/modelardb_manager/src/cluster.rs delete mode 100644 crates/modelardb_manager/src/error.rs delete mode 100644 crates/modelardb_manager/src/main.rs delete mode 100644 crates/modelardb_manager/src/remote.rs diff --git a/Cargo.lock b/Cargo.lock index 97ab260c..32074c80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3297,26 +3297,6 @@ dependencies = [ "tonic", ] -[[package]] -name = "modelardb_manager" -version = "0.1.0" -dependencies = [ - "arrow", - "arrow-flight", - "deltalake", - "futures", - "log", - "modelardb_storage", - "modelardb_types", - "prost", - "tempfile", - "tokio", - "tonic", - "tracing", - "tracing-subscriber", - "uuid", -] - [[package]] name = "modelardb_server" version = "0.1.0" diff --git a/crates/modelardb_manager/Cargo.toml b/crates/modelardb_manager/Cargo.toml deleted file mode 100644 index afe9912b..00000000 --- a/crates/modelardb_manager/Cargo.toml +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2023 The ModelarDB Contributors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[package] -name = "modelardb_manager" -version = "0.1.0" -license = "Apache-2.0" -edition = "2024" -authors = ["Soeren Kejser Jensen "] - -[[bin]] -name = "modelardbm" -path = "src/main.rs" - -[dependencies] -arrow-flight.workspace = true -arrow.workspace = true -deltalake.workspace = true -futures.workspace = true -modelardb_storage = { path = "../modelardb_storage" } -modelardb_types = { path = "../modelardb_types" } -prost.workspace = true -tokio = { workspace = true, features = ["rt-multi-thread", "signal"] } -tonic.workspace = true -uuid.workspace = true - -# Log is a dependency so the compile time filters for log and tracing can be set to the same values. -log = { workspace = true, features = ["max_level_debug", "release_max_level_info"] } -tracing = { workspace = true, features = ["max_level_debug", "release_max_level_info"] } -tracing-subscriber.workspace = true - -[dev-dependencies] -tempfile.workspace = true diff --git a/crates/modelardb_manager/src/cluster.rs b/crates/modelardb_manager/src/cluster.rs deleted file mode 100644 index aff0c1b2..00000000 --- a/crates/modelardb_manager/src/cluster.rs +++ /dev/null @@ -1,296 +0,0 @@ -/* Copyright 2023 The ModelarDB Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -//! Management of the cluster of nodes that are currently controlled by the manager. - -use std::collections::VecDeque; - -use arrow_flight::flight_service_client::FlightServiceClient; -use arrow_flight::{Action, Ticket}; -use futures::StreamExt; -use futures::stream::FuturesUnordered; -use log::info; -use modelardb_types::types::{Node, ServerMode}; -use tonic::Request; -use tonic::metadata::{Ascii, MetadataValue}; - -use crate::error::{ModelarDbManagerError, Result}; - -/// Stores the currently managed nodes in the cluster and allows for performing operations that need -/// to be applied to every single node in the cluster. -pub struct Cluster { - /// The nodes that are currently managed by the cluster. - nodes: Vec, - /// Queue of cloud nodes used to determine which cloud node should execute a query in - /// a round-robin fashion. - query_queue: VecDeque, -} - -impl Cluster { - pub fn new() -> Self { - Self { - nodes: vec![], - query_queue: VecDeque::new(), - } - } - - /// Checks if the node is already registered and adds it to the current nodes if not. If it - /// already exists, [`ModelarDbManagerError`] is returned. - pub fn register_node(&mut self, node: Node) -> Result<()> { - if self - .nodes - .iter() - .any(|n| n.url.to_lowercase() == node.url.to_lowercase()) - { - Err(ModelarDbManagerError::InvalidArgument(format!( - "A node with the url `{}` is already registered.", - node.url - ))) - } else { - // Also add it to the query queue if it is a cloud node. - if node.mode == ServerMode::Cloud { - self.query_queue.push_back(node.clone()); - } - - self.nodes.push(node); - - Ok(()) - } - } - - /// Remove the node with a url matching `url` from the current nodes, flush the node, and - /// finally kill the process running on the node. If no node with `url` exists, - /// [`ModelarDbManagerError`] is returned. - pub async fn remove_node(&mut self, url: &str, key: &MetadataValue) -> Result<()> { - if self - .nodes - .iter() - .any(|n| n.url.to_lowercase() == url.to_lowercase()) - { - self.nodes.retain(|n| n.url != url); - self.query_queue.retain(|n| n.url != url); - - // Flush the node and kill the process running on the node. - let mut flight_client = FlightServiceClient::connect(url.to_owned()).await?; - - let action = Action { - r#type: "KillNode".to_owned(), - body: vec![].into(), - }; - - // Add the key to the request metadata to indicate that the request is from the manager. - let mut request = Request::new(action); - request.metadata_mut().insert("x-manager-key", key.clone()); - - // TODO: Retry the request if the wrong error was returned. - // Since the process is killed, the error from the request is ignored. - let _ = flight_client.do_action(request).await; - - Ok(()) - } else { - Err(ModelarDbManagerError::InvalidArgument(format!( - "A node with the url `{url}` does not exist." - ))) - } - } - - /// Return the cloud node in the cluster that is currently most capable of running a query. If - /// there are no cloud nodes in the cluster, return [`ModelarDbManagerError`]. - pub fn query_node(&mut self) -> Result { - if let Some(query_node) = self.query_queue.pop_front() { - // Add the cloud node back to the queue. - self.query_queue.push_back(query_node.clone()); - - Ok(query_node) - } else { - Err(ModelarDbManagerError::InvalidState( - "There are no cloud nodes to execute the query in the cluster.".to_owned(), - )) - } - } - - /// For each node in the cluster, execute the given `sql` statement with the given `key` as - /// metadata. If the statement was successfully executed for each node, return [`Ok`], otherwise - /// return [`ModelarDbManagerError`]. - pub async fn cluster_do_get(&self, sql: &str, key: &MetadataValue) -> Result<()> { - let mut do_get_futures: FuturesUnordered<_> = self - .nodes - .iter() - .map(|node| self.connect_and_do_get(&node.url, sql, key)) - .collect(); - - // TODO: Fix issue where we return immediately if we encounter an error. If it is a - // connection error, we either need to retry later or remove the node. - // Run the futures concurrently and log when the statement has been executed on each node. - while let Some(result) = do_get_futures.next().await { - info!("Executed statement `{sql}` on node with url '{}'.", result?); - } - - Ok(()) - } - - /// Connect to the Apache Arrow Flight server given by `url` and execute the given `sql` - /// statement with the given `key` as metadata. If the statement was successfully executed, - /// return the url of the node to simplify logging, otherwise return [`ModelarDbManagerError`]. - async fn connect_and_do_get( - &self, - url: &str, - sql: &str, - key: &MetadataValue, - ) -> Result { - let mut flight_client = FlightServiceClient::connect(url.to_owned()).await?; - - // Add the key to the request metadata to indicate that the request is from the manager. - let mut request = Request::new(Ticket::new(sql.to_owned())); - request.metadata_mut().insert("x-manager-key", key.clone()); - - flight_client.do_get(request).await?; - - Ok(url.to_owned()) - } - - /// For each node in the cluster, execute the given `action` with the given `key` as metadata. - /// If the action was successfully executed for each node, return [`Ok`], otherwise return - /// [`ModelarDbManagerError`]. - pub async fn cluster_do_action( - &self, - action: Action, - key: &MetadataValue, - ) -> Result<()> { - let mut action_futures: FuturesUnordered<_> = self - .nodes - .iter() - .map(|node| self.connect_and_do_action(&node.url, action.clone(), key)) - .collect(); - - // Run the futures concurrently and log when the action has been executed on each node. - while let Some(result) = action_futures.next().await { - info!( - "Executed action `{}` on node with url '{}'.", - action.r#type, result? - ); - } - - Ok(()) - } - - /// Connect to the Apache Arrow Flight server given by `url` and make a request to do `action` - /// with the given `key` as metadata. If the action was successfully executed, return the url - /// of the node to simplify logging, otherwise return [`ModelarDbManagerError`]. - async fn connect_and_do_action( - &self, - url: &str, - action: Action, - key: &MetadataValue, - ) -> Result { - let mut flight_client = FlightServiceClient::connect(url.to_owned()).await?; - - // Add the key to the request metadata to indicate that the request is from the manager. - let mut request = Request::new(action); - request.metadata_mut().insert("x-manager-key", key.clone()); - - flight_client.do_action(request).await?; - - Ok(url.to_owned()) - } -} - -impl Default for Cluster { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -mod test { - use super::*; - - use uuid::Uuid; - - // Tests for Cluster. - #[test] - fn test_register_node() { - let node = Node::new("localhost".to_owned(), ServerMode::Edge); - let mut cluster = Cluster::new(); - - assert!(cluster.register_node(node.clone()).is_ok()); - assert!(cluster.nodes.contains(&node)); - assert!(cluster.query_queue.is_empty()); - - let cloud_node = Node::new("cloud".to_owned(), ServerMode::Cloud); - - assert!(cluster.register_node(cloud_node.clone()).is_ok()); - assert!(cluster.nodes.contains(&cloud_node)); - assert!(cluster.query_queue.contains(&cloud_node)); - } - - #[test] - fn test_register_already_registered_node() { - let node = Node::new("localhost".to_owned(), ServerMode::Edge); - let mut cluster = Cluster::new(); - - assert!(cluster.register_node(node.clone()).is_ok()); - - let result = cluster.register_node(node); - - assert_eq!( - result.unwrap_err().to_string(), - "Invalid Argument Error: A node with the url `localhost` is already registered." - ); - } - - #[tokio::test] - async fn test_remove_node_invalid_url() { - let mut cluster = Cluster::new(); - let result = cluster - .remove_node("invalid_url", &Uuid::new_v4().to_string().parse().unwrap()) - .await; - - assert_eq!( - result.unwrap_err().to_string(), - "Invalid Argument Error: A node with the url `invalid_url` does not exist." - ); - } - - #[test] - fn test_query_node_round_robin() { - let cloud_node_1 = Node::new("cloud_1".to_owned(), ServerMode::Cloud); - let cloud_node_2 = Node::new("cloud_2".to_owned(), ServerMode::Cloud); - let mut cluster = Cluster::new(); - - assert!(cluster.register_node(cloud_node_1.clone()).is_ok()); - assert!(cluster.register_node(cloud_node_2.clone()).is_ok()); - - assert_eq!(cluster.query_node().unwrap(), cloud_node_1); - assert_eq!(cluster.query_node().unwrap(), cloud_node_2); - assert_eq!(cluster.query_node().unwrap(), cloud_node_1); - assert_eq!(cluster.query_node().unwrap(), cloud_node_2); - } - - #[test] - fn test_query_node_no_cloud_nodes() { - let node = Node::new("localhost".to_owned(), ServerMode::Edge); - let mut cluster = Cluster::new(); - - assert!(cluster.register_node(node).is_ok()); - - let result = cluster.query_node(); - - assert_eq!( - result.unwrap_err().to_string(), - "Invalid State Error: There are no cloud nodes to execute the query in the cluster." - ); - } -} diff --git a/crates/modelardb_manager/src/error.rs b/crates/modelardb_manager/src/error.rs deleted file mode 100644 index bcbea113..00000000 --- a/crates/modelardb_manager/src/error.rs +++ /dev/null @@ -1,118 +0,0 @@ -/* Copyright 2024 The ModelarDB Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -//! The [`Error`] and [`Result`] types used throughout `modelardb_manager`. - -use std::error::Error; -use std::fmt::{Display, Formatter}; -use std::io::Error as IoError; -use std::result::Result as StdResult; - -use deltalake::errors::DeltaTableError; -use modelardb_storage::error::ModelarDbStorageError; -use modelardb_types::error::ModelarDbTypesError; -use tonic::Status as TonicStatusError; -use tonic::transport::Error as TonicTransportError; - -/// Result type used throughout `modelardb_manager`. -pub type Result = StdResult; - -/// Error type used throughout `modelardb_manager`. -#[derive(Debug)] -pub enum ModelarDbManagerError { - /// Error returned by Delta Lake. - DeltaLake(DeltaTableError), - /// Error returned when an invalid argument was passed. - InvalidArgument(String), - /// Error returned when an invalid state is encountered. - InvalidState(String), - /// Error returned from IO operations. - Io(IoError), - /// Error returned by modelardb_storage. - ModelarDbStorage(ModelarDbStorageError), - /// Error returned by modelardb_types. - ModelarDbTypes(ModelarDbTypesError), - /// Status returned by Tonic. - TonicStatus(Box), - /// Error returned by Tonic. - TonicTransport(TonicTransportError), -} - -impl Display for ModelarDbManagerError { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - match self { - Self::DeltaLake(reason) => write!(f, "Delta Lake Error: {reason}"), - Self::InvalidArgument(reason) => write!(f, "Invalid Argument Error: {reason}"), - Self::InvalidState(reason) => write!(f, "Invalid State Error: {reason}"), - Self::Io(reason) => write!(f, "Io Error: {reason}"), - Self::ModelarDbStorage(reason) => write!(f, "ModelarDB Storage Error: {reason}"), - Self::ModelarDbTypes(reason) => write!(f, "ModelarDB Types Error: {reason}"), - Self::TonicStatus(reason) => write!(f, "Tonic Status Error: {reason}"), - Self::TonicTransport(reason) => write!(f, "Tonic Transport Error: {reason}"), - } - } -} - -impl Error for ModelarDbManagerError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - // Return the error that caused self to occur if one exists. - match self { - Self::DeltaLake(reason) => Some(reason), - Self::InvalidArgument(_reason) => None, - Self::InvalidState(_reason) => None, - Self::Io(reason) => Some(reason), - Self::ModelarDbStorage(reason) => Some(reason), - Self::ModelarDbTypes(reason) => Some(reason), - Self::TonicStatus(reason) => Some(reason), - Self::TonicTransport(reason) => Some(reason), - } - } -} - -impl From for ModelarDbManagerError { - fn from(error: DeltaTableError) -> Self { - Self::DeltaLake(error) - } -} - -impl From for ModelarDbManagerError { - fn from(error: IoError) -> Self { - Self::Io(error) - } -} - -impl From for ModelarDbManagerError { - fn from(error: ModelarDbStorageError) -> Self { - Self::ModelarDbStorage(error) - } -} - -impl From for ModelarDbManagerError { - fn from(error: ModelarDbTypesError) -> Self { - Self::ModelarDbTypes(error) - } -} - -impl From for ModelarDbManagerError { - fn from(error: TonicStatusError) -> Self { - Self::TonicStatus(Box::new(error)) - } -} - -impl From for ModelarDbManagerError { - fn from(error: TonicTransportError) -> Self { - Self::TonicTransport(error) - } -} diff --git a/crates/modelardb_manager/src/main.rs b/crates/modelardb_manager/src/main.rs deleted file mode 100644 index a808228b..00000000 --- a/crates/modelardb_manager/src/main.rs +++ /dev/null @@ -1,82 +0,0 @@ -/* Copyright 2023 The ModelarDB Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -//! Implementation of ModelarDB manager's main function. - -mod cluster; -mod error; -mod remote; - -use std::env; -use std::sync::{Arc, LazyLock}; - -use modelardb_storage::data_folder::DataFolder; -use tokio::sync::RwLock; -use tonic::metadata::errors::InvalidMetadataValue; -use tonic::metadata::{Ascii, MetadataValue}; -use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; -use uuid::Uuid; - -use crate::cluster::Cluster; -use crate::error::{ModelarDbManagerError, Result}; -use crate::remote::start_apache_arrow_flight_server; - -/// The port of the Apache Arrow Flight Server. If the environment variable is not set, 9998 is used. -pub static PORT: LazyLock = - LazyLock::new(|| env::var("MODELARDBM_PORT").map_or(9998, |value| value.parse().unwrap())); - -/// Provides access to the managers components. -pub struct Context { - /// [`DataFolder`] for storing metadata and data in Apache Parquet files. - pub remote_data_folder: DataFolder, - /// Cluster of nodes currently controlled by the manager. - pub cluster: RwLock, - /// Key used to identify requests coming from the manager. - pub key: MetadataValue, -} - -/// Parse the command line arguments to extract the remote object store and start an Apache Arrow -/// Flight server. Returns [`ModelarDbManagerError`] if the command line arguments cannot be parsed, -/// if the metadata cannot be read from the Delta Lake, or if the Apache Arrow Flight server cannot -/// be started. -#[tokio::main] -async fn main() -> Result<()> { - // Initialize a tracing layer that logs events to stdout. - let stdout_log = tracing_subscriber::fmt::layer(); - tracing_subscriber::registry().with(stdout_log).init(); - - let remote_data_folder = DataFolder::open_remote_url("s3://test").await?; - - let cluster = Cluster::new(); - - // Retrieve and parse the key to a tonic metadata value since it is used in tonic requests. - let key = Uuid::new_v4() - .to_string() - .parse() - .map_err(|error: InvalidMetadataValue| { - ModelarDbManagerError::InvalidArgument(error.to_string()) - })?; - - // Create the Context. - let context = Arc::new(Context { - remote_data_folder, - cluster: RwLock::new(cluster), - key, - }); - - start_apache_arrow_flight_server(context, *PORT).await?; - - Ok(()) -} diff --git a/crates/modelardb_manager/src/remote.rs b/crates/modelardb_manager/src/remote.rs deleted file mode 100644 index 2c35d6fc..00000000 --- a/crates/modelardb_manager/src/remote.rs +++ /dev/null @@ -1,621 +0,0 @@ -/* Copyright 2023 The ModelarDB Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -//! Implementation of a request handler for Apache Arrow Flight in the form of [`FlightServiceHandler`]. -//! An Apache Arrow Flight server that process requests using [`FlightServiceHandler`] can be started -//! with [`start_apache_arrow_flight_server()`]. - -use std::error::Error; -use std::net::SocketAddr; -use std::pin::Pin; -use std::result::Result as StdResult; -use std::str; -use std::sync::Arc; - -use arrow::datatypes::Schema; -use arrow::ipc::writer::IpcWriteOptions; -use arrow_flight::flight_service_server::{FlightService, FlightServiceServer}; -use arrow_flight::{ - Action, ActionType, Criteria, Empty, FlightData, FlightDescriptor, FlightEndpoint, FlightInfo, - HandshakeRequest, HandshakeResponse, PollInfo, PutResult, Result as FlightResult, SchemaAsIpc, - SchemaResult, Ticket, -}; -use futures::{Stream, stream}; -use modelardb_storage::parser; -use modelardb_storage::parser::ModelarDbStatement; -use modelardb_types::types::{Table, TimeSeriesTableMetadata}; -use tonic::transport::Server; -use tonic::{Request, Response, Status, Streaming}; -use tracing::info; - -use crate::Context; -use crate::error::{ModelarDbManagerError, Result}; - -/// Start an Apache Arrow Flight server on 0.0.0.0:`port`. -pub async fn start_apache_arrow_flight_server(context: Arc, port: u16) -> Result<()> { - let localhost_with_port = "0.0.0.0:".to_owned() + &port.to_string(); - let localhost_with_port: SocketAddr = localhost_with_port.parse().map_err(|error| { - ModelarDbManagerError::InvalidArgument(format!( - "Unable to parse {localhost_with_port}: {error}" - )) - })?; - let handler = FlightServiceHandler::new(context); - let flight_service_server = FlightServiceServer::new(handler); - - info!("Starting Apache Arrow Flight on {}.", localhost_with_port); - - Server::builder() - .add_service(flight_service_server) - .serve(localhost_with_port) - .await - .map_err(|error| error.into()) -} - -/// Convert an `error` to a [`Status`] with [`tonic::Code::InvalidArgument`] as the code and `error` -/// converted to a [`String`] as the message. -pub fn error_to_status_invalid_argument(error: impl Error) -> Status { - Status::invalid_argument(error.to_string()) -} - -/// Convert an `error` to a [`Status`] with [`tonic::Code::Internal`] as the code and `error` -/// converted to a [`String`] as the message. -pub fn error_to_status_internal(error: impl Error) -> Status { - Status::internal(error.to_string()) -} - -/// Handler for processing Apache Arrow Flight requests. -/// [`FlightServiceHandler`] is based on the [Apache Arrow Flight examples] -/// published under Apache2. -/// -/// [Apache Arrow Flight examples]: https://github.com/apache/arrow-rs/blob/master/arrow-flight/examples -struct FlightServiceHandler { - /// Singleton that provides access to the system's components. - context: Arc, -} - -impl FlightServiceHandler { - pub fn new(context: Arc) -> FlightServiceHandler { - Self { context } - } - - /// Return the schema of the table with the name `table_name`. If the table does not exist or - /// the schema cannot be retrieved, return [`Status`]. - async fn table_schema(&self, table_name: &str) -> StdResult, Status> { - let data_folder = &self.context.remote_data_folder; - - if data_folder - .is_normal_table(table_name) - .await - .map_err(error_to_status_internal)? - { - let delta_table = self - .context - .remote_data_folder - .delta_table(table_name) - .await - .map_err(error_to_status_internal)?; - - let schema = delta_table - .snapshot() - .map_err(error_to_status_internal)? - .snapshot() - .arrow_schema(); - - Ok(schema) - } else if data_folder - .is_time_series_table(table_name) - .await - .map_err(error_to_status_internal)? - { - let time_series_table_metadata = data_folder - .time_series_table_metadata_for_time_series_table(table_name) - .await - .map_err(error_to_status_internal)?; - - Ok(time_series_table_metadata.query_schema) - } else { - Err(Status::invalid_argument(format!( - "Table with name '{table_name}' does not exist.", - ))) - } - } - - /// Return [`Ok`] if a table named `table_name` does not exist already in the metadata - /// database, otherwise return [`Status`]. - async fn check_if_table_exists(&self, table_name: &str) -> StdResult<(), Status> { - let existing_tables = self - .context - .remote_data_folder - .table_names() - .await - .map_err(error_to_status_internal)?; - - if existing_tables - .iter() - .any(|existing_table| existing_table == table_name) - { - let message = format!("Table with name '{table_name}' already exists."); - Err(Status::already_exists(message)) - } else { - Ok(()) - } - } - - /// Create a normal table, save it to the Delta Lake and create it for each node controlled by - /// the manager. If the normal table cannot be saved to the Delta Lake or created for each node, - /// return [`Status`]. - async fn save_and_create_cluster_normal_table( - &self, - table_name: &str, - schema: &Schema, - ) -> StdResult<(), Status> { - // Create an empty Delta Lake table. - self.context - .remote_data_folder - .create_normal_table(table_name, schema) - .await - .map_err(error_to_status_internal)?; - - // Persist the new normal table to the Delta Lake. - self.context - .remote_data_folder - .save_normal_table_metadata(table_name) - .await - .map_err(error_to_status_internal)?; - - // Register and save the table to each node in the cluster. - let protobuf_bytes = - modelardb_types::flight::encode_and_serialize_normal_table_metadata(table_name, schema) - .map_err(error_to_status_internal)?; - - let action = Action { - r#type: "CreateTable".to_owned(), - body: protobuf_bytes.into(), - }; - - self.context - .cluster - .read() - .await - .cluster_do_action(action, &self.context.key) - .await - .map_err(error_to_status_internal)?; - - info!("Created normal table '{}'.", table_name); - - Ok(()) - } - - /// Create a time series table, save it to the Delta Lake and create it for each node controlled - /// by the manager. If the time series table cannot be saved to the Delta Lake or created for - /// each node, return [`Status`]. - async fn save_and_create_cluster_time_series_table( - &self, - time_series_table_metadata: Arc, - ) -> StdResult<(), Status> { - // Create an empty Delta Lake table. - self.context - .remote_data_folder - .create_time_series_table(&time_series_table_metadata) - .await - .map_err(error_to_status_internal)?; - - // Persist the new time series table to the Delta Lake. - self.context - .remote_data_folder - .save_time_series_table_metadata(&time_series_table_metadata) - .await - .map_err(error_to_status_internal)?; - - // Register and save the time series table to each node in the cluster. - let protobuf_bytes = - modelardb_types::flight::encode_and_serialize_time_series_table_metadata( - &time_series_table_metadata, - ) - .map_err(error_to_status_internal)?; - - let action = Action { - r#type: "CreateTable".to_owned(), - body: protobuf_bytes.into(), - }; - - self.context - .cluster - .read() - .await - .cluster_do_action(action, &self.context.key) - .await - .map_err(error_to_status_internal)?; - - info!( - "Created time series table '{}'.", - time_series_table_metadata.name - ); - - Ok(()) - } - - /// Drop the table from the Delta Lake, the Delta Lake, and from each node controlled by the - /// manager. If the table does not exist or the table cannot be dropped from the remote data - /// folder and from each node, return [`Status`]. - async fn drop_cluster_table(&self, table_name: &str) -> StdResult<(), Status> { - // Drop the table from the remote data folder Delta Lake. This will return an error if the - // table does not exist. - self.context - .remote_data_folder - .drop_table_metadata(table_name) - .await - .map_err(error_to_status_internal)?; - - // Drop the table from the remote data folder Delta Lake. - self.context - .remote_data_folder - .drop_table(table_name) - .await - .map_err(error_to_status_internal)?; - - // Drop the table from the nodes controlled by the manager. - self.context - .cluster - .read() - .await - .cluster_do_get(&format!("DROP TABLE {table_name}"), &self.context.key) - .await - .map_err(error_to_status_internal)?; - - Ok(()) - } - - /// Truncate the table in the remote data folder and at each node controlled by the manager. If - /// the table does not exist or the table cannot be truncated in the remote data folder and at - /// each node, return [`Status`]. - async fn truncate_cluster_table(&self, table_name: &str) -> StdResult<(), Status> { - if self.check_if_table_exists(table_name).await.is_ok() { - return Err(Status::invalid_argument(format!( - "Table with name '{table_name}' does not exist.", - ))); - } - - // Truncate the table in the remote data folder Delta Lake. - self.context - .remote_data_folder - .truncate_table(table_name) - .await - .map_err(error_to_status_internal)?; - - // Truncate the table in the nodes controlled by the manager. - self.context - .cluster - .read() - .await - .cluster_do_get(&format!("TRUNCATE {table_name}"), &self.context.key) - .await - .map_err(error_to_status_internal)?; - - Ok(()) - } - - /// Vacuum the table in the remote data folder and at each node controlled by the manager. If - /// the table does not exist or the table cannot be vacuumed in the remote data folder - /// and at each node, return [`Status`]. - async fn vacuum_cluster_table( - &self, - table_name: &str, - maybe_retention_period_in_seconds: Option, - ) -> StdResult<(), Status> { - // Vacuum the table in the remote data folder Delta Lake. - self.context - .remote_data_folder - .vacuum_table(table_name, maybe_retention_period_in_seconds) - .await - .map_err(error_to_status_internal)?; - - // Vacuum the table in the nodes controlled by the manager. - let vacuum_sql = if let Some(retention_period) = maybe_retention_period_in_seconds { - format!("VACUUM {table_name} RETAIN {retention_period}") - } else { - format!("VACUUM {table_name}") - }; - - self.context - .cluster - .read() - .await - .cluster_do_get(&vacuum_sql, &self.context.key) - .await - .map_err(error_to_status_internal)?; - - Ok(()) - } -} - -#[tonic::async_trait] -impl FlightService for FlightServiceHandler { - type HandshakeStream = - Pin> + Send + Sync + 'static>>; - type ListFlightsStream = - Pin> + Send + Sync + 'static>>; - type DoGetStream = - Pin> + Send + Sync + 'static>>; - type DoPutStream = - Pin> + Send + Sync + 'static>>; - type DoExchangeStream = - Pin> + Send + Sync + 'static>>; - type DoActionStream = Pin< - Box> + Send + Sync + 'static>, - >; - type ListActionsStream = - Pin> + Send + Sync + 'static>>; - - /// Not implemented. - async fn handshake( - &self, - _request: Request>, - ) -> StdResult, Status> { - Err(Status::unimplemented("Not implemented.")) - } - - /// Provide the name of all tables in the catalog. - async fn list_flights( - &self, - _request: Request, - ) -> StdResult, Status> { - // Retrieve the table names from the Delta Lake. - let table_names = self - .context - .remote_data_folder - .table_names() - .await - .map_err(error_to_status_internal)?; - - let flight_descriptor = FlightDescriptor::new_path(table_names); - let flight_info = FlightInfo::new().with_descriptor(flight_descriptor); - - let output = stream::once(async { Ok(flight_info) }); - Ok(Response::new(Box::pin(output))) - } - - /// Given a query, return [`FlightInfo`] containing [`FlightEndpoints`](FlightEndpoint) - /// describing which cloud nodes should be used to execute the query. The query must be - /// provided in `FlightDescriptor.cmd`. - async fn get_flight_info( - &self, - request: Request, - ) -> StdResult, Status> { - let flight_descriptor = request.into_inner(); - - // Extract the query. - let query = str::from_utf8(&flight_descriptor.cmd) - .map_err(error_to_status_invalid_argument)? - .to_owned(); - - // Retrieve the cloud node that should execute the given query. - let mut cluster = self.context.cluster.write().await; - let cloud_node = cluster - .query_node() - .map_err(|error| Status::failed_precondition(error.to_string()))?; - - info!( - "Assigning query '{query}' to cloud node with url '{}'.", - cloud_node.url - ); - - // All data in the query result should be retrieved using a single endpoint. - let endpoint = FlightEndpoint::new() - .with_ticket(Ticket::new(query)) - .with_location(cloud_node.url); - - // schema is empty and total_records and total_bytes are -1 since we do not know anything - // about the result of the query at this point. - let flight_info = FlightInfo::new() - .with_descriptor(flight_descriptor) - .with_endpoint(endpoint) - .with_ordered(true); - - Ok(Response::new(flight_info)) - } - - /// Not implemented. - async fn poll_flight_info( - &self, - _request: Request, - ) -> StdResult, Status> { - Err(Status::unimplemented("Not implemented.")) - } - - /// Provide the schema of a table in the catalog. The name of the table must be provided as the - /// first element in `FlightDescriptor.path`. - async fn get_schema( - &self, - request: Request, - ) -> StdResult, Status> { - let flight_descriptor = request.into_inner(); - let table_name = flight_descriptor - .path - .first() - .ok_or_else(|| Status::invalid_argument("No table name in FlightDescriptor.path."))?; - - let schema = self.table_schema(table_name).await?; - - let options = IpcWriteOptions::default(); - let schema_as_ipc = SchemaAsIpc::new(&schema, &options); - let schema_result = schema_as_ipc.try_into().map_err(error_to_status_internal)?; - - Ok(Response::new(schema_result)) - } - - /// Execute a SQL statement provided in UTF-8 and return the schema of the result followed by - /// the result itself. Currently, CREATE TABLE, CREATE TIME SERIES TABLE, TRUNCATE, - /// DROP TABLE, and VACUUM are supported. - async fn do_get( - &self, - request: Request, - ) -> StdResult, Status> { - let ticket = request.into_inner(); - - // Extract the query. - let sql = str::from_utf8(&ticket.ticket) - .map_err(error_to_status_invalid_argument)? - .to_owned(); - - // Parse the query. - info!("Received SQL: '{}'.", sql); - - let modelardb_statement = parser::tokenize_and_parse_sql_statement(&sql) - .map_err(error_to_status_invalid_argument)?; - - // Execute the statement. - info!("Executing SQL: '{}'.", sql); - - match modelardb_statement { - ModelarDbStatement::CreateNormalTable { name, schema } => { - self.check_if_table_exists(&name).await?; - self.save_and_create_cluster_normal_table(&name, &schema) - .await?; - } - ModelarDbStatement::CreateTimeSeriesTable(time_series_table_metadata) => { - self.check_if_table_exists(&time_series_table_metadata.name) - .await?; - self.save_and_create_cluster_time_series_table(time_series_table_metadata) - .await?; - } - ModelarDbStatement::TruncateTable(table_names, ..) => { - for table_name in table_names { - self.truncate_cluster_table(&table_name).await?; - } - } - ModelarDbStatement::DropTable(table_names) => { - for table_name in table_names { - self.drop_cluster_table(&table_name).await?; - } - } - ModelarDbStatement::Vacuum(mut table_names, maybe_retention_period_in_seconds, ..) => { - // Vacuum all tables if no table names are provided. - if table_names.is_empty() { - table_names = self - .context - .remote_data_folder - .table_names() - .await - .map_err(error_to_status_internal)?; - } - - for table_name in table_names { - self.vacuum_cluster_table(&table_name, maybe_retention_period_in_seconds) - .await?; - } - } - // .. is not used so a compile error is raised if a new ModelarDbStatement is added. - ModelarDbStatement::Statement(_) | ModelarDbStatement::IncludeSelect(..) => { - return Err(Status::invalid_argument( - "Expected CREATE TABLE, CREATE TIME SERIES TABLE, TRUNCATE, or DROP TABLE.", - )); - } - }; - - // Confirm the SQL statement was executed by returning a stream with a schema but no data. - // stream::empty() cannot be used since do_get requires a schema in the response. - let options = IpcWriteOptions::default(); - let schema_as_flight_data = SchemaAsIpc::new(&Schema::empty(), &options).into(); - let output = stream::once(async { Ok(schema_as_flight_data) }); - - Ok(Response::new(Box::pin(output))) - } - - /// Not implemented. - async fn do_put( - &self, - _request: Request>, - ) -> StdResult, Status> { - Err(Status::unimplemented("Not implemented.")) - } - - /// Not implemented. - async fn do_exchange( - &self, - _request: Request>, - ) -> StdResult, Status> { - Err(Status::unimplemented("Not implemented.")) - } - - /// Perform a specific action based on the type of the action in `request`. Currently, the - /// following actions are supported: - /// * `CreateTable`: Create the table given in the [`TableMetadata`](protocol::TableMetadata) - /// protobuf message in the action body. The table is created for each node in the cluster of - /// nodes controlled by the manager. - /// * `NodeType`: Get the type of the node. The type is always `manager`. The type of the node - /// is returned as a string. - async fn do_action( - &self, - request: Request, - ) -> StdResult, Status> { - let action = request.into_inner(); - info!("Received request to perform action '{}'.", action.r#type); - - if action.r#type == "CreateTable" { - // Deserialize and extract the table metadata from the protobuf message in the action body. - let table_metadata = - modelardb_types::flight::deserialize_and_extract_table_metadata(&action.body) - .map_err(error_to_status_invalid_argument)?; - - match table_metadata { - Table::NormalTable(table_name, schema) => { - self.check_if_table_exists(&table_name).await?; - self.save_and_create_cluster_normal_table(&table_name, &schema) - .await?; - } - Table::TimeSeriesTable(metadata) => { - self.check_if_table_exists(&metadata.name).await?; - self.save_and_create_cluster_time_series_table(Arc::new(metadata)) - .await?; - } - } - - // Confirm the tables were created. - Ok(Response::new(Box::pin(stream::empty()))) - } else if action.r#type == "NodeType" { - let flight_result = FlightResult { - body: "manager".bytes().collect(), - }; - - Ok(Response::new(Box::pin(stream::once(async { - Ok(flight_result) - })))) - } else { - Err(Status::unimplemented("Action not implemented.")) - } - } - - /// Return all available actions, including both a name and a description for each action. - async fn list_actions( - &self, - _request: Request, - ) -> StdResult, Status> { - let create_tables_action = ActionType { - r#type: "CreateTable".to_owned(), - description: "Create the table given in the protobuf message in the action body." - .to_owned(), - }; - - let node_type_action = ActionType { - r#type: "NodeType".to_owned(), - description: "Get the type of the node.".to_owned(), - }; - - let output = stream::iter(vec![Ok(create_tables_action), Ok(node_type_action)]); - - Ok(Response::new(Box::pin(output))) - } -} From 3e8b56ffc7b91e781d3b85de6d1e4849b74b165d Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Thu, 20 Nov 2025 15:44:48 +0100 Subject: [PATCH 88/99] Remove unused dependency and build step --- .github/workflows/build-lint-test-and-upload.yml | 1 - Cargo.lock | 1 - crates/modelardb_server/Cargo.toml | 1 - crates/modelardb_server/src/context.rs | 4 ++-- crates/modelardb_types/src/types.rs | 3 +-- 5 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-lint-test-and-upload.yml b/.github/workflows/build-lint-test-and-upload.yml index c966c87c..caa62996 100644 --- a/.github/workflows/build-lint-test-and-upload.yml +++ b/.github/workflows/build-lint-test-and-upload.yml @@ -186,7 +186,6 @@ jobs: mv target/release/modelardb$SUFFIX modelardb_$RUST_TRIPLE/modelardb$SUFFIX mv target/release/modelardbb$SUFFIX modelardb_$RUST_TRIPLE/modelardbb$SUFFIX mv target/release/modelardbd$SUFFIX modelardb_$RUST_TRIPLE/modelardbd$SUFFIX - mv target/release/modelardbm$SUFFIX modelardb_$RUST_TRIPLE/modelardbm$SUFFIX # Upload Binaries. - name: Artifact Upload diff --git a/Cargo.lock b/Cargo.lock index 32074c80..3c024828 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3326,7 +3326,6 @@ dependencies = [ "tonic", "tracing", "tracing-subscriber", - "uuid", ] [[package]] diff --git a/crates/modelardb_server/Cargo.toml b/crates/modelardb_server/Cargo.toml index a56f44d6..1afc86cf 100644 --- a/crates/modelardb_server/Cargo.toml +++ b/crates/modelardb_server/Cargo.toml @@ -43,7 +43,6 @@ snmalloc-rs = { workspace = true, features = ["build_cc"] } tokio = { workspace = true, features = ["rt-multi-thread", "signal"] } tokio-stream.workspace = true tonic.workspace = true -uuid.workspace = true # Log is a dependency so the compile time filters for log and tracing can be set to the same values. log = { workspace = true, features = ["max_level_debug", "release_max_level_info"] } diff --git a/crates/modelardb_server/src/context.rs b/crates/modelardb_server/src/context.rs index 3c7bc54a..c7f004e4 100644 --- a/crates/modelardb_server/src/context.rs +++ b/crates/modelardb_server/src/context.rs @@ -41,8 +41,8 @@ pub struct Context { } impl Context { - /// Create the components needed in the [`Context`] and use them to create the [`Context`]. If a - /// metadata manager or storage engine could not be created, [`ModelarDbServerError`] is + /// Create the components needed in the [`Context`] and use them to create the [`Context`]. If the + /// configuration manager or storage engine could not be created, [`ModelarDbServerError`] is /// returned. pub async fn try_new(data_folders: DataFolders, cluster_mode: ClusterMode) -> Result { let configuration_manager = Arc::new(RwLock::new(ConfigurationManager::new(cluster_mode))); diff --git a/crates/modelardb_types/src/types.rs b/crates/modelardb_types/src/types.rs index ad434e4d..bc22c424 100644 --- a/crates/modelardb_types/src/types.rs +++ b/crates/modelardb_types/src/types.rs @@ -357,8 +357,7 @@ impl GeneratedColumn { } } -/// A single ModelarDB server that is controlled by the manager. The node can either be an edge node -/// or a cloud node. A node cannot be another manager. +/// A single ModelarDB server. The node can either be an edge node or a cloud node. #[derive(Debug, Clone, PartialEq)] pub struct Node { /// Apache Arrow Flight URL for the node. This URL uniquely identifies the node. From ef7506fd21d01a9576ff284221dd2c4748f5d2a8 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 26 Nov 2025 09:26:18 +0100 Subject: [PATCH 89/99] Refactor C-API to use int pointer for modelardb_type function --- crates/modelardb_embedded/src/capi.rs | 33 ++++++++------------------- 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/crates/modelardb_embedded/src/capi.rs b/crates/modelardb_embedded/src/capi.rs index 7a124fc5..b9e840b0 100644 --- a/crates/modelardb_embedded/src/capi.rs +++ b/crates/modelardb_embedded/src/capi.rs @@ -257,26 +257,17 @@ pub unsafe extern "C" fn modelardb_embedded_close( } } -/// Returns the ModelarDB type of the [`DataFolder`] or [`Client`] in `maybe_operations_ptr`. Assumes -/// `maybe_operations_ptr` points to a [`DataFolder`] or [`Client`]; `modelardb_type_array_ptr` is -/// a valid pointer to enough memory for an Apache Arrow C Data Interface Array; and -/// `modelardb_type_array_schema_ptr` is a valid pointer to enough memory for an Apache Arrow C -/// Data Interface Schema. Note that only a single value is written to the C Data Interface Array. +/// Returns the ModelarDB type of the [`DataFolder`] or [`Client`] in `maybe_operations_ptr`. +/// Assumes `maybe_operations_ptr` points to a [`DataFolder`] or [`Client`] and `modelardb_type_ptr` +/// is a valid pointer to enough memory for a 32-bit signed integer. #[unsafe(no_mangle)] pub unsafe extern "C" fn modelardb_embedded_modelardb_type( maybe_operations_ptr: *mut c_void, is_data_folder: bool, - modelardb_type_array_ptr: *mut FFI_ArrowArray, - modelardb_type_array_schema_ptr: *mut FFI_ArrowSchema, + modelardb_type_ptr: *mut c_int, ) -> c_int { - let maybe_unit = unsafe { - modelardb_type( - maybe_operations_ptr, - is_data_folder, - modelardb_type_array_ptr, - modelardb_type_array_schema_ptr, - ) - }; + let maybe_unit = + unsafe { modelardb_type(maybe_operations_ptr, is_data_folder, modelardb_type_ptr) }; set_error_and_return_code(maybe_unit) } @@ -285,20 +276,14 @@ pub unsafe extern "C" fn modelardb_embedded_modelardb_type( unsafe fn modelardb_type( maybe_operations_ptr: *mut c_void, is_data_folder: bool, - modelardb_type_array_ptr: *mut FFI_ArrowArray, - modelardb_type_array_schema_ptr: *mut FFI_ArrowSchema, + modelardb_type_ptr: *mut c_int, ) -> Result<()> { let modelardb = unsafe { c_void_to_operations(maybe_operations_ptr, is_data_folder)? }; let modelardb_type = TOKIO_RUNTIME.block_on(modelardb.modelardb_type())?; - let modelardb_type_str = format!("{:?}", modelardb_type); - - let modelardb_type_array = StringArray::from(vec![modelardb_type_str]); - let modelardb_type_array_data = modelardb_type_array.into_data(); - let (out_array, out_schema) = ffi::to_ffi(&modelardb_type_array_data)?; + let modelardb_type_int = modelardb_type as i32; - unsafe { modelardb_type_array_ptr.write(out_array) }; - unsafe { modelardb_type_array_schema_ptr.write(out_schema) }; + unsafe { modelardb_type_ptr.write(modelardb_type_int) }; Ok(()) } From b399dde01ed661e2e846ebe3168cbc0bf9b76e1f Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 26 Nov 2025 09:29:22 +0100 Subject: [PATCH 90/99] Refactor the Python API to match the use of int pointer in modelardb_type C-API --- .../bindings/python/modelardb/operations.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/crates/modelardb_embedded/bindings/python/modelardb/operations.py b/crates/modelardb_embedded/bindings/python/modelardb/operations.py index a05e67c9..424d0f22 100644 --- a/crates/modelardb_embedded/bindings/python/modelardb/operations.py +++ b/crates/modelardb_embedded/bindings/python/modelardb/operations.py @@ -134,8 +134,7 @@ def __find_library(build: str) -> str: int modelardb_embedded_modelardb_type(void* maybe_operations_ptr, bool is_data_folder, - struct ArrowArray* modelardb_type_array_ptr, - struct ArrowSchema* modelardb_type_array_schema_ptr); + int* modelardb_type_ptr); int modelardb_embedded_create(void* maybe_operations_ptr, bool is_data_folder, @@ -364,18 +363,16 @@ def modelardb_type(self) -> ModelarDBType: :return: The type of the ModelarDB instance. :rtype: ModelarDBType """ - modelardb_type_ffi = FFIArray.from_type(StringArray) + modelardb_type_int_ffi = ffi.new("int*") return_code = self.__library.modelardb_embedded_modelardb_type( self.__operations_ptr, self.__is_data_folder, - modelardb_type_ffi.array_ptr, - modelardb_type_ffi.schema_ptr, + modelardb_type_int_ffi ) self.__check_return_code_and_raise_error(return_code) - modelardb_type_str = modelardb_type_ffi.array().to_pylist()[0] - return ModelarDBType[modelardb_type_str] + return ModelarDBType(modelardb_type_int_ffi[0]) def create(self, table_name: str, table_type: NormalTable | TimeSeriesTable): """Creates a table with `table_name`, `schema`, and `error_bounds`. From 6ce3153bac93582086ec6a80e881ca860109368c Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Thu, 27 Nov 2025 20:26:09 +0100 Subject: [PATCH 91/99] Use release build instead of debug build in Docker cluster setup --- Dockerfile | 4 ++-- docker-compose-cluster.yml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index b22e34a0..2ff8b5d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ FROM rust:latest -ARG BUILD_PROFILE=dev +ARG BUILD_PROFILE=release WORKDIR /usr/src/app @@ -24,4 +24,4 @@ RUN apt-get update && apt-get install -y protobuf-compiler RUN cargo build --profile $BUILD_PROFILE -CMD ["target/debug/modelardbd", "edge", "data"] +CMD ["target/release/modelardbd", "edge", "data"] diff --git a/docker-compose-cluster.yml b/docker-compose-cluster.yml index 175557e5..d3206fa6 100644 --- a/docker-compose-cluster.yml +++ b/docker-compose-cluster.yml @@ -50,7 +50,7 @@ services: build: . image: modelardb container_name: modelardb-edge-1 - command: [ "target/debug/modelardbd", "edge", "data/edge", "s3://modelardb" ] + command: [ "target/release/modelardbd", "edge", "data/edge", "s3://modelardb" ] ports: - "9999:9999" environment: @@ -67,7 +67,7 @@ services: modelardb-edge-2: image: modelardb container_name: modelardb-edge-2 - command: [ "target/debug/modelardbd", "edge", "data/edge", "s3://modelardb" ] + command: [ "target/release/modelardbd", "edge", "data/edge", "s3://modelardb" ] ports: - "9998:9998" environment: @@ -84,7 +84,7 @@ services: modelardb-cloud-1: image: modelardb container_name: modelardb-cloud-1 - command: [ "target/debug/modelardbd", "cloud", "data/cloud", "s3://modelardb" ] + command: [ "target/release/modelardbd", "cloud", "data/cloud", "s3://modelardb" ] ports: - "9997:9997" environment: @@ -101,7 +101,7 @@ services: modelardb-cloud-2: image: modelardb container_name: modelardb-cloud-2 - command: [ "target/debug/modelardbd", "cloud", "data/cloud", "s3://modelardb" ] + command: [ "target/release/modelardbd", "cloud", "data/cloud", "s3://modelardb" ] ports: - "9996:9996" environment: From 592455b87e977dcb87440200928a2cfd4581fa22 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Fri, 28 Nov 2025 09:18:01 +0100 Subject: [PATCH 92/99] Make it possible to drop multiple cluster tables at the same time --- crates/modelardb_server/src/cluster.rs | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 71397052..fa350de1 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -197,19 +197,21 @@ impl Cluster { Ok(()) } - /// Drop the table with the given `table_name` in the remote data folder and in each peer - /// node. If the table could not be dropped, return [`ModelarDbServerError`]. - pub(crate) async fn drop_cluster_table(&self, table_name: &str) -> Result<()> { - // Drop the table from the remote data folder. - self.remote_data_folder - .drop_table_metadata(table_name) - .await?; + /// Drop the tables in `table_names` in the remote data folder and in each peer node. If the + /// tables could not be dropped, return [`ModelarDbServerError`]. + pub(crate) async fn drop_cluster_tables(&self, table_names: &[String]) -> Result<()> { + // Drop the tables from the remote data folder. + for table_name in table_names { + self.remote_data_folder + .drop_table_metadata(table_name) + .await?; - self.remote_data_folder.drop_table(table_name).await?; + self.remote_data_folder.drop_table(table_name).await?; + } - // Drop the table from each peer node. - self.cluster_do_get(&format!("DROP TABLE {table_name}")) - .await?; + // Drop the tables from each peer node. + let drop_sql = format!("DROP TABLE {}", table_names.join(", ")); + self.cluster_do_get(&drop_sql).await?; Ok(()) } From b7812e9b9f3847a39431db6f50b9b12652ee5bd1 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Fri, 28 Nov 2025 09:20:17 +0100 Subject: [PATCH 93/99] Refactor handling of drop table in server remote to match new cluster method --- crates/modelardb_server/src/remote.rs | 28 +++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/crates/modelardb_server/src/remote.rs b/crates/modelardb_server/src/remote.rs index bc191a1e..cdffbe96 100644 --- a/crates/modelardb_server/src/remote.rs +++ b/crates/modelardb_server/src/remote.rs @@ -368,31 +368,33 @@ impl FlightServiceHandler { Ok(()) } - /// Drop the table with the given `table_name`. If the node is running in a cluster, the table - /// is dropped in the remote data folder and locally in each node in the cluster. If not, the - /// table is only dropped locally. - async fn drop_table( + /// Drop the tables in `table_names`. If the node is running in a cluster, the tables are + /// dropped in the remote data folder and locally in each node in the cluster. If not, the + /// tables are only dropped locally. + async fn drop_tables( &self, - table_name: &str, + table_names: &[String], request_metadata: &MetadataMap, ) -> StdResult<(), Status> { let configuration_manager = self.context.configuration_manager.read().await; // If the cluster key is in the request, the request is from a peer node, which means the - // table has already been dropped in the remote data folder and propagated to all nodes. + // tables have already been dropped in the remote data folder and propagated to all nodes. if let ClusterMode::MultiNode(cluster) = &configuration_manager.cluster_mode && !cluster_key_in_request(cluster, request_metadata)? { cluster - .drop_cluster_table(table_name) + .drop_cluster_tables(table_names) .await .map_err(error_to_status_invalid_argument)?; } - self.context - .drop_table(table_name) - .await - .map_err(error_to_status_invalid_argument)?; + for table_name in table_names { + self.context + .drop_table(table_name) + .await + .map_err(error_to_status_invalid_argument)?; + } Ok(()) } @@ -691,9 +693,7 @@ impl FlightService for FlightServiceHandler { Ok(empty_record_batch_stream()) } ModelarDbStatement::DropTable(table_names) => { - for table_name in table_names { - self.drop_table(&table_name, request.metadata()).await?; - } + self.drop_tables(&table_names, request.metadata()).await?; Ok(empty_record_batch_stream()) } From 9ad7db17ed55f25f7b0df364ebefe709eb65942d Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Fri, 28 Nov 2025 09:51:20 +0100 Subject: [PATCH 94/99] Update Cluster documentation to only mention peer nodes --- crates/modelardb_server/src/cluster.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index fa350de1..df0d35a5 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -36,8 +36,8 @@ use tonic::metadata::{Ascii, MetadataValue}; use crate::context::Context; use crate::error::{ModelarDbServerError, Result}; -/// Stores the currently managed nodes in the cluster and allows for performing operations that need -/// to be applied to every single node in the cluster. +/// Stores the node that represents the local system and allows for performing operations that need +/// to be applied to every peer node in the cluster. #[derive(Clone)] pub(crate) struct Cluster { /// Node that represents the local system running `modelardbd`. @@ -263,7 +263,7 @@ impl Cluster { /// For each peer node in the cluster, execute the given `sql` statement with the cluster key /// as metadata. If the statement was successfully executed for each node, return [`Ok`], /// otherwise return [`ModelarDbServerError`]. - pub(crate) async fn cluster_do_get(&self, sql: &str) -> Result<()> { + async fn cluster_do_get(&self, sql: &str) -> Result<()> { let nodes = self.peer_nodes().await?; let mut do_get_futures: FuturesUnordered<_> = nodes @@ -301,7 +301,7 @@ impl Cluster { /// For each peer node in the cluster, execute the given `action` with the cluster key as /// metadata. If the action was successfully executed for each node, return [`Ok`], otherwise /// return [`ModelarDbServerError`]. - pub(crate) async fn cluster_do_action(&self, action: Action) -> Result<()> { + async fn cluster_do_action(&self, action: Action) -> Result<()> { let nodes = self.peer_nodes().await?; let mut action_futures: FuturesUnordered<_> = nodes From f3746e50e33780bfe19d58d5d93f6f82234b6194 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Fri, 28 Nov 2025 11:12:11 +0100 Subject: [PATCH 95/99] Remove deleted file from toctree --- crates/modelardb_embedded/bindings/python/docs/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/modelardb_embedded/bindings/python/docs/index.rst b/crates/modelardb_embedded/bindings/python/docs/index.rst index 603ab72d..9add5111 100644 --- a/crates/modelardb_embedded/bindings/python/docs/index.rst +++ b/crates/modelardb_embedded/bindings/python/docs/index.rst @@ -26,7 +26,6 @@ API api/modelardb api/error_bound api/ffi_array - api/node api/operations api/table From aacbf5214b5154fbb616434bdf3937ae48a2475b Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sun, 30 Nov 2025 09:04:29 +0100 Subject: [PATCH 96/99] Update based on comments from @chrthomsen --- crates/modelardb_storage/src/data_folder/cluster.rs | 4 ++-- docs/user/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/modelardb_storage/src/data_folder/cluster.rs b/crates/modelardb_storage/src/data_folder/cluster.rs index c4dd5eb5..808447ef 100644 --- a/crates/modelardb_storage/src/data_folder/cluster.rs +++ b/crates/modelardb_storage/src/data_folder/cluster.rs @@ -1,4 +1,4 @@ -/* Copyright 2025 The ModelarDB Contributors +/* Copyright 2023 The ModelarDB Contributors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ */ //! Management of the Delta Lake for the cluster. Metadata which is unique to the cluster, such as -//! the key and the nodes, are handled here. +//! the key and the nodes, is handled here. use std::str::FromStr; use std::sync::Arc; diff --git a/docs/user/README.md b/docs/user/README.md index 4ce57977..aff79437 100644 --- a/docs/user/README.md +++ b/docs/user/README.md @@ -55,7 +55,7 @@ together with other instances of `modelardbd` in a cluster architecture, dependi `modelardbd` can be deployed on a single node to manage data in a local folder. `modelardbd` can also be deployed in a distributed configuration across edge and cloud. In this configuration, the shared remote object store is responsible for keeping the database schema consistent across all `modelardbd` instances in the cluster. While the `modelardbd` -instances on the edge and in cloud provides the same functionality, the primary purpose of the instances on the edge is +instances on the edge and in cloud provide the same functionality, the primary purpose of the instances on the edge is to collect data and transfer it to an object store in the cloud while the primary purpose of the instances in the cloud is to execute queries on the data in the object store. From 0c76f276aecd11c7a3d50be6ea77b67b47b2080a Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Thu, 11 Dec 2025 13:57:30 +0100 Subject: [PATCH 97/99] Fix field access after merge --- crates/modelardb_server/src/cluster.rs | 2 +- crates/modelardb_server/src/configuration.rs | 2 +- crates/modelardb_server/src/main.rs | 2 +- crates/modelardb_server/src/remote.rs | 16 ++++++++-------- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index df0d35a5..94bde22f 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -702,7 +702,7 @@ mod test { /// [`ClusterMode::MultiNode`] and panic if not. async fn retrieve_and_create_tables(context: &Arc) -> Result<()> { if let ClusterMode::MultiNode(cluster) = - &context.configuration_manager.read().await.cluster_mode + context.configuration_manager.read().await.cluster_mode() { cluster.retrieve_and_create_tables(context).await } else { diff --git a/crates/modelardb_server/src/configuration.rs b/crates/modelardb_server/src/configuration.rs index 801bc3b6..16a6b29c 100644 --- a/crates/modelardb_server/src/configuration.rs +++ b/crates/modelardb_server/src/configuration.rs @@ -680,7 +680,7 @@ mod tests { let data_folders = DataFolders::new( local_data_folder.clone(), - Some(remote_data_folder), + Some(remote_data_folder.clone()), local_data_folder.clone(), ); diff --git a/crates/modelardb_server/src/main.rs b/crates/modelardb_server/src/main.rs index 9ffe41be..212a7913 100644 --- a/crates/modelardb_server/src/main.rs +++ b/crates/modelardb_server/src/main.rs @@ -139,7 +139,7 @@ fn setup_ctrl_c_handler(context: &Arc) { // If running in a cluster, remove the node from the remote data folder. let configuration_manager = ctrl_c_context.configuration_manager.read().await; - if let ClusterMode::MultiNode(cluster) = &configuration_manager.cluster_mode { + if let ClusterMode::MultiNode(cluster) = configuration_manager.cluster_mode() { cluster.remove_node().await.unwrap(); } diff --git a/crates/modelardb_server/src/remote.rs b/crates/modelardb_server/src/remote.rs index 5cbd7e73..0da9891b 100644 --- a/crates/modelardb_server/src/remote.rs +++ b/crates/modelardb_server/src/remote.rs @@ -322,7 +322,7 @@ impl FlightServiceHandler { // If the cluster key is in the request, the request is from a peer node, which means the // table has already been created in the remote data folder and propagated to all nodes. - if let ClusterMode::MultiNode(cluster) = &configuration_manager.cluster_mode + if let ClusterMode::MultiNode(cluster) = configuration_manager.cluster_mode() && !cluster_key_in_request(cluster, request_metadata)? { cluster @@ -351,7 +351,7 @@ impl FlightServiceHandler { // If the cluster key is in the request, the request is from a peer node, which means the // table has already been created in the remote data folder and propagated to all nodes. - if let ClusterMode::MultiNode(cluster) = &configuration_manager.cluster_mode + if let ClusterMode::MultiNode(cluster) = configuration_manager.cluster_mode() && !cluster_key_in_request(cluster, request_metadata)? { cluster @@ -380,7 +380,7 @@ impl FlightServiceHandler { // If the cluster key is in the request, the request is from a peer node, which means the // tables have already been dropped in the remote data folder and propagated to all nodes. - if let ClusterMode::MultiNode(cluster) = &configuration_manager.cluster_mode + if let ClusterMode::MultiNode(cluster) = configuration_manager.cluster_mode() && !cluster_key_in_request(cluster, request_metadata)? { cluster @@ -410,7 +410,7 @@ impl FlightServiceHandler { let configuration_manager = self.context.configuration_manager.read().await; if truncate_cluster { - if let ClusterMode::MultiNode(cluster) = &configuration_manager.cluster_mode { + if let ClusterMode::MultiNode(cluster) = configuration_manager.cluster_mode() { cluster .truncate_cluster_tables(table_names) .await @@ -442,7 +442,7 @@ impl FlightServiceHandler { let configuration_manager = self.context.configuration_manager.read().await; if vacuum_cluster { - if let ClusterMode::MultiNode(cluster) = &configuration_manager.cluster_mode { + if let ClusterMode::MultiNode(cluster) = configuration_manager.cluster_mode() { cluster .vacuum_cluster_tables(table_names, maybe_retention_period_in_seconds) .await @@ -563,7 +563,7 @@ impl FlightService for FlightServiceHandler { // Retrieve the cloud node that should execute the given query. let configuration_manager = self.context.configuration_manager.read().await; let cloud_node = - if let ClusterMode::MultiNode(cluster) = &configuration_manager.cluster_mode { + if let ClusterMode::MultiNode(cluster) = configuration_manager.cluster_mode() { cluster.query_node().await.map_err(error_to_status_internal) } else { Err(Status::internal("The node is not running in a cluster.")) @@ -879,7 +879,7 @@ impl FlightService for FlightServiceHandler { // If running in a cluster, remove the node from the remote data folder. let configuration_manager = self.context.configuration_manager.read().await; - if let ClusterMode::MultiNode(cluster) = &configuration_manager.cluster_mode { + if let ClusterMode::MultiNode(cluster) = configuration_manager.cluster_mode() { cluster .remove_node() .await @@ -960,7 +960,7 @@ impl FlightService for FlightServiceHandler { } else if action.r#type == "NodeType" { let configuration_manager = self.context.configuration_manager.read().await; - let node_type = match &configuration_manager.cluster_mode { + let node_type = match configuration_manager.cluster_mode() { ClusterMode::SingleNode => "SingleEdge", ClusterMode::MultiNode(cluster) => match cluster.node().mode { ServerMode::Edge => "ClusterEdge", From c9c2f3e7547183756aaf5077cb64357adcd414c5 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Fri, 12 Dec 2025 08:56:27 +0100 Subject: [PATCH 98/99] Wrap large enum variant in Box --- crates/modelardb_server/src/cluster.rs | 6 +++--- crates/modelardb_server/src/configuration.rs | 9 ++++++--- crates/modelardb_server/src/data_folders.rs | 4 ++-- crates/modelardb_server/src/main.rs | 3 +-- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 94bde22f..f0b21c2b 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -87,8 +87,8 @@ impl Cluster { /// Initialize the local database schema with the normal tables and time series tables from the /// cluster's database schema using the remote data folder. If the tables to create could not be - /// retrieved from the remote data folder, or the tables could not be created, - /// return [`ModelarDbServerError`]. + /// retrieved from the remote data folder, or the tables could not be created, return + /// [`ModelarDbServerError`]. pub(crate) async fn retrieve_and_create_tables(&self, context: &Arc) -> Result<()> { let local_data_folder = &context.data_folders.local_data_folder; @@ -726,7 +726,7 @@ mod test { Some(cluster.remote_data_folder.clone()), local_data_folder, ), - ClusterMode::MultiNode(cluster), + ClusterMode::MultiNode(Box::new(cluster)), ) .await .unwrap(), diff --git a/crates/modelardb_server/src/configuration.rs b/crates/modelardb_server/src/configuration.rs index 16a6b29c..dbdcc5db 100644 --- a/crates/modelardb_server/src/configuration.rs +++ b/crates/modelardb_server/src/configuration.rs @@ -690,9 +690,12 @@ mod tests { .unwrap(); let configuration_manager = Arc::new(RwLock::new( - ConfigurationManager::try_new(local_data_folder, ClusterMode::MultiNode(cluster)) - .await - .unwrap(), + ConfigurationManager::try_new( + local_data_folder, + ClusterMode::MultiNode(Box::new(cluster)), + ) + .await + .unwrap(), )); let storage_engine = Arc::new(RwLock::new( diff --git a/crates/modelardb_server/src/data_folders.rs b/crates/modelardb_server/src/data_folders.rs index 85b9c2be..c156af8e 100644 --- a/crates/modelardb_server/src/data_folders.rs +++ b/crates/modelardb_server/src/data_folders.rs @@ -84,7 +84,7 @@ impl DataFolders { let cluster = Cluster::try_new(node, remote_data_folder.clone()).await?; Ok(( - ClusterMode::MultiNode(cluster), + ClusterMode::MultiNode(Box::new(cluster)), Self::new( local_data_folder.clone(), Some(remote_data_folder), @@ -103,7 +103,7 @@ impl DataFolders { let cluster = Cluster::try_new(node, remote_data_folder.clone()).await?; Ok(( - ClusterMode::MultiNode(cluster), + ClusterMode::MultiNode(Box::new(cluster)), Self::new( local_data_folder, Some(remote_data_folder.clone()), diff --git a/crates/modelardb_server/src/main.rs b/crates/modelardb_server/src/main.rs index 212a7913..be10e170 100644 --- a/crates/modelardb_server/src/main.rs +++ b/crates/modelardb_server/src/main.rs @@ -42,11 +42,10 @@ pub static PORT: LazyLock = /// The different possible modes that a ModelarDB server can be deployed in, assigned when the /// server is started. -#[allow(clippy::large_enum_variant)] #[derive(Clone)] pub(crate) enum ClusterMode { SingleNode, - MultiNode(Cluster), + MultiNode(Box), } /// Setup tracing that prints to stdout, parse the command line arguments to extract From fd2ed1c8d9826554017feb47a9bc442fce722de4 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Fri, 12 Dec 2025 09:05:29 +0100 Subject: [PATCH 99/99] Update based on comments from @skejserjensen --- crates/modelardb_storage/src/parser.rs | 4 +--- docs/user/README.md | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/modelardb_storage/src/parser.rs b/crates/modelardb_storage/src/parser.rs index 9a5379e6..6d7fe21f 100644 --- a/crates/modelardb_storage/src/parser.rs +++ b/crates/modelardb_storage/src/parser.rs @@ -636,14 +636,12 @@ impl ModelarDbDialect { if Token::Comma == parser.peek_nth_token(0).token { parser.next_token(); } else { - break; + return Ok(table_names); }; } Err(error) => return Err(error), } } - - Ok(table_names) } } diff --git a/docs/user/README.md b/docs/user/README.md index 420ce3dc..60bc4880 100644 --- a/docs/user/README.md +++ b/docs/user/README.md @@ -139,7 +139,7 @@ mode the `modelardbd` instance will execute queries against the object store and modelardbd cloud path_to_local_data_folder s3://wind-turbine ``` -Note that the DBMS server uses `9999` as the default port. The port can be changed by specifying a different port with +Note that `modelardbd` uses `9999` as the default port. The port can be changed by specifying a different port with the following environment variable: ```shell