diff --git a/apps/framework-cli-e2e/test/utils/schema-definitions.ts b/apps/framework-cli-e2e/test/utils/schema-definitions.ts index ce622995e..16efc5984 100644 --- a/apps/framework-cli-e2e/test/utils/schema-definitions.ts +++ b/apps/framework-cli-e2e/test/utils/schema-definitions.ts @@ -57,6 +57,18 @@ export const TYPESCRIPT_TEST_SCHEMAS: ExpectedTableSchema[] = [ { name: "isDeleted", type: "Bool" }, ], }, + { + tableName: "NullEngineTest", + columns: [ + { name: "id", type: "String" }, + { name: "timestamp", type: /DateTime\('UTC'\)/ }, + { name: "value", type: "Float64" }, + { name: "category", type: "String" }, + { name: "version", type: "Float64" }, + { name: "isDeleted", type: "Bool" }, + ], + engine: "Null", + }, { tableName: "MergeTreeTestExpr", columns: [ @@ -458,6 +470,18 @@ export const PYTHON_TEST_SCHEMAS: ExpectedTableSchema[] = [ { name: "is_deleted", type: "Bool" }, ], }, + { + tableName: "NullEngineTest", + columns: [ + { name: "id", type: "String" }, + { name: "timestamp", type: /DateTime\('UTC'\)/ }, + { name: "value", type: "Int64" }, + { name: "category", type: "String" }, + { name: "version", type: "Int64" }, + { name: "is_deleted", type: "Bool" }, + ], + engine: "Null", + }, { tableName: "MergeTreeTestExpr", columns: [ diff --git a/apps/framework-cli/src/framework/core/partial_infrastructure_map.rs b/apps/framework-cli/src/framework/core/partial_infrastructure_map.rs index c2937c540..b3f6f0b14 100644 --- a/apps/framework-cli/src/framework/core/partial_infrastructure_map.rs +++ b/apps/framework-cli/src/framework/core/partial_infrastructure_map.rs @@ -159,6 +159,9 @@ struct DistributedConfig { #[derive(Debug, Deserialize)] #[serde(tag = "engine", rename_all = "camelCase")] enum EngineConfig { + #[serde(rename = "Null")] + Null {}, + #[serde(rename = "MergeTree")] MergeTree {}, @@ -747,6 +750,8 @@ impl PartialInfrastructureMap { partial_table: &PartialTable, ) -> Result { match &partial_table.engine_config { + Some(EngineConfig::Null {}) => Ok(ClickhouseEngine::Null), + Some(EngineConfig::MergeTree {}) => Ok(ClickhouseEngine::MergeTree), Some(EngineConfig::ReplacingMergeTree { ver, is_deleted }) => { diff --git a/apps/framework-cli/src/framework/python/generate.rs b/apps/framework-cli/src/framework/python/generate.rs index 312cd88f8..201f0d5b5 100644 --- a/apps/framework-cli/src/framework/python/generate.rs +++ b/apps/framework-cli/src/framework/python/generate.rs @@ -780,6 +780,10 @@ pub fn tables_to_python(tables: &[Table], life_cycle: Option) -> Stri crate::infrastructure::olap::clickhouse::queries::ClickhouseEngine::MergeTree => { writeln!(output, " engine=MergeTreeEngine(),").unwrap(); } + crate::infrastructure::olap::clickhouse::queries::ClickhouseEngine::Null => { + writeln!(output, " engine=NullEngine(),").unwrap(); + } + crate::infrastructure::olap::clickhouse::queries::ClickhouseEngine::ReplacingMergeTree { ver, is_deleted } => { // Emit ReplacingMergeTreeEngine with parameters if present write!(output, " engine=ReplacingMergeTreeEngine(").unwrap(); diff --git a/apps/framework-cli/src/framework/typescript/generate.rs b/apps/framework-cli/src/framework/typescript/generate.rs index ba37846f4..e2bc09063 100644 --- a/apps/framework-cli/src/framework/typescript/generate.rs +++ b/apps/framework-cli/src/framework/typescript/generate.rs @@ -685,6 +685,11 @@ pub fn tables_to_typescript(tables: &[Table], life_cycle: Option) -> writeln!(output, " }},").unwrap(); } } + crate::infrastructure::olap::clickhouse::queries::ClickhouseEngine::Null => { + // Table with engine Null : we expose the Null engine in TS + // (assuming you have ClickHouseEngines.Null in your TS enum) + writeln!(output, " engine: ClickHouseEngines.Null,").unwrap(); + } crate::infrastructure::olap::clickhouse::queries::ClickhouseEngine::MergeTree => { writeln!(output, " engine: ClickHouseEngines.MergeTree,").unwrap(); } diff --git a/apps/framework-cli/src/infrastructure/olap/clickhouse/diff_strategy.rs b/apps/framework-cli/src/infrastructure/olap/clickhouse/diff_strategy.rs index 46168bcc6..edb974146 100644 --- a/apps/framework-cli/src/infrastructure/olap/clickhouse/diff_strategy.rs +++ b/apps/framework-cli/src/infrastructure/olap/clickhouse/diff_strategy.rs @@ -393,6 +393,16 @@ impl ClickHouseTableDiffStrategy { // Skip population in production (user must handle manually) // Only populate in dev for new MVs with non-S3Queue sources if is_new && !has_s3queue_source && !is_production { + // If the target table uses the Null engine, skip truncation because there's nothing to truncate + let target_is_null_engine = tables.values().any(|table| { + matches!(table.engine, ClickhouseEngine::Null) + && Self::table_matches_mv_target( + table, + &mv_stmt.target_table, + &mv_stmt.target_database, + ) + }); + log::info!( "Adding population operation for materialized view '{}'", sql_resource.name @@ -408,7 +418,7 @@ impl ClickHouseTableDiffStrategy { .into_iter() .map(|t| t.qualified_name()) .collect(), - should_truncate: true, + should_truncate: !target_is_null_engine, }); } @@ -417,6 +427,30 @@ impl ClickHouseTableDiffStrategy { } } } + + /// Helper: does the given table correspond to the MV target (by name and optional database)? + fn table_matches_mv_target( + table: &Table, + target_table: &str, + target_database: &Option, + ) -> bool { + if table.name == target_table { + return true; + } + + match (&table.database, target_database) { + (Some(table_db), Some(target_db)) => { + table_db == target_db && table.name == target_table + || format!("{table_db}.{}", table.name) == format!("{target_db}.{target_table}") + } + (Some(table_db), None) => table_db.is_empty() && table.name == target_table, + (None, Some(target_db)) => { + format!("{target_db}.{target_table}") == table.name + || format!("{target_db}.{}", table.name) == target_table + } + _ => false, + } + } } impl TableDiffStrategy for ClickHouseTableDiffStrategy { diff --git a/apps/framework-cli/src/infrastructure/olap/clickhouse/queries.rs b/apps/framework-cli/src/infrastructure/olap/clickhouse/queries.rs index 19bcd760c..7ca9e9a72 100644 --- a/apps/framework-cli/src/infrastructure/olap/clickhouse/queries.rs +++ b/apps/framework-cli/src/infrastructure/olap/clickhouse/queries.rs @@ -234,6 +234,7 @@ impl BufferEngine { pub enum ClickhouseEngine { #[default] MergeTree, + Null, ReplacingMergeTree { // Optional version column for deduplication ver: Option, @@ -330,6 +331,7 @@ pub enum ClickhouseEngine { impl Into for ClickhouseEngine { fn into(self) -> String { match self { + ClickhouseEngine::Null => "Null".to_string(), ClickhouseEngine::MergeTree => "MergeTree".to_string(), ClickhouseEngine::ReplacingMergeTree { ver, is_deleted } => { Self::serialize_replacing_merge_tree(&ver, &is_deleted) @@ -422,6 +424,11 @@ impl<'a> TryFrom<&'a str> for ClickhouseEngine { type Error = &'a str; fn try_from(value: &'a str) -> Result { + // Try to parse Null engine + if value.eq_ignore_ascii_case("Null") { + return Ok(ClickhouseEngine::Null); + } + // Try to parse distributed variants first (SharedMergeTree, ReplicatedMergeTree) if let Some(engine) = Self::try_parse_distributed_engine(value) { return engine; @@ -775,6 +782,7 @@ impl ClickhouseEngine { let engine_name = value.strip_prefix("Shared").unwrap_or(value); match engine_name { + "Null" => Ok(ClickhouseEngine::Null), "MergeTree" => Ok(ClickhouseEngine::MergeTree), "ReplacingMergeTree" => Ok(ClickhouseEngine::ReplacingMergeTree { ver: None, @@ -1050,6 +1058,7 @@ impl ClickhouseEngine { /// Convert engine to string for proto storage (no sensitive data) pub fn to_proto_string(&self) -> String { match self { + ClickhouseEngine::Null => "Null".to_string(), ClickhouseEngine::MergeTree => "MergeTree".to_string(), ClickhouseEngine::ReplacingMergeTree { ver, is_deleted } => { Self::serialize_replacing_merge_tree(ver, is_deleted) @@ -1745,6 +1754,9 @@ impl ClickhouseEngine { // Without hashing "null", both would produce identical hashes. match self { + ClickhouseEngine::Null => { + hasher.update("Null".as_bytes()); + } ClickhouseEngine::MergeTree => { hasher.update("MergeTree".as_bytes()); } @@ -2249,6 +2261,7 @@ pub fn create_table_query( reg.register_escape_fn(no_escape); let engine = match &table.engine { + ClickhouseEngine::Null => "Null".to_string(), ClickhouseEngine::MergeTree => "MergeTree".to_string(), ClickhouseEngine::ReplacingMergeTree { ver, is_deleted } => build_replacing_merge_tree_ddl( ver, @@ -4927,6 +4940,16 @@ ENGINE = S3Queue('s3://my-bucket/data/*.csv', NOSIGN, 'CSV')"#; } } + #[test] + fn test_null_engine_roundtrip() { + let engine_str = "Null"; + let engine: ClickhouseEngine = engine_str.try_into().unwrap(); + assert!(matches!(engine, ClickhouseEngine::Null)); + + let serialized: String = engine.into(); + assert_eq!(serialized, "Null"); + } + #[test] fn test_buffer_engine_round_trip() { // Test Buffer engine with all parameters diff --git a/apps/framework-cli/src/infrastructure/olap/clickhouse/sql_parser.rs b/apps/framework-cli/src/infrastructure/olap/clickhouse/sql_parser.rs index 873008a0b..940cf6abb 100644 --- a/apps/framework-cli/src/infrastructure/olap/clickhouse/sql_parser.rs +++ b/apps/framework-cli/src/infrastructure/olap/clickhouse/sql_parser.rs @@ -1557,4 +1557,20 @@ pub mod tests { let indexes = extract_indexes_from_create_table(NESTED_OBJECTS_SQL).unwrap(); assert_eq!(indexes.len(), 0); } + + #[test] + fn test_extract_null_engine() { + // Test for engine = Null + let sql = "CREATE TABLE test (x Int32) ENGINE = Null"; + let result = extract_engine_from_create_table(sql); + assert_eq!(result, Some("Null".to_string())); + } + + #[test] + fn test_extract_null_engine_lowercase() { + // Test for engine = null (lowercase) + let sql = "create table test (x Int32) engine = null"; + let result = extract_engine_from_create_table(sql); + assert_eq!(result, Some("null".to_string())); + } } diff --git a/packages/py-moose-lib/moose_lib/blocks.py b/packages/py-moose-lib/moose_lib/blocks.py index 45f0b85dd..dfc160689 100644 --- a/packages/py-moose-lib/moose_lib/blocks.py +++ b/packages/py-moose-lib/moose_lib/blocks.py @@ -1,11 +1,12 @@ +import warnings +from abc import ABC from dataclasses import dataclass, field from enum import Enum -from typing import Dict, List, Optional, Any, Union -from abc import ABC -import warnings +from typing import Any, Dict, List, Optional, Union class ClickHouseEngines(Enum): + Null = "Null" MergeTree = "MergeTree" ReplacingMergeTree = "ReplacingMergeTree" SummingMergeTree = "SummingMergeTree" @@ -31,6 +32,11 @@ class EngineConfig(ABC): """Base class for engine configurations""" pass +@dataclass +class NullEngine(EngineConfig): + """Configuration for Null engine""" + pass + @dataclass class MergeTreeEngine(EngineConfig): """Configuration for MergeTree engine""" diff --git a/packages/py-moose-lib/moose_lib/dmv2/olap_table.py b/packages/py-moose-lib/moose_lib/dmv2/olap_table.py index 16d4097c7..39b23f3d2 100644 --- a/packages/py-moose-lib/moose_lib/dmv2/olap_table.py +++ b/packages/py-moose-lib/moose_lib/dmv2/olap_table.py @@ -6,20 +6,32 @@ """ import json import warnings +from dataclasses import dataclass +from typing import ( + Any, + Generic, + Iterator, + List, + Literal, + Optional, + Tuple, + TypeVar, + Union, +) + from clickhouse_connect import get_client from clickhouse_connect.driver.client import Client from clickhouse_connect.driver.exceptions import ClickHouseError -from dataclasses import dataclass from pydantic import BaseModel -from typing import List, Optional, Any, Literal, Union, Tuple, TypeVar, Generic, Iterator + from ..blocks import ClickHouseEngines, EngineConfig from ..commons import Logger from ..config.runtime import RuntimeClickHouseConfig +from ..data_models import Column, _to_columns, is_array_nested_type, is_nested_type from ..utilities.sql import quote_identifier -from .types import TypedMooseResource, T, Cols from ._registry import _tables -from ..data_models import Column, is_array_nested_type, is_nested_type, _to_columns from .life_cycle import LifeCycle +from .types import Cols, T, TypedMooseResource @dataclass @@ -159,11 +171,17 @@ def model_post_init(self, __context): # Validate that non-MergeTree engines don't have unsupported clauses if self.engine: - from ..blocks import S3Engine, S3QueueEngine, BufferEngine, DistributedEngine + from ..blocks import ( + BufferEngine, + DistributedEngine, + NullEngine, + S3Engine, + S3QueueEngine, + ) # S3QueueEngine, BufferEngine, and DistributedEngine don't support ORDER BY # Note: S3Engine DOES support ORDER BY (unlike S3Queue) - engines_without_order_by = (S3QueueEngine, BufferEngine, DistributedEngine) + engines_without_order_by = (NullEngine, S3QueueEngine, BufferEngine, DistributedEngine) if isinstance(self.engine, engines_without_order_by): engine_name = type(self.engine).__name__ @@ -174,7 +192,7 @@ def model_post_init(self, __context): ) # All non-MergeTree engines don't support SAMPLE BY - engines_without_sample_by = (S3Engine, S3QueueEngine, BufferEngine, DistributedEngine) + engines_without_sample_by = (NullEngine, S3Engine, S3QueueEngine, BufferEngine, DistributedEngine) if isinstance(self.engine, engines_without_sample_by): engine_name = type(self.engine).__name__ @@ -186,7 +204,7 @@ def model_post_init(self, __context): # Only S3QueueEngine, BufferEngine, and DistributedEngine don't support PARTITION BY # S3Engine DOES support PARTITION BY - engines_without_partition_by = (S3QueueEngine, BufferEngine, DistributedEngine) + engines_without_partition_by = (NullEngine, S3QueueEngine, BufferEngine, DistributedEngine) if isinstance(self.engine, engines_without_partition_by): engine_name = type(self.engine).__name__ @@ -237,9 +255,9 @@ def __init__(self, name: str, config: OlapConfig = OlapConfig(), **kwargs): # Validate cluster and explicit replication params are not both specified if config.cluster: from moose_lib.blocks import ( + ReplicatedAggregatingMergeTreeEngine, ReplicatedMergeTreeEngine, ReplicatedReplacingMergeTreeEngine, - ReplicatedAggregatingMergeTreeEngine, ReplicatedSummingMergeTreeEngine, ) diff --git a/packages/py-moose-lib/moose_lib/internal.py b/packages/py-moose-lib/moose_lib/internal.py index 1bd6ab7fb..a8a9ecae3 100644 --- a/packages/py-moose-lib/moose_lib/internal.py +++ b/packages/py-moose-lib/moose_lib/internal.py @@ -6,27 +6,30 @@ to convert the user-defined resources (from `dmv2.py`) into a serializable JSON format expected by the Moose infrastructure management system. """ -from importlib import import_module -from typing import Literal, Optional, List, Any, Dict, Union, TYPE_CHECKING -from pydantic import BaseModel, ConfigDict, AliasGenerator, Field import json -from .data_models import Column, _to_columns -from .blocks import EngineConfig, ClickHouseEngines +from importlib import import_module +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union + +from pydantic import AliasGenerator, BaseModel, ConfigDict, Field +from pydantic.alias_generators import to_camel +from pydantic.json_schema import JsonSchemaValue + from moose_lib.dmv2 import ( - get_tables, - get_streams, - get_ingest_apis, + OlapConfig, + OlapTable, + SqlResource, get_apis, + get_ingest_apis, get_sql_resources, - get_workflows, + get_streams, + get_tables, get_web_apps, - OlapTable, - OlapConfig, - SqlResource + get_workflows, ) from moose_lib.dmv2.stream import KafkaSchemaConfig -from pydantic.alias_generators import to_camel -from pydantic.json_schema import JsonSchemaValue + +from .blocks import ClickHouseEngines, EngineConfig +from .data_models import Column, _to_columns model_config = ConfigDict(alias_generator=AliasGenerator( serialization_alias=to_camel, @@ -63,6 +66,11 @@ class BaseEngineConfigDict(BaseModel): engine: str +class NullConfigDict(BaseEngineConfigDict): + """Configuration for Null engine.""" + engine: Literal["Null"] = "Null" + + class MergeTreeConfigDict(BaseEngineConfigDict): """Configuration for MergeTree engine.""" engine: Literal["MergeTree"] = "MergeTree" @@ -169,6 +177,7 @@ class DistributedConfigDict(BaseEngineConfigDict): # Discriminated union of all engine configurations EngineConfigDict = Union[ + NullConfigDict, MergeTreeConfigDict, ReplacingMergeTreeConfigDict, AggregatingMergeTreeConfigDict, @@ -441,8 +450,10 @@ def _convert_basic_engine_instance(engine: "EngineConfig") -> Optional[EngineCon EngineConfigDict if matched, None otherwise """ from moose_lib.blocks import ( - MergeTreeEngine, ReplacingMergeTreeEngine, - AggregatingMergeTreeEngine, SummingMergeTreeEngine + AggregatingMergeTreeEngine, + MergeTreeEngine, + ReplacingMergeTreeEngine, + SummingMergeTreeEngine, ) if isinstance(engine, MergeTreeEngine): @@ -469,8 +480,10 @@ def _convert_replicated_engine_instance(engine: "EngineConfig") -> Optional[Engi EngineConfigDict if matched, None otherwise """ from moose_lib.blocks import ( - ReplicatedMergeTreeEngine, ReplicatedReplacingMergeTreeEngine, - ReplicatedAggregatingMergeTreeEngine, ReplicatedSummingMergeTreeEngine + ReplicatedAggregatingMergeTreeEngine, + ReplicatedMergeTreeEngine, + ReplicatedReplacingMergeTreeEngine, + ReplicatedSummingMergeTreeEngine, ) if isinstance(engine, ReplicatedMergeTreeEngine): @@ -508,7 +521,13 @@ def _convert_engine_instance_to_config_dict(engine: "EngineConfig") -> EngineCon Returns: EngineConfigDict with engine-specific configuration """ - from moose_lib.blocks import S3QueueEngine, S3Engine, BufferEngine, DistributedEngine + from moose_lib.blocks import ( + BufferEngine, + DistributedEngine, + NullEngine, + S3Engine, + S3QueueEngine, + ) # Try S3Queue first if isinstance(engine, S3QueueEngine): @@ -560,6 +579,10 @@ def _convert_engine_instance_to_config_dict(engine: "EngineConfig") -> EngineCon policy_name=engine.policy_name ) + # Try NullEngine + if isinstance(engine, NullEngine): + return NullConfigDict() + # Try basic engines basic_config = _convert_basic_engine_instance(engine) if basic_config: @@ -619,6 +642,7 @@ def _convert_engine_to_config_dict(engine: Union[ClickHouseEngines, EngineConfig # Map engine names to specific config classes engine_map = { + "Null": NullConfigDict, "MergeTree": MergeTreeConfigDict, "ReplacingMergeTree": ReplacingMergeTreeConfigDict, "AggregatingMergeTree": AggregatingMergeTreeConfigDict, diff --git a/packages/py-moose-lib/tests/test_cluster_validation.py b/packages/py-moose-lib/tests/test_cluster_validation.py index 0073c804b..7d9e03471 100644 --- a/packages/py-moose-lib/tests/test_cluster_validation.py +++ b/packages/py-moose-lib/tests/test_cluster_validation.py @@ -1,7 +1,7 @@ """Tests for OlapTable cluster validation.""" import pytest -from moose_lib import OlapTable, OlapConfig, MergeTreeEngine, ReplicatedMergeTreeEngine +from moose_lib import MergeTreeEngine, OlapConfig, OlapTable, ReplicatedMergeTreeEngine from pydantic import BaseModel @@ -84,3 +84,5 @@ def test_replicated_engine_without_cluster_or_explicit_params_is_allowed(): ) assert table is not None + + diff --git a/packages/py-moose-lib/tests/test_s3queue_config.py b/packages/py-moose-lib/tests/test_s3queue_config.py index a9dacacb8..309d0d3ac 100644 --- a/packages/py-moose-lib/tests/test_s3queue_config.py +++ b/packages/py-moose-lib/tests/test_s3queue_config.py @@ -1,19 +1,24 @@ """Tests for S3Queue engine configuration with the new type hints.""" -import pytest -from pydantic import BaseModel -from datetime import datetime import warnings +from datetime import datetime -from moose_lib import OlapTable, OlapConfig, ClickHouseEngines -from moose_lib.blocks import S3QueueEngine, MergeTreeEngine, ReplacingMergeTreeEngine +import pytest +from moose_lib import ClickHouseEngines, OlapConfig, OlapTable +from moose_lib.blocks import ( + MergeTreeEngine, + NullEngine, + ReplacingMergeTreeEngine, + S3QueueEngine, +) from moose_lib.internal import ( - _convert_engine_to_config_dict, EngineConfigDict, - S3QueueConfigDict, MergeTreeConfigDict, - ReplacingMergeTreeConfigDict + ReplacingMergeTreeConfigDict, + S3QueueConfigDict, + _convert_engine_to_config_dict, ) +from pydantic import BaseModel class SampleEvent(BaseModel): @@ -95,6 +100,18 @@ def test_olap_table_with_mergetree_engines(): assert isinstance(table2.config.engine, ReplacingMergeTreeEngine) +def test_olap_table_with_null_engine(): + """Test creating OlapTable with NullEngine.""" + table = OlapTable[SampleEvent]( + "NullEngineTable", + OlapConfig( + engine=NullEngine() + # Note: NullEngine does not support order_by_fields + ) + ) + + assert isinstance(table.config.engine, NullEngine) + def test_engine_conversion_to_dict(): """Test conversion of engine configs to EngineConfigDict.""" # Create a mock table with S3QueueEngine @@ -289,7 +306,12 @@ def test_engine_config_validation(): def test_non_mergetree_engines_reject_unsupported_clauses(): """Test that non-MergeTree engines reject unsupported ORDER BY and SAMPLE BY clauses.""" - from moose_lib.blocks import S3Engine, S3QueueEngine, BufferEngine, DistributedEngine + from moose_lib.blocks import ( + BufferEngine, + DistributedEngine, + S3Engine, + S3QueueEngine, + ) # Test S3Engine DOES support ORDER BY (should not raise) config_s3_with_order_by = OlapConfig( diff --git a/packages/ts-moose-lib/src/blocks/helpers.ts b/packages/ts-moose-lib/src/blocks/helpers.ts index cec708054..2b91be357 100644 --- a/packages/ts-moose-lib/src/blocks/helpers.ts +++ b/packages/ts-moose-lib/src/blocks/helpers.ts @@ -35,6 +35,7 @@ export interface Blocks { } export enum ClickHouseEngines { + Null = "Null", MergeTree = "MergeTree", ReplacingMergeTree = "ReplacingMergeTree", SummingMergeTree = "SummingMergeTree", diff --git a/packages/ts-moose-lib/src/dmv2/internal.ts b/packages/ts-moose-lib/src/dmv2/internal.ts index 060adc12f..c3f12b9d3 100644 --- a/packages/ts-moose-lib/src/dmv2/internal.ts +++ b/packages/ts-moose-lib/src/dmv2/internal.ts @@ -65,6 +65,10 @@ const defaultRetentionPeriod = 60 * 60 * 24 * 7; /** * Engine-specific configuration types using discriminated union pattern */ +interface NullEngineConfig { + engine: "Null"; +} + interface MergeTreeEngineConfig { engine: "MergeTree"; } @@ -161,6 +165,7 @@ interface DistributedEngineConfig { * Union type for all supported engine configurations */ type EngineConfig = + | NullEngineConfig | MergeTreeEngineConfig | ReplacingMergeTreeEngineConfig | AggregatingMergeTreeEngineConfig @@ -406,6 +411,9 @@ function convertBasicEngineConfig( config: OlapConfig, ): EngineConfig | undefined { switch (engine) { + case ClickHouseEngines.Null: + return { engine: "Null" }; + case ClickHouseEngines.MergeTree: return { engine: "MergeTree" }; diff --git a/packages/ts-moose-lib/src/dmv2/sdk/olapTable.ts b/packages/ts-moose-lib/src/dmv2/sdk/olapTable.ts index 2ed387103..6b0516db9 100644 --- a/packages/ts-moose-lib/src/dmv2/sdk/olapTable.ts +++ b/packages/ts-moose-lib/src/dmv2/sdk/olapTable.ts @@ -235,6 +235,16 @@ export type BaseOlapConfig = ( cluster?: string; }; +/** + * Configuration for Null engine - discards all data and does not support ORDER BY/PARTITION BY/SAMPLE BY + */ +export type NullConfig = Omit< + BaseOlapConfig, + "orderByFields" | "orderByExpression" | "partitionBy" | "sampleByExpression" +> & { + engine: ClickHouseEngines.Null; +}; + /** * Configuration for MergeTree engine * @template T The data type of the records stored in the table. @@ -452,6 +462,7 @@ export type DistributedConfig = Omit< export type LegacyOlapConfig = BaseOlapConfig; type EngineConfig = + | NullConfig | MergeTreeConfig | ReplacingMergeTreeConfig | AggregatingMergeTreeConfig @@ -534,6 +545,33 @@ export class OlapTable extends TypedBase> { ); } + const isNullEngine = + "engine" in resolvedConfig && + resolvedConfig.engine === ClickHouseEngines.Null; + if (isNullEngine) { + if (hasFields || hasExpr) { + throw new Error( + `OlapTable ${name}: Null engine does not support ORDER BY clauses.`, + ); + } + if ( + "partitionBy" in resolvedConfig && + resolvedConfig.partitionBy !== undefined + ) { + throw new Error( + `OlapTable ${name}: Null engine does not support PARTITION BY clauses.`, + ); + } + if ( + "sampleByExpression" in resolvedConfig && + resolvedConfig.sampleByExpression !== undefined + ) { + throw new Error( + `OlapTable ${name}: Null engine does not support SAMPLE BY clauses.`, + ); + } + } + // Validate cluster and explicit replication params are not both specified const hasCluster = typeof (resolvedConfig as any).cluster === "string"; const hasKeeperPath = diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 20d90858f..fde755e67 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15357,7 +15357,7 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-module-utils@2.12.1(@typescript-eslint/parser@8.46.3(eslint@9.39.1(jiti@2.6.1))(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.1(jiti@2.6.1)): + eslint-module-utils@2.12.1(@typescript-eslint/parser@8.46.3(eslint@9.39.1(jiti@2.6.1))(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.1(jiti@2.6.1)))(eslint@9.39.1(jiti@2.6.1)): dependencies: debug: 3.2.7 optionalDependencies: @@ -15408,7 +15408,7 @@ snapshots: doctrine: 2.1.0 eslint: 9.39.1(jiti@2.6.1) eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.12.1(@typescript-eslint/parser@8.46.3(eslint@9.39.1(jiti@2.6.1))(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.1(jiti@2.6.1)) + eslint-module-utils: 2.12.1(@typescript-eslint/parser@8.46.3(eslint@9.39.1(jiti@2.6.1))(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.1(jiti@2.6.1)))(eslint@9.39.1(jiti@2.6.1)) hasown: 2.0.2 is-core-module: 2.16.1 is-glob: 4.0.3 diff --git a/templates/python-tests/src/ingest/engine_tests.py b/templates/python-tests/src/ingest/engine_tests.py index a93cce64e..d3f9a830b 100644 --- a/templates/python-tests/src/ingest/engine_tests.py +++ b/templates/python-tests/src/ingest/engine_tests.py @@ -11,6 +11,7 @@ ReplicatedReplacingMergeTreeEngine, ReplicatedAggregatingMergeTreeEngine, ReplicatedSummingMergeTreeEngine, + NullEngine, BufferEngine, # S3QueueEngine - requires S3 configuration, tested separately ) @@ -99,6 +100,14 @@ class EngineTestDataSample(BaseModel): ) ) +# Test Null engine (schema-only, discards writes) +null_engine_table = OlapTable[EngineTestData]( + "NullEngineTest", + OlapConfig( + engine=NullEngine(), + ) +) + # Test MergeTree with order_by_expression (equivalent to fields) merge_tree_table_expr = OlapTable[EngineTestData]( "MergeTreeTestExpr", @@ -274,6 +283,7 @@ class EngineTestDataSample(BaseModel): # can be properly instantiated and don't throw errors during table creation all_engine_test_tables = [ merge_tree_table, + null_engine_table, merge_tree_table_expr, replacing_merge_tree_basic_table, replacing_merge_tree_version_table, diff --git a/templates/typescript-tests/src/ingest/engineTests.ts b/templates/typescript-tests/src/ingest/engineTests.ts index 0782b441c..446a52d2e 100644 --- a/templates/typescript-tests/src/ingest/engineTests.ts +++ b/templates/typescript-tests/src/ingest/engineTests.ts @@ -64,6 +64,11 @@ export const MergeTreeTable = new OlapTable("MergeTreeTest", { orderByFields: ["id", "timestamp"], }); +// Test Null engine (schema-only, discards writes) +export const NullEngineTable = new OlapTable("NullEngineTest", { + engine: ClickHouseEngines.Null, +}); + // Test MergeTree with orderByExpression (equivalent to fields) export const MergeTreeTableExpr = new OlapTable( "MergeTreeTestExpr", @@ -247,6 +252,7 @@ export const BufferTable = new OlapTable("BufferTest", { */ export const allEngineTestTables = [ MergeTreeTable, + NullEngineTable, MergeTreeTableExpr, ReplacingMergeTreeBasicTable, ReplacingMergeTreeVersionTable,