From 1f2b7caee4be12080684172be27eafecda38fb55 Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 22 Nov 2025 04:15:33 +0100 Subject: [PATCH 1/8] Add null engine table implementation --- .../core/partial_infrastructure_map.rs | 5 ++ .../src/framework/python/generate.rs | 6 ++ .../src/framework/typescript/generate.rs | 5 ++ .../infrastructure/olap/clickhouse/queries.rs | 23 +++++++ .../olap/clickhouse/sql_parser.rs | 16 +++++ .../null-engine-example-py/.gitignore | 42 +++++++++++++ .../.vscode/extensions.json | 9 +++ .../.vscode/settings.json | 17 +++++ .../null-engine-example-py/README.md | 48 ++++++++++++++ .../null-engine-example-py/app/__init__.py | 0 .../app/apis/__init__.py | 0 .../app/ingest/__init__.py | 0 .../app/ingest/models.py | 26 ++++++++ .../null-engine-example-py/app/main.py | 2 + .../app/views/__init__.py | 0 .../app/workflows/__init__.py | 0 .../null-engine-example-py/moose.config.toml | 56 +++++++++++++++++ .../null-engine-example-py/requirements.txt | 7 +++ .../null-engine-example-py/setup.py | 14 +++++ .../template.config.toml | 26 ++++++++ packages/py-moose-lib/moose_lib/blocks.py | 12 +++- .../py-moose-lib/moose_lib/dmv2/olap_table.py | 34 +++++++--- packages/py-moose-lib/moose_lib/internal.py | 63 +++++++++++++------ .../tests/test_cluster_validation.py | 4 +- .../py-moose-lib/tests/test_s3queue_config.py | 40 +++++++++--- pnpm-lock.yaml | 4 +- 26 files changed, 417 insertions(+), 42 deletions(-) create mode 100644 examples/null-engine-example/null-engine-example-py/.gitignore create mode 100644 examples/null-engine-example/null-engine-example-py/.vscode/extensions.json create mode 100644 examples/null-engine-example/null-engine-example-py/.vscode/settings.json create mode 100644 examples/null-engine-example/null-engine-example-py/README.md create mode 100644 examples/null-engine-example/null-engine-example-py/app/__init__.py create mode 100644 examples/null-engine-example/null-engine-example-py/app/apis/__init__.py create mode 100644 examples/null-engine-example/null-engine-example-py/app/ingest/__init__.py create mode 100644 examples/null-engine-example/null-engine-example-py/app/ingest/models.py create mode 100644 examples/null-engine-example/null-engine-example-py/app/main.py create mode 100644 examples/null-engine-example/null-engine-example-py/app/views/__init__.py create mode 100644 examples/null-engine-example/null-engine-example-py/app/workflows/__init__.py create mode 100644 examples/null-engine-example/null-engine-example-py/moose.config.toml create mode 100644 examples/null-engine-example/null-engine-example-py/requirements.txt create mode 100644 examples/null-engine-example/null-engine-example-py/setup.py create mode 100644 examples/null-engine-example/null-engine-example-py/template.config.toml diff --git a/apps/framework-cli/src/framework/core/partial_infrastructure_map.rs b/apps/framework-cli/src/framework/core/partial_infrastructure_map.rs index c2937c5400..b3f6f0b148 100644 --- a/apps/framework-cli/src/framework/core/partial_infrastructure_map.rs +++ b/apps/framework-cli/src/framework/core/partial_infrastructure_map.rs @@ -159,6 +159,9 @@ struct DistributedConfig { #[derive(Debug, Deserialize)] #[serde(tag = "engine", rename_all = "camelCase")] enum EngineConfig { + #[serde(rename = "Null")] + Null {}, + #[serde(rename = "MergeTree")] MergeTree {}, @@ -747,6 +750,8 @@ impl PartialInfrastructureMap { partial_table: &PartialTable, ) -> Result { match &partial_table.engine_config { + Some(EngineConfig::Null {}) => Ok(ClickhouseEngine::Null), + Some(EngineConfig::MergeTree {}) => Ok(ClickhouseEngine::MergeTree), Some(EngineConfig::ReplacingMergeTree { ver, is_deleted }) => { diff --git a/apps/framework-cli/src/framework/python/generate.rs b/apps/framework-cli/src/framework/python/generate.rs index 312cd88f86..c0c943010d 100644 --- a/apps/framework-cli/src/framework/python/generate.rs +++ b/apps/framework-cli/src/framework/python/generate.rs @@ -780,6 +780,12 @@ pub fn tables_to_python(tables: &[Table], life_cycle: Option) -> Stri crate::infrastructure::olap::clickhouse::queries::ClickhouseEngine::MergeTree => { writeln!(output, " engine=MergeTreeEngine(),").unwrap(); } + crate::infrastructure::olap::clickhouse::queries::ClickhouseEngine::Null => { + // No explicit engine for Null in moose_lib, + // so we do not generate an `engine=...` parameter in OlapConfig. + // (If you want a real Null ENGINE later, we can adapt here.) + } + crate::infrastructure::olap::clickhouse::queries::ClickhouseEngine::ReplacingMergeTree { ver, is_deleted } => { // Emit ReplacingMergeTreeEngine with parameters if present write!(output, " engine=ReplacingMergeTreeEngine(").unwrap(); diff --git a/apps/framework-cli/src/framework/typescript/generate.rs b/apps/framework-cli/src/framework/typescript/generate.rs index ba37846f47..e2bc090636 100644 --- a/apps/framework-cli/src/framework/typescript/generate.rs +++ b/apps/framework-cli/src/framework/typescript/generate.rs @@ -685,6 +685,11 @@ pub fn tables_to_typescript(tables: &[Table], life_cycle: Option) -> writeln!(output, " }},").unwrap(); } } + crate::infrastructure::olap::clickhouse::queries::ClickhouseEngine::Null => { + // Table with engine Null : we expose the Null engine in TS + // (assuming you have ClickHouseEngines.Null in your TS enum) + writeln!(output, " engine: ClickHouseEngines.Null,").unwrap(); + } crate::infrastructure::olap::clickhouse::queries::ClickhouseEngine::MergeTree => { writeln!(output, " engine: ClickHouseEngines.MergeTree,").unwrap(); } diff --git a/apps/framework-cli/src/infrastructure/olap/clickhouse/queries.rs b/apps/framework-cli/src/infrastructure/olap/clickhouse/queries.rs index 19bcd760c7..7ca9e9a721 100644 --- a/apps/framework-cli/src/infrastructure/olap/clickhouse/queries.rs +++ b/apps/framework-cli/src/infrastructure/olap/clickhouse/queries.rs @@ -234,6 +234,7 @@ impl BufferEngine { pub enum ClickhouseEngine { #[default] MergeTree, + Null, ReplacingMergeTree { // Optional version column for deduplication ver: Option, @@ -330,6 +331,7 @@ pub enum ClickhouseEngine { impl Into for ClickhouseEngine { fn into(self) -> String { match self { + ClickhouseEngine::Null => "Null".to_string(), ClickhouseEngine::MergeTree => "MergeTree".to_string(), ClickhouseEngine::ReplacingMergeTree { ver, is_deleted } => { Self::serialize_replacing_merge_tree(&ver, &is_deleted) @@ -422,6 +424,11 @@ impl<'a> TryFrom<&'a str> for ClickhouseEngine { type Error = &'a str; fn try_from(value: &'a str) -> Result { + // Try to parse Null engine + if value.eq_ignore_ascii_case("Null") { + return Ok(ClickhouseEngine::Null); + } + // Try to parse distributed variants first (SharedMergeTree, ReplicatedMergeTree) if let Some(engine) = Self::try_parse_distributed_engine(value) { return engine; @@ -775,6 +782,7 @@ impl ClickhouseEngine { let engine_name = value.strip_prefix("Shared").unwrap_or(value); match engine_name { + "Null" => Ok(ClickhouseEngine::Null), "MergeTree" => Ok(ClickhouseEngine::MergeTree), "ReplacingMergeTree" => Ok(ClickhouseEngine::ReplacingMergeTree { ver: None, @@ -1050,6 +1058,7 @@ impl ClickhouseEngine { /// Convert engine to string for proto storage (no sensitive data) pub fn to_proto_string(&self) -> String { match self { + ClickhouseEngine::Null => "Null".to_string(), ClickhouseEngine::MergeTree => "MergeTree".to_string(), ClickhouseEngine::ReplacingMergeTree { ver, is_deleted } => { Self::serialize_replacing_merge_tree(ver, is_deleted) @@ -1745,6 +1754,9 @@ impl ClickhouseEngine { // Without hashing "null", both would produce identical hashes. match self { + ClickhouseEngine::Null => { + hasher.update("Null".as_bytes()); + } ClickhouseEngine::MergeTree => { hasher.update("MergeTree".as_bytes()); } @@ -2249,6 +2261,7 @@ pub fn create_table_query( reg.register_escape_fn(no_escape); let engine = match &table.engine { + ClickhouseEngine::Null => "Null".to_string(), ClickhouseEngine::MergeTree => "MergeTree".to_string(), ClickhouseEngine::ReplacingMergeTree { ver, is_deleted } => build_replacing_merge_tree_ddl( ver, @@ -4927,6 +4940,16 @@ ENGINE = S3Queue('s3://my-bucket/data/*.csv', NOSIGN, 'CSV')"#; } } + #[test] + fn test_null_engine_roundtrip() { + let engine_str = "Null"; + let engine: ClickhouseEngine = engine_str.try_into().unwrap(); + assert!(matches!(engine, ClickhouseEngine::Null)); + + let serialized: String = engine.into(); + assert_eq!(serialized, "Null"); + } + #[test] fn test_buffer_engine_round_trip() { // Test Buffer engine with all parameters diff --git a/apps/framework-cli/src/infrastructure/olap/clickhouse/sql_parser.rs b/apps/framework-cli/src/infrastructure/olap/clickhouse/sql_parser.rs index 873008a0b6..940cf6abb0 100644 --- a/apps/framework-cli/src/infrastructure/olap/clickhouse/sql_parser.rs +++ b/apps/framework-cli/src/infrastructure/olap/clickhouse/sql_parser.rs @@ -1557,4 +1557,20 @@ pub mod tests { let indexes = extract_indexes_from_create_table(NESTED_OBJECTS_SQL).unwrap(); assert_eq!(indexes.len(), 0); } + + #[test] + fn test_extract_null_engine() { + // Test for engine = Null + let sql = "CREATE TABLE test (x Int32) ENGINE = Null"; + let result = extract_engine_from_create_table(sql); + assert_eq!(result, Some("Null".to_string())); + } + + #[test] + fn test_extract_null_engine_lowercase() { + // Test for engine = null (lowercase) + let sql = "create table test (x Int32) engine = null"; + let result = extract_engine_from_create_table(sql); + assert_eq!(result, Some("null".to_string())); + } } diff --git a/examples/null-engine-example/null-engine-example-py/.gitignore b/examples/null-engine-example/null-engine-example-py/.gitignore new file mode 100644 index 0000000000..74052de968 --- /dev/null +++ b/examples/null-engine-example/null-engine-example-py/.gitignore @@ -0,0 +1,42 @@ +.moose +__pycache__ +*.pyc +*.pyo +*.pyd +.Python +env +.venv +venv +ENV +env.bak +.spyderproject +.ropeproject +.idea +*.ipynb_checkpoints +.pytest_cache +.mypy_cache +.hypothesis +.coverage +cover +*.cover +.DS_Store +.cache +*.so +*.egg +*.egg-info +dist +build +develop-eggs +downloads +eggs +lib +lib64 +parts +sdist +var +wheels +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + diff --git a/examples/null-engine-example/null-engine-example-py/.vscode/extensions.json b/examples/null-engine-example/null-engine-example-py/.vscode/extensions.json new file mode 100644 index 0000000000..dbc86167a6 --- /dev/null +++ b/examples/null-engine-example/null-engine-example-py/.vscode/extensions.json @@ -0,0 +1,9 @@ +{ + "recommendations": [ + "frigus02.vscode-sql-tagged-template-literals-syntax-only", + "mtxr.sqltools", + "ultram4rine.sqltools-clickhouse-driver", + "jeppeandersen.vscode-kafka", + "rangav.vscode-thunder-client" + ] +} diff --git a/examples/null-engine-example/null-engine-example-py/.vscode/settings.json b/examples/null-engine-example/null-engine-example-py/.vscode/settings.json new file mode 100644 index 0000000000..48a4637229 --- /dev/null +++ b/examples/null-engine-example/null-engine-example-py/.vscode/settings.json @@ -0,0 +1,17 @@ +{ + "sqltools.connections": [ + { + "server": "localhost", + "port": 18123, + "useHTTPS": false, + "database": "local", + "username": "panda", + "enableTls": false, + "password": "pandapass", + "driver": "ClickHouse", + "name": "moose clickhouse" + } + ], + "python.analysis.extraPaths": [".moose/versions"], + "python.analysis.typeCheckingMode": "basic" +} diff --git a/examples/null-engine-example/null-engine-example-py/README.md b/examples/null-engine-example/null-engine-example-py/README.md new file mode 100644 index 0000000000..a5de381060 --- /dev/null +++ b/examples/null-engine-example/null-engine-example-py/README.md @@ -0,0 +1,48 @@ +# Template: Python + +This is a Python-based Moose template that provides a foundation for building data-intensive applications using Python. + +[![PyPI Version](https://img.shields.io/pypi/v/moose-cli?logo=python)](https://pypi.org/project/moose-cli/) +[![Moose Community](https://img.shields.io/badge/slack-moose_community-purple.svg?logo=slack)](https://join.slack.com/t/moose-community/shared_invite/zt-2fjh5n3wz-cnOmM9Xe9DYAgQrNu8xKxg) +[![Docs](https://img.shields.io/badge/quick_start-docs-blue.svg)](https://docs.fiveonefour.com/moose/getting-started/quickstart) +[![MIT license](https://img.shields.io/badge/license-MIT-yellow.svg)](LICENSE) + +## Getting Started + +### Prerequisites + +* [Docker Desktop](https://www.docker.com/products/docker-desktop/) +* [Python](https://www.python.org/downloads/) (version 3.8+) +* [An Anthropic API Key](https://docs.anthropic.com/en/api/getting-started) +* [Cursor](https://www.cursor.com/) or [Claude Desktop](https://claude.ai/download) + +### Installation + +1. Install Moose CLI: `pip install moose-cli` +2. Create project: `moose init python` +3. Install dependencies: `cd && pip install -r requirements.txt` +4. Run Moose: `moose dev` + +You are ready to go! You can start editing the app by modifying primitives in the `app` subdirectory. + +## Learn More + +To learn more about Moose, take a look at the following resources: + +- [Moose Documentation](https://docs.fiveonefour.com/moose) - learn about Moose. +- [Sloan Documentation](https://docs.fiveonefour.com/sloan) - learn about Sloan, the MCP interface for data engineering. + +## Community + +You can join the Moose community [on Slack](https://join.slack.com/t/moose-community/shared_invite/zt-2fjh5n3wz-cnOmM9Xe9DYAgQrNu8xKxg). Check out the [MooseStack repo on GitHub](https://github.com/514-labs/moosestack). + +## Deploy on Boreal + +The easiest way to deploy your MooseStack Applications is to use [Boreal](https://www.fiveonefour.com/boreal) from 514 Labs, the creators of Moose. + +[Sign up](https://www.boreal.cloud/sign-up). + +## License + +This template is MIT licensed. + diff --git a/examples/null-engine-example/null-engine-example-py/app/__init__.py b/examples/null-engine-example/null-engine-example-py/app/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/null-engine-example/null-engine-example-py/app/apis/__init__.py b/examples/null-engine-example/null-engine-example-py/app/apis/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/null-engine-example/null-engine-example-py/app/ingest/__init__.py b/examples/null-engine-example/null-engine-example-py/app/ingest/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/null-engine-example/null-engine-example-py/app/ingest/models.py b/examples/null-engine-example/null-engine-example-py/app/ingest/models.py new file mode 100644 index 0000000000..e90be2cda3 --- /dev/null +++ b/examples/null-engine-example/null-engine-example-py/app/ingest/models.py @@ -0,0 +1,26 @@ +# This file was auto-generated by the framework. You can add data models or change the existing ones + +from datetime import datetime + +from moose_lib import ( + Key, + OlapConfig, +) +from moose_lib.blocks import NullEngine +from moose_lib.dmv2 import ( + OlapTable, +) +from pydantic import BaseModel + + +class Items(BaseModel): + order_id: Key[int] + id: Key[int] + updated_at: datetime + +items_null = OlapTable[Items]( + "items_null", + OlapConfig( + engine=NullEngine(), + ), +) \ No newline at end of file diff --git a/examples/null-engine-example/null-engine-example-py/app/main.py b/examples/null-engine-example/null-engine-example-py/app/main.py new file mode 100644 index 0000000000..2dfc8321a8 --- /dev/null +++ b/examples/null-engine-example/null-engine-example-py/app/main.py @@ -0,0 +1,2 @@ +# This file was auto-generated by the framework. You can add data models or change the existing ones +from app.ingest import models diff --git a/examples/null-engine-example/null-engine-example-py/app/views/__init__.py b/examples/null-engine-example/null-engine-example-py/app/views/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/null-engine-example/null-engine-example-py/app/workflows/__init__.py b/examples/null-engine-example/null-engine-example-py/app/workflows/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/null-engine-example/null-engine-example-py/moose.config.toml b/examples/null-engine-example/null-engine-example-py/moose.config.toml new file mode 100644 index 0000000000..fe19fc4af6 --- /dev/null +++ b/examples/null-engine-example/null-engine-example-py/moose.config.toml @@ -0,0 +1,56 @@ +language = "Python" + +[redpanda_config] +broker = "localhost:19092" +message_timeout_ms = 1000 +retention_ms = 30000 +replication_factor = 1 + +[clickhouse_config] +db_name = "local" +user = "panda" +password = "pandapass" +use_ssl = false +host = "localhost" +host_port = 18123 +native_port = 9000 + +[http_server_config] +host = "localhost" +port = 4000 +management_port = 5001 + +[redis_config] +url = "redis://127.0.0.1:6379" +key_prefix = "MS" + +[git_config] +main_branch_name = "main" + +[temporal_config] +db_user = "temporal" +db_password = "temporal" +db_port = 5432 +temporal_host = "localhost" +temporal_port = 7233 +temporal_version = "1.22.3" +admin_tools_version = "1.22.3" +ui_version = "2.21.3" +ui_port = 8080 +ui_cors_origins = "http://localhost:3000" +config_path = "config/dynamicconfig/development-sql.yaml" +postgresql_version = "13" +client_cert = "" +client_key = "" +ca_cert = "" +api_key = "" + +[supported_old_versions] + +[authentication] + +[features] +streaming_engine = true +workflows = true +data_model_v2 = true +apis = true diff --git a/examples/null-engine-example/null-engine-example-py/requirements.txt b/examples/null-engine-example/null-engine-example-py/requirements.txt new file mode 100644 index 0000000000..870d85e724 --- /dev/null +++ b/examples/null-engine-example/null-engine-example-py/requirements.txt @@ -0,0 +1,7 @@ +kafka-python-ng==2.2.2 +clickhouse-connect==0.7.16 +requests==2.32.4 +#moose-cli +#moose-lib +faker +sqlglot[rs]>=27.16.3 \ No newline at end of file diff --git a/examples/null-engine-example/null-engine-example-py/setup.py b/examples/null-engine-example/null-engine-example-py/setup.py new file mode 100644 index 0000000000..cdbfe7692b --- /dev/null +++ b/examples/null-engine-example/null-engine-example-py/setup.py @@ -0,0 +1,14 @@ + +import os + +from setuptools import setup + +requirements_path = os.path.join(os.path.dirname(__file__), "requirements.txt") +with open(requirements_path, "r") as f: + requirements = f.read().splitlines() + +setup( + name='null-engine-example-py', + version='0.0', + install_requires=requirements, +) diff --git a/examples/null-engine-example/null-engine-example-py/template.config.toml b/examples/null-engine-example/null-engine-example-py/template.config.toml new file mode 100644 index 0000000000..494d291588 --- /dev/null +++ b/examples/null-engine-example/null-engine-example-py/template.config.toml @@ -0,0 +1,26 @@ +language = "python" # Must be typescript or python +description = "Default Python project, seeded with foobar example components." +post_install_print = """ +Deploy on Boreal + +The easiest way to deploy your MooseStack Applications is to use Boreal +from the creators of MooseStack. + +https://boreal.cloud + +--------------------------------------------------------- + +📂 Go to your project directory: + $ cd {project_dir} + +🥄 Create a virtual environment (optional, recommended): + $ python3 -m venv .venv + $ source .venv/bin/activate + +📦 Install Dependencies: + $ pip install -r ./requirements.txt + +🛠️ Start Moose Server: + $ moose dev +""" +default_sloan_telemetry="standard" diff --git a/packages/py-moose-lib/moose_lib/blocks.py b/packages/py-moose-lib/moose_lib/blocks.py index 45f0b85dd6..dfc1606897 100644 --- a/packages/py-moose-lib/moose_lib/blocks.py +++ b/packages/py-moose-lib/moose_lib/blocks.py @@ -1,11 +1,12 @@ +import warnings +from abc import ABC from dataclasses import dataclass, field from enum import Enum -from typing import Dict, List, Optional, Any, Union -from abc import ABC -import warnings +from typing import Any, Dict, List, Optional, Union class ClickHouseEngines(Enum): + Null = "Null" MergeTree = "MergeTree" ReplacingMergeTree = "ReplacingMergeTree" SummingMergeTree = "SummingMergeTree" @@ -31,6 +32,11 @@ class EngineConfig(ABC): """Base class for engine configurations""" pass +@dataclass +class NullEngine(EngineConfig): + """Configuration for Null engine""" + pass + @dataclass class MergeTreeEngine(EngineConfig): """Configuration for MergeTree engine""" diff --git a/packages/py-moose-lib/moose_lib/dmv2/olap_table.py b/packages/py-moose-lib/moose_lib/dmv2/olap_table.py index 16d4097c7a..a8c5734792 100644 --- a/packages/py-moose-lib/moose_lib/dmv2/olap_table.py +++ b/packages/py-moose-lib/moose_lib/dmv2/olap_table.py @@ -6,20 +6,32 @@ """ import json import warnings +from dataclasses import dataclass +from typing import ( + Any, + Generic, + Iterator, + List, + Literal, + Optional, + Tuple, + TypeVar, + Union, +) + from clickhouse_connect import get_client from clickhouse_connect.driver.client import Client from clickhouse_connect.driver.exceptions import ClickHouseError -from dataclasses import dataclass from pydantic import BaseModel -from typing import List, Optional, Any, Literal, Union, Tuple, TypeVar, Generic, Iterator + from ..blocks import ClickHouseEngines, EngineConfig from ..commons import Logger from ..config.runtime import RuntimeClickHouseConfig +from ..data_models import Column, _to_columns, is_array_nested_type, is_nested_type from ..utilities.sql import quote_identifier -from .types import TypedMooseResource, T, Cols from ._registry import _tables -from ..data_models import Column, is_array_nested_type, is_nested_type, _to_columns from .life_cycle import LifeCycle +from .types import Cols, T, TypedMooseResource @dataclass @@ -159,11 +171,17 @@ def model_post_init(self, __context): # Validate that non-MergeTree engines don't have unsupported clauses if self.engine: - from ..blocks import S3Engine, S3QueueEngine, BufferEngine, DistributedEngine + from ..blocks import ( + BufferEngine, + DistributedEngine, + NullEngine, + S3Engine, + S3QueueEngine, + ) # S3QueueEngine, BufferEngine, and DistributedEngine don't support ORDER BY # Note: S3Engine DOES support ORDER BY (unlike S3Queue) - engines_without_order_by = (S3QueueEngine, BufferEngine, DistributedEngine) + engines_without_order_by = (NullEngine, S3QueueEngine, BufferEngine, DistributedEngine) if isinstance(self.engine, engines_without_order_by): engine_name = type(self.engine).__name__ @@ -174,7 +192,7 @@ def model_post_init(self, __context): ) # All non-MergeTree engines don't support SAMPLE BY - engines_without_sample_by = (S3Engine, S3QueueEngine, BufferEngine, DistributedEngine) + engines_without_sample_by = (NullEngine, S3Engine, S3QueueEngine, BufferEngine, DistributedEngine) if isinstance(self.engine, engines_without_sample_by): engine_name = type(self.engine).__name__ @@ -237,9 +255,9 @@ def __init__(self, name: str, config: OlapConfig = OlapConfig(), **kwargs): # Validate cluster and explicit replication params are not both specified if config.cluster: from moose_lib.blocks import ( + ReplicatedAggregatingMergeTreeEngine, ReplicatedMergeTreeEngine, ReplicatedReplacingMergeTreeEngine, - ReplicatedAggregatingMergeTreeEngine, ReplicatedSummingMergeTreeEngine, ) diff --git a/packages/py-moose-lib/moose_lib/internal.py b/packages/py-moose-lib/moose_lib/internal.py index 1bd6ab7fba..8d56b56442 100644 --- a/packages/py-moose-lib/moose_lib/internal.py +++ b/packages/py-moose-lib/moose_lib/internal.py @@ -6,27 +6,31 @@ to convert the user-defined resources (from `dmv2.py`) into a serializable JSON format expected by the Moose infrastructure management system. """ -from importlib import import_module -from typing import Literal, Optional, List, Any, Dict, Union, TYPE_CHECKING -from pydantic import BaseModel, ConfigDict, AliasGenerator, Field import json -from .data_models import Column, _to_columns -from .blocks import EngineConfig, ClickHouseEngines +from importlib import import_module +from tkinter import N +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union + +from pydantic import AliasGenerator, BaseModel, ConfigDict, Field +from pydantic.alias_generators import to_camel +from pydantic.json_schema import JsonSchemaValue + from moose_lib.dmv2 import ( - get_tables, - get_streams, - get_ingest_apis, + OlapConfig, + OlapTable, + SqlResource, get_apis, + get_ingest_apis, get_sql_resources, - get_workflows, + get_streams, + get_tables, get_web_apps, - OlapTable, - OlapConfig, - SqlResource + get_workflows, ) from moose_lib.dmv2.stream import KafkaSchemaConfig -from pydantic.alias_generators import to_camel -from pydantic.json_schema import JsonSchemaValue + +from .blocks import ClickHouseEngines, EngineConfig +from .data_models import Column, _to_columns model_config = ConfigDict(alias_generator=AliasGenerator( serialization_alias=to_camel, @@ -63,6 +67,11 @@ class BaseEngineConfigDict(BaseModel): engine: str +class NullConfigDict(BaseEngineConfigDict): + """Configuration for Null engine.""" + engine: Literal["Null"] = "Null" + + class MergeTreeConfigDict(BaseEngineConfigDict): """Configuration for MergeTree engine.""" engine: Literal["MergeTree"] = "MergeTree" @@ -169,6 +178,7 @@ class DistributedConfigDict(BaseEngineConfigDict): # Discriminated union of all engine configurations EngineConfigDict = Union[ + NullConfigDict, MergeTreeConfigDict, ReplacingMergeTreeConfigDict, AggregatingMergeTreeConfigDict, @@ -441,8 +451,10 @@ def _convert_basic_engine_instance(engine: "EngineConfig") -> Optional[EngineCon EngineConfigDict if matched, None otherwise """ from moose_lib.blocks import ( - MergeTreeEngine, ReplacingMergeTreeEngine, - AggregatingMergeTreeEngine, SummingMergeTreeEngine + AggregatingMergeTreeEngine, + MergeTreeEngine, + ReplacingMergeTreeEngine, + SummingMergeTreeEngine, ) if isinstance(engine, MergeTreeEngine): @@ -469,8 +481,10 @@ def _convert_replicated_engine_instance(engine: "EngineConfig") -> Optional[Engi EngineConfigDict if matched, None otherwise """ from moose_lib.blocks import ( - ReplicatedMergeTreeEngine, ReplicatedReplacingMergeTreeEngine, - ReplicatedAggregatingMergeTreeEngine, ReplicatedSummingMergeTreeEngine + ReplicatedAggregatingMergeTreeEngine, + ReplicatedMergeTreeEngine, + ReplicatedReplacingMergeTreeEngine, + ReplicatedSummingMergeTreeEngine, ) if isinstance(engine, ReplicatedMergeTreeEngine): @@ -508,7 +522,13 @@ def _convert_engine_instance_to_config_dict(engine: "EngineConfig") -> EngineCon Returns: EngineConfigDict with engine-specific configuration """ - from moose_lib.blocks import S3QueueEngine, S3Engine, BufferEngine, DistributedEngine + from moose_lib.blocks import ( + BufferEngine, + DistributedEngine, + NullEngine, + S3Engine, + S3QueueEngine, + ) # Try S3Queue first if isinstance(engine, S3QueueEngine): @@ -560,6 +580,10 @@ def _convert_engine_instance_to_config_dict(engine: "EngineConfig") -> EngineCon policy_name=engine.policy_name ) + # Try NullEngine + if isinstance(engine, NullEngine): + return NullConfigDict() + # Try basic engines basic_config = _convert_basic_engine_instance(engine) if basic_config: @@ -619,6 +643,7 @@ def _convert_engine_to_config_dict(engine: Union[ClickHouseEngines, EngineConfig # Map engine names to specific config classes engine_map = { + "NullEngine": NullConfigDict, "MergeTree": MergeTreeConfigDict, "ReplacingMergeTree": ReplacingMergeTreeConfigDict, "AggregatingMergeTree": AggregatingMergeTreeConfigDict, diff --git a/packages/py-moose-lib/tests/test_cluster_validation.py b/packages/py-moose-lib/tests/test_cluster_validation.py index 0073c804ba..7d9e034717 100644 --- a/packages/py-moose-lib/tests/test_cluster_validation.py +++ b/packages/py-moose-lib/tests/test_cluster_validation.py @@ -1,7 +1,7 @@ """Tests for OlapTable cluster validation.""" import pytest -from moose_lib import OlapTable, OlapConfig, MergeTreeEngine, ReplicatedMergeTreeEngine +from moose_lib import MergeTreeEngine, OlapConfig, OlapTable, ReplicatedMergeTreeEngine from pydantic import BaseModel @@ -84,3 +84,5 @@ def test_replicated_engine_without_cluster_or_explicit_params_is_allowed(): ) assert table is not None + + diff --git a/packages/py-moose-lib/tests/test_s3queue_config.py b/packages/py-moose-lib/tests/test_s3queue_config.py index a9dacacb83..309d0d3ac1 100644 --- a/packages/py-moose-lib/tests/test_s3queue_config.py +++ b/packages/py-moose-lib/tests/test_s3queue_config.py @@ -1,19 +1,24 @@ """Tests for S3Queue engine configuration with the new type hints.""" -import pytest -from pydantic import BaseModel -from datetime import datetime import warnings +from datetime import datetime -from moose_lib import OlapTable, OlapConfig, ClickHouseEngines -from moose_lib.blocks import S3QueueEngine, MergeTreeEngine, ReplacingMergeTreeEngine +import pytest +from moose_lib import ClickHouseEngines, OlapConfig, OlapTable +from moose_lib.blocks import ( + MergeTreeEngine, + NullEngine, + ReplacingMergeTreeEngine, + S3QueueEngine, +) from moose_lib.internal import ( - _convert_engine_to_config_dict, EngineConfigDict, - S3QueueConfigDict, MergeTreeConfigDict, - ReplacingMergeTreeConfigDict + ReplacingMergeTreeConfigDict, + S3QueueConfigDict, + _convert_engine_to_config_dict, ) +from pydantic import BaseModel class SampleEvent(BaseModel): @@ -95,6 +100,18 @@ def test_olap_table_with_mergetree_engines(): assert isinstance(table2.config.engine, ReplacingMergeTreeEngine) +def test_olap_table_with_null_engine(): + """Test creating OlapTable with NullEngine.""" + table = OlapTable[SampleEvent]( + "NullEngineTable", + OlapConfig( + engine=NullEngine() + # Note: NullEngine does not support order_by_fields + ) + ) + + assert isinstance(table.config.engine, NullEngine) + def test_engine_conversion_to_dict(): """Test conversion of engine configs to EngineConfigDict.""" # Create a mock table with S3QueueEngine @@ -289,7 +306,12 @@ def test_engine_config_validation(): def test_non_mergetree_engines_reject_unsupported_clauses(): """Test that non-MergeTree engines reject unsupported ORDER BY and SAMPLE BY clauses.""" - from moose_lib.blocks import S3Engine, S3QueueEngine, BufferEngine, DistributedEngine + from moose_lib.blocks import ( + BufferEngine, + DistributedEngine, + S3Engine, + S3QueueEngine, + ) # Test S3Engine DOES support ORDER BY (should not raise) config_s3_with_order_by = OlapConfig( diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 20d90858fb..fde755e678 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15357,7 +15357,7 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-module-utils@2.12.1(@typescript-eslint/parser@8.46.3(eslint@9.39.1(jiti@2.6.1))(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.1(jiti@2.6.1)): + eslint-module-utils@2.12.1(@typescript-eslint/parser@8.46.3(eslint@9.39.1(jiti@2.6.1))(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.1(jiti@2.6.1)))(eslint@9.39.1(jiti@2.6.1)): dependencies: debug: 3.2.7 optionalDependencies: @@ -15408,7 +15408,7 @@ snapshots: doctrine: 2.1.0 eslint: 9.39.1(jiti@2.6.1) eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.12.1(@typescript-eslint/parser@8.46.3(eslint@9.39.1(jiti@2.6.1))(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.1(jiti@2.6.1)) + eslint-module-utils: 2.12.1(@typescript-eslint/parser@8.46.3(eslint@9.39.1(jiti@2.6.1))(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.1(jiti@2.6.1)))(eslint@9.39.1(jiti@2.6.1)) hasown: 2.0.2 is-core-module: 2.16.1 is-glob: 4.0.3 From ef633d460f490ab74762fa3f2a0a2a62da089491 Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 22 Nov 2025 04:29:44 +0100 Subject: [PATCH 2/8] Add null engine in engines_without_partition_by and remove accidendal Tkinter import --- packages/py-moose-lib/moose_lib/dmv2/olap_table.py | 2 +- packages/py-moose-lib/moose_lib/internal.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/py-moose-lib/moose_lib/dmv2/olap_table.py b/packages/py-moose-lib/moose_lib/dmv2/olap_table.py index a8c5734792..39b23f3d2e 100644 --- a/packages/py-moose-lib/moose_lib/dmv2/olap_table.py +++ b/packages/py-moose-lib/moose_lib/dmv2/olap_table.py @@ -204,7 +204,7 @@ def model_post_init(self, __context): # Only S3QueueEngine, BufferEngine, and DistributedEngine don't support PARTITION BY # S3Engine DOES support PARTITION BY - engines_without_partition_by = (S3QueueEngine, BufferEngine, DistributedEngine) + engines_without_partition_by = (NullEngine, S3QueueEngine, BufferEngine, DistributedEngine) if isinstance(self.engine, engines_without_partition_by): engine_name = type(self.engine).__name__ diff --git a/packages/py-moose-lib/moose_lib/internal.py b/packages/py-moose-lib/moose_lib/internal.py index 8d56b56442..b52fd1d753 100644 --- a/packages/py-moose-lib/moose_lib/internal.py +++ b/packages/py-moose-lib/moose_lib/internal.py @@ -8,7 +8,6 @@ """ import json from importlib import import_module -from tkinter import N from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union from pydantic import AliasGenerator, BaseModel, ConfigDict, Field From 5d180ef81a2170851e53df0573d07d9996dc456b Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 22 Nov 2025 04:38:48 +0100 Subject: [PATCH 3/8] Remove import module in moose_lib and implemente null engine in tables_to_python in generate.rs --- apps/framework-cli/src/framework/python/generate.rs | 4 +--- packages/py-moose-lib/moose_lib/internal.py | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/apps/framework-cli/src/framework/python/generate.rs b/apps/framework-cli/src/framework/python/generate.rs index c0c943010d..201f0d5b58 100644 --- a/apps/framework-cli/src/framework/python/generate.rs +++ b/apps/framework-cli/src/framework/python/generate.rs @@ -781,9 +781,7 @@ pub fn tables_to_python(tables: &[Table], life_cycle: Option) -> Stri writeln!(output, " engine=MergeTreeEngine(),").unwrap(); } crate::infrastructure::olap::clickhouse::queries::ClickhouseEngine::Null => { - // No explicit engine for Null in moose_lib, - // so we do not generate an `engine=...` parameter in OlapConfig. - // (If you want a real Null ENGINE later, we can adapt here.) + writeln!(output, " engine=NullEngine(),").unwrap(); } crate::infrastructure::olap::clickhouse::queries::ClickhouseEngine::ReplacingMergeTree { ver, is_deleted } => { diff --git a/packages/py-moose-lib/moose_lib/internal.py b/packages/py-moose-lib/moose_lib/internal.py index b52fd1d753..7b2c687565 100644 --- a/packages/py-moose-lib/moose_lib/internal.py +++ b/packages/py-moose-lib/moose_lib/internal.py @@ -7,7 +7,6 @@ JSON format expected by the Moose infrastructure management system. """ import json -from importlib import import_module from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union from pydantic import AliasGenerator, BaseModel, ConfigDict, Field From 175ac2e95b87f02dd109f706d35b07cbf8041399 Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 22 Nov 2025 04:50:15 +0100 Subject: [PATCH 4/8] fix null key in engine_map dict --- packages/py-moose-lib/moose_lib/internal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/py-moose-lib/moose_lib/internal.py b/packages/py-moose-lib/moose_lib/internal.py index 7b2c687565..9af485e008 100644 --- a/packages/py-moose-lib/moose_lib/internal.py +++ b/packages/py-moose-lib/moose_lib/internal.py @@ -641,7 +641,7 @@ def _convert_engine_to_config_dict(engine: Union[ClickHouseEngines, EngineConfig # Map engine names to specific config classes engine_map = { - "NullEngine": NullConfigDict, + "Null": NullConfigDict, "MergeTree": MergeTreeConfigDict, "ReplacingMergeTree": ReplacingMergeTreeConfigDict, "AggregatingMergeTree": AggregatingMergeTreeConfigDict, From 11517f518941e12eebd53580ecb8ae0db0ec461f Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 22 Nov 2025 14:05:07 +0100 Subject: [PATCH 5/8] Reset import of import_module in internal.py --- packages/py-moose-lib/moose_lib/internal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/py-moose-lib/moose_lib/internal.py b/packages/py-moose-lib/moose_lib/internal.py index 9af485e008..b52fd1d753 100644 --- a/packages/py-moose-lib/moose_lib/internal.py +++ b/packages/py-moose-lib/moose_lib/internal.py @@ -7,6 +7,7 @@ JSON format expected by the Moose infrastructure management system. """ import json +from importlib import import_module from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union from pydantic import AliasGenerator, BaseModel, ConfigDict, Field @@ -641,7 +642,7 @@ def _convert_engine_to_config_dict(engine: Union[ClickHouseEngines, EngineConfig # Map engine names to specific config classes engine_map = { - "Null": NullConfigDict, + "NullEngine": NullConfigDict, "MergeTree": MergeTreeConfigDict, "ReplacingMergeTree": ReplacingMergeTreeConfigDict, "AggregatingMergeTree": AggregatingMergeTreeConfigDict, From 01c19ebaaf8f8a324866c8fd3045df94651685d6 Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 22 Nov 2025 14:22:41 +0100 Subject: [PATCH 6/8] fix null key in engine_map dict --- packages/py-moose-lib/moose_lib/internal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/py-moose-lib/moose_lib/internal.py b/packages/py-moose-lib/moose_lib/internal.py index b52fd1d753..a8a9ecae3f 100644 --- a/packages/py-moose-lib/moose_lib/internal.py +++ b/packages/py-moose-lib/moose_lib/internal.py @@ -642,7 +642,7 @@ def _convert_engine_to_config_dict(engine: Union[ClickHouseEngines, EngineConfig # Map engine names to specific config classes engine_map = { - "NullEngine": NullConfigDict, + "Null": NullConfigDict, "MergeTree": MergeTreeConfigDict, "ReplacingMergeTree": ReplacingMergeTreeConfigDict, "AggregatingMergeTree": AggregatingMergeTreeConfigDict, From 3dcbf7c5a6374a5abdbf05d77d4dd858da07764c Mon Sep 17 00:00:00 2001 From: admin Date: Mon, 24 Nov 2025 10:47:59 +0100 Subject: [PATCH 7/8] Add null engine TS implementation and python/TS automated e2e tests --- .../test/utils/schema-definitions.ts | 24 ++++++++ .../null-engine-example-py/.gitignore | 42 -------------- .../.vscode/extensions.json | 9 --- .../.vscode/settings.json | 17 ------ .../null-engine-example-py/README.md | 48 ---------------- .../null-engine-example-py/app/__init__.py | 0 .../app/apis/__init__.py | 0 .../app/ingest/__init__.py | 0 .../app/ingest/models.py | 26 --------- .../null-engine-example-py/app/main.py | 2 - .../app/views/__init__.py | 0 .../app/workflows/__init__.py | 0 .../null-engine-example-py/moose.config.toml | 56 ------------------- .../null-engine-example-py/requirements.txt | 7 --- .../null-engine-example-py/setup.py | 14 ----- .../template.config.toml | 26 --------- packages/ts-moose-lib/src/blocks/helpers.ts | 1 + packages/ts-moose-lib/src/dmv2/internal.ts | 8 +++ .../ts-moose-lib/src/dmv2/sdk/olapTable.ts | 38 +++++++++++++ .../python-tests/src/ingest/engine_tests.py | 10 ++++ .../src/ingest/engineTests.ts | 6 ++ 21 files changed, 87 insertions(+), 247 deletions(-) delete mode 100644 examples/null-engine-example/null-engine-example-py/.gitignore delete mode 100644 examples/null-engine-example/null-engine-example-py/.vscode/extensions.json delete mode 100644 examples/null-engine-example/null-engine-example-py/.vscode/settings.json delete mode 100644 examples/null-engine-example/null-engine-example-py/README.md delete mode 100644 examples/null-engine-example/null-engine-example-py/app/__init__.py delete mode 100644 examples/null-engine-example/null-engine-example-py/app/apis/__init__.py delete mode 100644 examples/null-engine-example/null-engine-example-py/app/ingest/__init__.py delete mode 100644 examples/null-engine-example/null-engine-example-py/app/ingest/models.py delete mode 100644 examples/null-engine-example/null-engine-example-py/app/main.py delete mode 100644 examples/null-engine-example/null-engine-example-py/app/views/__init__.py delete mode 100644 examples/null-engine-example/null-engine-example-py/app/workflows/__init__.py delete mode 100644 examples/null-engine-example/null-engine-example-py/moose.config.toml delete mode 100644 examples/null-engine-example/null-engine-example-py/requirements.txt delete mode 100644 examples/null-engine-example/null-engine-example-py/setup.py delete mode 100644 examples/null-engine-example/null-engine-example-py/template.config.toml diff --git a/apps/framework-cli-e2e/test/utils/schema-definitions.ts b/apps/framework-cli-e2e/test/utils/schema-definitions.ts index ce622995ee..16efc5984f 100644 --- a/apps/framework-cli-e2e/test/utils/schema-definitions.ts +++ b/apps/framework-cli-e2e/test/utils/schema-definitions.ts @@ -57,6 +57,18 @@ export const TYPESCRIPT_TEST_SCHEMAS: ExpectedTableSchema[] = [ { name: "isDeleted", type: "Bool" }, ], }, + { + tableName: "NullEngineTest", + columns: [ + { name: "id", type: "String" }, + { name: "timestamp", type: /DateTime\('UTC'\)/ }, + { name: "value", type: "Float64" }, + { name: "category", type: "String" }, + { name: "version", type: "Float64" }, + { name: "isDeleted", type: "Bool" }, + ], + engine: "Null", + }, { tableName: "MergeTreeTestExpr", columns: [ @@ -458,6 +470,18 @@ export const PYTHON_TEST_SCHEMAS: ExpectedTableSchema[] = [ { name: "is_deleted", type: "Bool" }, ], }, + { + tableName: "NullEngineTest", + columns: [ + { name: "id", type: "String" }, + { name: "timestamp", type: /DateTime\('UTC'\)/ }, + { name: "value", type: "Int64" }, + { name: "category", type: "String" }, + { name: "version", type: "Int64" }, + { name: "is_deleted", type: "Bool" }, + ], + engine: "Null", + }, { tableName: "MergeTreeTestExpr", columns: [ diff --git a/examples/null-engine-example/null-engine-example-py/.gitignore b/examples/null-engine-example/null-engine-example-py/.gitignore deleted file mode 100644 index 74052de968..0000000000 --- a/examples/null-engine-example/null-engine-example-py/.gitignore +++ /dev/null @@ -1,42 +0,0 @@ -.moose -__pycache__ -*.pyc -*.pyo -*.pyd -.Python -env -.venv -venv -ENV -env.bak -.spyderproject -.ropeproject -.idea -*.ipynb_checkpoints -.pytest_cache -.mypy_cache -.hypothesis -.coverage -cover -*.cover -.DS_Store -.cache -*.so -*.egg -*.egg-info -dist -build -develop-eggs -downloads -eggs -lib -lib64 -parts -sdist -var -wheels -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - diff --git a/examples/null-engine-example/null-engine-example-py/.vscode/extensions.json b/examples/null-engine-example/null-engine-example-py/.vscode/extensions.json deleted file mode 100644 index dbc86167a6..0000000000 --- a/examples/null-engine-example/null-engine-example-py/.vscode/extensions.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "recommendations": [ - "frigus02.vscode-sql-tagged-template-literals-syntax-only", - "mtxr.sqltools", - "ultram4rine.sqltools-clickhouse-driver", - "jeppeandersen.vscode-kafka", - "rangav.vscode-thunder-client" - ] -} diff --git a/examples/null-engine-example/null-engine-example-py/.vscode/settings.json b/examples/null-engine-example/null-engine-example-py/.vscode/settings.json deleted file mode 100644 index 48a4637229..0000000000 --- a/examples/null-engine-example/null-engine-example-py/.vscode/settings.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "sqltools.connections": [ - { - "server": "localhost", - "port": 18123, - "useHTTPS": false, - "database": "local", - "username": "panda", - "enableTls": false, - "password": "pandapass", - "driver": "ClickHouse", - "name": "moose clickhouse" - } - ], - "python.analysis.extraPaths": [".moose/versions"], - "python.analysis.typeCheckingMode": "basic" -} diff --git a/examples/null-engine-example/null-engine-example-py/README.md b/examples/null-engine-example/null-engine-example-py/README.md deleted file mode 100644 index a5de381060..0000000000 --- a/examples/null-engine-example/null-engine-example-py/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# Template: Python - -This is a Python-based Moose template that provides a foundation for building data-intensive applications using Python. - -[![PyPI Version](https://img.shields.io/pypi/v/moose-cli?logo=python)](https://pypi.org/project/moose-cli/) -[![Moose Community](https://img.shields.io/badge/slack-moose_community-purple.svg?logo=slack)](https://join.slack.com/t/moose-community/shared_invite/zt-2fjh5n3wz-cnOmM9Xe9DYAgQrNu8xKxg) -[![Docs](https://img.shields.io/badge/quick_start-docs-blue.svg)](https://docs.fiveonefour.com/moose/getting-started/quickstart) -[![MIT license](https://img.shields.io/badge/license-MIT-yellow.svg)](LICENSE) - -## Getting Started - -### Prerequisites - -* [Docker Desktop](https://www.docker.com/products/docker-desktop/) -* [Python](https://www.python.org/downloads/) (version 3.8+) -* [An Anthropic API Key](https://docs.anthropic.com/en/api/getting-started) -* [Cursor](https://www.cursor.com/) or [Claude Desktop](https://claude.ai/download) - -### Installation - -1. Install Moose CLI: `pip install moose-cli` -2. Create project: `moose init python` -3. Install dependencies: `cd && pip install -r requirements.txt` -4. Run Moose: `moose dev` - -You are ready to go! You can start editing the app by modifying primitives in the `app` subdirectory. - -## Learn More - -To learn more about Moose, take a look at the following resources: - -- [Moose Documentation](https://docs.fiveonefour.com/moose) - learn about Moose. -- [Sloan Documentation](https://docs.fiveonefour.com/sloan) - learn about Sloan, the MCP interface for data engineering. - -## Community - -You can join the Moose community [on Slack](https://join.slack.com/t/moose-community/shared_invite/zt-2fjh5n3wz-cnOmM9Xe9DYAgQrNu8xKxg). Check out the [MooseStack repo on GitHub](https://github.com/514-labs/moosestack). - -## Deploy on Boreal - -The easiest way to deploy your MooseStack Applications is to use [Boreal](https://www.fiveonefour.com/boreal) from 514 Labs, the creators of Moose. - -[Sign up](https://www.boreal.cloud/sign-up). - -## License - -This template is MIT licensed. - diff --git a/examples/null-engine-example/null-engine-example-py/app/__init__.py b/examples/null-engine-example/null-engine-example-py/app/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/examples/null-engine-example/null-engine-example-py/app/apis/__init__.py b/examples/null-engine-example/null-engine-example-py/app/apis/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/examples/null-engine-example/null-engine-example-py/app/ingest/__init__.py b/examples/null-engine-example/null-engine-example-py/app/ingest/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/examples/null-engine-example/null-engine-example-py/app/ingest/models.py b/examples/null-engine-example/null-engine-example-py/app/ingest/models.py deleted file mode 100644 index e90be2cda3..0000000000 --- a/examples/null-engine-example/null-engine-example-py/app/ingest/models.py +++ /dev/null @@ -1,26 +0,0 @@ -# This file was auto-generated by the framework. You can add data models or change the existing ones - -from datetime import datetime - -from moose_lib import ( - Key, - OlapConfig, -) -from moose_lib.blocks import NullEngine -from moose_lib.dmv2 import ( - OlapTable, -) -from pydantic import BaseModel - - -class Items(BaseModel): - order_id: Key[int] - id: Key[int] - updated_at: datetime - -items_null = OlapTable[Items]( - "items_null", - OlapConfig( - engine=NullEngine(), - ), -) \ No newline at end of file diff --git a/examples/null-engine-example/null-engine-example-py/app/main.py b/examples/null-engine-example/null-engine-example-py/app/main.py deleted file mode 100644 index 2dfc8321a8..0000000000 --- a/examples/null-engine-example/null-engine-example-py/app/main.py +++ /dev/null @@ -1,2 +0,0 @@ -# This file was auto-generated by the framework. You can add data models or change the existing ones -from app.ingest import models diff --git a/examples/null-engine-example/null-engine-example-py/app/views/__init__.py b/examples/null-engine-example/null-engine-example-py/app/views/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/examples/null-engine-example/null-engine-example-py/app/workflows/__init__.py b/examples/null-engine-example/null-engine-example-py/app/workflows/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/examples/null-engine-example/null-engine-example-py/moose.config.toml b/examples/null-engine-example/null-engine-example-py/moose.config.toml deleted file mode 100644 index fe19fc4af6..0000000000 --- a/examples/null-engine-example/null-engine-example-py/moose.config.toml +++ /dev/null @@ -1,56 +0,0 @@ -language = "Python" - -[redpanda_config] -broker = "localhost:19092" -message_timeout_ms = 1000 -retention_ms = 30000 -replication_factor = 1 - -[clickhouse_config] -db_name = "local" -user = "panda" -password = "pandapass" -use_ssl = false -host = "localhost" -host_port = 18123 -native_port = 9000 - -[http_server_config] -host = "localhost" -port = 4000 -management_port = 5001 - -[redis_config] -url = "redis://127.0.0.1:6379" -key_prefix = "MS" - -[git_config] -main_branch_name = "main" - -[temporal_config] -db_user = "temporal" -db_password = "temporal" -db_port = 5432 -temporal_host = "localhost" -temporal_port = 7233 -temporal_version = "1.22.3" -admin_tools_version = "1.22.3" -ui_version = "2.21.3" -ui_port = 8080 -ui_cors_origins = "http://localhost:3000" -config_path = "config/dynamicconfig/development-sql.yaml" -postgresql_version = "13" -client_cert = "" -client_key = "" -ca_cert = "" -api_key = "" - -[supported_old_versions] - -[authentication] - -[features] -streaming_engine = true -workflows = true -data_model_v2 = true -apis = true diff --git a/examples/null-engine-example/null-engine-example-py/requirements.txt b/examples/null-engine-example/null-engine-example-py/requirements.txt deleted file mode 100644 index 870d85e724..0000000000 --- a/examples/null-engine-example/null-engine-example-py/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -kafka-python-ng==2.2.2 -clickhouse-connect==0.7.16 -requests==2.32.4 -#moose-cli -#moose-lib -faker -sqlglot[rs]>=27.16.3 \ No newline at end of file diff --git a/examples/null-engine-example/null-engine-example-py/setup.py b/examples/null-engine-example/null-engine-example-py/setup.py deleted file mode 100644 index cdbfe7692b..0000000000 --- a/examples/null-engine-example/null-engine-example-py/setup.py +++ /dev/null @@ -1,14 +0,0 @@ - -import os - -from setuptools import setup - -requirements_path = os.path.join(os.path.dirname(__file__), "requirements.txt") -with open(requirements_path, "r") as f: - requirements = f.read().splitlines() - -setup( - name='null-engine-example-py', - version='0.0', - install_requires=requirements, -) diff --git a/examples/null-engine-example/null-engine-example-py/template.config.toml b/examples/null-engine-example/null-engine-example-py/template.config.toml deleted file mode 100644 index 494d291588..0000000000 --- a/examples/null-engine-example/null-engine-example-py/template.config.toml +++ /dev/null @@ -1,26 +0,0 @@ -language = "python" # Must be typescript or python -description = "Default Python project, seeded with foobar example components." -post_install_print = """ -Deploy on Boreal - -The easiest way to deploy your MooseStack Applications is to use Boreal -from the creators of MooseStack. - -https://boreal.cloud - ---------------------------------------------------------- - -📂 Go to your project directory: - $ cd {project_dir} - -🥄 Create a virtual environment (optional, recommended): - $ python3 -m venv .venv - $ source .venv/bin/activate - -📦 Install Dependencies: - $ pip install -r ./requirements.txt - -🛠️ Start Moose Server: - $ moose dev -""" -default_sloan_telemetry="standard" diff --git a/packages/ts-moose-lib/src/blocks/helpers.ts b/packages/ts-moose-lib/src/blocks/helpers.ts index cec7080544..2b91be3571 100644 --- a/packages/ts-moose-lib/src/blocks/helpers.ts +++ b/packages/ts-moose-lib/src/blocks/helpers.ts @@ -35,6 +35,7 @@ export interface Blocks { } export enum ClickHouseEngines { + Null = "Null", MergeTree = "MergeTree", ReplacingMergeTree = "ReplacingMergeTree", SummingMergeTree = "SummingMergeTree", diff --git a/packages/ts-moose-lib/src/dmv2/internal.ts b/packages/ts-moose-lib/src/dmv2/internal.ts index 060adc12fc..c3f12b9d36 100644 --- a/packages/ts-moose-lib/src/dmv2/internal.ts +++ b/packages/ts-moose-lib/src/dmv2/internal.ts @@ -65,6 +65,10 @@ const defaultRetentionPeriod = 60 * 60 * 24 * 7; /** * Engine-specific configuration types using discriminated union pattern */ +interface NullEngineConfig { + engine: "Null"; +} + interface MergeTreeEngineConfig { engine: "MergeTree"; } @@ -161,6 +165,7 @@ interface DistributedEngineConfig { * Union type for all supported engine configurations */ type EngineConfig = + | NullEngineConfig | MergeTreeEngineConfig | ReplacingMergeTreeEngineConfig | AggregatingMergeTreeEngineConfig @@ -406,6 +411,9 @@ function convertBasicEngineConfig( config: OlapConfig, ): EngineConfig | undefined { switch (engine) { + case ClickHouseEngines.Null: + return { engine: "Null" }; + case ClickHouseEngines.MergeTree: return { engine: "MergeTree" }; diff --git a/packages/ts-moose-lib/src/dmv2/sdk/olapTable.ts b/packages/ts-moose-lib/src/dmv2/sdk/olapTable.ts index 2ed3871038..6b0516db9d 100644 --- a/packages/ts-moose-lib/src/dmv2/sdk/olapTable.ts +++ b/packages/ts-moose-lib/src/dmv2/sdk/olapTable.ts @@ -235,6 +235,16 @@ export type BaseOlapConfig = ( cluster?: string; }; +/** + * Configuration for Null engine - discards all data and does not support ORDER BY/PARTITION BY/SAMPLE BY + */ +export type NullConfig = Omit< + BaseOlapConfig, + "orderByFields" | "orderByExpression" | "partitionBy" | "sampleByExpression" +> & { + engine: ClickHouseEngines.Null; +}; + /** * Configuration for MergeTree engine * @template T The data type of the records stored in the table. @@ -452,6 +462,7 @@ export type DistributedConfig = Omit< export type LegacyOlapConfig = BaseOlapConfig; type EngineConfig = + | NullConfig | MergeTreeConfig | ReplacingMergeTreeConfig | AggregatingMergeTreeConfig @@ -534,6 +545,33 @@ export class OlapTable extends TypedBase> { ); } + const isNullEngine = + "engine" in resolvedConfig && + resolvedConfig.engine === ClickHouseEngines.Null; + if (isNullEngine) { + if (hasFields || hasExpr) { + throw new Error( + `OlapTable ${name}: Null engine does not support ORDER BY clauses.`, + ); + } + if ( + "partitionBy" in resolvedConfig && + resolvedConfig.partitionBy !== undefined + ) { + throw new Error( + `OlapTable ${name}: Null engine does not support PARTITION BY clauses.`, + ); + } + if ( + "sampleByExpression" in resolvedConfig && + resolvedConfig.sampleByExpression !== undefined + ) { + throw new Error( + `OlapTable ${name}: Null engine does not support SAMPLE BY clauses.`, + ); + } + } + // Validate cluster and explicit replication params are not both specified const hasCluster = typeof (resolvedConfig as any).cluster === "string"; const hasKeeperPath = diff --git a/templates/python-tests/src/ingest/engine_tests.py b/templates/python-tests/src/ingest/engine_tests.py index a93cce64ee..d3f9a830b1 100644 --- a/templates/python-tests/src/ingest/engine_tests.py +++ b/templates/python-tests/src/ingest/engine_tests.py @@ -11,6 +11,7 @@ ReplicatedReplacingMergeTreeEngine, ReplicatedAggregatingMergeTreeEngine, ReplicatedSummingMergeTreeEngine, + NullEngine, BufferEngine, # S3QueueEngine - requires S3 configuration, tested separately ) @@ -99,6 +100,14 @@ class EngineTestDataSample(BaseModel): ) ) +# Test Null engine (schema-only, discards writes) +null_engine_table = OlapTable[EngineTestData]( + "NullEngineTest", + OlapConfig( + engine=NullEngine(), + ) +) + # Test MergeTree with order_by_expression (equivalent to fields) merge_tree_table_expr = OlapTable[EngineTestData]( "MergeTreeTestExpr", @@ -274,6 +283,7 @@ class EngineTestDataSample(BaseModel): # can be properly instantiated and don't throw errors during table creation all_engine_test_tables = [ merge_tree_table, + null_engine_table, merge_tree_table_expr, replacing_merge_tree_basic_table, replacing_merge_tree_version_table, diff --git a/templates/typescript-tests/src/ingest/engineTests.ts b/templates/typescript-tests/src/ingest/engineTests.ts index 0782b441c2..446a52d2e5 100644 --- a/templates/typescript-tests/src/ingest/engineTests.ts +++ b/templates/typescript-tests/src/ingest/engineTests.ts @@ -64,6 +64,11 @@ export const MergeTreeTable = new OlapTable("MergeTreeTest", { orderByFields: ["id", "timestamp"], }); +// Test Null engine (schema-only, discards writes) +export const NullEngineTable = new OlapTable("NullEngineTest", { + engine: ClickHouseEngines.Null, +}); + // Test MergeTree with orderByExpression (equivalent to fields) export const MergeTreeTableExpr = new OlapTable( "MergeTreeTestExpr", @@ -247,6 +252,7 @@ export const BufferTable = new OlapTable("BufferTest", { */ export const allEngineTestTables = [ MergeTreeTable, + NullEngineTable, MergeTreeTableExpr, ReplacingMergeTreeBasicTable, ReplacingMergeTreeVersionTable, From 88ef238babda240753afe86eddc982dc0a3b3d0e Mon Sep 17 00:00:00 2001 From: admin Date: Tue, 25 Nov 2025 09:40:48 +0100 Subject: [PATCH 8/8] avoid truncate on olap change of null engines tables defined as materialized view target table --- .../olap/clickhouse/diff_strategy.rs | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/apps/framework-cli/src/infrastructure/olap/clickhouse/diff_strategy.rs b/apps/framework-cli/src/infrastructure/olap/clickhouse/diff_strategy.rs index 46168bcc69..edb9741464 100644 --- a/apps/framework-cli/src/infrastructure/olap/clickhouse/diff_strategy.rs +++ b/apps/framework-cli/src/infrastructure/olap/clickhouse/diff_strategy.rs @@ -393,6 +393,16 @@ impl ClickHouseTableDiffStrategy { // Skip population in production (user must handle manually) // Only populate in dev for new MVs with non-S3Queue sources if is_new && !has_s3queue_source && !is_production { + // If the target table uses the Null engine, skip truncation because there's nothing to truncate + let target_is_null_engine = tables.values().any(|table| { + matches!(table.engine, ClickhouseEngine::Null) + && Self::table_matches_mv_target( + table, + &mv_stmt.target_table, + &mv_stmt.target_database, + ) + }); + log::info!( "Adding population operation for materialized view '{}'", sql_resource.name @@ -408,7 +418,7 @@ impl ClickHouseTableDiffStrategy { .into_iter() .map(|t| t.qualified_name()) .collect(), - should_truncate: true, + should_truncate: !target_is_null_engine, }); } @@ -417,6 +427,30 @@ impl ClickHouseTableDiffStrategy { } } } + + /// Helper: does the given table correspond to the MV target (by name and optional database)? + fn table_matches_mv_target( + table: &Table, + target_table: &str, + target_database: &Option, + ) -> bool { + if table.name == target_table { + return true; + } + + match (&table.database, target_database) { + (Some(table_db), Some(target_db)) => { + table_db == target_db && table.name == target_table + || format!("{table_db}.{}", table.name) == format!("{target_db}.{target_table}") + } + (Some(table_db), None) => table_db.is_empty() && table.name == target_table, + (None, Some(target_db)) => { + format!("{target_db}.{target_table}") == table.name + || format!("{target_db}.{}", table.name) == target_table + } + _ => false, + } + } } impl TableDiffStrategy for ClickHouseTableDiffStrategy {