allow ignoring partition by diff (#2946)

<!-- CURSOR_SUMMARY -->
> [!NOTE]
> Adds partition-by-aware table diffs and normalization-based ignoring
(TTL/partition_by) wired through planning, admin/remote plan, and
migration; updates APIs/types and tests accordingly.
> 
> - **Core infra map/diffing**:
> - Add `PartitionByChange`; include in `TableChange::Updated` and
`TableDiffStrategy::diff_table_update`.
> - Extend simple/table diffs to compute and carry
`partition_by_change`.
> - `diff_with_table_strategy(...)` now accepts `ignore_ops` and
normalizes tables before comparison.
> - **ClickHouse integration**:
> - Add `IgnorableOperation::ModifyPartitionBy` and
`normalize_table_for_diff` to strip ignored fields (table TTL, column
TTL, partition_by).
> - Update `diff_strategy` to take `partition_by_change` and trigger
drop+create on partition-by changes; add tests.
> - **Planning/CLI**:
> - Pass `ignore_ops` from project config (prod) into diffs in `plan.rs`
and admin `/admin/plan`.
> - Client-side `calculate_plan_diff_local(...)` accepts `ignore_ops`;
remote plan/migration paths updated accordingly.
> - Drift detection (`migrate.rs`) strips metadata and ignored fields
via normalization.
> - Remove prior operation-level ignore filtering from `MigrationPlan`
(and related tests).
> - **Webserver**:
> - Use `clickhouse::{create_client, check_ready}`; minor
health/ready/reality-check refactors.
> - **Tests**:
> - Update signatures/usages across tests; add tests for normalization
and partition-by behavior.
> 
> <sup>Written by [Cursor
Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit
ff98322. This will update automatically
on new commits. Configure
[here](https://cursor.com/dashboard?tab=bugbot).</sup>
<!-- /CURSOR_SUMMARY -->

Author: phiSgr

apps/framework-cli/src/cli/local_webserver.rs

@@ -39,11 +39,11 @@ use crate::framework::versions::Version;
 use crate::metrics::Metrics;
 use crate::utilities::auth::{get_claims, validate_jwt};
 
-use crate::infrastructure::stream::kafka;
-use crate::infrastructure::stream::kafka::models::ConfiguredProducer;
-
 use crate::framework::core::infrastructure::topic::{KafkaSchemaKind, SchemaRegistryReference};
 use crate::framework::typescript::bin::CliMessage;
+use crate::infrastructure::olap::clickhouse;
+use crate::infrastructure::stream::kafka;
+use crate::infrastructure::stream::kafka::models::ConfiguredProducer;
 use crate::project::{JwtConfig, Project};
 use crate::utilities::docker::DockerClient;
 use bytes::Buf;
@@ -850,9 +850,7 @@ async fn health_route(
 
     // Check ClickHouse connectivity only if OLAP is enabled
     if project.features.olap {
-        let olap_client = crate::infrastructure::olap::clickhouse::create_client(
-            project.clickhouse_config.clone(),
-        );
+        let olap_client = clickhouse::create_client(project.clickhouse_config.clone());
         match olap_client.client.query("SELECT 1").execute().await {
             Ok(_) => healthy.push("ClickHouse"),
             Err(e) => {
@@ -910,10 +908,8 @@ async fn ready_route(
 
     // ClickHouse: run a small query using the configured client (ensures HTTP pool is ready)
     if project.features.olap {
-        let ch = crate::infrastructure::olap::clickhouse::create_client(
-            project.clickhouse_config.clone(),
-        );
-        match crate::infrastructure::olap::clickhouse::check_ready(&ch).await {
+        let ch = clickhouse::create_client(project.clickhouse_config.clone());
+        match clickhouse::check_ready(&ch).await {
             Ok(_) => healthy.push("ClickHouse"),
             Err(e) => {
                 warn!("Ready check: ClickHouse not ready: {}", e);
@@ -999,9 +995,7 @@ async fn admin_reality_check_route(
     // Perform reality check (storage is guaranteed to be enabled at this point)
     let discrepancies = {
         // Create OLAP client and reality checker
-        let olap_client = crate::infrastructure::olap::clickhouse::create_client(
-            project.clickhouse_config.clone(),
-        );
+        let olap_client = clickhouse::create_client(project.clickhouse_config.clone());
         let reality_checker =
             crate::framework::core::infra_reality_checker::InfraRealityChecker::new(olap_client);
 
@@ -3119,8 +3113,7 @@ async fn admin_integrate_changes_route(
     }
 
     // Get reality check
-    let olap_client =
-        crate::infrastructure::olap::clickhouse::create_client(project.clickhouse_config.clone());
+    let olap_client = clickhouse::create_client(project.clickhouse_config.clone());
     let reality_checker =
         crate::framework::core::infra_reality_checker::InfraRealityChecker::new(olap_client);
 
@@ -3352,13 +3345,18 @@ async fn admin_plan_route(
 
     // Calculate the changes between the submitted infrastructure map and the current one
     // Use ClickHouse-specific strategy for table diffing
-    let clickhouse_strategy =
-        crate::infrastructure::olap::clickhouse::diff_strategy::ClickHouseTableDiffStrategy;
+    let clickhouse_strategy = clickhouse::diff_strategy::ClickHouseTableDiffStrategy;
+    let ignore_ops: &[clickhouse::IgnorableOperation] = if project.is_production {
+        &project.migration_config.ignore_operations
+    } else {
+        &[]
+    };
     let changes = current_infra_map.diff_with_table_strategy(
         &plan_request.infra_map,
         &clickhouse_strategy,
         true,
         project.is_production,
+        ignore_ops,
     );
 
     // Prepare the response

apps/framework-cli/src/cli/routines.rs

@@ -862,18 +862,20 @@ pub(crate) async fn get_remote_inframap_protobuf(
 /// # Arguments
 /// * `current_map` - The current infrastructure map (from server)
 /// * `target_map` - The target infrastructure map (from local project)
+/// * `ignore_ops` - Operations to ignore during comparison (e.g., ModifyPartitionBy)
 ///
 /// # Returns
 /// * `InfraChanges` - The calculated changes needed to go from current to target
 fn calculate_plan_diff_local(
     current_map: &InfrastructureMap,
     target_map: &InfrastructureMap,
+    ignore_ops: &[crate::infrastructure::olap::clickhouse::IgnorableOperation],
 ) -> crate::framework::core::infrastructure_map::InfraChanges {
     use crate::infrastructure::olap::clickhouse::diff_strategy::ClickHouseTableDiffStrategy;
 
     let clickhouse_strategy = ClickHouseTableDiffStrategy;
     // planning about action on prod env, respect_life_cycle is true
-    current_map.diff_with_table_strategy(target_map, &clickhouse_strategy, true, true)
+    current_map.diff_with_table_strategy(target_map, &clickhouse_strategy, true, true, ignore_ops)
 }
 
 /// Legacy implementation of remote_plan using the existing /admin/plan endpoint
@@ -1056,7 +1058,11 @@ pub async fn remote_plan(
     };
 
     // Calculate and display changes
-    let changes = calculate_plan_diff_local(&remote_infra_map, &local_infra_map);
+    let changes = calculate_plan_diff_local(
+        &remote_infra_map,
+        &local_infra_map,
+        &project.migration_config.ignore_operations,
+    );
 
     display::show_message_wrapper(
         MessageType::Success,
@@ -1149,7 +1155,11 @@ pub async fn remote_gen_migration(
         }
     };
 
-    let changes = calculate_plan_diff_local(&remote_infra_map, &local_infra_map);
+    let changes = calculate_plan_diff_local(
+        &remote_infra_map,
+        &local_infra_map,
+        &project.migration_config.ignore_operations,
+    );
 
     display::show_message_wrapper(
         MessageType::Success,
@@ -1159,12 +1169,9 @@ pub async fn remote_gen_migration(
         },
     );
 
-    let mut db_migration =
+    let db_migration =
         MigrationPlan::from_infra_plan(&changes, &project.clickhouse_config.db_name)?;
 
-    // Filter out ignored operations based on project config
-    db_migration.filter_ignored_operations(&project.migration_config.ignore_operations);
-
     Ok(MigrationPlanWithBeforeAfter {
         remote_state: remote_infra_map,
         local_infra_map,

apps/framework-cli/src/cli/routines/migrate.rs

@@ -100,8 +100,9 @@ fn strip_metadata_and_ignored_fields(
             let mut table = table.clone();
             table.metadata = None;
             // Also strip ignored fields
-            let table =
-                crate::framework::core::migration_plan::strip_ignored_fields(&table, ignore_ops);
+            let table = crate::infrastructure::olap::clickhouse::normalize_table_for_diff(
+                &table, ignore_ops,
+            );
             (name.clone(), table)
         })
         .collect()