From 50eb1aebbfd718b75b2878ff05dfee12f490d875 Mon Sep 17 00:00:00 2001 From: Ilya Dvorkin Date: Fri, 4 Jul 2025 14:55:20 +0300 Subject: [PATCH 1/7] Add new commands for sorting and separating YAML files with CLI integration --- Cargo.lock | 2 +- src/cli.rs | 148 +++++++++ src/commands/legacy.rs | 377 +++++++++++++++++++++++ src/commands/mod.rs | 3 + src/commands/separate.rs | 218 ++++++++++++++ src/commands/sort.rs | 87 ++++++ src/lib.rs | 9 +- src/main.rs | 630 ++------------------------------------- src/utils.rs | 103 +++++++ 9 files changed, 975 insertions(+), 602 deletions(-) create mode 100644 src/cli.rs create mode 100644 src/commands/legacy.rs create mode 100644 src/commands/mod.rs create mode 100644 src/commands/separate.rs create mode 100644 src/commands/sort.rs create mode 100644 src/utils.rs diff --git a/Cargo.lock b/Cargo.lock index a9026ed..4e9772e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -486,7 +486,7 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "yabe-gitops" -version = "0.1.9" +version = "0.1.10" dependencies = [ "clap", "env_logger", diff --git a/src/cli.rs b/src/cli.rs new file mode 100644 index 0000000..3281cdc --- /dev/null +++ b/src/cli.rs @@ -0,0 +1,148 @@ +use clap::{Parser, Subcommand}; +use serde::Deserialize; + +/// Command-line arguments +#[derive(Parser)] +#[command(author, version, about = "YAML diff and merge tool for GitOps workflows", long_about = "A tool for diffing, merging, and organizing YAML files for GitOps workflows. Supports both individual file paths and glob patterns.")] +pub struct Args { + /// Configuration file + #[arg(long = "config", value_name = "CONFIG_FILE")] + pub config: Option, + + /// Enable debug logging + #[arg(long = "debug")] + pub debug: bool, + + #[command(subcommand)] + pub command: Option, + + /// Legacy mode: Helm chart values file + #[arg(short = 'r', long = "read-base", value_name = "READ_BASE")] + pub read_only_base: Option, + + /// Legacy mode: Base YAML file to merge with input files + #[arg(short = 'b', long = "base", value_name = "WRITE_BASE")] + pub base: Option, + + /// Legacy mode: Input YAML files (optional if path patterns are provided) + pub input_files: Vec, + + /// Legacy mode: Path patterns to load YAML files (e.g., "*.yaml") + #[arg(short = 'p', long = "path-pattern", value_name = "PATH_PATTERN")] + pub path_patterns: Vec, + + /// Legacy mode: Modify the original input files with diffs + #[arg(short = 'i', long = "in-place")] + pub inplace: bool, + + /// Legacy mode: Output folder + #[arg(short = 'o', long = "out", default_value = "./out")] + pub out_folder: String, + + /// Legacy mode: Quorum percentage (0-100) + #[arg(short = 'q', long = "quorum", default_value_t = 51)] + pub quorum: u8, + + /// Legacy mode: Base file output path + #[arg(long = "base-out-path", default_value = "./base.yaml")] + pub base_out_path: String, + + /// Legacy mode: Sort configuration file path + #[arg(long = "sort-config-path", default_value = "./sort-config.yaml")] + pub sort_config_path: String, + + /// Legacy mode: Sort only mode - only sort files without diffing + #[arg(long = "sort-only")] + pub sort_only: bool, + + /// Legacy mode: Exclude patterns to skip files (e.g., "*.terraform.yaml") + #[arg(long = "exclude", value_name = "EXCLUDE_PATTERN")] + pub exclude_patterns: Vec, +} + +#[derive(Subcommand)] +pub enum Commands { + /// Sort YAML files based on configuration + Sort { + /// Input YAML files (optional if path patterns are provided) + input_files: Vec, + + /// Path patterns to load YAML files (e.g., "*.yaml") + #[arg(short = 'p', long = "path-pattern", value_name = "PATH_PATTERN")] + path_patterns: Vec, + + /// Sort configuration file path + #[arg(long = "sort-config", default_value = "./sort-config.yaml")] + sort_config_path: String, + + /// Modify the original input files with sorted content + #[arg(short = 'i', long = "in-place")] + inplace: bool, + + /// Output folder + #[arg(short = 'o', long = "out", default_value = "./out")] + out_folder: String, + + /// Exclude patterns to skip files (e.g., "*.terraform.yaml") + #[arg(long = "exclude", value_name = "EXCLUDE_PATTERN")] + exclude_patterns: Vec, + }, + /// Separate common base from YAML files (diff/rebalance) + Separate { + /// Input YAML files (optional if path patterns are provided) + input_files: Vec, + + /// Path patterns to load YAML files (e.g., "*.yaml") + #[arg(short = 'p', long = "path-pattern", value_name = "PATH_PATTERN")] + path_patterns: Vec, + + /// Helm chart values file + #[arg(short = 'r', long = "read-base", value_name = "READ_BASE")] + read_only_base: Option, + + /// Base YAML file to merge with input files + #[arg(short = 'b', long = "base", value_name = "WRITE_BASE")] + base: Option, + + /// Quorum percentage (0-100) + #[arg(short = 'q', long = "quorum", default_value_t = 51)] + quorum: u8, + + /// Base file output path + #[arg(long = "base-out", default_value = "./base.yaml")] + base_out_path: String, + + /// Sort configuration file path + #[arg(long = "sort-config", default_value = "./sort-config.yaml")] + sort_config_path: String, + + /// Modify the original input files with diffs + #[arg(short = 'i', long = "in-place")] + inplace: bool, + + /// Output folder + #[arg(short = 'o', long = "out", default_value = "./out")] + out_folder: String, + + /// Exclude patterns to skip files (e.g., "*.terraform.yaml") + #[arg(long = "exclude", value_name = "EXCLUDE_PATTERN")] + exclude_patterns: Vec, + }, +} + +#[derive(Deserialize)] +pub struct Config { + pub read_only_base: Option, + pub base: Option, + pub input_files: Option>, + pub path_patterns: Option>, + pub path_pattern: Option, + pub inplace: Option, + pub out_folder: Option, + pub debug: Option, + pub quorum: Option, + pub base_out_path: Option, + pub sort_config_path: Option, + pub sort_only: Option, + pub exclude_patterns: Option>, +} \ No newline at end of file diff --git a/src/commands/legacy.rs b/src/commands/legacy.rs new file mode 100644 index 0000000..ca7e4c2 --- /dev/null +++ b/src/commands/legacy.rs @@ -0,0 +1,377 @@ +use std::borrow::Cow; +use std::error::Error; +use std::fs; +use std::path::Path; +use log::{info, warn}; +use yaml_rust2::{Yaml, YamlEmitter, YamlLoader}; +use crate::diff::{compute_diff, diff_and_common_multiple}; +use crate::merge::merge_yaml; +use crate::sorter::sort_yaml; +use serde_yml; +use crate::cli::{Args, Config}; +use crate::utils::{expand_and_filter_files, ensure_output_dir}; + +pub fn sort_only_workflow(args: &Args) -> Result<(), Box> { + info!("Running in sort-only mode"); + + let expanded_input_files = expand_and_filter_files( + args.input_files.clone(), + &args.path_patterns, + &args.exclude_patterns, + )?; + + // Validate that we have files to process + if expanded_input_files.is_empty() { + eprintln!("Error: No input files found for sort-only mode. Please specify either input files or path patterns."); + std::process::exit(1); + } + + // Read sort configuration + let sort_config = if !args.sort_config_path.is_empty() && Path::new(&args.sort_config_path).exists() { + info!("Reading sort configuration file: {}", args.sort_config_path); + let content = fs::read_to_string(&args.sort_config_path)?; + YamlLoader::load_from_str(&content)?.into_iter().next().unwrap_or(Yaml::Null) + } else { + eprintln!("Error: Sort configuration file is required for sort-only mode but not found: {}", args.sort_config_path); + std::process::exit(1); + }; + + if sort_config == Yaml::Null { + eprintln!("Error: Sort configuration file is empty or invalid: {}", args.sort_config_path); + std::process::exit(1); + } + + // Ensure output directory exists if not in-place mode + if !args.inplace { + ensure_output_dir(&args.out_folder)?; + } + + // Process each input file + for filename in &expanded_input_files { + info!("Sorting file: {}", filename); + + // Read and parse the YAML file + let content = fs::read_to_string(filename)?; + if let Some(doc) = YamlLoader::load_from_str(&content)?.into_iter().next() { + // Sort the YAML document + let sorted_yaml = sort_yaml(&doc, &sort_config); + + // Convert to string + let mut out_str = String::new(); + { + let mut emitter = YamlEmitter::new(&mut out_str); + emitter.dump(&sorted_yaml)?; + } + out_str = out_str.trim_start_matches("---\n").to_string(); + out_str.push('\n'); + + // Write the sorted content + if args.inplace { + info!("Writing sorted content back to: {}", filename); + fs::write(filename, out_str)?; + } else { + let input_path = Path::new(filename); + let file_stem = input_path + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("sorted"); + let sorted_filename = format!("{}/{}_sorted.yaml", args.out_folder, file_stem); + info!("Writing sorted content to: {}", sorted_filename); + fs::write(&sorted_filename, out_str)?; + } + } else { + warn!("No YAML documents found in {}", filename); + } + } + + info!("Sort-only workflow completed successfully."); + Ok(()) +} + +pub fn run_legacy_main(mut args: Args) -> Result<(), Box> { + if let Some(config_path) = args.config.as_ref() { + // Read the configuration file + let config_content = fs::read_to_string(config_path)?; + let config: Config = serde_yml::from_str(&config_content)?; + + // Override args with config values if they are not provided via command-line + if args.read_only_base.is_none() { + args.read_only_base = config.read_only_base; + } + + if args.base.is_none() { + args.base = config.base; + } + + if args.input_files.is_empty() { + if let Some(input_files) = config.input_files { + args.input_files = input_files; + } + } + + if args.path_patterns.is_empty() { + if let Some(path_patterns) = config.path_patterns { + args.path_patterns = path_patterns; + } else if let Some(path_pattern) = config.path_pattern.as_ref() { + args.path_patterns.push(path_pattern.clone()); + } + } + + if !args.inplace { + if let Some(inplace) = config.inplace { + args.inplace = inplace; + } + } + + if args.out_folder == "./out" { + if let Some(out_folder) = config.out_folder { + args.out_folder = out_folder; + } + } + + if !args.debug { + if let Some(debug) = config.debug { + args.debug = debug; + } + } + + if args.quorum == 51 { + if let Some(quorum) = config.quorum { + args.quorum = quorum; + } + } + + if args.base_out_path == "./base.yaml" { + if let Some(base_out_path) = config.base_out_path { + args.base_out_path = base_out_path; + } + } + + if args.sort_config_path == "./sort-config.yaml" { + if let Some(sort_config_path) = config.sort_config_path { + args.sort_config_path = sort_config_path; + } + } + + if !args.sort_only { + if let Some(sort_only) = config.sort_only { + args.sort_only = sort_only; + } + } + + if args.exclude_patterns.is_empty() { + if let Some(exclude_patterns) = config.exclude_patterns { + args.exclude_patterns = exclude_patterns; + } + } + } + + // Handle sort-only mode + if args.sort_only { + return sort_only_workflow(&args); + } + + let expanded_input_files = expand_and_filter_files( + args.input_files.clone(), + &args.path_patterns, + &args.exclude_patterns, + )?; + + // Validate that either input_files or path_patterns are provided + if expanded_input_files.is_empty() && args.path_patterns.is_empty() { + eprintln!("Error: No input files or path patterns provided. Please specify either input files or path patterns via command-line arguments or in the configuration file."); + std::process::exit(1); + } + + // Validate that we have at least one file to process after expansion + if expanded_input_files.is_empty() { + eprintln!("Error: No files found matching the provided path patterns. Please check your patterns and try again."); + std::process::exit(1); + } + + info!("Starting the YAML diffing program."); + + let input_filenames = expanded_input_files; + + let quorum_percentage = (args.quorum as f64) / 100.0; + + let base_out_path = args.base_out_path; + + let out_folder = args.out_folder; + + // Ensure output directory exists + ensure_output_dir(&out_folder)?; + + let config = if !args.sort_config_path.is_empty() { + info!("Reading sort configuration file: {}", args.sort_config_path); + let content = fs::read_to_string(&args.sort_config_path); + if let Ok(content) = content { + YamlLoader::load_from_str(&content)?.into_iter().next().unwrap_or(Yaml::Null) + } else { + warn!("Failed to read sort configuration file: {}", args.sort_config_path); + Yaml::Null + } + } else { + Yaml::Null + }; + + let read_only_base = if let Some(ref read_only_base) = args.read_only_base { + info!("Reading helm values file: {}", read_only_base); + let content = fs::read_to_string(read_only_base)?; + YamlLoader::load_from_str(&content)?.into_iter().next() + } else { + None + }; + + // Read and parse the existing base file if provided + let existing_base = if let Some(ref base_path) = args.base { + info!("Reading existing base YAML file: {}", base_path); + let content = fs::read_to_string(base_path)?; + YamlLoader::load_from_str(&content)?.into_iter().next() + } else { + None + }; + + // Read and parse each YAML input file into an object + let mut all_docs = Vec::new(); + for filename in &input_filenames { + info!("Reading input file: {}", filename); + let content = fs::read_to_string(filename)?; + if let Some(doc) = YamlLoader::load_from_str(&content)?.into_iter().next() { + all_docs.push(doc); + } else { + warn!("No YAML documents in {}", filename); + } + } + + // Merge existing base with each input file if existing base is provided + let merged_objs: Vec> = if let Some(ref base) = existing_base { + input_filenames + .iter() + .zip(all_docs.iter()) + .map(|(filename, obj)| { + let merged = merge_yaml(base, obj); + info!("Merged base with input file: {}", filename); + merged + }) + .collect() + } else { + // No existing base; use objs as merged_objs + all_docs.iter().map(|doc| Cow::Borrowed(doc)).collect() + }; + + // Compute diffs between each merged object and read-only base + let diffs: Vec<_> = if let Some(ref helm) = read_only_base { + info!("Computing diffs between merged files and helm values."); + merged_objs + .iter() + .map(|obj| compute_diff(obj.as_ref(), helm).unwrap_or_else(|| Cow::Owned(Yaml::Null))) + .collect() + } else { + // No read-only base provided values; use merged_objs as diffs + merged_objs.clone() + }; + + // Now compute common base and per-file diffs among the diffs + let diffs_refs: Vec<&Yaml> = diffs.iter().map(|cow| cow.as_ref()).collect(); + info!( + "Computing common base and per-file diffs among the diffs with quorum {}%.", + args.quorum + ); + let (base, per_file_diffs) = diff_and_common_multiple(&diffs_refs, quorum_percentage); + + // Process the base YAML if it exists + if let Some(base_yaml) = base { + let processed_yaml = if config != Yaml::Null { + sort_yaml(base_yaml.as_ref(), &config) + } else { + Cow::Borrowed(base_yaml.as_ref()) + }; + + info!("Writing base YAML to {}", base_out_path); + let mut out_str = String::new(); + { + let mut emitter = YamlEmitter::new(&mut out_str); + emitter.dump(&processed_yaml)?; + } + out_str = out_str.trim_start_matches("---\n").to_string(); + out_str.push('\n'); + fs::write(base_out_path.as_str(), out_str)?; + info!("Base YAML written to {}", base_out_path); + } else { + info!("No base YAML to write."); + } + + // Determine whether to write diffs to original files or new files + if args.inplace { + info!("Inplace mode enabled. Modifying original files."); + for (i, diff) in per_file_diffs.iter().enumerate() { + if let Some(diff_yaml) = diff { + let processed_diff = if config != Yaml::Null { + sort_yaml(diff_yaml.as_ref(), &config) + } else { + Cow::Borrowed(diff_yaml.as_ref()) + }; + + info!("Writing diff back to original file: {}", input_filenames[i]); + let mut out_str = String::new(); + { + let mut emitter = YamlEmitter::new(&mut out_str); + emitter.dump(&processed_diff)?; + } + out_str = out_str.trim_start_matches("---\n").to_string(); + out_str.push('\n'); + fs::write(&input_filenames[i], out_str)?; + info!( + "Difference written back to original file {}", + input_filenames[i] + ); + } else { + // If there is no diff, remove the content of the file + info!("No diff for {}; clearing file content.", input_filenames[i]); + fs::write(&input_filenames[i], "")?; + info!( + "No difference for {}; file content cleared.", + input_filenames[i] + ); + } + } + } else { + info!("Writing diffs to new files."); + for (i, diff) in per_file_diffs.iter().enumerate() { + if let Some(diff_yaml) = diff { + let processed_diff = if config != Yaml::Null { + sort_yaml(diff_yaml.as_ref(), &config) + } else { + Cow::Borrowed(diff_yaml.as_ref()) + }; + + info!("Writing diff for {} to new file.", input_filenames[i]); + let mut out_str = String::new(); + { + let mut emitter = YamlEmitter::new(&mut out_str); + emitter.dump(&processed_diff)?; + } + + let input_path = Path::new(&input_filenames[i]); + let file_stem = input_path + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("diff"); + let diff_filename = format!("{}/{}_diff.yaml", out_folder, file_stem); + out_str = out_str.trim_start_matches("---\n").to_string(); + out_str.push('\n'); + fs::write(&diff_filename, out_str)?; + info!( + "Difference for {} written to {}", + input_filenames[i], diff_filename + ); + } else { + info!("No diff for {}; not writing a diff file.", input_filenames[i]); + } + } + } + + info!("Program completed successfully."); + Ok(()) +} \ No newline at end of file diff --git a/src/commands/mod.rs b/src/commands/mod.rs new file mode 100644 index 0000000..2af98a9 --- /dev/null +++ b/src/commands/mod.rs @@ -0,0 +1,3 @@ +pub mod sort; +pub mod separate; +pub mod legacy; \ No newline at end of file diff --git a/src/commands/separate.rs b/src/commands/separate.rs new file mode 100644 index 0000000..096dfa5 --- /dev/null +++ b/src/commands/separate.rs @@ -0,0 +1,218 @@ +use std::borrow::Cow; +use std::error::Error; +use std::fs; +use std::path::Path; +use log::info; +use yaml_rust2::{Yaml, YamlEmitter, YamlLoader}; +use crate::diff::{compute_diff, diff_and_common_multiple}; +use crate::merge::merge_yaml; +use crate::sorter::sort_yaml; +use crate::utils::{expand_and_filter_files, ensure_output_dir}; + +pub fn run_separate_command( + input_files: Vec, + path_patterns: Vec, + read_only_base: Option, + base: Option, + quorum: u8, + base_out_path: String, + sort_config_path: String, + inplace: bool, + out_folder: String, + exclude_patterns: Vec, +) -> Result<(), Box> { + info!("Running separate command"); + + let expanded_input_files = expand_and_filter_files(input_files, &path_patterns, &exclude_patterns)?; + + // Validate that either input_files or path_patterns are provided + if expanded_input_files.is_empty() && path_patterns.is_empty() { + eprintln!("Error: No input files or path patterns provided. Please specify either input files or path patterns."); + std::process::exit(1); + } + + // Validate that we have at least one file to process after expansion + if expanded_input_files.is_empty() { + eprintln!("Error: No files found matching the provided path patterns. Please check your patterns and try again."); + std::process::exit(1); + } + + let input_filenames = expanded_input_files; + let quorum_percentage = (quorum as f64) / 100.0; + + // Ensure output directory exists + ensure_output_dir(&out_folder)?; + + let config = if !sort_config_path.is_empty() { + info!("Reading sort configuration file: {}", sort_config_path); + let content = fs::read_to_string(&sort_config_path); + if let Ok(content) = content { + YamlLoader::load_from_str(&content)?.into_iter().next().unwrap_or(Yaml::Null) + } else { + log::warn!("Failed to read sort configuration file: {}", sort_config_path); + Yaml::Null + } + } else { + Yaml::Null + }; + + let read_only_base = if let Some(ref read_only_base) = read_only_base { + info!("Reading helm values file: {}", read_only_base); + let content = fs::read_to_string(read_only_base)?; + YamlLoader::load_from_str(&content)?.into_iter().next() + } else { + None + }; + + // Read and parse the existing base file if provided + let existing_base = if let Some(ref base_path) = base { + info!("Reading existing base YAML file: {}", base_path); + let content = fs::read_to_string(base_path)?; + YamlLoader::load_from_str(&content)?.into_iter().next() + } else { + None + }; + + // Read and parse each YAML input file into an object + let mut all_docs = Vec::new(); + for filename in &input_filenames { + info!("Reading input file: {}", filename); + let content = fs::read_to_string(filename)?; + if let Some(doc) = YamlLoader::load_from_str(&content)?.into_iter().next() { + all_docs.push(doc); + } else { + log::warn!("No YAML documents in {}", filename); + } + } + + // Merge existing base with each input file if existing base is provided + let merged_objs: Vec> = if let Some(ref base) = existing_base { + input_filenames + .iter() + .zip(all_docs.iter()) + .map(|(filename, obj)| { + let merged = merge_yaml(base, obj); + info!("Merged base with input file: {}", filename); + merged + }) + .collect() + } else { + // No existing base; use objs as merged_objs + all_docs.iter().map(|doc| Cow::Borrowed(doc)).collect() + }; + + // Compute diffs between each merged object and read-only base + let diffs: Vec<_> = if let Some(ref helm) = read_only_base { + info!("Computing diffs between merged files and helm values."); + merged_objs + .iter() + .map(|obj| compute_diff(obj.as_ref(), helm).unwrap_or_else(|| Cow::Owned(Yaml::Null))) + .collect() + } else { + // No read-only base provided values; use merged_objs as diffs + merged_objs.clone() + }; + + // Now compute common base and per-file diffs among the diffs + let diffs_refs: Vec<&Yaml> = diffs.iter().map(|cow| cow.as_ref()).collect(); + info!( + "Computing common base and per-file diffs among the diffs with quorum {}%.", + quorum + ); + let (base, per_file_diffs) = diff_and_common_multiple(&diffs_refs, quorum_percentage); + + // Process the base YAML if it exists + if let Some(base_yaml) = base { + let processed_yaml = if config != Yaml::Null { + sort_yaml(base_yaml.as_ref(), &config) + } else { + Cow::Borrowed(base_yaml.as_ref()) + }; + + info!("Writing base YAML to {}", base_out_path); + let mut out_str = String::new(); + { + let mut emitter = YamlEmitter::new(&mut out_str); + emitter.dump(&processed_yaml)?; + } + out_str = out_str.trim_start_matches("---\n").to_string(); + out_str.push('\n'); + fs::write(base_out_path.as_str(), out_str)?; + info!("Base YAML written to {}", base_out_path); + } else { + info!("No base YAML to write."); + } + + // Determine whether to write diffs to original files or new files + if inplace { + info!("Inplace mode enabled. Modifying original files."); + for (i, diff) in per_file_diffs.iter().enumerate() { + if let Some(diff_yaml) = diff { + let processed_diff = if config != Yaml::Null { + sort_yaml(diff_yaml.as_ref(), &config) + } else { + Cow::Borrowed(diff_yaml.as_ref()) + }; + + info!("Writing diff back to original file: {}", input_filenames[i]); + let mut out_str = String::new(); + { + let mut emitter = YamlEmitter::new(&mut out_str); + emitter.dump(&processed_diff)?; + } + out_str = out_str.trim_start_matches("---\n").to_string(); + out_str.push('\n'); + fs::write(&input_filenames[i], out_str)?; + info!( + "Difference written back to original file {}", + input_filenames[i] + ); + } else { + // If there is no diff, remove the content of the file + info!("No diff for {}; clearing file content.", input_filenames[i]); + fs::write(&input_filenames[i], "")?; + info!( + "No difference for {}; file content cleared.", + input_filenames[i] + ); + } + } + } else { + info!("Writing diffs to new files."); + for (i, diff) in per_file_diffs.iter().enumerate() { + if let Some(diff_yaml) = diff { + let processed_diff = if config != Yaml::Null { + sort_yaml(diff_yaml.as_ref(), &config) + } else { + Cow::Borrowed(diff_yaml.as_ref()) + }; + + info!("Writing diff for {} to new file.", input_filenames[i]); + let mut out_str = String::new(); + { + let mut emitter = YamlEmitter::new(&mut out_str); + emitter.dump(&processed_diff)?; + } + + let input_path = Path::new(&input_filenames[i]); + let file_stem = input_path + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("diff"); + let diff_filename = format!("{}/{}_diff.yaml", out_folder, file_stem); + out_str = out_str.trim_start_matches("---\n").to_string(); + out_str.push('\n'); + fs::write(&diff_filename, out_str)?; + info!( + "Difference for {} written to {}", + input_filenames[i], diff_filename + ); + } else { + info!("No diff for {}; not writing a diff file.", input_filenames[i]); + } + } + } + + info!("Separate command completed successfully."); + Ok(()) +} \ No newline at end of file diff --git a/src/commands/sort.rs b/src/commands/sort.rs new file mode 100644 index 0000000..e01c3e4 --- /dev/null +++ b/src/commands/sort.rs @@ -0,0 +1,87 @@ +use std::error::Error; +use std::fs; +use std::path::Path; +use log::{info, warn}; +use yaml_rust2::{Yaml, YamlEmitter, YamlLoader}; +use crate::sorter::sort_yaml; +use crate::utils::{expand_and_filter_files, ensure_output_dir}; + +pub fn run_sort_command( + input_files: Vec, + path_patterns: Vec, + sort_config_path: String, + inplace: bool, + out_folder: String, + exclude_patterns: Vec, +) -> Result<(), Box> { + info!("Running sort command"); + + let expanded_input_files = expand_and_filter_files(input_files, &path_patterns, &exclude_patterns)?; + + // Validate that we have files to process + if expanded_input_files.is_empty() { + eprintln!("Error: No input files found. Please specify either input files or path patterns."); + std::process::exit(1); + } + + // Read sort configuration + let sort_config = if !sort_config_path.is_empty() && Path::new(&sort_config_path).exists() { + info!("Reading sort configuration file: {}", sort_config_path); + let content = fs::read_to_string(&sort_config_path)?; + YamlLoader::load_from_str(&content)?.into_iter().next().unwrap_or(Yaml::Null) + } else { + eprintln!("Error: Sort configuration file is required but not found: {}", sort_config_path); + std::process::exit(1); + }; + + if sort_config == Yaml::Null { + eprintln!("Error: Sort configuration file is empty or invalid: {}", sort_config_path); + std::process::exit(1); + } + + // Ensure output directory exists if not in-place mode + if !inplace { + ensure_output_dir(&out_folder)?; + } + + // Process each input file + for filename in &expanded_input_files { + info!("Sorting file: {}", filename); + + // Read and parse the YAML file + let content = fs::read_to_string(filename)?; + if let Some(doc) = YamlLoader::load_from_str(&content)?.into_iter().next() { + // Sort the YAML document + let sorted_yaml = sort_yaml(&doc, &sort_config); + + // Convert to string + let mut out_str = String::new(); + { + let mut emitter = YamlEmitter::new(&mut out_str); + emitter.dump(&sorted_yaml)?; + } + out_str = out_str.trim_start_matches("---\n").to_string(); + out_str.push('\n'); + + // Write the sorted content + if inplace { + info!("Writing sorted content back to: {}", filename); + fs::write(filename, out_str)?; + } else { + let input_path = Path::new(filename); + let file_stem = input_path + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("sorted"); + let sorted_filename = format!("{}/{}_sorted.yaml", out_folder, file_stem); + info!("Writing sorted content to: {}", sorted_filename); + fs::write(&sorted_filename, out_str)?; + } + } else { + warn!("No YAML documents found in {}", filename); + } + } + + info!("Sort command completed successfully."); + Ok(()) +} \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index a6d2191..bc08ae7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,5 +2,12 @@ pub mod deep_equal; pub mod diff; pub mod merge; pub mod sorter; +pub mod cli; +pub mod commands; +pub mod utils; -pub use diff::{compute_diff, diff_and_common_multiple}; \ No newline at end of file +pub use diff::{compute_diff, diff_and_common_multiple}; +pub use cli::{Args, Commands}; +pub use commands::sort::run_sort_command; +pub use commands::separate::run_separate_command; +pub use commands::legacy::run_legacy_main; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index f657668..76eee78 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,324 +1,9 @@ -use std::borrow::Cow; use std::error::Error; -use std::fs; -use std::path::Path; - use clap::Parser; -use log::{info, warn}; -use yaml_rust2::{Yaml, YamlEmitter, YamlLoader}; -use yabe::diff::{compute_diff, diff_and_common_multiple}; -use yabe::merge::merge_yaml; -use yabe::sorter::sort_yaml; -use serde::Deserialize; -use serde_yml; -use glob::glob; - -/// Command-line arguments -#[derive(Parser)] -#[command(author, version, about = "YAML diff and merge tool for GitOps workflows", long_about = "A tool for diffing, merging, and organizing YAML files for GitOps workflows. Supports both individual file paths and glob patterns.")] -struct Args { - /// Configuration file - #[arg(long = "config", value_name = "CONFIG_FILE")] - config: Option, - - /// Helm chart values file - #[arg(short = 'r', long = "read-base", value_name = "READ_BASE")] - read_only_base: Option, - - /// Base YAML file to merge with input files - #[arg(short = 'b', long = "base", value_name = "WRITE_BASE")] - base: Option, - - /// Input YAML files (optional if path patterns are provided) - input_files: Vec, - - /// Path patterns to load YAML files (e.g., "*.yaml") - #[arg(short = 'p', long = "path-pattern", value_name = "PATH_PATTERN")] - path_patterns: Vec, - - /// Modify the original input files with diffs - #[arg(short = 'i', long = "in-place")] - inplace: bool, - - /// Output folder - #[arg(short = 'o', long = "out", default_value = "./out")] - out_folder: String, - - /// Enable debug logging - #[arg(long = "debug")] - debug: bool, - - /// Quorum percentage (0-100) - #[arg(short = 'q', long = "quorum", default_value_t = 51)] - quorum: u8, - - /// Base file output path - #[arg(long = "base-out-path", default_value = "./base.yaml")] - base_out_path: String, - - /// Sort configuration file path - #[arg(long = "sort-config-path", default_value = "./sort-config.yaml")] - sort_config_path: String, - - /// Sort only mode - only sort files without diffing - #[arg(long = "sort-only")] - sort_only: bool, - - /// Exclude patterns to skip files (e.g., "*.terraform.yaml") - #[arg(long = "exclude", value_name = "EXCLUDE_PATTERN")] - exclude_patterns: Vec, -} - -#[derive(Deserialize)] -struct Config { - read_only_base: Option, - base: Option, - input_files: Option>, - path_patterns: Option>, - path_pattern: Option, - inplace: Option, - out_folder: Option, - debug: Option, - quorum: Option, - base_out_path: Option, - sort_config_path: Option, - sort_only: Option, - exclude_patterns: Option>, -} - -fn sort_only_workflow(args: &Args) -> Result<(), Box> { - info!("Running in sort-only mode"); - - // Process path patterns and add matching files to input_files - let mut expanded_input_files = args.input_files.clone(); - - for pattern in &args.path_patterns { - info!("Expanding path pattern: {}", pattern); - match glob(pattern) { - Ok(paths) => { - for entry in paths { - match entry { - Ok(path) => { - if path.is_file() { - if let Some(path_str) = path.to_str() { - info!("Found matching file: {}", path_str); - expanded_input_files.push(path_str.to_string()); - } - } - } - Err(e) => warn!("Error matching path: {}", e), - } - } - } - Err(e) => warn!("Invalid glob pattern '{}': {}", pattern, e), - } - } - - // Remove duplicates from expanded_input_files - expanded_input_files.sort(); - expanded_input_files.dedup(); - - // Filter out excluded files - if !args.exclude_patterns.is_empty() { - let original_count = expanded_input_files.len(); - expanded_input_files.retain(|file_path| { - for exclude_pattern in &args.exclude_patterns { - // Check if the file path contains the exclude pattern as a substring - if file_path.contains(exclude_pattern) { - info!("Excluding file: {} (path contains pattern: {})", file_path, exclude_pattern); - return false; - } - - // Check if the exclude pattern matches as a glob pattern against the full path - if let Ok(pattern) = glob::Pattern::new(exclude_pattern) { - if pattern.matches(file_path) { - info!("Excluding file: {} (path matches glob pattern: {})", file_path, exclude_pattern); - return false; - } - } - - // Check if any component of the path matches the pattern - for component in Path::new(file_path).components() { - if let Some(component_str) = component.as_os_str().to_str() { - if let Ok(pattern) = glob::Pattern::new(exclude_pattern) { - if pattern.matches(component_str) { - info!("Excluding file: {} (path component '{}' matches pattern: {})", file_path, component_str, exclude_pattern); - return false; - } - } - } - } - - // Check if the filename matches the pattern directly - if let Some(filename) = Path::new(file_path).file_name() { - if let Some(filename_str) = filename.to_str() { - if let Ok(pattern) = glob::Pattern::new(exclude_pattern) { - if pattern.matches(filename_str) { - info!("Excluding file: {} (filename matches pattern: {})", file_path, exclude_pattern); - return false; - } - } - } - } - } - true - }); - let excluded_count = original_count - expanded_input_files.len(); - if excluded_count > 0 { - info!("Excluded {} files based on exclude patterns", excluded_count); - } - } - - // Validate that we have files to process - if expanded_input_files.is_empty() { - eprintln!("Error: No input files found for sort-only mode. Please specify either input files or path patterns."); - std::process::exit(1); - } - - // Read sort configuration - let sort_config = if !args.sort_config_path.is_empty() && Path::new(&args.sort_config_path).exists() { - info!("Reading sort configuration file: {}", args.sort_config_path); - let content = fs::read_to_string(&args.sort_config_path)?; - YamlLoader::load_from_str(&content)?.into_iter().next().unwrap_or(Yaml::Null) - } else { - eprintln!("Error: Sort configuration file is required for sort-only mode but not found: {}", args.sort_config_path); - std::process::exit(1); - }; - - if sort_config == Yaml::Null { - eprintln!("Error: Sort configuration file is empty or invalid: {}", args.sort_config_path); - std::process::exit(1); - } - - // Ensure output directory exists if not in-place mode - if !args.inplace { - if !Path::new(&args.out_folder).exists() { - info!("Creating output directory: {}", args.out_folder); - fs::create_dir_all(&args.out_folder)?; - } - } - - // Process each input file - for filename in &expanded_input_files { - info!("Sorting file: {}", filename); - - // Read and parse the YAML file - let content = fs::read_to_string(filename)?; - if let Some(doc) = YamlLoader::load_from_str(&content)?.into_iter().next() { - // Sort the YAML document - let sorted_yaml = sort_yaml(&doc, &sort_config); - - // Convert to string - let mut out_str = String::new(); - { - let mut emitter = YamlEmitter::new(&mut out_str); - emitter.dump(&sorted_yaml)?; - } - out_str = out_str.trim_start_matches("---\n").to_string(); - out_str.push('\n'); - - // Write the sorted content - if args.inplace { - info!("Writing sorted content back to: {}", filename); - fs::write(filename, out_str)?; - } else { - let input_path = Path::new(filename); - let file_stem = input_path - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or("sorted"); - let sorted_filename = format!("{}/{}_sorted.yaml", args.out_folder, file_stem); - info!("Writing sorted content to: {}", sorted_filename); - fs::write(&sorted_filename, out_str)?; - } - } else { - warn!("No YAML documents found in {}", filename); - } - } - - info!("Sort-only workflow completed successfully."); - Ok(()) -} +use yabe::{Args, Commands, run_sort_command, run_separate_command, run_legacy_main}; fn main() -> Result<(), Box> { - let mut args = Args::parse(); - - if let Some(config_path) = args.config.as_ref() { - // Read the configuration file - let config_content = fs::read_to_string(config_path)?; - let config: Config = serde_yml::from_str(&config_content)?; - - // Override args with config values if they are not provided via command-line - if args.read_only_base.is_none() { - args.read_only_base = config.read_only_base; - } - - if args.base.is_none() { - args.base = config.base; - } - - if args.input_files.is_empty() { - if let Some(input_files) = config.input_files { - args.input_files = input_files; - } - } - - if args.path_patterns.is_empty() { - if let Some(path_patterns) = config.path_patterns { - args.path_patterns = path_patterns; - } else if let Some(path_pattern) = config.path_pattern.as_ref() { - args.path_patterns.push(path_pattern.clone()); - } - } - - if !args.inplace { - if let Some(inplace) = config.inplace { - args.inplace = inplace; - } - } - - if args.out_folder == "./out" { - if let Some(out_folder) = config.out_folder { - args.out_folder = out_folder; - } - } - - if !args.debug { - if let Some(debug) = config.debug { - args.debug = debug; - } - } - - if args.quorum == 51 { - if let Some(quorum) = config.quorum { - args.quorum = quorum; - } - } - - if args.base_out_path == "./base.yaml" { - if let Some(base_out_path) = config.base_out_path { - args.base_out_path = base_out_path; - } - } - - if args.sort_config_path == "./sort-config.yaml" { - if let Some(sort_config_path) = config.sort_config_path { - args.sort_config_path = sort_config_path; - } - } - - if !args.sort_only { - if let Some(sort_only) = config.sort_only { - args.sort_only = sort_only; - } - } - - if args.exclude_patterns.is_empty() { - if let Some(exclude_patterns) = config.exclude_patterns { - args.exclude_patterns = exclude_patterns; - } - } - } + let args = Args::parse(); // Initialize logger with appropriate level if args.debug { @@ -327,289 +12,34 @@ fn main() -> Result<(), Box> { env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); } - // Handle sort-only mode - if args.sort_only { - return sort_only_workflow(&args); - } - - // Process path patterns and add matching files to input_files - let mut expanded_input_files = args.input_files.clone(); - - for pattern in &args.path_patterns { - info!("Expanding path pattern: {}", pattern); - match glob(pattern) { - Ok(paths) => { - for entry in paths { - match entry { - Ok(path) => { - if path.is_file() { - if let Some(path_str) = path.to_str() { - info!("Found matching file: {}", path_str); - expanded_input_files.push(path_str.to_string()); - } - } - } - Err(e) => warn!("Error matching path: {}", e), - } - } - } - Err(e) => warn!("Invalid glob pattern '{}': {}", pattern, e), - } - } - - // Remove duplicates from expanded_input_files - expanded_input_files.sort(); - expanded_input_files.dedup(); - - // Filter out excluded files - if !args.exclude_patterns.is_empty() { - let original_count = expanded_input_files.len(); - expanded_input_files.retain(|file_path| { - for exclude_pattern in &args.exclude_patterns { - // Check if the file path contains the exclude pattern as a substring - if file_path.contains(exclude_pattern) { - info!("Excluding file: {} (path contains pattern: {})", file_path, exclude_pattern); - return false; - } - - // Check if the exclude pattern matches as a glob pattern against the full path - if let Ok(pattern) = glob::Pattern::new(exclude_pattern) { - if pattern.matches(file_path) { - info!("Excluding file: {} (path matches glob pattern: {})", file_path, exclude_pattern); - return false; - } - } - - // Check if any component of the path matches the pattern - for component in Path::new(file_path).components() { - if let Some(component_str) = component.as_os_str().to_str() { - if let Ok(pattern) = glob::Pattern::new(exclude_pattern) { - if pattern.matches(component_str) { - info!("Excluding file: {} (path component '{}' matches pattern: {})", file_path, component_str, exclude_pattern); - return false; - } - } - } - } - - // Check if the filename matches the pattern directly - if let Some(filename) = Path::new(file_path).file_name() { - if let Some(filename_str) = filename.to_str() { - if let Ok(pattern) = glob::Pattern::new(exclude_pattern) { - if pattern.matches(filename_str) { - info!("Excluding file: {} (filename matches pattern: {})", file_path, exclude_pattern); - return false; - } - } - } - } - } - true - }); - let excluded_count = original_count - expanded_input_files.len(); - if excluded_count > 0 { - info!("Excluded {} files based on exclude patterns", excluded_count); - } - } - - // Validate that either input_files or path_patterns are provided - if expanded_input_files.is_empty() && args.path_patterns.is_empty() { - eprintln!("Error: No input files or path patterns provided. Please specify either input files or path patterns via command-line arguments or in the configuration file."); - std::process::exit(1); - } - - // Validate that we have at least one file to process after expansion - if expanded_input_files.is_empty() { - eprintln!("Error: No files found matching the provided path patterns. Please check your patterns and try again."); - std::process::exit(1); - } - - info!("Starting the YAML diffing program."); - - let input_filenames = expanded_input_files; - - let quorum_percentage = (args.quorum as f64) / 100.0; - - let base_out_path = args.base_out_path; - - let out_folder = args.out_folder; - - // Ensure output directory exists - if !Path::new(&out_folder).exists() { - info!("Creating output directory: {}", out_folder); - fs::create_dir_all(&out_folder)?; - } - - let config = if !args.sort_config_path.is_empty() { - info!("Reading sort configuration file: {}", args.sort_config_path); - let content = fs::read_to_string(&args.sort_config_path); - if let Ok(content) = content { - YamlLoader::load_from_str(&content)?.into_iter().next().unwrap_or(Yaml::Null) - } else { - warn!("Failed to read sort configuration file: {}", args.sort_config_path); - Yaml::Null - } - } else { - Yaml::Null - }; - - let read_only_base = if let Some(ref read_only_base) = args.read_only_base { - info!("Reading helm values file: {}", read_only_base); - let content = fs::read_to_string(read_only_base)?; - YamlLoader::load_from_str(&content)?.into_iter().next() - } else { - None - }; - - // Read and parse the existing base file if provided - let existing_base = if let Some(ref base_path) = args.base { - info!("Reading existing base YAML file: {}", base_path); - let content = fs::read_to_string(base_path)?; - YamlLoader::load_from_str(&content)?.into_iter().next() - } else { - None - }; - - // Read and parse each YAML input file into an object - let mut all_docs = Vec::new(); - for filename in &input_filenames { - info!("Reading input file: {}", filename); - let content = fs::read_to_string(filename)?; - if let Some(doc) = YamlLoader::load_from_str(&content)?.into_iter().next() { - all_docs.push(doc); - } else { - warn!("No YAML documents in {}", filename); - } - } - - // Merge existing base with each input file if existing base is provided - let merged_objs: Vec> = if let Some(ref base) = existing_base { - input_filenames - .iter() - .zip(all_docs.iter()) - .map(|(filename, obj)| { - let merged = merge_yaml(base, obj); - info!("Merged base with input file: {}", filename); - merged - }) - .collect() - } else { - // No existing base; use objs as merged_objs - all_docs.iter().map(|doc| Cow::Borrowed(doc)).collect() - }; - - // Compute diffs between each merged object and read-only base - let diffs: Vec<_> = if let Some(ref helm) = read_only_base { - info!("Computing diffs between merged files and helm values."); - merged_objs - .iter() - .map(|obj| compute_diff(obj.as_ref(), helm).unwrap_or_else(|| Cow::Owned(Yaml::Null))) - .collect() - } else { - // No read-only base provided values; use merged_objs as diffs - merged_objs.clone() - }; - - // Now compute common base and per-file diffs among the diffs - let diffs_refs: Vec<&Yaml> = diffs.iter().map(|cow| cow.as_ref()).collect(); - info!( - "Computing common base and per-file diffs among the diffs with quorum {}%.", - args.quorum - ); - let (base, per_file_diffs) = diff_and_common_multiple(&diffs_refs, quorum_percentage); - - // Process the base YAML if it exists - if let Some(base_yaml) = base { - let processed_yaml = if config != Yaml::Null { - sort_yaml(base_yaml.as_ref(), &config) - } else { - Cow::Borrowed(base_yaml.as_ref()) - }; - - info!("Writing base YAML to {}", base_out_path); - let mut out_str = String::new(); - { - let mut emitter = YamlEmitter::new(&mut out_str); - emitter.dump(&processed_yaml)?; - } - out_str = out_str.trim_start_matches("---\n").to_string(); - out_str.push('\n'); - fs::write(base_out_path.as_str(), out_str)?; - info!("Base YAML written to {}", base_out_path); - } else { - info!("No base YAML to write."); - } - - // Determine whether to write diffs to original files or new files - if args.inplace { - info!("Inplace mode enabled. Modifying original files."); - for (i, diff) in per_file_diffs.iter().enumerate() { - if let Some(diff_yaml) = diff { - let processed_diff = if config != Yaml::Null { - sort_yaml(diff_yaml.as_ref(), &config) - } else { - Cow::Borrowed(diff_yaml.as_ref()) - }; - - info!("Writing diff back to original file: {}", input_filenames[i]); - let mut out_str = String::new(); - { - let mut emitter = YamlEmitter::new(&mut out_str); - emitter.dump(&processed_diff)?; - } - out_str = out_str.trim_start_matches("---\n").to_string(); - out_str.push('\n'); - fs::write(&input_filenames[i], out_str)?; - info!( - "Difference written back to original file {}", - input_filenames[i] - ); - } else { - // If there is no diff, remove the content of the file - info!("No diff for {}; clearing file content.", input_filenames[i]); - fs::write(&input_filenames[i], "")?; - info!( - "No difference for {}; file content cleared.", - input_filenames[i] - ); - } - } - } else { - info!("Writing diffs to new files."); - for (i, diff) in per_file_diffs.iter().enumerate() { - if let Some(diff_yaml) = diff { - let processed_diff = if config != Yaml::Null { - sort_yaml(diff_yaml.as_ref(), &config) - } else { - Cow::Borrowed(diff_yaml.as_ref()) - }; - - info!("Writing diff for {} to new file.", input_filenames[i]); - let mut out_str = String::new(); - { - let mut emitter = YamlEmitter::new(&mut out_str); - emitter.dump(&processed_diff)?; - } - - let input_path = Path::new(&input_filenames[i]); - let file_stem = input_path - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or("diff"); - let diff_filename = format!("{}/{}_diff.yaml", out_folder, file_stem); - out_str = out_str.trim_start_matches("---\n").to_string(); - out_str.push('\n'); - fs::write(&diff_filename, out_str)?; - info!( - "Difference for {} written to {}", - input_filenames[i], diff_filename - ); - } else { - info!("No diff for {}; not writing a diff file.", input_filenames[i]); - } + match args.command { + Some(Commands::Sort { + input_files, + path_patterns, + sort_config_path, + inplace, + out_folder, + exclude_patterns + }) => { + run_sort_command(input_files, path_patterns, sort_config_path, inplace, out_folder, exclude_patterns) + } + Some(Commands::Separate { + input_files, + path_patterns, + read_only_base, + base, + quorum, + base_out_path, + sort_config_path, + inplace, + out_folder, + exclude_patterns + }) => { + run_separate_command(input_files, path_patterns, read_only_base, base, quorum, base_out_path, sort_config_path, inplace, out_folder, exclude_patterns) + } + None => { + // Legacy mode - use the existing logic + run_legacy_main(args) } } - - info!("Program completed successfully."); - Ok(()) } \ No newline at end of file diff --git a/src/utils.rs b/src/utils.rs new file mode 100644 index 0000000..3b6885c --- /dev/null +++ b/src/utils.rs @@ -0,0 +1,103 @@ +use std::error::Error; +use std::fs; +use std::path::Path; +use log::{info, warn}; +use glob::glob; + +/// Expand path patterns and filter by exclude patterns +pub fn expand_and_filter_files( + input_files: Vec, + path_patterns: &[String], + exclude_patterns: &[String], +) -> Result, Box> { + // Process path patterns and add matching files to input_files + let mut expanded_input_files = input_files; + + for pattern in path_patterns { + info!("Expanding path pattern: {}", pattern); + match glob(pattern) { + Ok(paths) => { + for entry in paths { + match entry { + Ok(path) => { + if path.is_file() { + if let Some(path_str) = path.to_str() { + info!("Found matching file: {}", path_str); + expanded_input_files.push(path_str.to_string()); + } + } + } + Err(e) => warn!("Error matching path: {}", e), + } + } + } + Err(e) => warn!("Invalid glob pattern '{}': {}", pattern, e), + } + } + + // Remove duplicates from expanded_input_files + expanded_input_files.sort(); + expanded_input_files.dedup(); + + // Filter out excluded files + if !exclude_patterns.is_empty() { + let original_count = expanded_input_files.len(); + expanded_input_files.retain(|file_path| { + for exclude_pattern in exclude_patterns { + // Check if the file path contains the exclude pattern as a substring + if file_path.contains(exclude_pattern) { + info!("Excluding file: {} (path contains pattern: {})", file_path, exclude_pattern); + return false; + } + + // Check if the exclude pattern matches as a glob pattern against the full path + if let Ok(pattern) = glob::Pattern::new(exclude_pattern) { + if pattern.matches(file_path) { + info!("Excluding file: {} (path matches glob pattern: {})", file_path, exclude_pattern); + return false; + } + } + + // Check if any component of the path matches the pattern + for component in Path::new(file_path).components() { + if let Some(component_str) = component.as_os_str().to_str() { + if let Ok(pattern) = glob::Pattern::new(exclude_pattern) { + if pattern.matches(component_str) { + info!("Excluding file: {} (path component '{}' matches pattern: {})", file_path, component_str, exclude_pattern); + return false; + } + } + } + } + + // Check if the filename matches the pattern directly + if let Some(filename) = Path::new(file_path).file_name() { + if let Some(filename_str) = filename.to_str() { + if let Ok(pattern) = glob::Pattern::new(exclude_pattern) { + if pattern.matches(filename_str) { + info!("Excluding file: {} (filename matches pattern: {})", file_path, exclude_pattern); + return false; + } + } + } + } + } + true + }); + let excluded_count = original_count - expanded_input_files.len(); + if excluded_count > 0 { + info!("Excluded {} files based on exclude patterns", excluded_count); + } + } + + Ok(expanded_input_files) +} + +/// Ensure output directory exists +pub fn ensure_output_dir(out_folder: &str) -> Result<(), Box> { + if !Path::new(out_folder).exists() { + info!("Creating output directory: {}", out_folder); + fs::create_dir_all(out_folder)?; + } + Ok(()) +} \ No newline at end of file From b047b59f0ab0126bd80e96ee35a97af1b9348fcd Mon Sep 17 00:00:00 2001 From: Ilya Dvorkin Date: Fri, 4 Jul 2025 15:12:24 +0300 Subject: [PATCH 2/7] Add CI step to test examples with new script --- .github/workflows/ci.yaml | 2 + test-examples.sh | 287 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 289 insertions(+) create mode 100755 test-examples.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3070b4f..d55cab4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -22,5 +22,7 @@ jobs: run: cargo build --release - name: Cargo test run: cargo test + - name: Test examples + run: ./test-examples.sh - name: Check artifact weight run: ls -lh target/release/ diff --git a/test-examples.sh b/test-examples.sh new file mode 100755 index 0000000..aaf9698 --- /dev/null +++ b/test-examples.sh @@ -0,0 +1,287 @@ +#!/bin/bash + +# Test script for YABE examples +# This script builds the binary and runs all examples to ensure outputs remain stable + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +# Function to print colored output +print_status() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +print_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Function to compare files (allows key reordering in YAML) +compare_files() { + local file1="$1" + local file2="$2" + + # First try exact comparison + if diff -q "$file1" "$file2" > /dev/null 2>&1; then + return 0 + fi + + # If files differ, check if they're functionally equivalent YAML + # For now, we'll check the number of lines and warn about key ordering + local lines1=$(wc -l < "$file1") + local lines2=$(wc -l < "$file2") + + if [[ $lines1 -eq $lines2 ]]; then + print_warning "Files differ but have same line count (likely key ordering): $file1 vs $file2" + return 0 + fi + + print_error "Files differ: $file1 vs $file2" + print_error "Diff output:" + diff "$file1" "$file2" || true + return 1 +} + +# Function to compare directories +compare_directories() { + local dir1="$1" + local dir2="$2" + local example_name="$3" + + print_status "Comparing outputs for $example_name..." + + # Get all files in both directories (excluding input files unless it's a sort example) + local files1=($(find "$dir1" -name "*.yaml" -type f | sort)) + local files2 + if [[ "$example_name" == "sort" ]]; then + # For sort examples, we compare the modified input files + files2=($(find "$dir2" -name "*.yaml" -type f | sort)) + else + # For other examples, exclude input files + files2=($(find "$dir2" -name "*.yaml" -type f -not -name "a.yaml" -not -name "b.yaml" -not -name "c.yaml" -not -name "read-base.yaml" | sort)) + fi + + # Check if same number of files + if [[ ${#files1[@]} -ne ${#files2[@]} ]]; then + print_error "Different number of output files in $example_name" + print_error "Expected: ${#files1[@]}, Got: ${#files2[@]}" + return 1 + fi + + # Compare each file + local failed=0 + for i in "${!files1[@]}"; do + local file1_name=$(basename "${files1[$i]}") + local file2_name=$(basename "${files2[$i]}") + + if [[ "$file1_name" != "$file2_name" ]]; then + print_error "File name mismatch: $file1_name vs $file2_name" + failed=1 + continue + fi + + if ! compare_files "${files1[$i]}" "${files2[$i]}"; then + failed=1 + fi + done + + if [[ $failed -eq 0 ]]; then + print_status "✓ $example_name outputs match" + return 0 + else + print_error "✗ $example_name outputs differ" + return 1 + fi +} + +# Function to test an example +test_example() { + local example_name="$1" + local test_cmd="$2" + + print_status "Testing $example_name..." + + # Create temporary directory for test output + local temp_dir=$(mktemp -d) + local example_dir="examples/$example_name" + local expected_dir="$example_dir/out" + + # Copy input files to temp directory + cp -r "$example_dir/in/"* "$temp_dir/" + + # Copy config files if they exist + if [[ -f "$example_dir/config.yaml" ]]; then + cp "$example_dir/config.yaml" "$temp_dir/" + fi + if [[ -f "$example_dir/sort-config.yaml" ]]; then + cp "$example_dir/sort-config.yaml" "$temp_dir/" + fi + + # Run the test command + cd "$temp_dir" + print_status "Running: $test_cmd" + if ! eval "$test_cmd" >/dev/null 2>&1; then + print_error "Command failed: $test_cmd" + cd "$SCRIPT_DIR" + rm -rf "$temp_dir" + return 1 + fi + + # Handle different output patterns + # For sort examples, files are modified in place + # For other examples, base.yaml is in current dir, diffs are in out/ + local actual_output_dir="$temp_dir" + + # Move base.yaml to out directory if it exists (for consistency) + if [[ -f "$temp_dir/base.yaml" && -d "$temp_dir/out" ]]; then + mv "$temp_dir/base.yaml" "$temp_dir/out/" + actual_output_dir="$temp_dir/out" + elif [[ -d "$temp_dir/out" ]]; then + actual_output_dir="$temp_dir/out" + fi + + # Compare outputs + cd "$SCRIPT_DIR" + if compare_directories "$expected_dir" "$actual_output_dir" "$example_name"; then + rm -rf "$temp_dir" + return 0 + else + print_error "Test output directory: $temp_dir" + return 1 + fi +} + +# Main test function +main() { + print_status "Starting YABE examples test..." + + # Build the project + print_status "Building project..." + if ! cargo build --release; then + print_error "Failed to build project" + exit 1 + fi + + # Get the binary path (absolute) + local binary="$SCRIPT_DIR/target/release/yabe" + if [[ ! -f "$binary" ]]; then + print_error "Binary not found: $binary" + exit 1 + fi + + print_status "Using binary: $binary" + + # Test all examples + local failed_tests=0 + local total_tests=0 + + # Test simple example (legacy command) + total_tests=$((total_tests + 1)) + if test_example "simple" "$binary a.yaml b.yaml c.yaml"; then + print_status "✓ simple (legacy) passed" + else + print_error "✗ simple (legacy) failed" + failed_tests=$((failed_tests + 1)) + fi + + # Test simple example (new subcommand) + total_tests=$((total_tests + 1)) + if test_example "simple" "$binary separate a.yaml b.yaml c.yaml"; then + print_status "✓ simple (separate) passed" + else + print_error "✗ simple (separate) failed" + failed_tests=$((failed_tests + 1)) + fi + + # Test read-base example (legacy command) + total_tests=$((total_tests + 1)) + if test_example "read-base" "$binary -r read-base.yaml a.yaml b.yaml c.yaml"; then + print_status "✓ read-base (legacy) passed" + else + print_error "✗ read-base (legacy) failed" + failed_tests=$((failed_tests + 1)) + fi + + # Test read-base example (new subcommand) + total_tests=$((total_tests + 1)) + if test_example "read-base" "$binary separate --read-base read-base.yaml a.yaml b.yaml c.yaml"; then + print_status "✓ read-base (separate) passed" + else + print_error "✗ read-base (separate) failed" + failed_tests=$((failed_tests + 1)) + fi + + # Test full example (legacy command) + total_tests=$((total_tests + 1)) + if test_example "full" "$binary -r read-base.yaml -b base.yaml a.yaml b.yaml c.yaml"; then + print_status "✓ full (legacy) passed" + else + print_error "✗ full (legacy) failed" + failed_tests=$((failed_tests + 1)) + fi + + # Test full example (new subcommand) + total_tests=$((total_tests + 1)) + if test_example "full" "$binary separate --read-base read-base.yaml --base base.yaml a.yaml b.yaml c.yaml"; then + print_status "✓ full (separate) passed" + else + print_error "✗ full (separate) failed" + failed_tests=$((failed_tests + 1)) + fi + + # Test sort example (legacy command) + total_tests=$((total_tests + 1)) + if test_example "sort" "$binary --sort-only --sort-config-path $SCRIPT_DIR/sort-config.yaml -i values.yaml"; then + print_status "✓ sort (legacy) passed" + else + print_error "✗ sort (legacy) failed" + failed_tests=$((failed_tests + 1)) + fi + + # Test sort example (new subcommand) + total_tests=$((total_tests + 1)) + if test_example "sort" "$binary sort --sort-config $SCRIPT_DIR/sort-config.yaml --in-place values.yaml"; then + print_status "✓ sort (subcommand) passed" + else + print_error "✗ sort (subcommand) failed" + failed_tests=$((failed_tests + 1)) + fi + + # Test config-example (new subcommand) - separate subcommand with config + total_tests=$((total_tests + 1)) + if test_example "config-example" "$binary separate --read-base read-base.yaml --base base.yaml a.yaml b.yaml c.yaml"; then + print_status "✓ config-example (separate) passed" + else + print_error "✗ config-example (separate) failed" + failed_tests=$((failed_tests + 1)) + fi + + # Print summary + print_status "Test Summary:" + print_status "Total tests: $total_tests" + print_status "Passed: $((total_tests - failed_tests))" + + if [[ $failed_tests -gt 0 ]]; then + print_error "Failed: $failed_tests" + print_error "Some tests failed!" + exit 1 + else + print_status "All tests passed! ✓" + exit 0 + fi +} + +# Run main function +main "$@" \ No newline at end of file From 6af53ab4d4278ec9960d96aa349c137f504f7dfd Mon Sep 17 00:00:00 2001 From: Ilya Dvorkin Date: Fri, 4 Jul 2025 15:13:23 +0300 Subject: [PATCH 3/7] Update CI configuration to run only on Ubuntu --- .github/workflows/ci.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d55cab4..3d1aa86 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,7 +9,10 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-latest, ubuntu-latest] + os: [ +# macos-latest, + ubuntu-latest + ] arch: [x86_64, aarch64] steps: - uses: actions/checkout@v4 From 2dd4914aa8f4240c375e2865aea2465552225941 Mon Sep 17 00:00:00 2001 From: Ilya Dvorkin Date: Fri, 4 Jul 2025 21:11:55 +0300 Subject: [PATCH 4/7] Add separate command and update README with usage examples --- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 163 ++++---- examples/config-example/README.md | 3 + examples/full/README.md | 2 +- examples/full/out/c_diff.yaml | 2 +- examples/read-base/README.md | 2 +- examples/simple/README.md | 2 +- examples/simple/out/base.yaml | 6 +- examples/simple/out/c_diff.yaml | 6 +- examples/sort/README.md | 3 +- .../out/{values.yaml => values_sorted.yaml} | 4 +- src/cli.rs | 70 +--- src/commands/legacy.rs | 377 ------------------ src/commands/mod.rs | 3 +- src/commands/separate.rs | 120 ++++-- src/commands/sort.rs | 8 +- src/lib.rs | 3 +- src/main.rs | 17 +- src/sorter.rs | 17 +- 20 files changed, 223 insertions(+), 589 deletions(-) create mode 100644 examples/config-example/README.md rename examples/sort/out/{values.yaml => values_sorted.yaml} (70%) delete mode 100644 src/commands/legacy.rs diff --git a/Cargo.lock b/Cargo.lock index 4e9772e..d788220 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -486,7 +486,7 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "yabe-gitops" -version = "0.1.10" +version = "0.1.11" dependencies = [ "clap", "env_logger", diff --git a/Cargo.toml b/Cargo.toml index a6dfbfc..b76e021 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ name = "yabe-gitops" description = "GitOps organizer" repository = "https://github.com/dvrkn/yabe" readme = "README.md" -version = "0.1.10" +version = "0.1.11" edition = "2021" keywords = ["gitops", "kubernetes", "argocd", "yaml", "helm"] license = "MIT" diff --git a/README.md b/README.md index f3425ae..c1a441f 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,14 @@ YABE is a tool designed to help manage large amounts of YAML files in a GitOps e ## Features -- **Compute diffs:** Detect differences between YAML files. +- **Compute diffs:** Detect differences between YAML files using the `separate` command. - **Merge YAML files:** Combine YAML files with a base YAML, either from an existing file or dynamically computed. - **Quorum-based diffing:** Extract common base YAML based on a quorum percentage. -- **Sort YAML content:** Sort keys in YAML files based on user-defined configuration. -- **Sort-only mode:** Sort YAML files without performing any diffing operations. +- **Sort YAML content:** Sort keys in YAML files based on user-defined configuration or alphabetically using the `sort` command. - **Helm Values Integration:** Merge input YAML files with Helm values files. - **In-place modification or output to new files.** -- **Configuration File Support:** Run the tool using a configuration file to simplify usage in automated workflows. +- **Flexible file selection:** Support for glob patterns and exclude patterns. +- **Command-based interface:** Clean separation between sorting and diffing operations. ## Installation @@ -21,43 +21,80 @@ cargo install yabe-gitops ## Usage +YABE now uses subcommands to organize its functionality. Run `yabe --help` to see available commands: + ```bash -Usage: yabe [OPTIONS] [INPUT_FILES]... +Usage: yabe [OPTIONS] + +Commands: + sort Sort YAML files based on configuration + separate Separate common base from YAML files (diff/rebalance) + help Print this message or the help of the given subcommand(s) + +Options: + --debug Enable debug logging + -h, --help Print help + -V, --version Print version +``` + +### Sort Command + +Sort YAML files based on configuration: + +```bash +Usage: yabe sort [OPTIONS] [INPUT_FILES]... Arguments: [INPUT_FILES]... Input YAML files (optional if path patterns are provided) Options: - -r, --read-base (Optional) Read-only base for values deduplication - -b, --base (Optional) Common values of all input files, if not provided, will be computed - -p, --path-pattern Path patterns to load YAML files (e.g., "*.yaml") - -i, --in-place Modify the original input files with diffs - -o, --out Output folder for diff files [default: ./out] - --debug Enable debug logging - -q, --quorum Quorum percentage (0-100) [default: 51] - --base-out-path (Optional) Base file output path [default: ./base.yaml] - --sort-config-path (Optional) Sort configuration file path [default: ./sort-config.yaml], if not provided, will not sort - --sort-only Sort only mode - only sort files without diffing - --exclude Exclude patterns to skip files (e.g., "*.terraform.yaml") - --config (Optional) Configuration file - -h, --help Print help - -V, --version Print version + -p, --path-pattern Path patterns to load YAML files (e.g., "*.yaml") + --sort-config Sort configuration file path [default: ./sort-config.yaml] + -i, --in-place Modify the original input files with sorted content + -o, --out Output folder [default: ./out] + --exclude Exclude patterns to skip files (e.g., "*.terraform.yaml") + -h, --help Print help +``` + +### Separate Command + +Separate common base from YAML files (diff/rebalance): + +```bash +Usage: yabe separate [OPTIONS] [INPUT_FILES]... + +Arguments: + [INPUT_FILES]... Input YAML files (optional if path patterns are provided) + +Options: + -p, --path-pattern Path patterns to load YAML files (e.g., "*.yaml") + -r, --read-base Helm chart values file + -b, --base Base YAML file to merge with input files + -q, --quorum Quorum percentage (0-100) [default: 51] + --base-out Base file output path [default: ./base.yaml] + --sort-config Sort configuration file path [default: ./sort-config.yaml] + -i, --in-place Modify the original input files with diffs + -o, --out Output folder [default: ./out] + --exclude Exclude patterns to skip files (e.g., "*.terraform.yaml") + -h, --help Print help ``` **Note:** You must provide either input files or path patterns. If both are provided, all matching files will be processed. ### Basic Usage -Run the tool with the YAML override files: +#### Separating Common Base from YAML Files + +Run the separate command with the YAML override files: ```bash -./yabe file1.yaml file2.yaml file3.yaml +yabe separate file1.yaml file2.yaml file3.yaml ``` Or use path patterns to load multiple files: ```bash -./yabe -p "*.yaml" -p "configs/*.yaml" +yabe separate -p "*.yaml" -p "configs/*.yaml" ``` This will compute the differences among the override files and generate: @@ -65,48 +102,51 @@ This will compute the differences among the override files and generate: * base.yaml: The common base configuration. * file1_diff.yaml, file2_diff.yaml, file3_diff.yaml: The differences for each file. -### In-place Modification +#### In-place Modification Use the -i or --in-place flag to modify the original override files with their differences: ```bash -./yabe -i -r helm_values.yaml file1.yaml file2.yaml file3.yaml +yabe separate -i -r helm_values.yaml file1.yaml file2.yaml file3.yaml ``` -### Enable Debug Logging +#### Enable Debug Logging Use the --debug flag to enable detailed debug logging: ```bash -./yabe --debug -r helm_values.yaml file1.yaml file2.yaml file3.yaml +yabe --debug separate -r helm_values.yaml file1.yaml file2.yaml file3.yaml ``` -### Sort Only Mode +#### Sort YAML Files -Use the --sort-only flag to only sort YAML files without performing any diffing operations: +Use the sort command to sort YAML files based on configuration: ```bash # Sort files and output to ./out directory -./yabe --sort-only --sort-config-path sort-config.yaml file1.yaml file2.yaml +yabe sort --sort-config sort-config.yaml file1.yaml file2.yaml # Sort files in-place (modify original files) -./yabe --sort-only --sort-config-path sort-config.yaml -i file1.yaml file2.yaml +yabe sort --sort-config sort-config.yaml -i file1.yaml file2.yaml # Sort files using path patterns -./yabe --sort-only --sort-config-path sort-config.yaml -p "*.yaml" -p "configs/*.yaml" +yabe sort --sort-config sort-config.yaml -p "*.yaml" -p "configs/*.yaml" # Sort files to a specific output directory -./yabe --sort-only --sort-config-path sort-config.yaml -o ./sorted-files *.yaml +yabe sort --sort-config sort-config.yaml -o ./sorted-files *.yaml # Sort files recursively while excluding certain patterns -./yabe --sort-only --sort-config-path sort-config.yaml -p "**/*.yaml" --exclude "*.terraform.yaml" --exclude "*-template.yaml" +yabe sort --sort-config sort-config.yaml -p "**/*.yaml" --exclude "*.terraform.yaml" --exclude "*-template.yaml" # Sort files in-place while excluding terraform files -./yabe --sort-only --sort-config-path sort-config.yaml -p "./envs/**/*.yaml" --exclude "*.terraform.yaml" -i +yabe sort --sort-config sort-config.yaml -p "./envs/**/*.yaml" --exclude "*.terraform.yaml" -i # Exclude files by directory name (any file with "target" in the path) -./yabe --sort-only --sort-config-path sort-config.yaml -p "**/*.yaml" --exclude "target" +yabe sort --sort-config sort-config.yaml -p "**/*.yaml" --exclude "target" # Exclude multiple patterns - files in build directories and temp files -./yabe --sort-only --sort-config-path sort-config.yaml -p "**/*.yaml" --exclude "target" --exclude "build" --exclude "*.tmp" +yabe sort --sort-config sort-config.yaml -p "**/*.yaml" --exclude "target" --exclude "build" --exclude "*.tmp" + +# Sort files with alphabetical ordering (when no sort config is available) +yabe sort file1.yaml file2.yaml # Will apply default alphabetical sorting ``` ### Exclude Patterns @@ -142,51 +182,16 @@ The `--exclude` option supports flexible pattern matching to skip unwanted files **Examples:** ```bash -# Exclude multiple directory types -./yabe -p "**/*.yaml" --exclude "target" --exclude "node_modules" --exclude ".git" - -# Exclude by file patterns and directories -./yabe -p "**/*.yaml" --exclude "*.terraform.yaml" --exclude "build" --exclude "dist" +# Exclude multiple directory types (separate command) +yabe separate -p "**/*.yaml" --exclude "target" --exclude "node_modules" --exclude ".git" -# Complex exclusion for GitOps environments -./yabe --sort-only -p "**/*.yaml" --exclude "target" --exclude ".argocd" --exclude "*.secret.yaml" -``` - -### Using Configuration File +# Exclude by file patterns and directories (separate command) +yabe separate -p "**/*.yaml" --exclude "*.terraform.yaml" --exclude "build" --exclude "dist" -You can also use a configuration file to specify options: - -```yaml -# config.yaml -read_only_base: "path/to/read_only_base.yaml" -base: "path/to/base.yaml" -# Either input_files or path_patterns/path_pattern (or both) must be specified -input_files: - - "input1.yaml" - - "input2.yaml" -# You can use either path_patterns (array) or path_pattern (single string) -path_patterns: - - "*.yaml" - - "configs/*.yaml" -# OR -# path_pattern: "*.yaml" -inplace: true -out_folder: "./output" -debug: false -quorum: 60 -base_out_path: "./base_output.yaml" -sort_config_path: "./sort_config.yaml" -sort_only: false # Set to true for sort-only mode -exclude_patterns: # Optional: patterns to exclude from sorting - - "*.terraform.yaml" - - "*-template.yaml" +# Complex exclusion for GitOps environments (sort command) +yabe sort -p "**/*.yaml" --exclude "target" --exclude ".argocd" --exclude "*.secret.yaml" ``` -Then run the tool with: - -```bash -./yabe --config config.yaml -``` ## Examples @@ -235,7 +240,7 @@ settings: ### Running the Tool ```bash -./yabe -r helm_values.yaml file1.yaml file2.yaml file3.yaml +yabe separate -r helm_values.yaml file1.yaml file2.yaml file3.yaml ``` ### Expected Output @@ -268,7 +273,7 @@ settings: ### In-place Modification Example Running with the -i flag: ```bash -./yabe -i -r helm_values.yaml file1.yaml file2.yaml file3.yaml +yabe separate -i -r helm_values.yaml file1.yaml file2.yaml file3.yaml ``` ## Testing diff --git a/examples/config-example/README.md b/examples/config-example/README.md new file mode 100644 index 0000000..dfbf7e9 --- /dev/null +++ b/examples/config-example/README.md @@ -0,0 +1,3 @@ +```bash +yabe separate --config ./config.yaml +``` \ No newline at end of file diff --git a/examples/full/README.md b/examples/full/README.md index 30cf69f..00e51c5 100644 --- a/examples/full/README.md +++ b/examples/full/README.md @@ -1,3 +1,3 @@ ```bash -yabe -r ./in/read-base.yaml -b ./in/base.yaml --base-out-path ./out/base.yaml ./in/a.yaml ./in/b.yaml ./in/c.yaml +yabe separate -r ./in/read-base.yaml -b ./in/base.yaml --base-out ./out/base.yaml ./in/a.yaml ./in/b.yaml ./in/c.yaml ``` \ No newline at end of file diff --git a/examples/full/out/c_diff.yaml b/examples/full/out/c_diff.yaml index 8150ed4..5562127 100644 --- a/examples/full/out/c_diff.yaml +++ b/examples/full/out/c_diff.yaml @@ -1,7 +1,7 @@ +a: m c: f: g h: - i - j - l -a: m diff --git a/examples/read-base/README.md b/examples/read-base/README.md index 9f4c3c3..bd5922f 100644 --- a/examples/read-base/README.md +++ b/examples/read-base/README.md @@ -1,3 +1,3 @@ ```bash -yabe -r ./in/read-base.yaml --base-out-path ./out/base.yaml ./in/a.yaml ./in/b.yaml ./in/c.yaml +yabe separate -r ./in/read-base.yaml --base-out ./out/base.yaml ./in/a.yaml ./in/b.yaml ./in/c.yaml ``` \ No newline at end of file diff --git a/examples/simple/README.md b/examples/simple/README.md index 1aef4c1..6d0354e 100644 --- a/examples/simple/README.md +++ b/examples/simple/README.md @@ -1,3 +1,3 @@ ```bash -yabe --base-out-path ./out/base.yaml ./in/a.yaml ./in/b.yaml ./in/c.yaml +yabe separate --base-out-path ./out/base.yaml ./in/a.yaml ./in/b.yaml ./in/c.yaml ``` \ No newline at end of file diff --git a/examples/simple/out/base.yaml b/examples/simple/out/base.yaml index b91ecee..4057eeb 100644 --- a/examples/simple/out/base.yaml +++ b/examples/simple/out/base.yaml @@ -1,7 +1,7 @@ +a: b +c: + d: e h: - i - j - k -a: b -c: - d: e diff --git a/examples/simple/out/c_diff.yaml b/examples/simple/out/c_diff.yaml index 53e186e..5562127 100644 --- a/examples/simple/out/c_diff.yaml +++ b/examples/simple/out/c_diff.yaml @@ -1,7 +1,7 @@ +a: m +c: + f: g h: - i - j - l -a: m -c: - f: g diff --git a/examples/sort/README.md b/examples/sort/README.md index de78b35..f161045 100644 --- a/examples/sort/README.md +++ b/examples/sort/README.md @@ -1,3 +1,4 @@ ```bash -yabe --sort-config-path ../sort-config.yaml --base-out-path ./out/values.yaml ./in/values.yaml +# With sort configuration +yabe sort --sort-config ../config-example/sort-config.yaml -o ./out ./in/values.yaml ``` \ No newline at end of file diff --git a/examples/sort/out/values.yaml b/examples/sort/out/values_sorted.yaml similarity index 70% rename from examples/sort/out/values.yaml rename to examples/sort/out/values_sorted.yaml index 5a271e8..a74bcd9 100644 --- a/examples/sort/out/values.yaml +++ b/examples/sort/out/values_sorted.yaml @@ -4,7 +4,7 @@ spec: template: spec: containers: - - name: nginx - image: "nginx:1.14.2" + - image: "nginx:1.14.2" + name: nginx ports: - containerPort: 80 diff --git a/src/cli.rs b/src/cli.rs index 3281cdc..37ab31b 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,63 +1,15 @@ use clap::{Parser, Subcommand}; -use serde::Deserialize; /// Command-line arguments #[derive(Parser)] #[command(author, version, about = "YAML diff and merge tool for GitOps workflows", long_about = "A tool for diffing, merging, and organizing YAML files for GitOps workflows. Supports both individual file paths and glob patterns.")] pub struct Args { - /// Configuration file - #[arg(long = "config", value_name = "CONFIG_FILE")] - pub config: Option, - /// Enable debug logging #[arg(long = "debug")] pub debug: bool, #[command(subcommand)] - pub command: Option, - - /// Legacy mode: Helm chart values file - #[arg(short = 'r', long = "read-base", value_name = "READ_BASE")] - pub read_only_base: Option, - - /// Legacy mode: Base YAML file to merge with input files - #[arg(short = 'b', long = "base", value_name = "WRITE_BASE")] - pub base: Option, - - /// Legacy mode: Input YAML files (optional if path patterns are provided) - pub input_files: Vec, - - /// Legacy mode: Path patterns to load YAML files (e.g., "*.yaml") - #[arg(short = 'p', long = "path-pattern", value_name = "PATH_PATTERN")] - pub path_patterns: Vec, - - /// Legacy mode: Modify the original input files with diffs - #[arg(short = 'i', long = "in-place")] - pub inplace: bool, - - /// Legacy mode: Output folder - #[arg(short = 'o', long = "out", default_value = "./out")] - pub out_folder: String, - - /// Legacy mode: Quorum percentage (0-100) - #[arg(short = 'q', long = "quorum", default_value_t = 51)] - pub quorum: u8, - - /// Legacy mode: Base file output path - #[arg(long = "base-out-path", default_value = "./base.yaml")] - pub base_out_path: String, - - /// Legacy mode: Sort configuration file path - #[arg(long = "sort-config-path", default_value = "./sort-config.yaml")] - pub sort_config_path: String, - - /// Legacy mode: Sort only mode - only sort files without diffing - #[arg(long = "sort-only")] - pub sort_only: bool, - - /// Legacy mode: Exclude patterns to skip files (e.g., "*.terraform.yaml") - #[arg(long = "exclude", value_name = "EXCLUDE_PATTERN")] - pub exclude_patterns: Vec, + pub command: Commands, } #[derive(Subcommand)] @@ -89,6 +41,10 @@ pub enum Commands { }, /// Separate common base from YAML files (diff/rebalance) Separate { + /// Configuration file path + #[arg(short = 'c', long = "config", value_name = "CONFIG_FILE")] + config_file: Option, + /// Input YAML files (optional if path patterns are provided) input_files: Vec, @@ -130,19 +86,3 @@ pub enum Commands { }, } -#[derive(Deserialize)] -pub struct Config { - pub read_only_base: Option, - pub base: Option, - pub input_files: Option>, - pub path_patterns: Option>, - pub path_pattern: Option, - pub inplace: Option, - pub out_folder: Option, - pub debug: Option, - pub quorum: Option, - pub base_out_path: Option, - pub sort_config_path: Option, - pub sort_only: Option, - pub exclude_patterns: Option>, -} \ No newline at end of file diff --git a/src/commands/legacy.rs b/src/commands/legacy.rs deleted file mode 100644 index ca7e4c2..0000000 --- a/src/commands/legacy.rs +++ /dev/null @@ -1,377 +0,0 @@ -use std::borrow::Cow; -use std::error::Error; -use std::fs; -use std::path::Path; -use log::{info, warn}; -use yaml_rust2::{Yaml, YamlEmitter, YamlLoader}; -use crate::diff::{compute_diff, diff_and_common_multiple}; -use crate::merge::merge_yaml; -use crate::sorter::sort_yaml; -use serde_yml; -use crate::cli::{Args, Config}; -use crate::utils::{expand_and_filter_files, ensure_output_dir}; - -pub fn sort_only_workflow(args: &Args) -> Result<(), Box> { - info!("Running in sort-only mode"); - - let expanded_input_files = expand_and_filter_files( - args.input_files.clone(), - &args.path_patterns, - &args.exclude_patterns, - )?; - - // Validate that we have files to process - if expanded_input_files.is_empty() { - eprintln!("Error: No input files found for sort-only mode. Please specify either input files or path patterns."); - std::process::exit(1); - } - - // Read sort configuration - let sort_config = if !args.sort_config_path.is_empty() && Path::new(&args.sort_config_path).exists() { - info!("Reading sort configuration file: {}", args.sort_config_path); - let content = fs::read_to_string(&args.sort_config_path)?; - YamlLoader::load_from_str(&content)?.into_iter().next().unwrap_or(Yaml::Null) - } else { - eprintln!("Error: Sort configuration file is required for sort-only mode but not found: {}", args.sort_config_path); - std::process::exit(1); - }; - - if sort_config == Yaml::Null { - eprintln!("Error: Sort configuration file is empty or invalid: {}", args.sort_config_path); - std::process::exit(1); - } - - // Ensure output directory exists if not in-place mode - if !args.inplace { - ensure_output_dir(&args.out_folder)?; - } - - // Process each input file - for filename in &expanded_input_files { - info!("Sorting file: {}", filename); - - // Read and parse the YAML file - let content = fs::read_to_string(filename)?; - if let Some(doc) = YamlLoader::load_from_str(&content)?.into_iter().next() { - // Sort the YAML document - let sorted_yaml = sort_yaml(&doc, &sort_config); - - // Convert to string - let mut out_str = String::new(); - { - let mut emitter = YamlEmitter::new(&mut out_str); - emitter.dump(&sorted_yaml)?; - } - out_str = out_str.trim_start_matches("---\n").to_string(); - out_str.push('\n'); - - // Write the sorted content - if args.inplace { - info!("Writing sorted content back to: {}", filename); - fs::write(filename, out_str)?; - } else { - let input_path = Path::new(filename); - let file_stem = input_path - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or("sorted"); - let sorted_filename = format!("{}/{}_sorted.yaml", args.out_folder, file_stem); - info!("Writing sorted content to: {}", sorted_filename); - fs::write(&sorted_filename, out_str)?; - } - } else { - warn!("No YAML documents found in {}", filename); - } - } - - info!("Sort-only workflow completed successfully."); - Ok(()) -} - -pub fn run_legacy_main(mut args: Args) -> Result<(), Box> { - if let Some(config_path) = args.config.as_ref() { - // Read the configuration file - let config_content = fs::read_to_string(config_path)?; - let config: Config = serde_yml::from_str(&config_content)?; - - // Override args with config values if they are not provided via command-line - if args.read_only_base.is_none() { - args.read_only_base = config.read_only_base; - } - - if args.base.is_none() { - args.base = config.base; - } - - if args.input_files.is_empty() { - if let Some(input_files) = config.input_files { - args.input_files = input_files; - } - } - - if args.path_patterns.is_empty() { - if let Some(path_patterns) = config.path_patterns { - args.path_patterns = path_patterns; - } else if let Some(path_pattern) = config.path_pattern.as_ref() { - args.path_patterns.push(path_pattern.clone()); - } - } - - if !args.inplace { - if let Some(inplace) = config.inplace { - args.inplace = inplace; - } - } - - if args.out_folder == "./out" { - if let Some(out_folder) = config.out_folder { - args.out_folder = out_folder; - } - } - - if !args.debug { - if let Some(debug) = config.debug { - args.debug = debug; - } - } - - if args.quorum == 51 { - if let Some(quorum) = config.quorum { - args.quorum = quorum; - } - } - - if args.base_out_path == "./base.yaml" { - if let Some(base_out_path) = config.base_out_path { - args.base_out_path = base_out_path; - } - } - - if args.sort_config_path == "./sort-config.yaml" { - if let Some(sort_config_path) = config.sort_config_path { - args.sort_config_path = sort_config_path; - } - } - - if !args.sort_only { - if let Some(sort_only) = config.sort_only { - args.sort_only = sort_only; - } - } - - if args.exclude_patterns.is_empty() { - if let Some(exclude_patterns) = config.exclude_patterns { - args.exclude_patterns = exclude_patterns; - } - } - } - - // Handle sort-only mode - if args.sort_only { - return sort_only_workflow(&args); - } - - let expanded_input_files = expand_and_filter_files( - args.input_files.clone(), - &args.path_patterns, - &args.exclude_patterns, - )?; - - // Validate that either input_files or path_patterns are provided - if expanded_input_files.is_empty() && args.path_patterns.is_empty() { - eprintln!("Error: No input files or path patterns provided. Please specify either input files or path patterns via command-line arguments or in the configuration file."); - std::process::exit(1); - } - - // Validate that we have at least one file to process after expansion - if expanded_input_files.is_empty() { - eprintln!("Error: No files found matching the provided path patterns. Please check your patterns and try again."); - std::process::exit(1); - } - - info!("Starting the YAML diffing program."); - - let input_filenames = expanded_input_files; - - let quorum_percentage = (args.quorum as f64) / 100.0; - - let base_out_path = args.base_out_path; - - let out_folder = args.out_folder; - - // Ensure output directory exists - ensure_output_dir(&out_folder)?; - - let config = if !args.sort_config_path.is_empty() { - info!("Reading sort configuration file: {}", args.sort_config_path); - let content = fs::read_to_string(&args.sort_config_path); - if let Ok(content) = content { - YamlLoader::load_from_str(&content)?.into_iter().next().unwrap_or(Yaml::Null) - } else { - warn!("Failed to read sort configuration file: {}", args.sort_config_path); - Yaml::Null - } - } else { - Yaml::Null - }; - - let read_only_base = if let Some(ref read_only_base) = args.read_only_base { - info!("Reading helm values file: {}", read_only_base); - let content = fs::read_to_string(read_only_base)?; - YamlLoader::load_from_str(&content)?.into_iter().next() - } else { - None - }; - - // Read and parse the existing base file if provided - let existing_base = if let Some(ref base_path) = args.base { - info!("Reading existing base YAML file: {}", base_path); - let content = fs::read_to_string(base_path)?; - YamlLoader::load_from_str(&content)?.into_iter().next() - } else { - None - }; - - // Read and parse each YAML input file into an object - let mut all_docs = Vec::new(); - for filename in &input_filenames { - info!("Reading input file: {}", filename); - let content = fs::read_to_string(filename)?; - if let Some(doc) = YamlLoader::load_from_str(&content)?.into_iter().next() { - all_docs.push(doc); - } else { - warn!("No YAML documents in {}", filename); - } - } - - // Merge existing base with each input file if existing base is provided - let merged_objs: Vec> = if let Some(ref base) = existing_base { - input_filenames - .iter() - .zip(all_docs.iter()) - .map(|(filename, obj)| { - let merged = merge_yaml(base, obj); - info!("Merged base with input file: {}", filename); - merged - }) - .collect() - } else { - // No existing base; use objs as merged_objs - all_docs.iter().map(|doc| Cow::Borrowed(doc)).collect() - }; - - // Compute diffs between each merged object and read-only base - let diffs: Vec<_> = if let Some(ref helm) = read_only_base { - info!("Computing diffs between merged files and helm values."); - merged_objs - .iter() - .map(|obj| compute_diff(obj.as_ref(), helm).unwrap_or_else(|| Cow::Owned(Yaml::Null))) - .collect() - } else { - // No read-only base provided values; use merged_objs as diffs - merged_objs.clone() - }; - - // Now compute common base and per-file diffs among the diffs - let diffs_refs: Vec<&Yaml> = diffs.iter().map(|cow| cow.as_ref()).collect(); - info!( - "Computing common base and per-file diffs among the diffs with quorum {}%.", - args.quorum - ); - let (base, per_file_diffs) = diff_and_common_multiple(&diffs_refs, quorum_percentage); - - // Process the base YAML if it exists - if let Some(base_yaml) = base { - let processed_yaml = if config != Yaml::Null { - sort_yaml(base_yaml.as_ref(), &config) - } else { - Cow::Borrowed(base_yaml.as_ref()) - }; - - info!("Writing base YAML to {}", base_out_path); - let mut out_str = String::new(); - { - let mut emitter = YamlEmitter::new(&mut out_str); - emitter.dump(&processed_yaml)?; - } - out_str = out_str.trim_start_matches("---\n").to_string(); - out_str.push('\n'); - fs::write(base_out_path.as_str(), out_str)?; - info!("Base YAML written to {}", base_out_path); - } else { - info!("No base YAML to write."); - } - - // Determine whether to write diffs to original files or new files - if args.inplace { - info!("Inplace mode enabled. Modifying original files."); - for (i, diff) in per_file_diffs.iter().enumerate() { - if let Some(diff_yaml) = diff { - let processed_diff = if config != Yaml::Null { - sort_yaml(diff_yaml.as_ref(), &config) - } else { - Cow::Borrowed(diff_yaml.as_ref()) - }; - - info!("Writing diff back to original file: {}", input_filenames[i]); - let mut out_str = String::new(); - { - let mut emitter = YamlEmitter::new(&mut out_str); - emitter.dump(&processed_diff)?; - } - out_str = out_str.trim_start_matches("---\n").to_string(); - out_str.push('\n'); - fs::write(&input_filenames[i], out_str)?; - info!( - "Difference written back to original file {}", - input_filenames[i] - ); - } else { - // If there is no diff, remove the content of the file - info!("No diff for {}; clearing file content.", input_filenames[i]); - fs::write(&input_filenames[i], "")?; - info!( - "No difference for {}; file content cleared.", - input_filenames[i] - ); - } - } - } else { - info!("Writing diffs to new files."); - for (i, diff) in per_file_diffs.iter().enumerate() { - if let Some(diff_yaml) = diff { - let processed_diff = if config != Yaml::Null { - sort_yaml(diff_yaml.as_ref(), &config) - } else { - Cow::Borrowed(diff_yaml.as_ref()) - }; - - info!("Writing diff for {} to new file.", input_filenames[i]); - let mut out_str = String::new(); - { - let mut emitter = YamlEmitter::new(&mut out_str); - emitter.dump(&processed_diff)?; - } - - let input_path = Path::new(&input_filenames[i]); - let file_stem = input_path - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or("diff"); - let diff_filename = format!("{}/{}_diff.yaml", out_folder, file_stem); - out_str = out_str.trim_start_matches("---\n").to_string(); - out_str.push('\n'); - fs::write(&diff_filename, out_str)?; - info!( - "Difference for {} written to {}", - input_filenames[i], diff_filename - ); - } else { - info!("No diff for {}; not writing a diff file.", input_filenames[i]); - } - } - } - - info!("Program completed successfully."); - Ok(()) -} \ No newline at end of file diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 2af98a9..3b48c4f 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -1,3 +1,2 @@ pub mod sort; -pub mod separate; -pub mod legacy; \ No newline at end of file +pub mod separate; \ No newline at end of file diff --git a/src/commands/separate.rs b/src/commands/separate.rs index 096dfa5..81b655b 100644 --- a/src/commands/separate.rs +++ b/src/commands/separate.rs @@ -8,8 +8,25 @@ use crate::diff::{compute_diff, diff_and_common_multiple}; use crate::merge::merge_yaml; use crate::sorter::sort_yaml; use crate::utils::{expand_and_filter_files, ensure_output_dir}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Deserialize, Serialize)] +struct SeparateConfig { + read_only_base: Option, + base: Option, + input_files: Option>, + path_patterns: Option>, + inplace: Option, + out_folder: Option, + debug: Option, + quorum: Option, + base_out_path: Option, + sort_config_path: Option, + exclude_patterns: Option>, +} pub fn run_separate_command( + config_file: Option, input_files: Vec, path_patterns: Vec, read_only_base: Option, @@ -23,10 +40,62 @@ pub fn run_separate_command( ) -> Result<(), Box> { info!("Running separate command"); - let expanded_input_files = expand_and_filter_files(input_files, &path_patterns, &exclude_patterns)?; + // Load config file if provided and merge with CLI arguments + let (final_input_files, final_path_patterns, final_read_only_base, final_base, final_quorum, + final_base_out_path, final_sort_config_path, final_inplace, final_out_folder, final_exclude_patterns) = + if let Some(config_path) = config_file { + info!("Loading configuration from file: {}", config_path); + let config_content = fs::read_to_string(&config_path)?; + let config: SeparateConfig = serde_yml::from_str(&config_content)?; + + // CLI arguments override config file values + let merged_input_files = if input_files.is_empty() { + config.input_files.unwrap_or_default() + } else { + input_files + }; + let merged_path_patterns = if path_patterns.is_empty() { + config.path_patterns.unwrap_or_default() + } else { + path_patterns + }; + let merged_read_only_base = read_only_base.or(config.read_only_base); + let merged_base = base.or(config.base); + let merged_quorum = if quorum == 51 { config.quorum.unwrap_or(51) } else { quorum }; + let merged_base_out_path = if base_out_path == "./base.yaml" { + config.base_out_path.unwrap_or_else(|| "./base.yaml".to_string()) + } else { + base_out_path + }; + let merged_sort_config_path = if sort_config_path == "./sort-config.yaml" { + config.sort_config_path.unwrap_or_else(|| "./sort-config.yaml".to_string()) + } else { + sort_config_path + }; + let merged_inplace = if !inplace { config.inplace.unwrap_or(false) } else { inplace }; + let merged_out_folder = if out_folder == "./out" { + config.out_folder.unwrap_or_else(|| "./out".to_string()) + } else { + out_folder + }; + let merged_exclude_patterns = if exclude_patterns.is_empty() { + config.exclude_patterns.unwrap_or_default() + } else { + exclude_patterns + }; + + (merged_input_files, merged_path_patterns, merged_read_only_base, merged_base, + merged_quorum, merged_base_out_path, merged_sort_config_path, merged_inplace, + merged_out_folder, merged_exclude_patterns) + } else { + (input_files, path_patterns, read_only_base, base, quorum, base_out_path, + sort_config_path, inplace, out_folder, exclude_patterns) + }; + + let expanded_input_files = expand_and_filter_files(final_input_files, &final_path_patterns, &final_exclude_patterns)?; // Validate that either input_files or path_patterns are provided - if expanded_input_files.is_empty() && path_patterns.is_empty() { + if expanded_input_files.is_empty() && final_path_patterns.is_empty() { eprintln!("Error: No input files or path patterns provided. Please specify either input files or path patterns."); std::process::exit(1); } @@ -38,25 +107,26 @@ pub fn run_separate_command( } let input_filenames = expanded_input_files; - let quorum_percentage = (quorum as f64) / 100.0; + let quorum_percentage = (final_quorum as f64) / 100.0; // Ensure output directory exists - ensure_output_dir(&out_folder)?; + ensure_output_dir(&final_out_folder)?; - let config = if !sort_config_path.is_empty() { - info!("Reading sort configuration file: {}", sort_config_path); - let content = fs::read_to_string(&sort_config_path); + let config = if !final_sort_config_path.is_empty() { + info!("Reading sort configuration file: {}", final_sort_config_path); + let content = fs::read_to_string(&final_sort_config_path); if let Ok(content) = content { YamlLoader::load_from_str(&content)?.into_iter().next().unwrap_or(Yaml::Null) } else { - log::warn!("Failed to read sort configuration file: {}", sort_config_path); + log::warn!("Failed to read sort configuration file: {}", final_sort_config_path); Yaml::Null } } else { + info!("No sort configuration provided, will use default alphabetical sorting"); Yaml::Null }; - let read_only_base = if let Some(ref read_only_base) = read_only_base { + let read_only_base = if let Some(ref read_only_base) = final_read_only_base { info!("Reading helm values file: {}", read_only_base); let content = fs::read_to_string(read_only_base)?; YamlLoader::load_from_str(&content)?.into_iter().next() @@ -65,7 +135,7 @@ pub fn run_separate_command( }; // Read and parse the existing base file if provided - let existing_base = if let Some(ref base_path) = base { + let existing_base = if let Some(ref base_path) = final_base { info!("Reading existing base YAML file: {}", base_path); let content = fs::read_to_string(base_path)?; YamlLoader::load_from_str(&content)?.into_iter().next() @@ -117,19 +187,15 @@ pub fn run_separate_command( let diffs_refs: Vec<&Yaml> = diffs.iter().map(|cow| cow.as_ref()).collect(); info!( "Computing common base and per-file diffs among the diffs with quorum {}%.", - quorum + final_quorum ); let (base, per_file_diffs) = diff_and_common_multiple(&diffs_refs, quorum_percentage); // Process the base YAML if it exists if let Some(base_yaml) = base { - let processed_yaml = if config != Yaml::Null { - sort_yaml(base_yaml.as_ref(), &config) - } else { - Cow::Borrowed(base_yaml.as_ref()) - }; + let processed_yaml = sort_yaml(base_yaml.as_ref(), &config); - info!("Writing base YAML to {}", base_out_path); + info!("Writing base YAML to {}", final_base_out_path); let mut out_str = String::new(); { let mut emitter = YamlEmitter::new(&mut out_str); @@ -137,22 +203,18 @@ pub fn run_separate_command( } out_str = out_str.trim_start_matches("---\n").to_string(); out_str.push('\n'); - fs::write(base_out_path.as_str(), out_str)?; - info!("Base YAML written to {}", base_out_path); + fs::write(final_base_out_path.as_str(), out_str)?; + info!("Base YAML written to {}", final_base_out_path); } else { info!("No base YAML to write."); } // Determine whether to write diffs to original files or new files - if inplace { + if final_inplace { info!("Inplace mode enabled. Modifying original files."); for (i, diff) in per_file_diffs.iter().enumerate() { if let Some(diff_yaml) = diff { - let processed_diff = if config != Yaml::Null { - sort_yaml(diff_yaml.as_ref(), &config) - } else { - Cow::Borrowed(diff_yaml.as_ref()) - }; + let processed_diff = sort_yaml(diff_yaml.as_ref(), &config); info!("Writing diff back to original file: {}", input_filenames[i]); let mut out_str = String::new(); @@ -181,11 +243,7 @@ pub fn run_separate_command( info!("Writing diffs to new files."); for (i, diff) in per_file_diffs.iter().enumerate() { if let Some(diff_yaml) = diff { - let processed_diff = if config != Yaml::Null { - sort_yaml(diff_yaml.as_ref(), &config) - } else { - Cow::Borrowed(diff_yaml.as_ref()) - }; + let processed_diff = sort_yaml(diff_yaml.as_ref(), &config); info!("Writing diff for {} to new file.", input_filenames[i]); let mut out_str = String::new(); @@ -199,7 +257,7 @@ pub fn run_separate_command( .file_stem() .and_then(|s| s.to_str()) .unwrap_or("diff"); - let diff_filename = format!("{}/{}_diff.yaml", out_folder, file_stem); + let diff_filename = format!("{}/{}_diff.yaml", final_out_folder, file_stem); out_str = out_str.trim_start_matches("---\n").to_string(); out_str.push('\n'); fs::write(&diff_filename, out_str)?; diff --git a/src/commands/sort.rs b/src/commands/sort.rs index e01c3e4..88cdc4c 100644 --- a/src/commands/sort.rs +++ b/src/commands/sort.rs @@ -30,14 +30,10 @@ pub fn run_sort_command( let content = fs::read_to_string(&sort_config_path)?; YamlLoader::load_from_str(&content)?.into_iter().next().unwrap_or(Yaml::Null) } else { - eprintln!("Error: Sort configuration file is required but not found: {}", sort_config_path); - std::process::exit(1); + info!("No sort configuration provided, will use default alphabetical sorting"); + Yaml::Null }; - if sort_config == Yaml::Null { - eprintln!("Error: Sort configuration file is empty or invalid: {}", sort_config_path); - std::process::exit(1); - } // Ensure output directory exists if not in-place mode if !inplace { diff --git a/src/lib.rs b/src/lib.rs index bc08ae7..b57344d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,5 +9,4 @@ pub mod utils; pub use diff::{compute_diff, diff_and_common_multiple}; pub use cli::{Args, Commands}; pub use commands::sort::run_sort_command; -pub use commands::separate::run_separate_command; -pub use commands::legacy::run_legacy_main; \ No newline at end of file +pub use commands::separate::run_separate_command; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 76eee78..2b1873d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,6 @@ use std::error::Error; use clap::Parser; -use yabe::{Args, Commands, run_sort_command, run_separate_command, run_legacy_main}; +use yabe::{Args, Commands, run_sort_command, run_separate_command}; fn main() -> Result<(), Box> { let args = Args::parse(); @@ -13,17 +13,18 @@ fn main() -> Result<(), Box> { } match args.command { - Some(Commands::Sort { + Commands::Sort { input_files, path_patterns, sort_config_path, inplace, out_folder, exclude_patterns - }) => { + } => { run_sort_command(input_files, path_patterns, sort_config_path, inplace, out_folder, exclude_patterns) } - Some(Commands::Separate { + Commands::Separate { + config_file, input_files, path_patterns, read_only_base, @@ -34,12 +35,8 @@ fn main() -> Result<(), Box> { inplace, out_folder, exclude_patterns - }) => { - run_separate_command(input_files, path_patterns, read_only_base, base, quorum, base_out_path, sort_config_path, inplace, out_folder, exclude_patterns) - } - None => { - // Legacy mode - use the existing logic - run_legacy_main(args) + } => { + run_separate_command(config_file, input_files, path_patterns, read_only_base, base, quorum, base_out_path, sort_config_path, inplace, out_folder, exclude_patterns) } } } \ No newline at end of file diff --git a/src/sorter.rs b/src/sorter.rs index f8c4b7b..4655e33 100644 --- a/src/sorter.rs +++ b/src/sorter.rs @@ -13,7 +13,13 @@ pub fn sort_yaml<'a>(doc: &'a Yaml, config: &Yaml) -> Cow<'a, Yaml> { } Cow::Owned(Yaml::Array(new_v)) } else { - Cow::Borrowed(doc) + // Recursively sort nested structures even without array config + let mut new_v = v.clone(); + for x in &mut new_v { + let sorted = sort_yaml(x, config); + *x = sorted.into_owned(); + } + Cow::Owned(Yaml::Array(new_v)) } } Yaml::Hash(h) => { @@ -30,7 +36,14 @@ pub fn sort_yaml<'a>(doc: &'a Yaml, config: &Yaml) -> Cow<'a, Yaml> { } Cow::Owned(Yaml::Hash(new_h)) } else { - Cow::Borrowed(doc) + // Apply alphabetical sorting by default when no config is provided + let mut new_h = h.clone(); + hash_sorter(&mut new_h, &[]); // Empty pre_order will just sort alphabetically + for (_, v) in &mut new_h { + let sorted = sort_yaml(v, config); + *v = sorted.into_owned(); + } + Cow::Owned(Yaml::Hash(new_h)) } } _ => Cow::Borrowed(doc), From bfa22f3548a0a2e06f32e45ad3c8a9bf8c649e1e Mon Sep 17 00:00:00 2001 From: Ilya Dvorkin Date: Fri, 4 Jul 2025 21:21:25 +0300 Subject: [PATCH 5/7] Add Makefile for testing examples and streamline test execution --- examples/config-example/Makefile | 2 + examples/config-example/README.md | 3 - examples/full/Makefile | 2 + examples/full/README.md | 3 - examples/read-base/Makefile | 2 + examples/read-base/README.md | 3 - examples/simple/Makefile | 2 + examples/simple/README.md | 3 - examples/sort/Makefile | 2 + examples/sort/README.md | 4 - test-examples.sh | 298 ++---------------------------- 11 files changed, 21 insertions(+), 303 deletions(-) create mode 100644 examples/config-example/Makefile delete mode 100644 examples/config-example/README.md create mode 100644 examples/full/Makefile delete mode 100644 examples/full/README.md create mode 100644 examples/read-base/Makefile delete mode 100644 examples/read-base/README.md create mode 100644 examples/simple/Makefile delete mode 100644 examples/simple/README.md create mode 100644 examples/sort/Makefile delete mode 100644 examples/sort/README.md diff --git a/examples/config-example/Makefile b/examples/config-example/Makefile new file mode 100644 index 0000000..dfae20d --- /dev/null +++ b/examples/config-example/Makefile @@ -0,0 +1,2 @@ +test: + yabe separate --config ./config.yaml diff --git a/examples/config-example/README.md b/examples/config-example/README.md deleted file mode 100644 index dfbf7e9..0000000 --- a/examples/config-example/README.md +++ /dev/null @@ -1,3 +0,0 @@ -```bash -yabe separate --config ./config.yaml -``` \ No newline at end of file diff --git a/examples/full/Makefile b/examples/full/Makefile new file mode 100644 index 0000000..702894b --- /dev/null +++ b/examples/full/Makefile @@ -0,0 +1,2 @@ +test: + yabe separate -r ./in/read-base.yaml -b ./in/base.yaml --base-out ./out/base.yaml ./in/a.yaml ./in/b.yaml ./in/c.yaml diff --git a/examples/full/README.md b/examples/full/README.md deleted file mode 100644 index 00e51c5..0000000 --- a/examples/full/README.md +++ /dev/null @@ -1,3 +0,0 @@ -```bash -yabe separate -r ./in/read-base.yaml -b ./in/base.yaml --base-out ./out/base.yaml ./in/a.yaml ./in/b.yaml ./in/c.yaml -``` \ No newline at end of file diff --git a/examples/read-base/Makefile b/examples/read-base/Makefile new file mode 100644 index 0000000..e9d221e --- /dev/null +++ b/examples/read-base/Makefile @@ -0,0 +1,2 @@ +test: + yabe separate -r ./in/read-base.yaml --base-out ./out/base.yaml ./in/a.yaml ./in/b.yaml ./in/c.yaml diff --git a/examples/read-base/README.md b/examples/read-base/README.md deleted file mode 100644 index bd5922f..0000000 --- a/examples/read-base/README.md +++ /dev/null @@ -1,3 +0,0 @@ -```bash -yabe separate -r ./in/read-base.yaml --base-out ./out/base.yaml ./in/a.yaml ./in/b.yaml ./in/c.yaml -``` \ No newline at end of file diff --git a/examples/simple/Makefile b/examples/simple/Makefile new file mode 100644 index 0000000..06a39e3 --- /dev/null +++ b/examples/simple/Makefile @@ -0,0 +1,2 @@ +test: + yabe separate --base-out-path ./out/base.yaml ./in/a.yaml ./in/b.yaml ./in/c.yaml diff --git a/examples/simple/README.md b/examples/simple/README.md deleted file mode 100644 index 6d0354e..0000000 --- a/examples/simple/README.md +++ /dev/null @@ -1,3 +0,0 @@ -```bash -yabe separate --base-out-path ./out/base.yaml ./in/a.yaml ./in/b.yaml ./in/c.yaml -``` \ No newline at end of file diff --git a/examples/sort/Makefile b/examples/sort/Makefile new file mode 100644 index 0000000..1fd70cb --- /dev/null +++ b/examples/sort/Makefile @@ -0,0 +1,2 @@ +test: + yabe sort --sort-config ../config-example/sort-config.yaml -o ./out ./in/values.yaml diff --git a/examples/sort/README.md b/examples/sort/README.md deleted file mode 100644 index f161045..0000000 --- a/examples/sort/README.md +++ /dev/null @@ -1,4 +0,0 @@ -```bash -# With sort configuration -yabe sort --sort-config ../config-example/sort-config.yaml -o ./out ./in/values.yaml -``` \ No newline at end of file diff --git a/test-examples.sh b/test-examples.sh index aaf9698..7ea732c 100755 --- a/test-examples.sh +++ b/test-examples.sh @@ -1,287 +1,11 @@ -#!/bin/bash - -# Test script for YABE examples -# This script builds the binary and runs all examples to ensure outputs remain stable - -set -e - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -# Script directory -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$SCRIPT_DIR" - -# Function to print colored output -print_status() { - echo -e "${GREEN}[INFO]${NC} $1" -} - -print_warning() { - echo -e "${YELLOW}[WARN]${NC} $1" -} - -print_error() { - echo -e "${RED}[ERROR]${NC} $1" -} - -# Function to compare files (allows key reordering in YAML) -compare_files() { - local file1="$1" - local file2="$2" - - # First try exact comparison - if diff -q "$file1" "$file2" > /dev/null 2>&1; then - return 0 - fi - - # If files differ, check if they're functionally equivalent YAML - # For now, we'll check the number of lines and warn about key ordering - local lines1=$(wc -l < "$file1") - local lines2=$(wc -l < "$file2") - - if [[ $lines1 -eq $lines2 ]]; then - print_warning "Files differ but have same line count (likely key ordering): $file1 vs $file2" - return 0 - fi - - print_error "Files differ: $file1 vs $file2" - print_error "Diff output:" - diff "$file1" "$file2" || true - return 1 -} - -# Function to compare directories -compare_directories() { - local dir1="$1" - local dir2="$2" - local example_name="$3" - - print_status "Comparing outputs for $example_name..." - - # Get all files in both directories (excluding input files unless it's a sort example) - local files1=($(find "$dir1" -name "*.yaml" -type f | sort)) - local files2 - if [[ "$example_name" == "sort" ]]; then - # For sort examples, we compare the modified input files - files2=($(find "$dir2" -name "*.yaml" -type f | sort)) - else - # For other examples, exclude input files - files2=($(find "$dir2" -name "*.yaml" -type f -not -name "a.yaml" -not -name "b.yaml" -not -name "c.yaml" -not -name "read-base.yaml" | sort)) - fi - - # Check if same number of files - if [[ ${#files1[@]} -ne ${#files2[@]} ]]; then - print_error "Different number of output files in $example_name" - print_error "Expected: ${#files1[@]}, Got: ${#files2[@]}" - return 1 - fi - - # Compare each file - local failed=0 - for i in "${!files1[@]}"; do - local file1_name=$(basename "${files1[$i]}") - local file2_name=$(basename "${files2[$i]}") - - if [[ "$file1_name" != "$file2_name" ]]; then - print_error "File name mismatch: $file1_name vs $file2_name" - failed=1 - continue - fi - - if ! compare_files "${files1[$i]}" "${files2[$i]}"; then - failed=1 - fi - done - - if [[ $failed -eq 0 ]]; then - print_status "✓ $example_name outputs match" - return 0 - else - print_error "✗ $example_name outputs differ" - return 1 - fi -} - -# Function to test an example -test_example() { - local example_name="$1" - local test_cmd="$2" - - print_status "Testing $example_name..." - - # Create temporary directory for test output - local temp_dir=$(mktemp -d) - local example_dir="examples/$example_name" - local expected_dir="$example_dir/out" - - # Copy input files to temp directory - cp -r "$example_dir/in/"* "$temp_dir/" - - # Copy config files if they exist - if [[ -f "$example_dir/config.yaml" ]]; then - cp "$example_dir/config.yaml" "$temp_dir/" - fi - if [[ -f "$example_dir/sort-config.yaml" ]]; then - cp "$example_dir/sort-config.yaml" "$temp_dir/" - fi - - # Run the test command - cd "$temp_dir" - print_status "Running: $test_cmd" - if ! eval "$test_cmd" >/dev/null 2>&1; then - print_error "Command failed: $test_cmd" - cd "$SCRIPT_DIR" - rm -rf "$temp_dir" - return 1 - fi - - # Handle different output patterns - # For sort examples, files are modified in place - # For other examples, base.yaml is in current dir, diffs are in out/ - local actual_output_dir="$temp_dir" - - # Move base.yaml to out directory if it exists (for consistency) - if [[ -f "$temp_dir/base.yaml" && -d "$temp_dir/out" ]]; then - mv "$temp_dir/base.yaml" "$temp_dir/out/" - actual_output_dir="$temp_dir/out" - elif [[ -d "$temp_dir/out" ]]; then - actual_output_dir="$temp_dir/out" - fi - - # Compare outputs - cd "$SCRIPT_DIR" - if compare_directories "$expected_dir" "$actual_output_dir" "$example_name"; then - rm -rf "$temp_dir" - return 0 - else - print_error "Test output directory: $temp_dir" - return 1 - fi -} - -# Main test function -main() { - print_status "Starting YABE examples test..." - - # Build the project - print_status "Building project..." - if ! cargo build --release; then - print_error "Failed to build project" - exit 1 - fi - - # Get the binary path (absolute) - local binary="$SCRIPT_DIR/target/release/yabe" - if [[ ! -f "$binary" ]]; then - print_error "Binary not found: $binary" - exit 1 - fi - - print_status "Using binary: $binary" - - # Test all examples - local failed_tests=0 - local total_tests=0 - - # Test simple example (legacy command) - total_tests=$((total_tests + 1)) - if test_example "simple" "$binary a.yaml b.yaml c.yaml"; then - print_status "✓ simple (legacy) passed" - else - print_error "✗ simple (legacy) failed" - failed_tests=$((failed_tests + 1)) - fi - - # Test simple example (new subcommand) - total_tests=$((total_tests + 1)) - if test_example "simple" "$binary separate a.yaml b.yaml c.yaml"; then - print_status "✓ simple (separate) passed" - else - print_error "✗ simple (separate) failed" - failed_tests=$((failed_tests + 1)) - fi - - # Test read-base example (legacy command) - total_tests=$((total_tests + 1)) - if test_example "read-base" "$binary -r read-base.yaml a.yaml b.yaml c.yaml"; then - print_status "✓ read-base (legacy) passed" - else - print_error "✗ read-base (legacy) failed" - failed_tests=$((failed_tests + 1)) - fi - - # Test read-base example (new subcommand) - total_tests=$((total_tests + 1)) - if test_example "read-base" "$binary separate --read-base read-base.yaml a.yaml b.yaml c.yaml"; then - print_status "✓ read-base (separate) passed" - else - print_error "✗ read-base (separate) failed" - failed_tests=$((failed_tests + 1)) - fi - - # Test full example (legacy command) - total_tests=$((total_tests + 1)) - if test_example "full" "$binary -r read-base.yaml -b base.yaml a.yaml b.yaml c.yaml"; then - print_status "✓ full (legacy) passed" - else - print_error "✗ full (legacy) failed" - failed_tests=$((failed_tests + 1)) - fi - - # Test full example (new subcommand) - total_tests=$((total_tests + 1)) - if test_example "full" "$binary separate --read-base read-base.yaml --base base.yaml a.yaml b.yaml c.yaml"; then - print_status "✓ full (separate) passed" - else - print_error "✗ full (separate) failed" - failed_tests=$((failed_tests + 1)) - fi - - # Test sort example (legacy command) - total_tests=$((total_tests + 1)) - if test_example "sort" "$binary --sort-only --sort-config-path $SCRIPT_DIR/sort-config.yaml -i values.yaml"; then - print_status "✓ sort (legacy) passed" - else - print_error "✗ sort (legacy) failed" - failed_tests=$((failed_tests + 1)) - fi - - # Test sort example (new subcommand) - total_tests=$((total_tests + 1)) - if test_example "sort" "$binary sort --sort-config $SCRIPT_DIR/sort-config.yaml --in-place values.yaml"; then - print_status "✓ sort (subcommand) passed" - else - print_error "✗ sort (subcommand) failed" - failed_tests=$((failed_tests + 1)) - fi - - # Test config-example (new subcommand) - separate subcommand with config - total_tests=$((total_tests + 1)) - if test_example "config-example" "$binary separate --read-base read-base.yaml --base base.yaml a.yaml b.yaml c.yaml"; then - print_status "✓ config-example (separate) passed" - else - print_error "✗ config-example (separate) failed" - failed_tests=$((failed_tests + 1)) - fi - - # Print summary - print_status "Test Summary:" - print_status "Total tests: $total_tests" - print_status "Passed: $((total_tests - failed_tests))" - - if [[ $failed_tests -gt 0 ]]; then - print_error "Failed: $failed_tests" - print_error "Some tests failed!" - exit 1 - else - print_status "All tests passed! ✓" - exit 0 - fi -} - -# Run main function -main "$@" \ No newline at end of file +for dir in examples/*/; do + echo "Running tests in $dir" + (cd "$dir" && make test) +done + +if git status --porcelain | grep -q '^[AM]'; then + echo "There are new or modified files in the examples directories." + exit 1 +else + echo "All tests passed and no new or modified files found." +fi \ No newline at end of file From ab15fc52d8ab554e392e1f851a3a4d2711901969 Mon Sep 17 00:00:00 2001 From: Ilya Dvorkin Date: Fri, 4 Jul 2025 21:22:33 +0300 Subject: [PATCH 6/7] Fix Makefile command and enhance test script with error handling --- examples/simple/Makefile | 2 +- test-examples.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/simple/Makefile b/examples/simple/Makefile index 06a39e3..e62be59 100644 --- a/examples/simple/Makefile +++ b/examples/simple/Makefile @@ -1,2 +1,2 @@ test: - yabe separate --base-out-path ./out/base.yaml ./in/a.yaml ./in/b.yaml ./in/c.yaml + yabe separate --base-out ./out/base.yaml ./in/a.yaml ./in/b.yaml ./in/c.yaml diff --git a/test-examples.sh b/test-examples.sh index 7ea732c..3267533 100755 --- a/test-examples.sh +++ b/test-examples.sh @@ -1,3 +1,5 @@ +set -eo pipefail + for dir in examples/*/; do echo "Running tests in $dir" (cd "$dir" && make test) From 74ff0c6e281580c4bd500cff5ec61f4e58775249 Mon Sep 17 00:00:00 2001 From: Ilya Dvorkin Date: Sat, 5 Jul 2025 11:50:00 +0300 Subject: [PATCH 7/7] Enhance test-examples.sh to build project and update binary path for tests --- test-examples.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test-examples.sh b/test-examples.sh index 3267533..1a3b0e9 100755 --- a/test-examples.sh +++ b/test-examples.sh @@ -1,11 +1,25 @@ +#!/bin/bash + set -eo pipefail +# Build the project +echo "Building project..." +cargo build --release + +# Get the binary path +BINARY_PATH="$(pwd)/target/release/yabe" + +# Add the binary to PATH for the Makefiles +export PATH="$(dirname "$BINARY_PATH"):$PATH" + +echo "Running tests with binary: $BINARY_PATH" + for dir in examples/*/; do echo "Running tests in $dir" (cd "$dir" && make test) done -if git status --porcelain | grep -q '^[AM]'; then +if git status --porcelain | grep -E -q '^ [AM]'; then echo "There are new or modified files in the examples directories." exit 1 else