diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3070b4f..3d1aa86 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,7 +9,10 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-latest, ubuntu-latest] + os: [ +# macos-latest, + ubuntu-latest + ] arch: [x86_64, aarch64] steps: - uses: actions/checkout@v4 @@ -22,5 +25,7 @@ jobs: run: cargo build --release - name: Cargo test run: cargo test + - name: Test examples + run: ./test-examples.sh - name: Check artifact weight run: ls -lh target/release/ diff --git a/Cargo.lock b/Cargo.lock index a9026ed..d788220 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -486,7 +486,7 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "yabe-gitops" -version = "0.1.9" +version = "0.1.11" dependencies = [ "clap", "env_logger", diff --git a/Cargo.toml b/Cargo.toml index a6dfbfc..b76e021 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ name = "yabe-gitops" description = "GitOps organizer" repository = "https://github.com/dvrkn/yabe" readme = "README.md" -version = "0.1.10" +version = "0.1.11" edition = "2021" keywords = ["gitops", "kubernetes", "argocd", "yaml", "helm"] license = "MIT" diff --git a/README.md b/README.md index f3425ae..c1a441f 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,14 @@ YABE is a tool designed to help manage large amounts of YAML files in a GitOps e ## Features -- **Compute diffs:** Detect differences between YAML files. +- **Compute diffs:** Detect differences between YAML files using the `separate` command. - **Merge YAML files:** Combine YAML files with a base YAML, either from an existing file or dynamically computed. - **Quorum-based diffing:** Extract common base YAML based on a quorum percentage. -- **Sort YAML content:** Sort keys in YAML files based on user-defined configuration. -- **Sort-only mode:** Sort YAML files without performing any diffing operations. +- **Sort YAML content:** Sort keys in YAML files based on user-defined configuration or alphabetically using the `sort` command. - **Helm Values Integration:** Merge input YAML files with Helm values files. - **In-place modification or output to new files.** -- **Configuration File Support:** Run the tool using a configuration file to simplify usage in automated workflows. +- **Flexible file selection:** Support for glob patterns and exclude patterns. +- **Command-based interface:** Clean separation between sorting and diffing operations. ## Installation @@ -21,43 +21,80 @@ cargo install yabe-gitops ## Usage +YABE now uses subcommands to organize its functionality. Run `yabe --help` to see available commands: + ```bash -Usage: yabe [OPTIONS] [INPUT_FILES]... +Usage: yabe [OPTIONS] + +Commands: + sort Sort YAML files based on configuration + separate Separate common base from YAML files (diff/rebalance) + help Print this message or the help of the given subcommand(s) + +Options: + --debug Enable debug logging + -h, --help Print help + -V, --version Print version +``` + +### Sort Command + +Sort YAML files based on configuration: + +```bash +Usage: yabe sort [OPTIONS] [INPUT_FILES]... Arguments: [INPUT_FILES]... Input YAML files (optional if path patterns are provided) Options: - -r, --read-base (Optional) Read-only base for values deduplication - -b, --base (Optional) Common values of all input files, if not provided, will be computed - -p, --path-pattern Path patterns to load YAML files (e.g., "*.yaml") - -i, --in-place Modify the original input files with diffs - -o, --out Output folder for diff files [default: ./out] - --debug Enable debug logging - -q, --quorum Quorum percentage (0-100) [default: 51] - --base-out-path (Optional) Base file output path [default: ./base.yaml] - --sort-config-path (Optional) Sort configuration file path [default: ./sort-config.yaml], if not provided, will not sort - --sort-only Sort only mode - only sort files without diffing - --exclude Exclude patterns to skip files (e.g., "*.terraform.yaml") - --config (Optional) Configuration file - -h, --help Print help - -V, --version Print version + -p, --path-pattern Path patterns to load YAML files (e.g., "*.yaml") + --sort-config Sort configuration file path [default: ./sort-config.yaml] + -i, --in-place Modify the original input files with sorted content + -o, --out Output folder [default: ./out] + --exclude Exclude patterns to skip files (e.g., "*.terraform.yaml") + -h, --help Print help +``` + +### Separate Command + +Separate common base from YAML files (diff/rebalance): + +```bash +Usage: yabe separate [OPTIONS] [INPUT_FILES]... + +Arguments: + [INPUT_FILES]... Input YAML files (optional if path patterns are provided) + +Options: + -p, --path-pattern Path patterns to load YAML files (e.g., "*.yaml") + -r, --read-base Helm chart values file + -b, --base Base YAML file to merge with input files + -q, --quorum Quorum percentage (0-100) [default: 51] + --base-out Base file output path [default: ./base.yaml] + --sort-config Sort configuration file path [default: ./sort-config.yaml] + -i, --in-place Modify the original input files with diffs + -o, --out Output folder [default: ./out] + --exclude Exclude patterns to skip files (e.g., "*.terraform.yaml") + -h, --help Print help ``` **Note:** You must provide either input files or path patterns. If both are provided, all matching files will be processed. ### Basic Usage -Run the tool with the YAML override files: +#### Separating Common Base from YAML Files + +Run the separate command with the YAML override files: ```bash -./yabe file1.yaml file2.yaml file3.yaml +yabe separate file1.yaml file2.yaml file3.yaml ``` Or use path patterns to load multiple files: ```bash -./yabe -p "*.yaml" -p "configs/*.yaml" +yabe separate -p "*.yaml" -p "configs/*.yaml" ``` This will compute the differences among the override files and generate: @@ -65,48 +102,51 @@ This will compute the differences among the override files and generate: * base.yaml: The common base configuration. * file1_diff.yaml, file2_diff.yaml, file3_diff.yaml: The differences for each file. -### In-place Modification +#### In-place Modification Use the -i or --in-place flag to modify the original override files with their differences: ```bash -./yabe -i -r helm_values.yaml file1.yaml file2.yaml file3.yaml +yabe separate -i -r helm_values.yaml file1.yaml file2.yaml file3.yaml ``` -### Enable Debug Logging +#### Enable Debug Logging Use the --debug flag to enable detailed debug logging: ```bash -./yabe --debug -r helm_values.yaml file1.yaml file2.yaml file3.yaml +yabe --debug separate -r helm_values.yaml file1.yaml file2.yaml file3.yaml ``` -### Sort Only Mode +#### Sort YAML Files -Use the --sort-only flag to only sort YAML files without performing any diffing operations: +Use the sort command to sort YAML files based on configuration: ```bash # Sort files and output to ./out directory -./yabe --sort-only --sort-config-path sort-config.yaml file1.yaml file2.yaml +yabe sort --sort-config sort-config.yaml file1.yaml file2.yaml # Sort files in-place (modify original files) -./yabe --sort-only --sort-config-path sort-config.yaml -i file1.yaml file2.yaml +yabe sort --sort-config sort-config.yaml -i file1.yaml file2.yaml # Sort files using path patterns -./yabe --sort-only --sort-config-path sort-config.yaml -p "*.yaml" -p "configs/*.yaml" +yabe sort --sort-config sort-config.yaml -p "*.yaml" -p "configs/*.yaml" # Sort files to a specific output directory -./yabe --sort-only --sort-config-path sort-config.yaml -o ./sorted-files *.yaml +yabe sort --sort-config sort-config.yaml -o ./sorted-files *.yaml # Sort files recursively while excluding certain patterns -./yabe --sort-only --sort-config-path sort-config.yaml -p "**/*.yaml" --exclude "*.terraform.yaml" --exclude "*-template.yaml" +yabe sort --sort-config sort-config.yaml -p "**/*.yaml" --exclude "*.terraform.yaml" --exclude "*-template.yaml" # Sort files in-place while excluding terraform files -./yabe --sort-only --sort-config-path sort-config.yaml -p "./envs/**/*.yaml" --exclude "*.terraform.yaml" -i +yabe sort --sort-config sort-config.yaml -p "./envs/**/*.yaml" --exclude "*.terraform.yaml" -i # Exclude files by directory name (any file with "target" in the path) -./yabe --sort-only --sort-config-path sort-config.yaml -p "**/*.yaml" --exclude "target" +yabe sort --sort-config sort-config.yaml -p "**/*.yaml" --exclude "target" # Exclude multiple patterns - files in build directories and temp files -./yabe --sort-only --sort-config-path sort-config.yaml -p "**/*.yaml" --exclude "target" --exclude "build" --exclude "*.tmp" +yabe sort --sort-config sort-config.yaml -p "**/*.yaml" --exclude "target" --exclude "build" --exclude "*.tmp" + +# Sort files with alphabetical ordering (when no sort config is available) +yabe sort file1.yaml file2.yaml # Will apply default alphabetical sorting ``` ### Exclude Patterns @@ -142,51 +182,16 @@ The `--exclude` option supports flexible pattern matching to skip unwanted files **Examples:** ```bash -# Exclude multiple directory types -./yabe -p "**/*.yaml" --exclude "target" --exclude "node_modules" --exclude ".git" - -# Exclude by file patterns and directories -./yabe -p "**/*.yaml" --exclude "*.terraform.yaml" --exclude "build" --exclude "dist" +# Exclude multiple directory types (separate command) +yabe separate -p "**/*.yaml" --exclude "target" --exclude "node_modules" --exclude ".git" -# Complex exclusion for GitOps environments -./yabe --sort-only -p "**/*.yaml" --exclude "target" --exclude ".argocd" --exclude "*.secret.yaml" -``` - -### Using Configuration File +# Exclude by file patterns and directories (separate command) +yabe separate -p "**/*.yaml" --exclude "*.terraform.yaml" --exclude "build" --exclude "dist" -You can also use a configuration file to specify options: - -```yaml -# config.yaml -read_only_base: "path/to/read_only_base.yaml" -base: "path/to/base.yaml" -# Either input_files or path_patterns/path_pattern (or both) must be specified -input_files: - - "input1.yaml" - - "input2.yaml" -# You can use either path_patterns (array) or path_pattern (single string) -path_patterns: - - "*.yaml" - - "configs/*.yaml" -# OR -# path_pattern: "*.yaml" -inplace: true -out_folder: "./output" -debug: false -quorum: 60 -base_out_path: "./base_output.yaml" -sort_config_path: "./sort_config.yaml" -sort_only: false # Set to true for sort-only mode -exclude_patterns: # Optional: patterns to exclude from sorting - - "*.terraform.yaml" - - "*-template.yaml" +# Complex exclusion for GitOps environments (sort command) +yabe sort -p "**/*.yaml" --exclude "target" --exclude ".argocd" --exclude "*.secret.yaml" ``` -Then run the tool with: - -```bash -./yabe --config config.yaml -``` ## Examples @@ -235,7 +240,7 @@ settings: ### Running the Tool ```bash -./yabe -r helm_values.yaml file1.yaml file2.yaml file3.yaml +yabe separate -r helm_values.yaml file1.yaml file2.yaml file3.yaml ``` ### Expected Output @@ -268,7 +273,7 @@ settings: ### In-place Modification Example Running with the -i flag: ```bash -./yabe -i -r helm_values.yaml file1.yaml file2.yaml file3.yaml +yabe separate -i -r helm_values.yaml file1.yaml file2.yaml file3.yaml ``` ## Testing diff --git a/examples/config-example/Makefile b/examples/config-example/Makefile new file mode 100644 index 0000000..dfae20d --- /dev/null +++ b/examples/config-example/Makefile @@ -0,0 +1,2 @@ +test: + yabe separate --config ./config.yaml diff --git a/examples/full/Makefile b/examples/full/Makefile new file mode 100644 index 0000000..702894b --- /dev/null +++ b/examples/full/Makefile @@ -0,0 +1,2 @@ +test: + yabe separate -r ./in/read-base.yaml -b ./in/base.yaml --base-out ./out/base.yaml ./in/a.yaml ./in/b.yaml ./in/c.yaml diff --git a/examples/full/README.md b/examples/full/README.md deleted file mode 100644 index 30cf69f..0000000 --- a/examples/full/README.md +++ /dev/null @@ -1,3 +0,0 @@ -```bash -yabe -r ./in/read-base.yaml -b ./in/base.yaml --base-out-path ./out/base.yaml ./in/a.yaml ./in/b.yaml ./in/c.yaml -``` \ No newline at end of file diff --git a/examples/full/out/c_diff.yaml b/examples/full/out/c_diff.yaml index 8150ed4..5562127 100644 --- a/examples/full/out/c_diff.yaml +++ b/examples/full/out/c_diff.yaml @@ -1,7 +1,7 @@ +a: m c: f: g h: - i - j - l -a: m diff --git a/examples/read-base/Makefile b/examples/read-base/Makefile new file mode 100644 index 0000000..e9d221e --- /dev/null +++ b/examples/read-base/Makefile @@ -0,0 +1,2 @@ +test: + yabe separate -r ./in/read-base.yaml --base-out ./out/base.yaml ./in/a.yaml ./in/b.yaml ./in/c.yaml diff --git a/examples/read-base/README.md b/examples/read-base/README.md deleted file mode 100644 index 9f4c3c3..0000000 --- a/examples/read-base/README.md +++ /dev/null @@ -1,3 +0,0 @@ -```bash -yabe -r ./in/read-base.yaml --base-out-path ./out/base.yaml ./in/a.yaml ./in/b.yaml ./in/c.yaml -``` \ No newline at end of file diff --git a/examples/simple/Makefile b/examples/simple/Makefile new file mode 100644 index 0000000..e62be59 --- /dev/null +++ b/examples/simple/Makefile @@ -0,0 +1,2 @@ +test: + yabe separate --base-out ./out/base.yaml ./in/a.yaml ./in/b.yaml ./in/c.yaml diff --git a/examples/simple/README.md b/examples/simple/README.md deleted file mode 100644 index 1aef4c1..0000000 --- a/examples/simple/README.md +++ /dev/null @@ -1,3 +0,0 @@ -```bash -yabe --base-out-path ./out/base.yaml ./in/a.yaml ./in/b.yaml ./in/c.yaml -``` \ No newline at end of file diff --git a/examples/simple/out/base.yaml b/examples/simple/out/base.yaml index b91ecee..4057eeb 100644 --- a/examples/simple/out/base.yaml +++ b/examples/simple/out/base.yaml @@ -1,7 +1,7 @@ +a: b +c: + d: e h: - i - j - k -a: b -c: - d: e diff --git a/examples/simple/out/c_diff.yaml b/examples/simple/out/c_diff.yaml index 53e186e..5562127 100644 --- a/examples/simple/out/c_diff.yaml +++ b/examples/simple/out/c_diff.yaml @@ -1,7 +1,7 @@ +a: m +c: + f: g h: - i - j - l -a: m -c: - f: g diff --git a/examples/sort/Makefile b/examples/sort/Makefile new file mode 100644 index 0000000..1fd70cb --- /dev/null +++ b/examples/sort/Makefile @@ -0,0 +1,2 @@ +test: + yabe sort --sort-config ../config-example/sort-config.yaml -o ./out ./in/values.yaml diff --git a/examples/sort/README.md b/examples/sort/README.md deleted file mode 100644 index de78b35..0000000 --- a/examples/sort/README.md +++ /dev/null @@ -1,3 +0,0 @@ -```bash -yabe --sort-config-path ../sort-config.yaml --base-out-path ./out/values.yaml ./in/values.yaml -``` \ No newline at end of file diff --git a/examples/sort/out/values.yaml b/examples/sort/out/values_sorted.yaml similarity index 70% rename from examples/sort/out/values.yaml rename to examples/sort/out/values_sorted.yaml index 5a271e8..a74bcd9 100644 --- a/examples/sort/out/values.yaml +++ b/examples/sort/out/values_sorted.yaml @@ -4,7 +4,7 @@ spec: template: spec: containers: - - name: nginx - image: "nginx:1.14.2" + - image: "nginx:1.14.2" + name: nginx ports: - containerPort: 80 diff --git a/src/cli.rs b/src/cli.rs new file mode 100644 index 0000000..37ab31b --- /dev/null +++ b/src/cli.rs @@ -0,0 +1,88 @@ +use clap::{Parser, Subcommand}; + +/// Command-line arguments +#[derive(Parser)] +#[command(author, version, about = "YAML diff and merge tool for GitOps workflows", long_about = "A tool for diffing, merging, and organizing YAML files for GitOps workflows. Supports both individual file paths and glob patterns.")] +pub struct Args { + /// Enable debug logging + #[arg(long = "debug")] + pub debug: bool, + + #[command(subcommand)] + pub command: Commands, +} + +#[derive(Subcommand)] +pub enum Commands { + /// Sort YAML files based on configuration + Sort { + /// Input YAML files (optional if path patterns are provided) + input_files: Vec, + + /// Path patterns to load YAML files (e.g., "*.yaml") + #[arg(short = 'p', long = "path-pattern", value_name = "PATH_PATTERN")] + path_patterns: Vec, + + /// Sort configuration file path + #[arg(long = "sort-config", default_value = "./sort-config.yaml")] + sort_config_path: String, + + /// Modify the original input files with sorted content + #[arg(short = 'i', long = "in-place")] + inplace: bool, + + /// Output folder + #[arg(short = 'o', long = "out", default_value = "./out")] + out_folder: String, + + /// Exclude patterns to skip files (e.g., "*.terraform.yaml") + #[arg(long = "exclude", value_name = "EXCLUDE_PATTERN")] + exclude_patterns: Vec, + }, + /// Separate common base from YAML files (diff/rebalance) + Separate { + /// Configuration file path + #[arg(short = 'c', long = "config", value_name = "CONFIG_FILE")] + config_file: Option, + + /// Input YAML files (optional if path patterns are provided) + input_files: Vec, + + /// Path patterns to load YAML files (e.g., "*.yaml") + #[arg(short = 'p', long = "path-pattern", value_name = "PATH_PATTERN")] + path_patterns: Vec, + + /// Helm chart values file + #[arg(short = 'r', long = "read-base", value_name = "READ_BASE")] + read_only_base: Option, + + /// Base YAML file to merge with input files + #[arg(short = 'b', long = "base", value_name = "WRITE_BASE")] + base: Option, + + /// Quorum percentage (0-100) + #[arg(short = 'q', long = "quorum", default_value_t = 51)] + quorum: u8, + + /// Base file output path + #[arg(long = "base-out", default_value = "./base.yaml")] + base_out_path: String, + + /// Sort configuration file path + #[arg(long = "sort-config", default_value = "./sort-config.yaml")] + sort_config_path: String, + + /// Modify the original input files with diffs + #[arg(short = 'i', long = "in-place")] + inplace: bool, + + /// Output folder + #[arg(short = 'o', long = "out", default_value = "./out")] + out_folder: String, + + /// Exclude patterns to skip files (e.g., "*.terraform.yaml") + #[arg(long = "exclude", value_name = "EXCLUDE_PATTERN")] + exclude_patterns: Vec, + }, +} + diff --git a/src/commands/mod.rs b/src/commands/mod.rs new file mode 100644 index 0000000..3b48c4f --- /dev/null +++ b/src/commands/mod.rs @@ -0,0 +1,2 @@ +pub mod sort; +pub mod separate; \ No newline at end of file diff --git a/src/commands/separate.rs b/src/commands/separate.rs new file mode 100644 index 0000000..81b655b --- /dev/null +++ b/src/commands/separate.rs @@ -0,0 +1,276 @@ +use std::borrow::Cow; +use std::error::Error; +use std::fs; +use std::path::Path; +use log::info; +use yaml_rust2::{Yaml, YamlEmitter, YamlLoader}; +use crate::diff::{compute_diff, diff_and_common_multiple}; +use crate::merge::merge_yaml; +use crate::sorter::sort_yaml; +use crate::utils::{expand_and_filter_files, ensure_output_dir}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Deserialize, Serialize)] +struct SeparateConfig { + read_only_base: Option, + base: Option, + input_files: Option>, + path_patterns: Option>, + inplace: Option, + out_folder: Option, + debug: Option, + quorum: Option, + base_out_path: Option, + sort_config_path: Option, + exclude_patterns: Option>, +} + +pub fn run_separate_command( + config_file: Option, + input_files: Vec, + path_patterns: Vec, + read_only_base: Option, + base: Option, + quorum: u8, + base_out_path: String, + sort_config_path: String, + inplace: bool, + out_folder: String, + exclude_patterns: Vec, +) -> Result<(), Box> { + info!("Running separate command"); + + // Load config file if provided and merge with CLI arguments + let (final_input_files, final_path_patterns, final_read_only_base, final_base, final_quorum, + final_base_out_path, final_sort_config_path, final_inplace, final_out_folder, final_exclude_patterns) = + if let Some(config_path) = config_file { + info!("Loading configuration from file: {}", config_path); + let config_content = fs::read_to_string(&config_path)?; + let config: SeparateConfig = serde_yml::from_str(&config_content)?; + + // CLI arguments override config file values + let merged_input_files = if input_files.is_empty() { + config.input_files.unwrap_or_default() + } else { + input_files + }; + let merged_path_patterns = if path_patterns.is_empty() { + config.path_patterns.unwrap_or_default() + } else { + path_patterns + }; + let merged_read_only_base = read_only_base.or(config.read_only_base); + let merged_base = base.or(config.base); + let merged_quorum = if quorum == 51 { config.quorum.unwrap_or(51) } else { quorum }; + let merged_base_out_path = if base_out_path == "./base.yaml" { + config.base_out_path.unwrap_or_else(|| "./base.yaml".to_string()) + } else { + base_out_path + }; + let merged_sort_config_path = if sort_config_path == "./sort-config.yaml" { + config.sort_config_path.unwrap_or_else(|| "./sort-config.yaml".to_string()) + } else { + sort_config_path + }; + let merged_inplace = if !inplace { config.inplace.unwrap_or(false) } else { inplace }; + let merged_out_folder = if out_folder == "./out" { + config.out_folder.unwrap_or_else(|| "./out".to_string()) + } else { + out_folder + }; + let merged_exclude_patterns = if exclude_patterns.is_empty() { + config.exclude_patterns.unwrap_or_default() + } else { + exclude_patterns + }; + + (merged_input_files, merged_path_patterns, merged_read_only_base, merged_base, + merged_quorum, merged_base_out_path, merged_sort_config_path, merged_inplace, + merged_out_folder, merged_exclude_patterns) + } else { + (input_files, path_patterns, read_only_base, base, quorum, base_out_path, + sort_config_path, inplace, out_folder, exclude_patterns) + }; + + let expanded_input_files = expand_and_filter_files(final_input_files, &final_path_patterns, &final_exclude_patterns)?; + + // Validate that either input_files or path_patterns are provided + if expanded_input_files.is_empty() && final_path_patterns.is_empty() { + eprintln!("Error: No input files or path patterns provided. Please specify either input files or path patterns."); + std::process::exit(1); + } + + // Validate that we have at least one file to process after expansion + if expanded_input_files.is_empty() { + eprintln!("Error: No files found matching the provided path patterns. Please check your patterns and try again."); + std::process::exit(1); + } + + let input_filenames = expanded_input_files; + let quorum_percentage = (final_quorum as f64) / 100.0; + + // Ensure output directory exists + ensure_output_dir(&final_out_folder)?; + + let config = if !final_sort_config_path.is_empty() { + info!("Reading sort configuration file: {}", final_sort_config_path); + let content = fs::read_to_string(&final_sort_config_path); + if let Ok(content) = content { + YamlLoader::load_from_str(&content)?.into_iter().next().unwrap_or(Yaml::Null) + } else { + log::warn!("Failed to read sort configuration file: {}", final_sort_config_path); + Yaml::Null + } + } else { + info!("No sort configuration provided, will use default alphabetical sorting"); + Yaml::Null + }; + + let read_only_base = if let Some(ref read_only_base) = final_read_only_base { + info!("Reading helm values file: {}", read_only_base); + let content = fs::read_to_string(read_only_base)?; + YamlLoader::load_from_str(&content)?.into_iter().next() + } else { + None + }; + + // Read and parse the existing base file if provided + let existing_base = if let Some(ref base_path) = final_base { + info!("Reading existing base YAML file: {}", base_path); + let content = fs::read_to_string(base_path)?; + YamlLoader::load_from_str(&content)?.into_iter().next() + } else { + None + }; + + // Read and parse each YAML input file into an object + let mut all_docs = Vec::new(); + for filename in &input_filenames { + info!("Reading input file: {}", filename); + let content = fs::read_to_string(filename)?; + if let Some(doc) = YamlLoader::load_from_str(&content)?.into_iter().next() { + all_docs.push(doc); + } else { + log::warn!("No YAML documents in {}", filename); + } + } + + // Merge existing base with each input file if existing base is provided + let merged_objs: Vec> = if let Some(ref base) = existing_base { + input_filenames + .iter() + .zip(all_docs.iter()) + .map(|(filename, obj)| { + let merged = merge_yaml(base, obj); + info!("Merged base with input file: {}", filename); + merged + }) + .collect() + } else { + // No existing base; use objs as merged_objs + all_docs.iter().map(|doc| Cow::Borrowed(doc)).collect() + }; + + // Compute diffs between each merged object and read-only base + let diffs: Vec<_> = if let Some(ref helm) = read_only_base { + info!("Computing diffs between merged files and helm values."); + merged_objs + .iter() + .map(|obj| compute_diff(obj.as_ref(), helm).unwrap_or_else(|| Cow::Owned(Yaml::Null))) + .collect() + } else { + // No read-only base provided values; use merged_objs as diffs + merged_objs.clone() + }; + + // Now compute common base and per-file diffs among the diffs + let diffs_refs: Vec<&Yaml> = diffs.iter().map(|cow| cow.as_ref()).collect(); + info!( + "Computing common base and per-file diffs among the diffs with quorum {}%.", + final_quorum + ); + let (base, per_file_diffs) = diff_and_common_multiple(&diffs_refs, quorum_percentage); + + // Process the base YAML if it exists + if let Some(base_yaml) = base { + let processed_yaml = sort_yaml(base_yaml.as_ref(), &config); + + info!("Writing base YAML to {}", final_base_out_path); + let mut out_str = String::new(); + { + let mut emitter = YamlEmitter::new(&mut out_str); + emitter.dump(&processed_yaml)?; + } + out_str = out_str.trim_start_matches("---\n").to_string(); + out_str.push('\n'); + fs::write(final_base_out_path.as_str(), out_str)?; + info!("Base YAML written to {}", final_base_out_path); + } else { + info!("No base YAML to write."); + } + + // Determine whether to write diffs to original files or new files + if final_inplace { + info!("Inplace mode enabled. Modifying original files."); + for (i, diff) in per_file_diffs.iter().enumerate() { + if let Some(diff_yaml) = diff { + let processed_diff = sort_yaml(diff_yaml.as_ref(), &config); + + info!("Writing diff back to original file: {}", input_filenames[i]); + let mut out_str = String::new(); + { + let mut emitter = YamlEmitter::new(&mut out_str); + emitter.dump(&processed_diff)?; + } + out_str = out_str.trim_start_matches("---\n").to_string(); + out_str.push('\n'); + fs::write(&input_filenames[i], out_str)?; + info!( + "Difference written back to original file {}", + input_filenames[i] + ); + } else { + // If there is no diff, remove the content of the file + info!("No diff for {}; clearing file content.", input_filenames[i]); + fs::write(&input_filenames[i], "")?; + info!( + "No difference for {}; file content cleared.", + input_filenames[i] + ); + } + } + } else { + info!("Writing diffs to new files."); + for (i, diff) in per_file_diffs.iter().enumerate() { + if let Some(diff_yaml) = diff { + let processed_diff = sort_yaml(diff_yaml.as_ref(), &config); + + info!("Writing diff for {} to new file.", input_filenames[i]); + let mut out_str = String::new(); + { + let mut emitter = YamlEmitter::new(&mut out_str); + emitter.dump(&processed_diff)?; + } + + let input_path = Path::new(&input_filenames[i]); + let file_stem = input_path + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("diff"); + let diff_filename = format!("{}/{}_diff.yaml", final_out_folder, file_stem); + out_str = out_str.trim_start_matches("---\n").to_string(); + out_str.push('\n'); + fs::write(&diff_filename, out_str)?; + info!( + "Difference for {} written to {}", + input_filenames[i], diff_filename + ); + } else { + info!("No diff for {}; not writing a diff file.", input_filenames[i]); + } + } + } + + info!("Separate command completed successfully."); + Ok(()) +} \ No newline at end of file diff --git a/src/commands/sort.rs b/src/commands/sort.rs new file mode 100644 index 0000000..88cdc4c --- /dev/null +++ b/src/commands/sort.rs @@ -0,0 +1,83 @@ +use std::error::Error; +use std::fs; +use std::path::Path; +use log::{info, warn}; +use yaml_rust2::{Yaml, YamlEmitter, YamlLoader}; +use crate::sorter::sort_yaml; +use crate::utils::{expand_and_filter_files, ensure_output_dir}; + +pub fn run_sort_command( + input_files: Vec, + path_patterns: Vec, + sort_config_path: String, + inplace: bool, + out_folder: String, + exclude_patterns: Vec, +) -> Result<(), Box> { + info!("Running sort command"); + + let expanded_input_files = expand_and_filter_files(input_files, &path_patterns, &exclude_patterns)?; + + // Validate that we have files to process + if expanded_input_files.is_empty() { + eprintln!("Error: No input files found. Please specify either input files or path patterns."); + std::process::exit(1); + } + + // Read sort configuration + let sort_config = if !sort_config_path.is_empty() && Path::new(&sort_config_path).exists() { + info!("Reading sort configuration file: {}", sort_config_path); + let content = fs::read_to_string(&sort_config_path)?; + YamlLoader::load_from_str(&content)?.into_iter().next().unwrap_or(Yaml::Null) + } else { + info!("No sort configuration provided, will use default alphabetical sorting"); + Yaml::Null + }; + + + // Ensure output directory exists if not in-place mode + if !inplace { + ensure_output_dir(&out_folder)?; + } + + // Process each input file + for filename in &expanded_input_files { + info!("Sorting file: {}", filename); + + // Read and parse the YAML file + let content = fs::read_to_string(filename)?; + if let Some(doc) = YamlLoader::load_from_str(&content)?.into_iter().next() { + // Sort the YAML document + let sorted_yaml = sort_yaml(&doc, &sort_config); + + // Convert to string + let mut out_str = String::new(); + { + let mut emitter = YamlEmitter::new(&mut out_str); + emitter.dump(&sorted_yaml)?; + } + out_str = out_str.trim_start_matches("---\n").to_string(); + out_str.push('\n'); + + // Write the sorted content + if inplace { + info!("Writing sorted content back to: {}", filename); + fs::write(filename, out_str)?; + } else { + let input_path = Path::new(filename); + let file_stem = input_path + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("sorted"); + let sorted_filename = format!("{}/{}_sorted.yaml", out_folder, file_stem); + info!("Writing sorted content to: {}", sorted_filename); + fs::write(&sorted_filename, out_str)?; + } + } else { + warn!("No YAML documents found in {}", filename); + } + } + + info!("Sort command completed successfully."); + Ok(()) +} \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index a6d2191..b57344d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,5 +2,11 @@ pub mod deep_equal; pub mod diff; pub mod merge; pub mod sorter; +pub mod cli; +pub mod commands; +pub mod utils; -pub use diff::{compute_diff, diff_and_common_multiple}; \ No newline at end of file +pub use diff::{compute_diff, diff_and_common_multiple}; +pub use cli::{Args, Commands}; +pub use commands::sort::run_sort_command; +pub use commands::separate::run_separate_command; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index f657668..2b1873d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,324 +1,9 @@ -use std::borrow::Cow; use std::error::Error; -use std::fs; -use std::path::Path; - use clap::Parser; -use log::{info, warn}; -use yaml_rust2::{Yaml, YamlEmitter, YamlLoader}; -use yabe::diff::{compute_diff, diff_and_common_multiple}; -use yabe::merge::merge_yaml; -use yabe::sorter::sort_yaml; -use serde::Deserialize; -use serde_yml; -use glob::glob; - -/// Command-line arguments -#[derive(Parser)] -#[command(author, version, about = "YAML diff and merge tool for GitOps workflows", long_about = "A tool for diffing, merging, and organizing YAML files for GitOps workflows. Supports both individual file paths and glob patterns.")] -struct Args { - /// Configuration file - #[arg(long = "config", value_name = "CONFIG_FILE")] - config: Option, - - /// Helm chart values file - #[arg(short = 'r', long = "read-base", value_name = "READ_BASE")] - read_only_base: Option, - - /// Base YAML file to merge with input files - #[arg(short = 'b', long = "base", value_name = "WRITE_BASE")] - base: Option, - - /// Input YAML files (optional if path patterns are provided) - input_files: Vec, - - /// Path patterns to load YAML files (e.g., "*.yaml") - #[arg(short = 'p', long = "path-pattern", value_name = "PATH_PATTERN")] - path_patterns: Vec, - - /// Modify the original input files with diffs - #[arg(short = 'i', long = "in-place")] - inplace: bool, - - /// Output folder - #[arg(short = 'o', long = "out", default_value = "./out")] - out_folder: String, - - /// Enable debug logging - #[arg(long = "debug")] - debug: bool, - - /// Quorum percentage (0-100) - #[arg(short = 'q', long = "quorum", default_value_t = 51)] - quorum: u8, - - /// Base file output path - #[arg(long = "base-out-path", default_value = "./base.yaml")] - base_out_path: String, - - /// Sort configuration file path - #[arg(long = "sort-config-path", default_value = "./sort-config.yaml")] - sort_config_path: String, - - /// Sort only mode - only sort files without diffing - #[arg(long = "sort-only")] - sort_only: bool, - - /// Exclude patterns to skip files (e.g., "*.terraform.yaml") - #[arg(long = "exclude", value_name = "EXCLUDE_PATTERN")] - exclude_patterns: Vec, -} - -#[derive(Deserialize)] -struct Config { - read_only_base: Option, - base: Option, - input_files: Option>, - path_patterns: Option>, - path_pattern: Option, - inplace: Option, - out_folder: Option, - debug: Option, - quorum: Option, - base_out_path: Option, - sort_config_path: Option, - sort_only: Option, - exclude_patterns: Option>, -} - -fn sort_only_workflow(args: &Args) -> Result<(), Box> { - info!("Running in sort-only mode"); - - // Process path patterns and add matching files to input_files - let mut expanded_input_files = args.input_files.clone(); - - for pattern in &args.path_patterns { - info!("Expanding path pattern: {}", pattern); - match glob(pattern) { - Ok(paths) => { - for entry in paths { - match entry { - Ok(path) => { - if path.is_file() { - if let Some(path_str) = path.to_str() { - info!("Found matching file: {}", path_str); - expanded_input_files.push(path_str.to_string()); - } - } - } - Err(e) => warn!("Error matching path: {}", e), - } - } - } - Err(e) => warn!("Invalid glob pattern '{}': {}", pattern, e), - } - } - - // Remove duplicates from expanded_input_files - expanded_input_files.sort(); - expanded_input_files.dedup(); - - // Filter out excluded files - if !args.exclude_patterns.is_empty() { - let original_count = expanded_input_files.len(); - expanded_input_files.retain(|file_path| { - for exclude_pattern in &args.exclude_patterns { - // Check if the file path contains the exclude pattern as a substring - if file_path.contains(exclude_pattern) { - info!("Excluding file: {} (path contains pattern: {})", file_path, exclude_pattern); - return false; - } - - // Check if the exclude pattern matches as a glob pattern against the full path - if let Ok(pattern) = glob::Pattern::new(exclude_pattern) { - if pattern.matches(file_path) { - info!("Excluding file: {} (path matches glob pattern: {})", file_path, exclude_pattern); - return false; - } - } - - // Check if any component of the path matches the pattern - for component in Path::new(file_path).components() { - if let Some(component_str) = component.as_os_str().to_str() { - if let Ok(pattern) = glob::Pattern::new(exclude_pattern) { - if pattern.matches(component_str) { - info!("Excluding file: {} (path component '{}' matches pattern: {})", file_path, component_str, exclude_pattern); - return false; - } - } - } - } - - // Check if the filename matches the pattern directly - if let Some(filename) = Path::new(file_path).file_name() { - if let Some(filename_str) = filename.to_str() { - if let Ok(pattern) = glob::Pattern::new(exclude_pattern) { - if pattern.matches(filename_str) { - info!("Excluding file: {} (filename matches pattern: {})", file_path, exclude_pattern); - return false; - } - } - } - } - } - true - }); - let excluded_count = original_count - expanded_input_files.len(); - if excluded_count > 0 { - info!("Excluded {} files based on exclude patterns", excluded_count); - } - } - - // Validate that we have files to process - if expanded_input_files.is_empty() { - eprintln!("Error: No input files found for sort-only mode. Please specify either input files or path patterns."); - std::process::exit(1); - } - - // Read sort configuration - let sort_config = if !args.sort_config_path.is_empty() && Path::new(&args.sort_config_path).exists() { - info!("Reading sort configuration file: {}", args.sort_config_path); - let content = fs::read_to_string(&args.sort_config_path)?; - YamlLoader::load_from_str(&content)?.into_iter().next().unwrap_or(Yaml::Null) - } else { - eprintln!("Error: Sort configuration file is required for sort-only mode but not found: {}", args.sort_config_path); - std::process::exit(1); - }; - - if sort_config == Yaml::Null { - eprintln!("Error: Sort configuration file is empty or invalid: {}", args.sort_config_path); - std::process::exit(1); - } - - // Ensure output directory exists if not in-place mode - if !args.inplace { - if !Path::new(&args.out_folder).exists() { - info!("Creating output directory: {}", args.out_folder); - fs::create_dir_all(&args.out_folder)?; - } - } - - // Process each input file - for filename in &expanded_input_files { - info!("Sorting file: {}", filename); - - // Read and parse the YAML file - let content = fs::read_to_string(filename)?; - if let Some(doc) = YamlLoader::load_from_str(&content)?.into_iter().next() { - // Sort the YAML document - let sorted_yaml = sort_yaml(&doc, &sort_config); - - // Convert to string - let mut out_str = String::new(); - { - let mut emitter = YamlEmitter::new(&mut out_str); - emitter.dump(&sorted_yaml)?; - } - out_str = out_str.trim_start_matches("---\n").to_string(); - out_str.push('\n'); - - // Write the sorted content - if args.inplace { - info!("Writing sorted content back to: {}", filename); - fs::write(filename, out_str)?; - } else { - let input_path = Path::new(filename); - let file_stem = input_path - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or("sorted"); - let sorted_filename = format!("{}/{}_sorted.yaml", args.out_folder, file_stem); - info!("Writing sorted content to: {}", sorted_filename); - fs::write(&sorted_filename, out_str)?; - } - } else { - warn!("No YAML documents found in {}", filename); - } - } - - info!("Sort-only workflow completed successfully."); - Ok(()) -} +use yabe::{Args, Commands, run_sort_command, run_separate_command}; fn main() -> Result<(), Box> { - let mut args = Args::parse(); - - if let Some(config_path) = args.config.as_ref() { - // Read the configuration file - let config_content = fs::read_to_string(config_path)?; - let config: Config = serde_yml::from_str(&config_content)?; - - // Override args with config values if they are not provided via command-line - if args.read_only_base.is_none() { - args.read_only_base = config.read_only_base; - } - - if args.base.is_none() { - args.base = config.base; - } - - if args.input_files.is_empty() { - if let Some(input_files) = config.input_files { - args.input_files = input_files; - } - } - - if args.path_patterns.is_empty() { - if let Some(path_patterns) = config.path_patterns { - args.path_patterns = path_patterns; - } else if let Some(path_pattern) = config.path_pattern.as_ref() { - args.path_patterns.push(path_pattern.clone()); - } - } - - if !args.inplace { - if let Some(inplace) = config.inplace { - args.inplace = inplace; - } - } - - if args.out_folder == "./out" { - if let Some(out_folder) = config.out_folder { - args.out_folder = out_folder; - } - } - - if !args.debug { - if let Some(debug) = config.debug { - args.debug = debug; - } - } - - if args.quorum == 51 { - if let Some(quorum) = config.quorum { - args.quorum = quorum; - } - } - - if args.base_out_path == "./base.yaml" { - if let Some(base_out_path) = config.base_out_path { - args.base_out_path = base_out_path; - } - } - - if args.sort_config_path == "./sort-config.yaml" { - if let Some(sort_config_path) = config.sort_config_path { - args.sort_config_path = sort_config_path; - } - } - - if !args.sort_only { - if let Some(sort_only) = config.sort_only { - args.sort_only = sort_only; - } - } - - if args.exclude_patterns.is_empty() { - if let Some(exclude_patterns) = config.exclude_patterns { - args.exclude_patterns = exclude_patterns; - } - } - } + let args = Args::parse(); // Initialize logger with appropriate level if args.debug { @@ -327,289 +12,31 @@ fn main() -> Result<(), Box> { env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); } - // Handle sort-only mode - if args.sort_only { - return sort_only_workflow(&args); - } - - // Process path patterns and add matching files to input_files - let mut expanded_input_files = args.input_files.clone(); - - for pattern in &args.path_patterns { - info!("Expanding path pattern: {}", pattern); - match glob(pattern) { - Ok(paths) => { - for entry in paths { - match entry { - Ok(path) => { - if path.is_file() { - if let Some(path_str) = path.to_str() { - info!("Found matching file: {}", path_str); - expanded_input_files.push(path_str.to_string()); - } - } - } - Err(e) => warn!("Error matching path: {}", e), - } - } - } - Err(e) => warn!("Invalid glob pattern '{}': {}", pattern, e), - } - } - - // Remove duplicates from expanded_input_files - expanded_input_files.sort(); - expanded_input_files.dedup(); - - // Filter out excluded files - if !args.exclude_patterns.is_empty() { - let original_count = expanded_input_files.len(); - expanded_input_files.retain(|file_path| { - for exclude_pattern in &args.exclude_patterns { - // Check if the file path contains the exclude pattern as a substring - if file_path.contains(exclude_pattern) { - info!("Excluding file: {} (path contains pattern: {})", file_path, exclude_pattern); - return false; - } - - // Check if the exclude pattern matches as a glob pattern against the full path - if let Ok(pattern) = glob::Pattern::new(exclude_pattern) { - if pattern.matches(file_path) { - info!("Excluding file: {} (path matches glob pattern: {})", file_path, exclude_pattern); - return false; - } - } - - // Check if any component of the path matches the pattern - for component in Path::new(file_path).components() { - if let Some(component_str) = component.as_os_str().to_str() { - if let Ok(pattern) = glob::Pattern::new(exclude_pattern) { - if pattern.matches(component_str) { - info!("Excluding file: {} (path component '{}' matches pattern: {})", file_path, component_str, exclude_pattern); - return false; - } - } - } - } - - // Check if the filename matches the pattern directly - if let Some(filename) = Path::new(file_path).file_name() { - if let Some(filename_str) = filename.to_str() { - if let Ok(pattern) = glob::Pattern::new(exclude_pattern) { - if pattern.matches(filename_str) { - info!("Excluding file: {} (filename matches pattern: {})", file_path, exclude_pattern); - return false; - } - } - } - } - } - true - }); - let excluded_count = original_count - expanded_input_files.len(); - if excluded_count > 0 { - info!("Excluded {} files based on exclude patterns", excluded_count); - } - } - - // Validate that either input_files or path_patterns are provided - if expanded_input_files.is_empty() && args.path_patterns.is_empty() { - eprintln!("Error: No input files or path patterns provided. Please specify either input files or path patterns via command-line arguments or in the configuration file."); - std::process::exit(1); - } - - // Validate that we have at least one file to process after expansion - if expanded_input_files.is_empty() { - eprintln!("Error: No files found matching the provided path patterns. Please check your patterns and try again."); - std::process::exit(1); - } - - info!("Starting the YAML diffing program."); - - let input_filenames = expanded_input_files; - - let quorum_percentage = (args.quorum as f64) / 100.0; - - let base_out_path = args.base_out_path; - - let out_folder = args.out_folder; - - // Ensure output directory exists - if !Path::new(&out_folder).exists() { - info!("Creating output directory: {}", out_folder); - fs::create_dir_all(&out_folder)?; - } - - let config = if !args.sort_config_path.is_empty() { - info!("Reading sort configuration file: {}", args.sort_config_path); - let content = fs::read_to_string(&args.sort_config_path); - if let Ok(content) = content { - YamlLoader::load_from_str(&content)?.into_iter().next().unwrap_or(Yaml::Null) - } else { - warn!("Failed to read sort configuration file: {}", args.sort_config_path); - Yaml::Null - } - } else { - Yaml::Null - }; - - let read_only_base = if let Some(ref read_only_base) = args.read_only_base { - info!("Reading helm values file: {}", read_only_base); - let content = fs::read_to_string(read_only_base)?; - YamlLoader::load_from_str(&content)?.into_iter().next() - } else { - None - }; - - // Read and parse the existing base file if provided - let existing_base = if let Some(ref base_path) = args.base { - info!("Reading existing base YAML file: {}", base_path); - let content = fs::read_to_string(base_path)?; - YamlLoader::load_from_str(&content)?.into_iter().next() - } else { - None - }; - - // Read and parse each YAML input file into an object - let mut all_docs = Vec::new(); - for filename in &input_filenames { - info!("Reading input file: {}", filename); - let content = fs::read_to_string(filename)?; - if let Some(doc) = YamlLoader::load_from_str(&content)?.into_iter().next() { - all_docs.push(doc); - } else { - warn!("No YAML documents in {}", filename); - } - } - - // Merge existing base with each input file if existing base is provided - let merged_objs: Vec> = if let Some(ref base) = existing_base { - input_filenames - .iter() - .zip(all_docs.iter()) - .map(|(filename, obj)| { - let merged = merge_yaml(base, obj); - info!("Merged base with input file: {}", filename); - merged - }) - .collect() - } else { - // No existing base; use objs as merged_objs - all_docs.iter().map(|doc| Cow::Borrowed(doc)).collect() - }; - - // Compute diffs between each merged object and read-only base - let diffs: Vec<_> = if let Some(ref helm) = read_only_base { - info!("Computing diffs between merged files and helm values."); - merged_objs - .iter() - .map(|obj| compute_diff(obj.as_ref(), helm).unwrap_or_else(|| Cow::Owned(Yaml::Null))) - .collect() - } else { - // No read-only base provided values; use merged_objs as diffs - merged_objs.clone() - }; - - // Now compute common base and per-file diffs among the diffs - let diffs_refs: Vec<&Yaml> = diffs.iter().map(|cow| cow.as_ref()).collect(); - info!( - "Computing common base and per-file diffs among the diffs with quorum {}%.", - args.quorum - ); - let (base, per_file_diffs) = diff_and_common_multiple(&diffs_refs, quorum_percentage); - - // Process the base YAML if it exists - if let Some(base_yaml) = base { - let processed_yaml = if config != Yaml::Null { - sort_yaml(base_yaml.as_ref(), &config) - } else { - Cow::Borrowed(base_yaml.as_ref()) - }; - - info!("Writing base YAML to {}", base_out_path); - let mut out_str = String::new(); - { - let mut emitter = YamlEmitter::new(&mut out_str); - emitter.dump(&processed_yaml)?; - } - out_str = out_str.trim_start_matches("---\n").to_string(); - out_str.push('\n'); - fs::write(base_out_path.as_str(), out_str)?; - info!("Base YAML written to {}", base_out_path); - } else { - info!("No base YAML to write."); - } - - // Determine whether to write diffs to original files or new files - if args.inplace { - info!("Inplace mode enabled. Modifying original files."); - for (i, diff) in per_file_diffs.iter().enumerate() { - if let Some(diff_yaml) = diff { - let processed_diff = if config != Yaml::Null { - sort_yaml(diff_yaml.as_ref(), &config) - } else { - Cow::Borrowed(diff_yaml.as_ref()) - }; - - info!("Writing diff back to original file: {}", input_filenames[i]); - let mut out_str = String::new(); - { - let mut emitter = YamlEmitter::new(&mut out_str); - emitter.dump(&processed_diff)?; - } - out_str = out_str.trim_start_matches("---\n").to_string(); - out_str.push('\n'); - fs::write(&input_filenames[i], out_str)?; - info!( - "Difference written back to original file {}", - input_filenames[i] - ); - } else { - // If there is no diff, remove the content of the file - info!("No diff for {}; clearing file content.", input_filenames[i]); - fs::write(&input_filenames[i], "")?; - info!( - "No difference for {}; file content cleared.", - input_filenames[i] - ); - } - } - } else { - info!("Writing diffs to new files."); - for (i, diff) in per_file_diffs.iter().enumerate() { - if let Some(diff_yaml) = diff { - let processed_diff = if config != Yaml::Null { - sort_yaml(diff_yaml.as_ref(), &config) - } else { - Cow::Borrowed(diff_yaml.as_ref()) - }; - - info!("Writing diff for {} to new file.", input_filenames[i]); - let mut out_str = String::new(); - { - let mut emitter = YamlEmitter::new(&mut out_str); - emitter.dump(&processed_diff)?; - } - - let input_path = Path::new(&input_filenames[i]); - let file_stem = input_path - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or("diff"); - let diff_filename = format!("{}/{}_diff.yaml", out_folder, file_stem); - out_str = out_str.trim_start_matches("---\n").to_string(); - out_str.push('\n'); - fs::write(&diff_filename, out_str)?; - info!( - "Difference for {} written to {}", - input_filenames[i], diff_filename - ); - } else { - info!("No diff for {}; not writing a diff file.", input_filenames[i]); - } + match args.command { + Commands::Sort { + input_files, + path_patterns, + sort_config_path, + inplace, + out_folder, + exclude_patterns + } => { + run_sort_command(input_files, path_patterns, sort_config_path, inplace, out_folder, exclude_patterns) + } + Commands::Separate { + config_file, + input_files, + path_patterns, + read_only_base, + base, + quorum, + base_out_path, + sort_config_path, + inplace, + out_folder, + exclude_patterns + } => { + run_separate_command(config_file, input_files, path_patterns, read_only_base, base, quorum, base_out_path, sort_config_path, inplace, out_folder, exclude_patterns) } } - - info!("Program completed successfully."); - Ok(()) } \ No newline at end of file diff --git a/src/sorter.rs b/src/sorter.rs index f8c4b7b..4655e33 100644 --- a/src/sorter.rs +++ b/src/sorter.rs @@ -13,7 +13,13 @@ pub fn sort_yaml<'a>(doc: &'a Yaml, config: &Yaml) -> Cow<'a, Yaml> { } Cow::Owned(Yaml::Array(new_v)) } else { - Cow::Borrowed(doc) + // Recursively sort nested structures even without array config + let mut new_v = v.clone(); + for x in &mut new_v { + let sorted = sort_yaml(x, config); + *x = sorted.into_owned(); + } + Cow::Owned(Yaml::Array(new_v)) } } Yaml::Hash(h) => { @@ -30,7 +36,14 @@ pub fn sort_yaml<'a>(doc: &'a Yaml, config: &Yaml) -> Cow<'a, Yaml> { } Cow::Owned(Yaml::Hash(new_h)) } else { - Cow::Borrowed(doc) + // Apply alphabetical sorting by default when no config is provided + let mut new_h = h.clone(); + hash_sorter(&mut new_h, &[]); // Empty pre_order will just sort alphabetically + for (_, v) in &mut new_h { + let sorted = sort_yaml(v, config); + *v = sorted.into_owned(); + } + Cow::Owned(Yaml::Hash(new_h)) } } _ => Cow::Borrowed(doc), diff --git a/src/utils.rs b/src/utils.rs new file mode 100644 index 0000000..3b6885c --- /dev/null +++ b/src/utils.rs @@ -0,0 +1,103 @@ +use std::error::Error; +use std::fs; +use std::path::Path; +use log::{info, warn}; +use glob::glob; + +/// Expand path patterns and filter by exclude patterns +pub fn expand_and_filter_files( + input_files: Vec, + path_patterns: &[String], + exclude_patterns: &[String], +) -> Result, Box> { + // Process path patterns and add matching files to input_files + let mut expanded_input_files = input_files; + + for pattern in path_patterns { + info!("Expanding path pattern: {}", pattern); + match glob(pattern) { + Ok(paths) => { + for entry in paths { + match entry { + Ok(path) => { + if path.is_file() { + if let Some(path_str) = path.to_str() { + info!("Found matching file: {}", path_str); + expanded_input_files.push(path_str.to_string()); + } + } + } + Err(e) => warn!("Error matching path: {}", e), + } + } + } + Err(e) => warn!("Invalid glob pattern '{}': {}", pattern, e), + } + } + + // Remove duplicates from expanded_input_files + expanded_input_files.sort(); + expanded_input_files.dedup(); + + // Filter out excluded files + if !exclude_patterns.is_empty() { + let original_count = expanded_input_files.len(); + expanded_input_files.retain(|file_path| { + for exclude_pattern in exclude_patterns { + // Check if the file path contains the exclude pattern as a substring + if file_path.contains(exclude_pattern) { + info!("Excluding file: {} (path contains pattern: {})", file_path, exclude_pattern); + return false; + } + + // Check if the exclude pattern matches as a glob pattern against the full path + if let Ok(pattern) = glob::Pattern::new(exclude_pattern) { + if pattern.matches(file_path) { + info!("Excluding file: {} (path matches glob pattern: {})", file_path, exclude_pattern); + return false; + } + } + + // Check if any component of the path matches the pattern + for component in Path::new(file_path).components() { + if let Some(component_str) = component.as_os_str().to_str() { + if let Ok(pattern) = glob::Pattern::new(exclude_pattern) { + if pattern.matches(component_str) { + info!("Excluding file: {} (path component '{}' matches pattern: {})", file_path, component_str, exclude_pattern); + return false; + } + } + } + } + + // Check if the filename matches the pattern directly + if let Some(filename) = Path::new(file_path).file_name() { + if let Some(filename_str) = filename.to_str() { + if let Ok(pattern) = glob::Pattern::new(exclude_pattern) { + if pattern.matches(filename_str) { + info!("Excluding file: {} (filename matches pattern: {})", file_path, exclude_pattern); + return false; + } + } + } + } + } + true + }); + let excluded_count = original_count - expanded_input_files.len(); + if excluded_count > 0 { + info!("Excluded {} files based on exclude patterns", excluded_count); + } + } + + Ok(expanded_input_files) +} + +/// Ensure output directory exists +pub fn ensure_output_dir(out_folder: &str) -> Result<(), Box> { + if !Path::new(out_folder).exists() { + info!("Creating output directory: {}", out_folder); + fs::create_dir_all(out_folder)?; + } + Ok(()) +} \ No newline at end of file diff --git a/test-examples.sh b/test-examples.sh new file mode 100755 index 0000000..1a3b0e9 --- /dev/null +++ b/test-examples.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -eo pipefail + +# Build the project +echo "Building project..." +cargo build --release + +# Get the binary path +BINARY_PATH="$(pwd)/target/release/yabe" + +# Add the binary to PATH for the Makefiles +export PATH="$(dirname "$BINARY_PATH"):$PATH" + +echo "Running tests with binary: $BINARY_PATH" + +for dir in examples/*/; do + echo "Running tests in $dir" + (cd "$dir" && make test) +done + +if git status --porcelain | grep -E -q '^ [AM]'; then + echo "There are new or modified files in the examples directories." + exit 1 +else + echo "All tests passed and no new or modified files found." +fi \ No newline at end of file