From eaf682257bcbdea08db6b24dc7ca77d4fbce29e4 Mon Sep 17 00:00:00 2001 From: Tim Eijgelshoven Date: Fri, 20 Jun 2025 17:13:58 -0700 Subject: [PATCH 1/4] feat: Add optional function to return manifest of generated files --- .../for_production_example_unix_test.go | 6 +- integration-tests/required_version_test.go | 5 +- manifest/manifest.go | 38 ++++++++ manifest/manifest_test.go | 75 ++++++++++++++++ templates/template_processor.go | 86 +++++++++++++----- templates/template_processor_test.go | 89 +++++++++++++++++++ 6 files changed, 272 insertions(+), 27 deletions(-) create mode 100644 manifest/manifest.go create mode 100644 manifest/manifest_test.go diff --git a/integration-tests/for_production_example_unix_test.go b/integration-tests/for_production_example_unix_test.go index 78606c82..61a3805b 100644 --- a/integration-tests/for_production_example_unix_test.go +++ b/integration-tests/for_production_example_unix_test.go @@ -4,7 +4,7 @@ package integration_tests import ( - "io/ioutil" + "os" "path/filepath" "testing" @@ -19,9 +19,9 @@ func TestForProductionTerragruntArchitectureBoilerplateExample(t *testing.T) { forProductionExamplePath := "../examples/for-production/terragrunt-architecture-catalog" - outputBasePath, err := ioutil.TempDir("", "boilerplate-for-production-output") + outputBasePath, err := os.MkdirTemp("", "boilerplate-for-production-output") require.NoError(t, err) - //defer os.RemoveAll(outputBasePath) + defer os.RemoveAll(outputBasePath) templateFolder, err := filepath.Abs(filepath.Join(forProductionExamplePath, "blueprints", "reference-architecture")) require.NoError(t, err) diff --git a/integration-tests/required_version_test.go b/integration-tests/required_version_test.go index 2023cad6..ea7c10ab 100644 --- a/integration-tests/required_version_test.go +++ b/integration-tests/required_version_test.go @@ -1,13 +1,12 @@ package integration_tests import ( - "io/ioutil" "os" "testing" "github.com/gruntwork-io/boilerplate/cli" "github.com/gruntwork-io/boilerplate/config" - "github.com/gruntwork-io/go-commons/errors" + "github.com/gruntwork-io/boilerplate/errors" "github.com/gruntwork-io/go-commons/version" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -58,7 +57,7 @@ func TestRequiredVersionUnderTest(t *testing.T) { func runRequiredVersionExample(t *testing.T, templateFolder string) error { app := cli.CreateBoilerplateCli() - outputPath, err := ioutil.TempDir("", "boilerplate-test-output-reqver") + outputPath, err := os.MkdirTemp("", "boilerplate-test-output-reqver") require.NoError(t, err) defer os.RemoveAll(outputPath) diff --git a/manifest/manifest.go b/manifest/manifest.go new file mode 100644 index 00000000..7600c540 --- /dev/null +++ b/manifest/manifest.go @@ -0,0 +1,38 @@ +package manifest + +import ( + "path/filepath" +) + +// GeneratedFile represents a file that was generated by boilerplate +type GeneratedFile struct { + Path string `json:"path"` +} + +// Manifest represents the complete list of files generated by boilerplate +type Manifest struct { + OutputDir string + Files []GeneratedFile +} + +// NewManifest creates a new manifest +func NewManifest(outputDir string) *Manifest { + return &Manifest{ + OutputDir: outputDir, + Files: []GeneratedFile{}, + } +} + +// AddFile adds a generated file to the manifest +func (m *Manifest) AddFile(outputPath string) error { + // Convert to relative path from output directory + relPath, err := filepath.Rel(m.OutputDir, outputPath) + if err != nil { + relPath = outputPath + } + + m.Files = append(m.Files, GeneratedFile{ + Path: relPath, + }) + return nil +} diff --git a/manifest/manifest_test.go b/manifest/manifest_test.go new file mode 100644 index 00000000..ab02a2cf --- /dev/null +++ b/manifest/manifest_test.go @@ -0,0 +1,75 @@ +package manifest + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewManifest(t *testing.T) { + t.Parallel() + + outputDir := "/test/output" + + manifest := NewManifest(outputDir) + + assert.Equal(t, outputDir, manifest.OutputDir) + assert.Empty(t, manifest.Files) +} + +func TestAddFile(t *testing.T) { + t.Parallel() + + outputDir := "/test/output" + manifest := NewManifest(outputDir) + + // Test adding a file in the output directory + outputPath := filepath.Join(outputDir, "subdir", "file.txt") + + err := manifest.AddFile(outputPath) + + assert.NoError(t, err) + assert.Len(t, manifest.Files, 1) + assert.Equal(t, "subdir/file.txt", manifest.Files[0].Path) +} + +func TestAddFileOutsideOutputDir(t *testing.T) { + t.Parallel() + + outputDir := "/test/output" + manifest := NewManifest(outputDir) + + // Test adding a file outside the output directory + outputPath := "/other/dir/file.txt" + + err := manifest.AddFile(outputPath) + + assert.NoError(t, err) + assert.Len(t, manifest.Files, 1) + expectedRelPath, _ := filepath.Rel(outputDir, outputPath) + assert.Equal(t, expectedRelPath, manifest.Files[0].Path) +} + +func TestAddMultipleFiles(t *testing.T) { + t.Parallel() + + outputDir := "/test/output" + manifest := NewManifest(outputDir) + + // Add first file + err1 := manifest.AddFile( + filepath.Join(outputDir, "file1.txt"), + ) + + // Add second file + err2 := manifest.AddFile( + filepath.Join(outputDir, "subdir", "file2.txt"), + ) + + assert.NoError(t, err1) + assert.NoError(t, err2) + assert.Len(t, manifest.Files, 2) + assert.Equal(t, "file1.txt", manifest.Files[0].Path) + assert.Equal(t, "subdir/file2.txt", manifest.Files[1].Path) +} diff --git a/templates/template_processor.go b/templates/template_processor.go index 5008abc3..bd658400 100644 --- a/templates/template_processor.go +++ b/templates/template_processor.go @@ -11,6 +11,8 @@ import ( "github.com/gruntwork-io/go-commons/collections" + "github.com/gruntwork-io/boilerplate/manifest" + "github.com/gruntwork-io/boilerplate/config" "github.com/gruntwork-io/boilerplate/errors" getter_helper "github.com/gruntwork-io/boilerplate/getter-helper" @@ -23,12 +25,23 @@ import ( // The name of the variable that contains the current value of the loop in each iteration of for_each const eachVarName = "__each__" +// Maintaining original function name and signature for backwards compatability +func ProcessTemplate(options, rootOpts *options.BoilerplateOptions, thisDep variables.Dependency) error { + _, err := processTemplateInternal(options, rootOpts, thisDep, false) + return err +} + +// ProcessTemplateWithManifest works like ProcessTemplate but also returns the manifest of generated files +func ProcessTemplateWithManifest(options, rootOpts *options.BoilerplateOptions, thisDep variables.Dependency) (*manifest.Manifest, error) { + return processTemplateInternal(options, rootOpts, thisDep, true) +} + // Process the boilerplate template specified in the given options and use the existing variables. This function will // download remote templates to a temporary working directory, which is cleaned up at the end of the function. This // function will load any missing variables (either from command line options or by prompting the user), execute all the // dependent boilerplate templates, and then execute this template. Note that we pass in rootOptions so that template // dependencies can inspect properties of the root template. -func ProcessTemplate(options, rootOpts *options.BoilerplateOptions, thisDep variables.Dependency) error { +func processTemplateInternal(options, rootOpts *options.BoilerplateOptions, thisDep variables.Dependency, returnManifest bool) (*manifest.Manifest, error) { // If TemplateFolder is already set, use that directly as it is a local template. Otherwise, download to a temporary // working directory. if options.TemplateFolder == "" { @@ -38,7 +51,7 @@ func ProcessTemplate(options, rootOpts *options.BoilerplateOptions, thisDep vari os.RemoveAll(workingDir) }() if err != nil { - return err + return nil, err } // Set the TemplateFolder of the options to the download dir @@ -47,56 +60,64 @@ func ProcessTemplate(options, rootOpts *options.BoilerplateOptions, thisDep vari rootBoilerplateConfig, err := config.LoadBoilerplateConfig(rootOpts) if err != nil { - return err + return nil, err } if err := config.EnforceRequiredVersion(rootBoilerplateConfig); err != nil { - return err + return nil, err } boilerplateConfig, err := config.LoadBoilerplateConfig(options) if err != nil { - return err + return nil, err } if err := config.EnforceRequiredVersion(boilerplateConfig); err != nil { - return err + return nil, err } vars, err := config.GetVariables(options, boilerplateConfig, rootBoilerplateConfig, thisDep) if err != nil { - return err + return nil, err } err = os.MkdirAll(options.OutputFolder, 0o777) if err != nil { - return errors.WithStackTrace(err) + return nil, errors.WithStackTrace(err) } err = processHooks(boilerplateConfig.Hooks.BeforeHooks, options, vars) if err != nil { - return err + return nil, err } err = processDependencies(boilerplateConfig.Dependencies, options, boilerplateConfig.GetVariablesMap(), vars) if err != nil { - return err + return nil, err } partials, err := processPartials(boilerplateConfig.Partials, options, vars) if err != nil { - return err + return nil, err } - err = processTemplateFolder(boilerplateConfig, options, vars, partials) + var fileManifest *manifest.Manifest + if returnManifest { + fileManifest = manifest.NewManifest(options.OutputFolder) + } + + err = processTemplateFolder(boilerplateConfig, options, vars, partials, fileManifest) if err != nil { - return err + return nil, err } err = processHooks(boilerplateConfig.Hooks.AfterHooks, options, vars) if err != nil { - return err + return nil, err } - return nil + if returnManifest { + return fileManifest, nil + } + return nil, nil } func processPartials(partials []string, opts *options.BoilerplateOptions, vars map[string]interface{}) ([]string, error) { @@ -621,6 +642,7 @@ func processTemplateFolder( opts *options.BoilerplateOptions, variables map[string]interface{}, partials []string, + fileManifest *manifest.Manifest, ) error { util.Logger.Printf("Processing templates in %s and outputting generated files to %s", opts.TemplateFolder, opts.OutputFolder) @@ -643,7 +665,7 @@ func processTemplateFolder( return createOutputDir(path, opts, variables) } else { engine := determineTemplateEngine(processedEngines, path) - return processFile(path, opts, variables, partials, engine) + return processFile(path, opts, variables, partials, engine, fileManifest) } }) } @@ -656,6 +678,7 @@ func processFile( variables map[string]interface{}, partials []string, engine variables.TemplateEngineType, + fileManifest *manifest.Manifest, ) error { isText, err := util.IsTextFile(path) if err != nil { @@ -663,9 +686,9 @@ func processFile( } if isText { - return processTemplate(path, opts, variables, partials, engine) + return processTemplate(path, opts, variables, partials, engine, fileManifest) } else { - return copyFile(path, opts, variables) + return copyFile(path, opts, variables, fileManifest) } } @@ -714,14 +737,24 @@ func outPath(file string, opts *options.BoilerplateOptions, variables map[string } // Copy the given file, which is in options.TemplateFolder, to options.OutputFolder -func copyFile(file string, opts *options.BoilerplateOptions, variables map[string]interface{}) error { +func copyFile(file string, opts *options.BoilerplateOptions, variables map[string]interface{}, fileManifest *manifest.Manifest) error { destination, err := outPath(file, opts, variables) if err != nil { return err } util.Logger.Printf("Copying %s to %s", file, destination) - return util.CopyFile(file, destination) + if err := util.CopyFile(file, destination); err != nil { + return err + } + + if fileManifest != nil { + if err := fileManifest.AddFile(destination); err != nil { + return err + } + } + + return nil } // Run the template at templatePath, which is in templateFolder, through the Go template engine with the given @@ -732,6 +765,7 @@ func processTemplate( vars map[string]interface{}, partials []string, engine variables.TemplateEngineType, + fileManifest *manifest.Manifest, ) error { destination, err := outPath(templatePath, opts, vars) if err != nil { @@ -754,7 +788,17 @@ func processTemplate( destination = strings.TrimSuffix(destination, ".jsonnet") } - return util.WriteFileWithSamePermissions(templatePath, destination, []byte(out)) + if err := util.WriteFileWithSamePermissions(templatePath, destination, []byte(out)); err != nil { + return err + } + + if fileManifest != nil { + if err := fileManifest.AddFile(destination); err != nil { + return err + } + } + + return nil } // Return true if this is a path that should not be copied diff --git a/templates/template_processor_test.go b/templates/template_processor_test.go index 697237f1..3cf936f9 100644 --- a/templates/template_processor_test.go +++ b/templates/template_processor_test.go @@ -93,6 +93,95 @@ func TestCloneOptionsForDependency(t *testing.T) { } } +func TestProcessTemplateWithManifest(t *testing.T) { + t.Parallel() + + // Create a temporary directory for the test + templateDir, err := os.MkdirTemp("", "template-test") + require.NoError(t, err) + defer os.RemoveAll(templateDir) + + // Create a temporary output directory + outputDir, err := os.MkdirTemp("", "output-test") + require.NoError(t, err) + defer os.RemoveAll(outputDir) + + // Create a simple template file + templateFile := filepath.Join(templateDir, "test.txt") + err = os.WriteFile(templateFile, []byte("This is a test!"), 0644) + require.NoError(t, err) + + // Create an empty boilerplate.yml file + boilerplateYml := filepath.Join(templateDir, "boilerplate.yml") + err = os.WriteFile(boilerplateYml, []byte{}, 0644) + require.NoError(t, err) + + // Set up options + opts := &options.BoilerplateOptions{ + TemplateFolder: templateDir, + OutputFolder: outputDir, + NonInteractive: true, + OnMissingKey: options.ExitWithError, + OnMissingConfig: options.Exit, + } + + // Call ProcessTemplateWithManifest + manifest, err := ProcessTemplateWithManifest(opts, opts, variables.Dependency{}) + require.NoError(t, err) + require.NotNil(t, manifest) + + // Verify the manifest contains the expected file + require.Len(t, manifest.Files, 1) + assert.Equal(t, "test.txt", manifest.Files[0].Path) + + // Verify the file content + content, err := os.ReadFile(filepath.Join(outputDir, "test.txt")) + require.NoError(t, err) + assert.Equal(t, "This is a test!", string(content)) +} + +func TestProcessTemplate(t *testing.T) { + t.Parallel() + + // Create a temporary directory for the test + templateDir, err := os.MkdirTemp("", "template-test") + require.NoError(t, err) + defer os.RemoveAll(templateDir) + + // Create a temporary output directory + outputDir, err := os.MkdirTemp("", "output-test") + require.NoError(t, err) + defer os.RemoveAll(outputDir) + + // Create a simple template file + templateFile := filepath.Join(templateDir, "test.txt") + err = os.WriteFile(templateFile, []byte("This is a test!"), 0644) + require.NoError(t, err) + + // Create an empty boilerplate.yml file + boilerplateYml := filepath.Join(templateDir, "boilerplate.yml") + err = os.WriteFile(boilerplateYml, []byte{}, 0644) + require.NoError(t, err) + + // Set up options + opts := &options.BoilerplateOptions{ + TemplateFolder: templateDir, + OutputFolder: outputDir, + NonInteractive: true, + OnMissingKey: options.ExitWithError, + OnMissingConfig: options.Exit, + } + + // Call the original ProcessTemplate function + err = ProcessTemplate(opts, opts, variables.Dependency{}) + require.NoError(t, err) + + // Verify the file content + content, err := os.ReadFile(filepath.Join(outputDir, "test.txt")) + require.NoError(t, err) + assert.Equal(t, "This is a test!", string(content)) +} + func TestCloneVariablesForDependency(t *testing.T) { t.Parallel() From 009aa041ccd8f910a701fabdd429d7de57b3f46f Mon Sep 17 00:00:00 2001 From: Tim Eijgelshoven Date: Fri, 25 Jul 2025 19:17:03 -0400 Subject: [PATCH 2/4] feat: add manifest functionality and cli flag --- README.md | 10 ++++++- cli/boilerplate_cli.go | 24 +++++++++++++++- integration-tests/examples_test.go | 43 ++++++++++++++++++++++++++++ options/options.go | 3 ++ templates/template_processor.go | 30 +++++++------------ templates/template_processor_test.go | 20 +++++++------ 6 files changed, 99 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index c680c05b..a3bf6c70 100644 --- a/README.md +++ b/README.md @@ -134,13 +134,14 @@ You can find older versions on the [Releases Page](https://github.com/gruntwork- you to run arbitrary scripts. 1. **Cross-platform**: Boilerplate is easy to install (it's a standalone binary) and works on all major platforms (Mac, Linux, Windows). +1. **File manifest**: Optionally generate a JSON manifest of all created files for integration with other tools. ## Working with boilerplate When you run Boilerplate, it performs the following steps: 1. Read the `boilerplate.yml` file in the folder specified by the `--template-url` option to find all defined - varaibles. + variables. 1. Gather values for the variables from any `--var` and `--var-file` options that were passed in and prompting the user for the rest (unless the `--non-interactive` flag is specified). 1. Copy each file from `--template-url` to `--output-folder`, running each non-binary file through the Go @@ -185,6 +186,7 @@ The `boilerplate` binary supports the following options: * `--no-shell`: If this flag is set, no `shell` helpers will execute. They will instead return the text "replace-me". * `--disable-dependency-prompt` (optional): Do not prompt for confirmation to include dependencies. Has the same effect as --non-interactive, without disabling variable prompts. Default: `false`. +* `--output-manifest` (optional): Write a JSON list of all generated files to `boilerplate-manifest.json` in the current working directory. * `--help`: Show the help text and exit. * `--version`: Show the version and exit. @@ -214,6 +216,12 @@ Generate a project in ~/output from the templates in this repo's `include` examp boilerplate --template-url "git@github.com:gruntwork-io/boilerplate.git//examples/for-learning-and-testing/include?ref=main" --output-folder ~/output --var-file vars.yml ``` +Generate a project and create a manifest file listing all generated files: + +``` +boilerplate --template-url ~/templates --output-folder ~/output --output-manifest +``` + #### The boilerplate.yml file diff --git a/cli/boilerplate_cli.go b/cli/boilerplate_cli.go index 5fc92fb5..7031887f 100644 --- a/cli/boilerplate_cli.go +++ b/cli/boilerplate_cli.go @@ -1,7 +1,9 @@ package cli import ( + "encoding/json" "fmt" + "os" "github.com/gruntwork-io/go-commons/entrypoint" "github.com/gruntwork-io/go-commons/version" @@ -96,6 +98,10 @@ func CreateBoilerplateCli() *cli.App { Name: options.OptDisableDependencyPrompt, Usage: fmt.Sprintf("Do not prompt for confirmation to include dependencies. Has the same effect as --%s, without disabling variable prompts.", options.OptNonInteractive), }, + &cli.BoolFlag{ + Name: options.OptOutputManifest, + Usage: "Write a JSON list of all generated files to boilerplate-manifest.json in the current working directory.", + }, } // We pass JSON/YAML content to various CLI flags, such as --var, and this JSON/YAML content may contain commas or @@ -123,5 +129,21 @@ func runApp(cliContext *cli.Context) error { // The root boilerplate.yml is not itself a dependency, so we pass an empty Dependency. emptyDep := variables.Dependency{} - return templates.ProcessTemplate(opts, opts, emptyDep) + generatedFiles, err := templates.ProcessTemplate(opts, opts, emptyDep) + if err != nil { + return err + } + + if opts.OutputManifest { + data, err := json.Marshal(generatedFiles) + if err != nil { + return err + } + err = os.WriteFile("boilerplate-manifest.json", data, 0644) + if err != nil { + return err + } + } + + return nil } diff --git a/integration-tests/examples_test.go b/integration-tests/examples_test.go index fdd0e6e9..d3e43390 100644 --- a/integration-tests/examples_test.go +++ b/integration-tests/examples_test.go @@ -150,3 +150,46 @@ func testExample(t *testing.T, templateFolder string, outputFolder string, varFi assertDirectoriesEqual(t, expectedOutputFolder, outputFolder) } } +func TestExampleWithManifest(t *testing.T) { + t.Parallel() + + templateFolder := "../examples/for-learning-and-testing/website" + varFile := "../test-fixtures/examples-var-files/website/vars.yml" + + outputFolder, err := os.MkdirTemp("", "boilerplate-manifest-test") + require.NoError(t, err) + defer os.RemoveAll(outputFolder) + + // Change to output directory to test manifest file creation + originalDir, err := os.Getwd() + require.NoError(t, err) + defer os.Chdir(originalDir) + + err = os.Chdir(outputFolder) + require.NoError(t, err) + + app := cli.CreateBoilerplateCli() + args := []string{ + "boilerplate", + "--template-url", path.Join(originalDir, templateFolder), + "--output-folder", "output", + "--var-file", path.Join(originalDir, varFile), + "--output-manifest", + "--non-interactive", + } + + err = app.Run(args) + require.NoError(t, err) + + // Check manifest file was created + manifestPath := "boilerplate-manifest.json" + require.FileExists(t, manifestPath) + + // Read and verify manifest content + manifestContent, err := os.ReadFile(manifestPath) + require.NoError(t, err) + + manifestStr := string(manifestContent) + assert.Contains(t, manifestStr, "index.html") + assert.Contains(t, manifestStr, "logo.png") +} \ No newline at end of file diff --git a/options/options.go b/options/options.go index 98bf7805..789a823d 100644 --- a/options/options.go +++ b/options/options.go @@ -20,6 +20,7 @@ const OptMissingConfigAction = "missing-config-action" const OptNoHooks = "no-hooks" const OptNoShell = "no-shell" const OptDisableDependencyPrompt = "disable-dependency-prompt" +const OptOutputManifest = "output-manifest" // The command-line options for the boilerplate app type BoilerplateOptions struct { @@ -40,6 +41,7 @@ type BoilerplateOptions struct { ExecuteAllShellCommands bool // Track shell command confirmations similar to hooks ShellCommandAnswers map[string]bool + OutputManifest bool } // Validate that the options have reasonable values and return an error if they don't @@ -102,6 +104,7 @@ func ParseOptions(cliContext *cli.Context) (*BoilerplateOptions, error) { DisableDependencyPrompt: cliContext.Bool(OptDisableDependencyPrompt), ExecuteAllShellCommands: false, ShellCommandAnswers: make(map[string]bool), + OutputManifest: cliContext.Bool(OptOutputManifest), } if err := options.Validate(); err != nil { diff --git a/templates/template_processor.go b/templates/template_processor.go index bd658400..b1eb87bb 100644 --- a/templates/template_processor.go +++ b/templates/template_processor.go @@ -25,23 +25,13 @@ import ( // The name of the variable that contains the current value of the loop in each iteration of for_each const eachVarName = "__each__" -// Maintaining original function name and signature for backwards compatability -func ProcessTemplate(options, rootOpts *options.BoilerplateOptions, thisDep variables.Dependency) error { - _, err := processTemplateInternal(options, rootOpts, thisDep, false) - return err -} - -// ProcessTemplateWithManifest works like ProcessTemplate but also returns the manifest of generated files -func ProcessTemplateWithManifest(options, rootOpts *options.BoilerplateOptions, thisDep variables.Dependency) (*manifest.Manifest, error) { - return processTemplateInternal(options, rootOpts, thisDep, true) -} - // Process the boilerplate template specified in the given options and use the existing variables. This function will // download remote templates to a temporary working directory, which is cleaned up at the end of the function. This // function will load any missing variables (either from command line options or by prompting the user), execute all the // dependent boilerplate templates, and then execute this template. Note that we pass in rootOptions so that template // dependencies can inspect properties of the root template. -func processTemplateInternal(options, rootOpts *options.BoilerplateOptions, thisDep variables.Dependency, returnManifest bool) (*manifest.Manifest, error) { +// Returns a list of generated file paths relative to the output directory. +func ProcessTemplate(options, rootOpts *options.BoilerplateOptions, thisDep variables.Dependency) ([]string, error) { // If TemplateFolder is already set, use that directly as it is a local template. Otherwise, download to a temporary // working directory. if options.TemplateFolder == "" { @@ -99,10 +89,7 @@ func processTemplateInternal(options, rootOpts *options.BoilerplateOptions, this return nil, err } - var fileManifest *manifest.Manifest - if returnManifest { - fileManifest = manifest.NewManifest(options.OutputFolder) - } + fileManifest := manifest.NewManifest(options.OutputFolder) err = processTemplateFolder(boilerplateConfig, options, vars, partials, fileManifest) if err != nil { @@ -114,10 +101,12 @@ func processTemplateInternal(options, rootOpts *options.BoilerplateOptions, this return nil, err } - if returnManifest { - return fileManifest, nil + var paths []string + for _, file := range fileManifest.Files { + paths = append(paths, file.Path) } - return nil, nil + + return paths, nil } func processPartials(partials []string, opts *options.BoilerplateOptions, vars map[string]interface{}) ([]string, error) { @@ -401,7 +390,8 @@ func processDependency( } util.Logger.Printf("Processing dependency %s, with template folder %s and output folder %s", dependency.Name, dependencyOptions.TemplateFolder, dependencyOptions.OutputFolder) - return ProcessTemplate(dependencyOptions, opts, dependency) + _, err = ProcessTemplate(dependencyOptions, opts, dependency) + return err } forEach := dependency.ForEach diff --git a/templates/template_processor_test.go b/templates/template_processor_test.go index 3cf936f9..6496dbb6 100644 --- a/templates/template_processor_test.go +++ b/templates/template_processor_test.go @@ -93,7 +93,7 @@ func TestCloneOptionsForDependency(t *testing.T) { } } -func TestProcessTemplateWithManifest(t *testing.T) { +func TestProcessTemplateReturnsFiles(t *testing.T) { t.Parallel() // Create a temporary directory for the test @@ -125,14 +125,14 @@ func TestProcessTemplateWithManifest(t *testing.T) { OnMissingConfig: options.Exit, } - // Call ProcessTemplateWithManifest - manifest, err := ProcessTemplateWithManifest(opts, opts, variables.Dependency{}) + // Call ProcessTemplate + files, err := ProcessTemplate(opts, opts, variables.Dependency{}) require.NoError(t, err) - require.NotNil(t, manifest) + require.NotNil(t, files) - // Verify the manifest contains the expected file - require.Len(t, manifest.Files, 1) - assert.Equal(t, "test.txt", manifest.Files[0].Path) + // Verify the files list contains the expected file + require.Len(t, files, 1) + assert.Equal(t, "test.txt", files[0]) // Verify the file content content, err := os.ReadFile(filepath.Join(outputDir, "test.txt")) @@ -172,9 +172,11 @@ func TestProcessTemplate(t *testing.T) { OnMissingConfig: options.Exit, } - // Call the original ProcessTemplate function - err = ProcessTemplate(opts, opts, variables.Dependency{}) + // Call ProcessTemplate function + files, err := ProcessTemplate(opts, opts, variables.Dependency{}) require.NoError(t, err) + require.Len(t, files, 1) + assert.Equal(t, "test.txt", files[0]) // Verify the file content content, err := os.ReadFile(filepath.Join(outputDir, "test.txt")) From 5584201662daa91a033b6070ead05ffd38fb333a Mon Sep 17 00:00:00 2001 From: Tim Eijgelshoven Date: Mon, 28 Jul 2025 19:43:17 -0400 Subject: [PATCH 3/4] feat: add versioning to file manifest --- cli/boilerplate_cli.go | 14 +++-- integration-tests/examples_test.go | 9 ++- manifest/manifest.go | 75 +++++++++++++++++------ manifest/manifest_test.go | 96 +++++++++++++++--------------- templates/template_processor.go | 45 +++++++------- 5 files changed, 142 insertions(+), 97 deletions(-) diff --git a/cli/boilerplate_cli.go b/cli/boilerplate_cli.go index 7031887f..e9335dc6 100644 --- a/cli/boilerplate_cli.go +++ b/cli/boilerplate_cli.go @@ -1,14 +1,13 @@ package cli import ( - "encoding/json" "fmt" - "os" "github.com/gruntwork-io/go-commons/entrypoint" "github.com/gruntwork-io/go-commons/version" "github.com/urfave/cli/v2" + "github.com/gruntwork-io/boilerplate/manifest" "github.com/gruntwork-io/boilerplate/options" "github.com/gruntwork-io/boilerplate/templates" "github.com/gruntwork-io/boilerplate/variables" @@ -135,11 +134,14 @@ func runApp(cliContext *cli.Context) error { } if opts.OutputManifest { - data, err := json.Marshal(generatedFiles) - if err != nil { - return err + // Convert file paths to GeneratedFile structs + files := make([]manifest.GeneratedFile, 0, len(generatedFiles)) + for _, path := range generatedFiles { + files = append(files, manifest.GeneratedFile{Path: path}) } - err = os.WriteFile("boilerplate-manifest.json", data, 0644) + + // Update versioned manifest + err = manifest.UpdateVersionedManifest(opts.OutputFolder, opts.TemplateUrl, opts.Vars, files) if err != nil { return err } diff --git a/integration-tests/examples_test.go b/integration-tests/examples_test.go index d3e43390..291dbb49 100644 --- a/integration-tests/examples_test.go +++ b/integration-tests/examples_test.go @@ -181,8 +181,8 @@ func TestExampleWithManifest(t *testing.T) { err = app.Run(args) require.NoError(t, err) - // Check manifest file was created - manifestPath := "boilerplate-manifest.json" + // Check manifest file was created in output directory + manifestPath := "output/boilerplate-manifest.json" require.FileExists(t, manifestPath) // Read and verify manifest content @@ -190,6 +190,11 @@ func TestExampleWithManifest(t *testing.T) { require.NoError(t, err) manifestStr := string(manifestContent) + // Verify versioned manifest structure + assert.Contains(t, manifestStr, "latest_version") + assert.Contains(t, manifestStr, "versions") assert.Contains(t, manifestStr, "index.html") assert.Contains(t, manifestStr, "logo.png") + assert.Contains(t, manifestStr, "template_url") + assert.Contains(t, manifestStr, "boilerplate_version") } \ No newline at end of file diff --git a/manifest/manifest.go b/manifest/manifest.go index 7600c540..edba376a 100644 --- a/manifest/manifest.go +++ b/manifest/manifest.go @@ -1,7 +1,12 @@ package manifest import ( + "encoding/json" + "os" "path/filepath" + "time" + + "github.com/gruntwork-io/go-commons/version" ) // GeneratedFile represents a file that was generated by boilerplate @@ -9,30 +14,62 @@ type GeneratedFile struct { Path string `json:"path"` } -// Manifest represents the complete list of files generated by boilerplate -type Manifest struct { - OutputDir string - Files []GeneratedFile +// ManifestEntry represents a single version entry in the versioned manifest +type ManifestEntry struct { + Timestamp string `json:"timestamp"` + TemplateURL string `json:"template_url"` + BoilerplateVersion string `json:"boilerplate_version"` + Variables map[string]interface{} `json:"variables"` + OutputDir string `json:"output_dir"` + Files []GeneratedFile `json:"files"` } -// NewManifest creates a new manifest -func NewManifest(outputDir string) *Manifest { - return &Manifest{ - OutputDir: outputDir, - Files: []GeneratedFile{}, - } +// VersionedManifest represents the complete versioned manifest +type VersionedManifest struct { + LatestVersion string `json:"latest_version"` + Versions map[string]ManifestEntry `json:"versions"` } -// AddFile adds a generated file to the manifest -func (m *Manifest) AddFile(outputPath string) error { - // Convert to relative path from output directory - relPath, err := filepath.Rel(m.OutputDir, outputPath) + + +// UpdateVersionedManifest updates the versioned manifest with a new entry +func UpdateVersionedManifest(outputDir, templateURL string, variables map[string]interface{}, files []GeneratedFile) error { + manifestPath := filepath.Join(outputDir, "boilerplate-manifest.json") + + // Read existing manifest or create new one + manifest := &VersionedManifest{ + Versions: make(map[string]ManifestEntry), + } + + if data, err := os.ReadFile(manifestPath); err == nil { + if err := json.Unmarshal(data, manifest); err != nil { + // If unmarshal fails, start with empty manifest + manifest = &VersionedManifest{ + Versions: make(map[string]ManifestEntry), + } + } + } + + // Create new version entry + timestamp := time.Now().UTC().Format(time.RFC3339) + entry := ManifestEntry{ + Timestamp: timestamp, + TemplateURL: templateURL, + BoilerplateVersion: version.GetVersion(), + Variables: variables, + OutputDir: outputDir, + Files: files, + } + + // Add to manifest + manifest.Versions[timestamp] = entry + manifest.LatestVersion = timestamp + + // Write back to file + data, err := json.MarshalIndent(manifest, "", " ") if err != nil { - relPath = outputPath + return err } - m.Files = append(m.Files, GeneratedFile{ - Path: relPath, - }) - return nil + return os.WriteFile(manifestPath, data, 0644) } diff --git a/manifest/manifest_test.go b/manifest/manifest_test.go index ab02a2cf..d87f81c8 100644 --- a/manifest/manifest_test.go +++ b/manifest/manifest_test.go @@ -1,75 +1,77 @@ package manifest import ( + "encoding/json" + "os" "path/filepath" "testing" + "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestNewManifest(t *testing.T) { +func TestUpdateVersionedManifestMultipleVersions(t *testing.T) { t.Parallel() - outputDir := "/test/output" + // Create temporary directory + tempDir, err := os.MkdirTemp("", "manifest-test") + require.NoError(t, err) + defer os.RemoveAll(tempDir) - manifest := NewManifest(outputDir) + // First update + err = UpdateVersionedManifest(tempDir, "template1", map[string]interface{}{"var1": "value1"}, []GeneratedFile{{Path: "file1.txt"}}) + require.NoError(t, err) - assert.Equal(t, outputDir, manifest.OutputDir) - assert.Empty(t, manifest.Files) -} + // Wait a moment to ensure different timestamps + time.Sleep(time.Second * 1) -func TestAddFile(t *testing.T) { - t.Parallel() + // Second update + err = UpdateVersionedManifest(tempDir, "template2", map[string]interface{}{"var2": "value2"}, []GeneratedFile{{Path: "file2.txt"}}) + require.NoError(t, err) - outputDir := "/test/output" - manifest := NewManifest(outputDir) + // Read manifest + manifestPath := filepath.Join(tempDir, "boilerplate-manifest.json") + data, err := os.ReadFile(manifestPath) + require.NoError(t, err) - // Test adding a file in the output directory - outputPath := filepath.Join(outputDir, "subdir", "file.txt") + var manifest VersionedManifest + err = json.Unmarshal(data, &manifest) + require.NoError(t, err) - err := manifest.AddFile(outputPath) + // Verify we have 2 versions + assert.Len(t, manifest.Versions, 2) + assert.NotEmpty(t, manifest.LatestVersion) - assert.NoError(t, err) - assert.Len(t, manifest.Files, 1) - assert.Equal(t, "subdir/file.txt", manifest.Files[0].Path) + // Verify latest version points to second update + latestEntry := manifest.Versions[manifest.LatestVersion] + assert.Equal(t, "template2", latestEntry.TemplateURL) + assert.Equal(t, map[string]interface{}{"var2": "value2"}, latestEntry.Variables) } -func TestAddFileOutsideOutputDir(t *testing.T) { +func TestUpdateVersionedManifestInvalidExistingFile(t *testing.T) { t.Parallel() - outputDir := "/test/output" - manifest := NewManifest(outputDir) + // Create temporary directory + tempDir, err := os.MkdirTemp("", "manifest-test") + require.NoError(t, err) + defer os.RemoveAll(tempDir) - // Test adding a file outside the output directory - outputPath := "/other/dir/file.txt" - - err := manifest.AddFile(outputPath) + // Create invalid JSON file + manifestPath := filepath.Join(tempDir, "boilerplate-manifest.json") + err = os.WriteFile(manifestPath, []byte("invalid json"), 0644) + require.NoError(t, err) + // Update should still work (creates new manifest) + err = UpdateVersionedManifest(tempDir, "template", map[string]interface{}{}, []GeneratedFile{}) assert.NoError(t, err) - assert.Len(t, manifest.Files, 1) - expectedRelPath, _ := filepath.Rel(outputDir, outputPath) - assert.Equal(t, expectedRelPath, manifest.Files[0].Path) -} -func TestAddMultipleFiles(t *testing.T) { - t.Parallel() - - outputDir := "/test/output" - manifest := NewManifest(outputDir) - - // Add first file - err1 := manifest.AddFile( - filepath.Join(outputDir, "file1.txt"), - ) + // Verify manifest was recreated + data, err := os.ReadFile(manifestPath) + require.NoError(t, err) - // Add second file - err2 := manifest.AddFile( - filepath.Join(outputDir, "subdir", "file2.txt"), - ) - - assert.NoError(t, err1) - assert.NoError(t, err2) - assert.Len(t, manifest.Files, 2) - assert.Equal(t, "file1.txt", manifest.Files[0].Path) - assert.Equal(t, "subdir/file2.txt", manifest.Files[1].Path) + var manifest VersionedManifest + err = json.Unmarshal(data, &manifest) + assert.NoError(t, err) + assert.Len(t, manifest.Versions, 1) } diff --git a/templates/template_processor.go b/templates/template_processor.go index b1eb87bb..295576ad 100644 --- a/templates/template_processor.go +++ b/templates/template_processor.go @@ -11,8 +11,6 @@ import ( "github.com/gruntwork-io/go-commons/collections" - "github.com/gruntwork-io/boilerplate/manifest" - "github.com/gruntwork-io/boilerplate/config" "github.com/gruntwork-io/boilerplate/errors" getter_helper "github.com/gruntwork-io/boilerplate/getter-helper" @@ -89,9 +87,9 @@ func ProcessTemplate(options, rootOpts *options.BoilerplateOptions, thisDep vari return nil, err } - fileManifest := manifest.NewManifest(options.OutputFolder) + var generatedFilePaths []string - err = processTemplateFolder(boilerplateConfig, options, vars, partials, fileManifest) + err = processTemplateFolder(boilerplateConfig, options, vars, partials, &generatedFilePaths) if err != nil { return nil, err } @@ -101,12 +99,7 @@ func ProcessTemplate(options, rootOpts *options.BoilerplateOptions, thisDep vari return nil, err } - var paths []string - for _, file := range fileManifest.Files { - paths = append(paths, file.Path) - } - - return paths, nil + return generatedFilePaths, nil } func processPartials(partials []string, opts *options.BoilerplateOptions, vars map[string]interface{}) ([]string, error) { @@ -632,7 +625,7 @@ func processTemplateFolder( opts *options.BoilerplateOptions, variables map[string]interface{}, partials []string, - fileManifest *manifest.Manifest, + generatedFilePaths *[]string, ) error { util.Logger.Printf("Processing templates in %s and outputting generated files to %s", opts.TemplateFolder, opts.OutputFolder) @@ -655,7 +648,7 @@ func processTemplateFolder( return createOutputDir(path, opts, variables) } else { engine := determineTemplateEngine(processedEngines, path) - return processFile(path, opts, variables, partials, engine, fileManifest) + return processFile(path, opts, variables, partials, engine, generatedFilePaths) } }) } @@ -668,7 +661,7 @@ func processFile( variables map[string]interface{}, partials []string, engine variables.TemplateEngineType, - fileManifest *manifest.Manifest, + generatedFilePaths *[]string, ) error { isText, err := util.IsTextFile(path) if err != nil { @@ -676,9 +669,9 @@ func processFile( } if isText { - return processTemplate(path, opts, variables, partials, engine, fileManifest) + return processTemplate(path, opts, variables, partials, engine, generatedFilePaths) } else { - return copyFile(path, opts, variables, fileManifest) + return copyFile(path, opts, variables, generatedFilePaths) } } @@ -727,7 +720,7 @@ func outPath(file string, opts *options.BoilerplateOptions, variables map[string } // Copy the given file, which is in options.TemplateFolder, to options.OutputFolder -func copyFile(file string, opts *options.BoilerplateOptions, variables map[string]interface{}, fileManifest *manifest.Manifest) error { +func copyFile(file string, opts *options.BoilerplateOptions, variables map[string]interface{}, generatedFilePaths *[]string) error { destination, err := outPath(file, opts, variables) if err != nil { return err @@ -738,10 +731,13 @@ func copyFile(file string, opts *options.BoilerplateOptions, variables map[strin return err } - if fileManifest != nil { - if err := fileManifest.AddFile(destination); err != nil { - return err + if generatedFilePaths != nil { + // Convert to relative path from output directory + relPath, err := filepath.Rel(opts.OutputFolder, destination) + if err != nil { + relPath = destination } + *generatedFilePaths = append(*generatedFilePaths, relPath) } return nil @@ -755,7 +751,7 @@ func processTemplate( vars map[string]interface{}, partials []string, engine variables.TemplateEngineType, - fileManifest *manifest.Manifest, + generatedFilePaths *[]string, ) error { destination, err := outPath(templatePath, opts, vars) if err != nil { @@ -782,10 +778,13 @@ func processTemplate( return err } - if fileManifest != nil { - if err := fileManifest.AddFile(destination); err != nil { - return err + if generatedFilePaths != nil { + // Convert to relative path from output directory + relPath, err := filepath.Rel(opts.OutputFolder, destination) + if err != nil { + relPath = destination } + *generatedFilePaths = append(*generatedFilePaths, relPath) } return nil From 579755018b25a87057b5fd96a11eac5011388a24 Mon Sep 17 00:00:00 2001 From: Tim Eijgelshoven Date: Mon, 28 Jul 2025 19:50:42 -0400 Subject: [PATCH 4/4] fix: update README with versioned manifest info --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a3bf6c70..a7b78cd6 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ You can find older versions on the [Releases Page](https://github.com/gruntwork- you to run arbitrary scripts. 1. **Cross-platform**: Boilerplate is easy to install (it's a standalone binary) and works on all major platforms (Mac, Linux, Windows). -1. **File manifest**: Optionally generate a JSON manifest of all created files for integration with other tools. +1. **Versioned file manifest**: Optionally generate a versioned JSON manifest that tracks all created files across multiple template generations. Each time boilerplate runs with `--output-manifest`, it adds a new version entry to the existing manifest file (or creates one if none exists), preserving the history of all previous generations for integration with other tools. ## Working with boilerplate @@ -186,7 +186,7 @@ The `boilerplate` binary supports the following options: * `--no-shell`: If this flag is set, no `shell` helpers will execute. They will instead return the text "replace-me". * `--disable-dependency-prompt` (optional): Do not prompt for confirmation to include dependencies. Has the same effect as --non-interactive, without disabling variable prompts. Default: `false`. -* `--output-manifest` (optional): Write a JSON list of all generated files to `boilerplate-manifest.json` in the current working directory. +* `--output-manifest` (optional): Write a versioned JSON manifest of all generated files to `boilerplate-manifest.json` in the output directory. If a manifest file already exists, a new version entry will be added to track the current generation alongside previous ones. * `--help`: Show the help text and exit. * `--version`: Show the version and exit. @@ -216,7 +216,7 @@ Generate a project in ~/output from the templates in this repo's `include` examp boilerplate --template-url "git@github.com:gruntwork-io/boilerplate.git//examples/for-learning-and-testing/include?ref=main" --output-folder ~/output --var-file vars.yml ``` -Generate a project and create a manifest file listing all generated files: +Generate a project and create a versioned manifest file listing all generated files: ``` boilerplate --template-url ~/templates --output-folder ~/output --output-manifest