diff --git a/README.md b/README.md index c680c05..a7b78cd 100644 --- a/README.md +++ b/README.md @@ -134,13 +134,14 @@ You can find older versions on the [Releases Page](https://github.com/gruntwork- you to run arbitrary scripts. 1. **Cross-platform**: Boilerplate is easy to install (it's a standalone binary) and works on all major platforms (Mac, Linux, Windows). +1. **Versioned file manifest**: Optionally generate a versioned JSON manifest that tracks all created files across multiple template generations. Each time boilerplate runs with `--output-manifest`, it adds a new version entry to the existing manifest file (or creates one if none exists), preserving the history of all previous generations for integration with other tools. ## Working with boilerplate When you run Boilerplate, it performs the following steps: 1. Read the `boilerplate.yml` file in the folder specified by the `--template-url` option to find all defined - varaibles. + variables. 1. Gather values for the variables from any `--var` and `--var-file` options that were passed in and prompting the user for the rest (unless the `--non-interactive` flag is specified). 1. Copy each file from `--template-url` to `--output-folder`, running each non-binary file through the Go @@ -185,6 +186,7 @@ The `boilerplate` binary supports the following options: * `--no-shell`: If this flag is set, no `shell` helpers will execute. They will instead return the text "replace-me". * `--disable-dependency-prompt` (optional): Do not prompt for confirmation to include dependencies. Has the same effect as --non-interactive, without disabling variable prompts. Default: `false`. +* `--output-manifest` (optional): Write a versioned JSON manifest of all generated files to `boilerplate-manifest.json` in the output directory. If a manifest file already exists, a new version entry will be added to track the current generation alongside previous ones. * `--help`: Show the help text and exit. * `--version`: Show the version and exit. @@ -214,6 +216,12 @@ Generate a project in ~/output from the templates in this repo's `include` examp boilerplate --template-url "git@github.com:gruntwork-io/boilerplate.git//examples/for-learning-and-testing/include?ref=main" --output-folder ~/output --var-file vars.yml ``` +Generate a project and create a versioned manifest file listing all generated files: + +``` +boilerplate --template-url ~/templates --output-folder ~/output --output-manifest +``` + #### The boilerplate.yml file diff --git a/cli/boilerplate_cli.go b/cli/boilerplate_cli.go index 5fc92fb..e9335dc 100644 --- a/cli/boilerplate_cli.go +++ b/cli/boilerplate_cli.go @@ -7,6 +7,7 @@ import ( "github.com/gruntwork-io/go-commons/version" "github.com/urfave/cli/v2" + "github.com/gruntwork-io/boilerplate/manifest" "github.com/gruntwork-io/boilerplate/options" "github.com/gruntwork-io/boilerplate/templates" "github.com/gruntwork-io/boilerplate/variables" @@ -96,6 +97,10 @@ func CreateBoilerplateCli() *cli.App { Name: options.OptDisableDependencyPrompt, Usage: fmt.Sprintf("Do not prompt for confirmation to include dependencies. Has the same effect as --%s, without disabling variable prompts.", options.OptNonInteractive), }, + &cli.BoolFlag{ + Name: options.OptOutputManifest, + Usage: "Write a JSON list of all generated files to boilerplate-manifest.json in the current working directory.", + }, } // We pass JSON/YAML content to various CLI flags, such as --var, and this JSON/YAML content may contain commas or @@ -123,5 +128,24 @@ func runApp(cliContext *cli.Context) error { // The root boilerplate.yml is not itself a dependency, so we pass an empty Dependency. emptyDep := variables.Dependency{} - return templates.ProcessTemplate(opts, opts, emptyDep) + generatedFiles, err := templates.ProcessTemplate(opts, opts, emptyDep) + if err != nil { + return err + } + + if opts.OutputManifest { + // Convert file paths to GeneratedFile structs + files := make([]manifest.GeneratedFile, 0, len(generatedFiles)) + for _, path := range generatedFiles { + files = append(files, manifest.GeneratedFile{Path: path}) + } + + // Update versioned manifest + err = manifest.UpdateVersionedManifest(opts.OutputFolder, opts.TemplateUrl, opts.Vars, files) + if err != nil { + return err + } + } + + return nil } diff --git a/integration-tests/examples_test.go b/integration-tests/examples_test.go index fdd0e6e..291dbb4 100644 --- a/integration-tests/examples_test.go +++ b/integration-tests/examples_test.go @@ -150,3 +150,51 @@ func testExample(t *testing.T, templateFolder string, outputFolder string, varFi assertDirectoriesEqual(t, expectedOutputFolder, outputFolder) } } +func TestExampleWithManifest(t *testing.T) { + t.Parallel() + + templateFolder := "../examples/for-learning-and-testing/website" + varFile := "../test-fixtures/examples-var-files/website/vars.yml" + + outputFolder, err := os.MkdirTemp("", "boilerplate-manifest-test") + require.NoError(t, err) + defer os.RemoveAll(outputFolder) + + // Change to output directory to test manifest file creation + originalDir, err := os.Getwd() + require.NoError(t, err) + defer os.Chdir(originalDir) + + err = os.Chdir(outputFolder) + require.NoError(t, err) + + app := cli.CreateBoilerplateCli() + args := []string{ + "boilerplate", + "--template-url", path.Join(originalDir, templateFolder), + "--output-folder", "output", + "--var-file", path.Join(originalDir, varFile), + "--output-manifest", + "--non-interactive", + } + + err = app.Run(args) + require.NoError(t, err) + + // Check manifest file was created in output directory + manifestPath := "output/boilerplate-manifest.json" + require.FileExists(t, manifestPath) + + // Read and verify manifest content + manifestContent, err := os.ReadFile(manifestPath) + require.NoError(t, err) + + manifestStr := string(manifestContent) + // Verify versioned manifest structure + assert.Contains(t, manifestStr, "latest_version") + assert.Contains(t, manifestStr, "versions") + assert.Contains(t, manifestStr, "index.html") + assert.Contains(t, manifestStr, "logo.png") + assert.Contains(t, manifestStr, "template_url") + assert.Contains(t, manifestStr, "boilerplate_version") +} \ No newline at end of file diff --git a/integration-tests/for_production_example_unix_test.go b/integration-tests/for_production_example_unix_test.go index 78606c8..61a3805 100644 --- a/integration-tests/for_production_example_unix_test.go +++ b/integration-tests/for_production_example_unix_test.go @@ -4,7 +4,7 @@ package integration_tests import ( - "io/ioutil" + "os" "path/filepath" "testing" @@ -19,9 +19,9 @@ func TestForProductionTerragruntArchitectureBoilerplateExample(t *testing.T) { forProductionExamplePath := "../examples/for-production/terragrunt-architecture-catalog" - outputBasePath, err := ioutil.TempDir("", "boilerplate-for-production-output") + outputBasePath, err := os.MkdirTemp("", "boilerplate-for-production-output") require.NoError(t, err) - //defer os.RemoveAll(outputBasePath) + defer os.RemoveAll(outputBasePath) templateFolder, err := filepath.Abs(filepath.Join(forProductionExamplePath, "blueprints", "reference-architecture")) require.NoError(t, err) diff --git a/integration-tests/required_version_test.go b/integration-tests/required_version_test.go index 2023cad..ea7c10a 100644 --- a/integration-tests/required_version_test.go +++ b/integration-tests/required_version_test.go @@ -1,13 +1,12 @@ package integration_tests import ( - "io/ioutil" "os" "testing" "github.com/gruntwork-io/boilerplate/cli" "github.com/gruntwork-io/boilerplate/config" - "github.com/gruntwork-io/go-commons/errors" + "github.com/gruntwork-io/boilerplate/errors" "github.com/gruntwork-io/go-commons/version" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -58,7 +57,7 @@ func TestRequiredVersionUnderTest(t *testing.T) { func runRequiredVersionExample(t *testing.T, templateFolder string) error { app := cli.CreateBoilerplateCli() - outputPath, err := ioutil.TempDir("", "boilerplate-test-output-reqver") + outputPath, err := os.MkdirTemp("", "boilerplate-test-output-reqver") require.NoError(t, err) defer os.RemoveAll(outputPath) diff --git a/manifest/manifest.go b/manifest/manifest.go new file mode 100644 index 0000000..edba376 --- /dev/null +++ b/manifest/manifest.go @@ -0,0 +1,75 @@ +package manifest + +import ( + "encoding/json" + "os" + "path/filepath" + "time" + + "github.com/gruntwork-io/go-commons/version" +) + +// GeneratedFile represents a file that was generated by boilerplate +type GeneratedFile struct { + Path string `json:"path"` +} + +// ManifestEntry represents a single version entry in the versioned manifest +type ManifestEntry struct { + Timestamp string `json:"timestamp"` + TemplateURL string `json:"template_url"` + BoilerplateVersion string `json:"boilerplate_version"` + Variables map[string]interface{} `json:"variables"` + OutputDir string `json:"output_dir"` + Files []GeneratedFile `json:"files"` +} + +// VersionedManifest represents the complete versioned manifest +type VersionedManifest struct { + LatestVersion string `json:"latest_version"` + Versions map[string]ManifestEntry `json:"versions"` +} + + + +// UpdateVersionedManifest updates the versioned manifest with a new entry +func UpdateVersionedManifest(outputDir, templateURL string, variables map[string]interface{}, files []GeneratedFile) error { + manifestPath := filepath.Join(outputDir, "boilerplate-manifest.json") + + // Read existing manifest or create new one + manifest := &VersionedManifest{ + Versions: make(map[string]ManifestEntry), + } + + if data, err := os.ReadFile(manifestPath); err == nil { + if err := json.Unmarshal(data, manifest); err != nil { + // If unmarshal fails, start with empty manifest + manifest = &VersionedManifest{ + Versions: make(map[string]ManifestEntry), + } + } + } + + // Create new version entry + timestamp := time.Now().UTC().Format(time.RFC3339) + entry := ManifestEntry{ + Timestamp: timestamp, + TemplateURL: templateURL, + BoilerplateVersion: version.GetVersion(), + Variables: variables, + OutputDir: outputDir, + Files: files, + } + + // Add to manifest + manifest.Versions[timestamp] = entry + manifest.LatestVersion = timestamp + + // Write back to file + data, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return err + } + + return os.WriteFile(manifestPath, data, 0644) +} diff --git a/manifest/manifest_test.go b/manifest/manifest_test.go new file mode 100644 index 0000000..d87f81c --- /dev/null +++ b/manifest/manifest_test.go @@ -0,0 +1,77 @@ +package manifest + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestUpdateVersionedManifestMultipleVersions(t *testing.T) { + t.Parallel() + + // Create temporary directory + tempDir, err := os.MkdirTemp("", "manifest-test") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + // First update + err = UpdateVersionedManifest(tempDir, "template1", map[string]interface{}{"var1": "value1"}, []GeneratedFile{{Path: "file1.txt"}}) + require.NoError(t, err) + + // Wait a moment to ensure different timestamps + time.Sleep(time.Second * 1) + + // Second update + err = UpdateVersionedManifest(tempDir, "template2", map[string]interface{}{"var2": "value2"}, []GeneratedFile{{Path: "file2.txt"}}) + require.NoError(t, err) + + // Read manifest + manifestPath := filepath.Join(tempDir, "boilerplate-manifest.json") + data, err := os.ReadFile(manifestPath) + require.NoError(t, err) + + var manifest VersionedManifest + err = json.Unmarshal(data, &manifest) + require.NoError(t, err) + + // Verify we have 2 versions + assert.Len(t, manifest.Versions, 2) + assert.NotEmpty(t, manifest.LatestVersion) + + // Verify latest version points to second update + latestEntry := manifest.Versions[manifest.LatestVersion] + assert.Equal(t, "template2", latestEntry.TemplateURL) + assert.Equal(t, map[string]interface{}{"var2": "value2"}, latestEntry.Variables) +} + +func TestUpdateVersionedManifestInvalidExistingFile(t *testing.T) { + t.Parallel() + + // Create temporary directory + tempDir, err := os.MkdirTemp("", "manifest-test") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + // Create invalid JSON file + manifestPath := filepath.Join(tempDir, "boilerplate-manifest.json") + err = os.WriteFile(manifestPath, []byte("invalid json"), 0644) + require.NoError(t, err) + + // Update should still work (creates new manifest) + err = UpdateVersionedManifest(tempDir, "template", map[string]interface{}{}, []GeneratedFile{}) + assert.NoError(t, err) + + // Verify manifest was recreated + data, err := os.ReadFile(manifestPath) + require.NoError(t, err) + + var manifest VersionedManifest + err = json.Unmarshal(data, &manifest) + assert.NoError(t, err) + assert.Len(t, manifest.Versions, 1) +} diff --git a/options/options.go b/options/options.go index 98bf780..789a823 100644 --- a/options/options.go +++ b/options/options.go @@ -20,6 +20,7 @@ const OptMissingConfigAction = "missing-config-action" const OptNoHooks = "no-hooks" const OptNoShell = "no-shell" const OptDisableDependencyPrompt = "disable-dependency-prompt" +const OptOutputManifest = "output-manifest" // The command-line options for the boilerplate app type BoilerplateOptions struct { @@ -40,6 +41,7 @@ type BoilerplateOptions struct { ExecuteAllShellCommands bool // Track shell command confirmations similar to hooks ShellCommandAnswers map[string]bool + OutputManifest bool } // Validate that the options have reasonable values and return an error if they don't @@ -102,6 +104,7 @@ func ParseOptions(cliContext *cli.Context) (*BoilerplateOptions, error) { DisableDependencyPrompt: cliContext.Bool(OptDisableDependencyPrompt), ExecuteAllShellCommands: false, ShellCommandAnswers: make(map[string]bool), + OutputManifest: cliContext.Bool(OptOutputManifest), } if err := options.Validate(); err != nil { diff --git a/templates/template_processor.go b/templates/template_processor.go index 5008abc..295576a 100644 --- a/templates/template_processor.go +++ b/templates/template_processor.go @@ -28,7 +28,8 @@ const eachVarName = "__each__" // function will load any missing variables (either from command line options or by prompting the user), execute all the // dependent boilerplate templates, and then execute this template. Note that we pass in rootOptions so that template // dependencies can inspect properties of the root template. -func ProcessTemplate(options, rootOpts *options.BoilerplateOptions, thisDep variables.Dependency) error { +// Returns a list of generated file paths relative to the output directory. +func ProcessTemplate(options, rootOpts *options.BoilerplateOptions, thisDep variables.Dependency) ([]string, error) { // If TemplateFolder is already set, use that directly as it is a local template. Otherwise, download to a temporary // working directory. if options.TemplateFolder == "" { @@ -38,7 +39,7 @@ func ProcessTemplate(options, rootOpts *options.BoilerplateOptions, thisDep vari os.RemoveAll(workingDir) }() if err != nil { - return err + return nil, err } // Set the TemplateFolder of the options to the download dir @@ -47,56 +48,58 @@ func ProcessTemplate(options, rootOpts *options.BoilerplateOptions, thisDep vari rootBoilerplateConfig, err := config.LoadBoilerplateConfig(rootOpts) if err != nil { - return err + return nil, err } if err := config.EnforceRequiredVersion(rootBoilerplateConfig); err != nil { - return err + return nil, err } boilerplateConfig, err := config.LoadBoilerplateConfig(options) if err != nil { - return err + return nil, err } if err := config.EnforceRequiredVersion(boilerplateConfig); err != nil { - return err + return nil, err } vars, err := config.GetVariables(options, boilerplateConfig, rootBoilerplateConfig, thisDep) if err != nil { - return err + return nil, err } err = os.MkdirAll(options.OutputFolder, 0o777) if err != nil { - return errors.WithStackTrace(err) + return nil, errors.WithStackTrace(err) } err = processHooks(boilerplateConfig.Hooks.BeforeHooks, options, vars) if err != nil { - return err + return nil, err } err = processDependencies(boilerplateConfig.Dependencies, options, boilerplateConfig.GetVariablesMap(), vars) if err != nil { - return err + return nil, err } partials, err := processPartials(boilerplateConfig.Partials, options, vars) if err != nil { - return err + return nil, err } - err = processTemplateFolder(boilerplateConfig, options, vars, partials) + var generatedFilePaths []string + + err = processTemplateFolder(boilerplateConfig, options, vars, partials, &generatedFilePaths) if err != nil { - return err + return nil, err } err = processHooks(boilerplateConfig.Hooks.AfterHooks, options, vars) if err != nil { - return err + return nil, err } - return nil + return generatedFilePaths, nil } func processPartials(partials []string, opts *options.BoilerplateOptions, vars map[string]interface{}) ([]string, error) { @@ -380,7 +383,8 @@ func processDependency( } util.Logger.Printf("Processing dependency %s, with template folder %s and output folder %s", dependency.Name, dependencyOptions.TemplateFolder, dependencyOptions.OutputFolder) - return ProcessTemplate(dependencyOptions, opts, dependency) + _, err = ProcessTemplate(dependencyOptions, opts, dependency) + return err } forEach := dependency.ForEach @@ -621,6 +625,7 @@ func processTemplateFolder( opts *options.BoilerplateOptions, variables map[string]interface{}, partials []string, + generatedFilePaths *[]string, ) error { util.Logger.Printf("Processing templates in %s and outputting generated files to %s", opts.TemplateFolder, opts.OutputFolder) @@ -643,7 +648,7 @@ func processTemplateFolder( return createOutputDir(path, opts, variables) } else { engine := determineTemplateEngine(processedEngines, path) - return processFile(path, opts, variables, partials, engine) + return processFile(path, opts, variables, partials, engine, generatedFilePaths) } }) } @@ -656,6 +661,7 @@ func processFile( variables map[string]interface{}, partials []string, engine variables.TemplateEngineType, + generatedFilePaths *[]string, ) error { isText, err := util.IsTextFile(path) if err != nil { @@ -663,9 +669,9 @@ func processFile( } if isText { - return processTemplate(path, opts, variables, partials, engine) + return processTemplate(path, opts, variables, partials, engine, generatedFilePaths) } else { - return copyFile(path, opts, variables) + return copyFile(path, opts, variables, generatedFilePaths) } } @@ -714,14 +720,27 @@ func outPath(file string, opts *options.BoilerplateOptions, variables map[string } // Copy the given file, which is in options.TemplateFolder, to options.OutputFolder -func copyFile(file string, opts *options.BoilerplateOptions, variables map[string]interface{}) error { +func copyFile(file string, opts *options.BoilerplateOptions, variables map[string]interface{}, generatedFilePaths *[]string) error { destination, err := outPath(file, opts, variables) if err != nil { return err } util.Logger.Printf("Copying %s to %s", file, destination) - return util.CopyFile(file, destination) + if err := util.CopyFile(file, destination); err != nil { + return err + } + + if generatedFilePaths != nil { + // Convert to relative path from output directory + relPath, err := filepath.Rel(opts.OutputFolder, destination) + if err != nil { + relPath = destination + } + *generatedFilePaths = append(*generatedFilePaths, relPath) + } + + return nil } // Run the template at templatePath, which is in templateFolder, through the Go template engine with the given @@ -732,6 +751,7 @@ func processTemplate( vars map[string]interface{}, partials []string, engine variables.TemplateEngineType, + generatedFilePaths *[]string, ) error { destination, err := outPath(templatePath, opts, vars) if err != nil { @@ -754,7 +774,20 @@ func processTemplate( destination = strings.TrimSuffix(destination, ".jsonnet") } - return util.WriteFileWithSamePermissions(templatePath, destination, []byte(out)) + if err := util.WriteFileWithSamePermissions(templatePath, destination, []byte(out)); err != nil { + return err + } + + if generatedFilePaths != nil { + // Convert to relative path from output directory + relPath, err := filepath.Rel(opts.OutputFolder, destination) + if err != nil { + relPath = destination + } + *generatedFilePaths = append(*generatedFilePaths, relPath) + } + + return nil } // Return true if this is a path that should not be copied diff --git a/templates/template_processor_test.go b/templates/template_processor_test.go index 697237f..6496dbb 100644 --- a/templates/template_processor_test.go +++ b/templates/template_processor_test.go @@ -93,6 +93,97 @@ func TestCloneOptionsForDependency(t *testing.T) { } } +func TestProcessTemplateReturnsFiles(t *testing.T) { + t.Parallel() + + // Create a temporary directory for the test + templateDir, err := os.MkdirTemp("", "template-test") + require.NoError(t, err) + defer os.RemoveAll(templateDir) + + // Create a temporary output directory + outputDir, err := os.MkdirTemp("", "output-test") + require.NoError(t, err) + defer os.RemoveAll(outputDir) + + // Create a simple template file + templateFile := filepath.Join(templateDir, "test.txt") + err = os.WriteFile(templateFile, []byte("This is a test!"), 0644) + require.NoError(t, err) + + // Create an empty boilerplate.yml file + boilerplateYml := filepath.Join(templateDir, "boilerplate.yml") + err = os.WriteFile(boilerplateYml, []byte{}, 0644) + require.NoError(t, err) + + // Set up options + opts := &options.BoilerplateOptions{ + TemplateFolder: templateDir, + OutputFolder: outputDir, + NonInteractive: true, + OnMissingKey: options.ExitWithError, + OnMissingConfig: options.Exit, + } + + // Call ProcessTemplate + files, err := ProcessTemplate(opts, opts, variables.Dependency{}) + require.NoError(t, err) + require.NotNil(t, files) + + // Verify the files list contains the expected file + require.Len(t, files, 1) + assert.Equal(t, "test.txt", files[0]) + + // Verify the file content + content, err := os.ReadFile(filepath.Join(outputDir, "test.txt")) + require.NoError(t, err) + assert.Equal(t, "This is a test!", string(content)) +} + +func TestProcessTemplate(t *testing.T) { + t.Parallel() + + // Create a temporary directory for the test + templateDir, err := os.MkdirTemp("", "template-test") + require.NoError(t, err) + defer os.RemoveAll(templateDir) + + // Create a temporary output directory + outputDir, err := os.MkdirTemp("", "output-test") + require.NoError(t, err) + defer os.RemoveAll(outputDir) + + // Create a simple template file + templateFile := filepath.Join(templateDir, "test.txt") + err = os.WriteFile(templateFile, []byte("This is a test!"), 0644) + require.NoError(t, err) + + // Create an empty boilerplate.yml file + boilerplateYml := filepath.Join(templateDir, "boilerplate.yml") + err = os.WriteFile(boilerplateYml, []byte{}, 0644) + require.NoError(t, err) + + // Set up options + opts := &options.BoilerplateOptions{ + TemplateFolder: templateDir, + OutputFolder: outputDir, + NonInteractive: true, + OnMissingKey: options.ExitWithError, + OnMissingConfig: options.Exit, + } + + // Call ProcessTemplate function + files, err := ProcessTemplate(opts, opts, variables.Dependency{}) + require.NoError(t, err) + require.Len(t, files, 1) + assert.Equal(t, "test.txt", files[0]) + + // Verify the file content + content, err := os.ReadFile(filepath.Join(outputDir, "test.txt")) + require.NoError(t, err) + assert.Equal(t, "This is a test!", string(content)) +} + func TestCloneVariablesForDependency(t *testing.T) { t.Parallel()