From 62a08182049364dabda440310f73018c45e3e7ab Mon Sep 17 00:00:00 2001 From: Austin Moses Date: Thu, 9 Apr 2026 12:05:36 -0600 Subject: [PATCH] Update backend formatting and add --format flag --- pkg/cmd/run.go | 124 ++++++++++++++++++++++++-------- pkg/controllers/secrets.go | 24 ++++--- pkg/controllers/secrets_test.go | 42 +++++++++++ tests/e2e/run-mount.sh | 30 ++++++-- 4 files changed, 176 insertions(+), 44 deletions(-) mode change 100755 => 100644 tests/e2e/run-mount.sh diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index ac91b2a0..60c3b6d7 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -123,32 +123,17 @@ doppler run --mount secrets.json -- cat secrets.json`, utils.HandleError(errors.New("invalid passphrase")) } - if !enableFallback { - flags := []string{"fallback", "fallback-only", "offline", "fallback-readonly", "no-exit-on-write-failure", "passphrase"} - for _, flag := range flags { - if cmd.Flags().Changed(flag) { - utils.LogWarning(fmt.Sprintf("--%s has no effect when the fallback file is disabled", flag)) - } - } - } - - fallbackOpts := controllers.FallbackOptions{ - Enable: enableFallback, - Path: fallbackPath, - LegacyPath: legacyFallbackPath, - Readonly: fallbackReadonly, - Exclusive: fallbackOnly, - ExclusiveFlag: fallbackFlag, - ExitOnWriteFailure: exitOnWriteFailure, - Passphrase: passphrase, - } - mountPath := cmd.Flag("mount").Value.String() - mountFormatString := cmd.Flag("mount-format").Value.String() + // --format is the primary flag, --mount-format is a deprecated alias + mountFormatString := cmd.Flag("format").Value.String() + if cmd.Flags().Changed("mount-format") && !cmd.Flags().Changed("format") { + // Use --mount-format value if specified and --format was not + mountFormatString = cmd.Flag("mount-format").Value.String() + } mountTemplate := cmd.Flag("mount-template").Value.String() maxReads := utils.GetIntFlag(cmd, "mount-max-reads", 32) // only auto-detect the format if it hasn't been explicitly specified - shouldAutoDetectFormat := !cmd.Flags().Changed("mount-format") + shouldAutoDetectFormat := !cmd.Flags().Changed("format") && !cmd.Flags().Changed("mount-format") shouldMountFile := mountPath != "" shouldMountTemplate := mountTemplate != "" @@ -156,7 +141,7 @@ doppler run --mount secrets.json -- cat secrets.json`, if mountFormatVal, ok := models.SecretsMountFormatMap[mountFormatString]; ok { mountFormat = mountFormatVal } else { - utils.HandleError(fmt.Errorf("Invalid mount format. Valid formats are %s", models.SecretsMountFormats)) + utils.HandleError(fmt.Errorf("Invalid format. Valid formats are %s", models.SecretsMountFormats)) } if preserveEnv != "false" { @@ -197,14 +182,66 @@ doppler run --mount secrets.json -- cat secrets.json`, if shouldMountTemplate { if mountFormat != models.TemplateMountFormat { - utils.HandleError(errors.New("--mount-template can only be used with --mount-format=template")) + utils.HandleError(errors.New("--mount-template can only be used with --format=template")) } templateBody = controllers.ReadTemplateFile(mountTemplate) } else if mountFormat == models.TemplateMountFormat { - utils.HandleError(errors.New("--mount-template must be specified when using --mount-format=template")) + utils.HandleError(errors.New("--mount-template must be specified when using --format=template")) + } + } + + // Determine the API format for mounting + // Template format requires JSON for client-side rendering + // JSON format uses the standard FetchSecrets path with caching + // Other formats (env, docker, etc.) use backend formatting via http.DownloadSecrets + var mountAPIFormat models.SecretsFormat + if shouldMountFile && mountFormat != models.TemplateMountFormat { + // Find the matching SecretsFormat enum (same pattern as secrets download) + for _, val := range models.SecretsFormatList { + if val.String() == mountFormat { + mountAPIFormat = val + break + } + } + } + + // For non-JSON mount formats, disable caching/fallback + if shouldMountFile && mountAPIFormat != models.JSON && mountFormat != models.TemplateMountFormat { + enableFallback = false + enableCache = false + fallbackPath = "" + legacyFallbackPath = "" + metadataPath = "" + + flags := []string{"fallback", "fallback-only", "offline", "fallback-readonly", "no-exit-on-write-failure", "passphrase"} + for _, flag := range flags { + if cmd.Flags().Changed(flag) { + utils.LogWarning(fmt.Sprintf("--%s has no effect when mount format is %s", flag, mountFormat)) + } } } + // Warn about fallback flags that have no effect when fallback is disabled (non-mount case) + if !enableFallback && !shouldMountFile { + flags := []string{"fallback", "fallback-only", "offline", "fallback-readonly", "no-exit-on-write-failure", "passphrase"} + for _, flag := range flags { + if cmd.Flags().Changed(flag) { + utils.LogWarning(fmt.Sprintf("--%s has no effect when the fallback file is disabled", flag)) + } + } + } + + fallbackOpts := controllers.FallbackOptions{ + Enable: enableFallback, + Path: fallbackPath, + LegacyPath: legacyFallbackPath, + Readonly: fallbackReadonly, + Exclusive: fallbackOnly, + ExclusiveFlag: fallbackFlag, + ExitOnWriteFailure: exitOnWriteFailure, + Passphrase: passphrase, + } + mountOptions := controllers.MountOptions{ Enable: shouldMountFile, Format: mountFormat, @@ -252,8 +289,25 @@ doppler run --mount secrets.json -- cat secrets.json`, } startProcess := func() { - // ensure we can fetch the new secrets before restarting the process - secrets, fromCache := controllers.FetchSecrets(localConfig, enableCache, fallbackOpts, metadataPath, nameTransformer, dynamicSecretsTTL, format, secretsToInclude) + var secrets map[string]string + var fromCache bool + + // For non-JSON, non-template mount formats, use backend formatting + useBackendFormatting := shouldMountFile && mountAPIFormat != models.JSON && mountFormat != models.TemplateMountFormat + if useBackendFormatting { + var apiError http.Error + _, _, formattedBytes, apiError := http.DownloadSecrets(localConfig.APIHost.Value, utils.GetBool(localConfig.VerifyTLS.Value, true), localConfig.Token.Value, localConfig.EnclaveProject.Value, localConfig.EnclaveConfig.Value, mountAPIFormat, nameTransformer, "", dynamicSecretsTTL, secretsToInclude) + if !apiError.IsNil() { + utils.HandleError(apiError.Unwrap(), apiError.Message) + } + mountOptions.FormattedBytes = formattedBytes + secrets = map[string]string{} + fromCache = false + } else { + // For JSON and template formats, use the standard FetchSecrets path with caching + secrets, fromCache = controllers.FetchSecrets(localConfig, enableCache, fallbackOpts, metadataPath, nameTransformer, dynamicSecretsTTL, format, secretsToInclude) + } + secretsFetchedAt := time.Now() if secretsFetchedAt.After(lastSecretsFetch) { lastSecretsFetch = secretsFetchedAt @@ -262,7 +316,10 @@ doppler run --mount secrets.json -- cat secrets.json`, watchedValuesMayBeStale = false } - controllers.ValidateSecrets(secrets, secretsToInclude, exitOnMissingIncludedSecrets, mountOptions) + // Validation requires parsed secrets; skip for backend-formatted mount (raw bytes) + if !useBackendFormatting { + controllers.ValidateSecrets(secrets, secretsToInclude, exitOnMissingIncludedSecrets, mountOptions) + } isRestart := c != nil // terminate the old process @@ -656,13 +713,18 @@ func init() { runCmd.Flags().Bool("no-liveness-ping", false, "disable the periodic liveness ping") // secrets mount flags runCmd.Flags().String("mount", "", "write secrets to an ephemeral file, accessible at DOPPLER_CLI_SECRETS_PATH. when enabled, secrets are NOT injected into the environment") - runCmd.Flags().String("mount-format", "json", fmt.Sprintf("file format to use. if not specified, will be auto-detected from mount name. one of %v", models.SecretsMountFormats)) - err = runCmd.RegisterFlagCompletionFunc("mount-format", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return []string{projectTemplateFileName}, cobra.ShellCompDirectiveDefault + runCmd.Flags().String("format", "json", fmt.Sprintf("file format to use. if not specified, will be auto-detected from mount name. one of %v", models.SecretsMountFormats)) + err = runCmd.RegisterFlagCompletionFunc("format", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return models.SecretsMountFormats, cobra.ShellCompDirectiveDefault }) if err != nil { utils.HandleError(err) } + // --mount-format is a deprecated alias for --format + runCmd.Flags().String("mount-format", "json", fmt.Sprintf("file format to use. one of %v", models.SecretsMountFormats)) + if err := runCmd.Flags().MarkDeprecated("mount-format", "use --format instead"); err != nil { + utils.HandleError(err) + } runCmd.Flags().String("mount-template", "", "template file to use. secrets will be rendered into this template before mount. see 'doppler secrets substitute' for more info.") runCmd.Flags().Int("mount-max-reads", 0, "maximum number of times the mounted secrets file can be read (0 for unlimited)") runCmd.Flags().StringSliceVar(&secretsToInclude, "only-secrets", []string{}, "only include the specified secrets") diff --git a/pkg/controllers/secrets.go b/pkg/controllers/secrets.go index bd23ac85..1af410d6 100644 --- a/pkg/controllers/secrets.go +++ b/pkg/controllers/secrets.go @@ -77,11 +77,12 @@ type FallbackOptions struct { } type MountOptions struct { - Enable bool - Format string - Path string - Template string - MaxReads int + Enable bool + Format string + Path string + Template string + MaxReads int + FormattedBytes []byte // Pre-formatted bytes from the backend (for non-JSON formats) } func GetSecrets(config models.ScopedOptions) (map[string]models.ComputedSecret, Error) { @@ -378,9 +379,16 @@ func PrepareSecrets(dopplerSecrets map[string]string, originalEnv []string, pres secrets = dopplerSecrets env = originalEnv - secretsBytes, err := SecretsToBytes(secrets, mountOptions.Format, mountOptions.Template) - if !err.IsNil() { - utils.HandleError(err.Unwrap(), err.Message) + var secretsBytes []byte + // Use pre-formatted bytes from the backend if available (for non-JSON formats) + if mountOptions.FormattedBytes != nil { + secretsBytes = mountOptions.FormattedBytes + } else { + var err Error + secretsBytes, err = SecretsToBytes(secrets, mountOptions.Format, mountOptions.Template) + if !err.IsNil() { + utils.HandleError(err.Unwrap(), err.Message) + } } absMountPath, handler, err := MountSecrets(secretsBytes, mountOptions.Path, mountOptions.MaxReads) if !err.IsNil() { diff --git a/pkg/controllers/secrets_test.go b/pkg/controllers/secrets_test.go index ce03712a..53dfc2f0 100644 --- a/pkg/controllers/secrets_test.go +++ b/pkg/controllers/secrets_test.go @@ -166,3 +166,45 @@ func TestMountSecretsBrokenPipe(t *testing.T) { assert.NoError(t, readErr, "mount should survive broken pipe") assert.Equal(t, secrets, content) } + +func TestPrepareSecretsUsesFormattedBytes(t *testing.T) { + if !utils.SupportsNamedPipes { + t.Skip("Named pipes not supported on this platform") + } + + // Pre-formatted bytes from the backend (simulating what http.DownloadSecrets returns) + // This tests that PrepareSecrets uses FormattedBytes directly instead of calling SecretsToBytes + formattedBytes := []byte("SECRET=value\\nwith_escaped_newline") + + mountPath := filepath.Join(t.TempDir(), "formatted_secrets") + mountOptions := MountOptions{ + Enable: true, + Format: "docker", + Path: mountPath, + FormattedBytes: formattedBytes, + MaxReads: 1, + } + + // Pass empty secrets map since FormattedBytes should be used instead + env, cleanup := PrepareSecrets(map[string]string{}, []string{}, "false", mountOptions) + if cleanup != nil { + defer cleanup() + } + + // Verify DOPPLER_CLI_SECRETS_PATH is set + var secretsPath string + for _, e := range env { + if strings.HasPrefix(e, "DOPPLER_CLI_SECRETS_PATH=") { + secretsPath = strings.TrimPrefix(e, "DOPPLER_CLI_SECRETS_PATH=") + break + } + } + assert.NotEmpty(t, secretsPath, "DOPPLER_CLI_SECRETS_PATH should be set") + + time.Sleep(50 * time.Millisecond) + + // Read the mounted file and verify it contains the pre-formatted bytes + content, readErr := os.ReadFile(secretsPath) + assert.NoError(t, readErr) + assert.Equal(t, formattedBytes, content, "mounted file should contain pre-formatted bytes from backend") +} diff --git a/tests/e2e/run-mount.sh b/tests/e2e/run-mount.sh old mode 100755 new mode 100644 index 2c10e290..5283af94 --- a/tests/e2e/run-mount.sh +++ b/tests/e2e/run-mount.sh @@ -75,7 +75,7 @@ fi beforeEach -# verify specified format is used +# verify specified format is used (--mount-format, deprecated) EXPECTED_SECRETS='{"DOPPLER_CONFIG":"e2e","DOPPLER_ENVIRONMENT":"e2e","DOPPLER_PROJECT":"cli","FOO":"bar","HOME":"123","TEST":"abc"}' actual="$("$DOPPLER_BINARY" run --mount secrets.env --mount-format json --command "cat \$DOPPLER_CLI_SECRETS_PATH")" if [[ "$actual" != "$(echo -e "$EXPECTED_SECRETS")" ]]; then @@ -85,6 +85,16 @@ fi beforeEach +# verify specified format is used (--format, new flag) +EXPECTED_SECRETS='{"DOPPLER_CONFIG":"e2e","DOPPLER_ENVIRONMENT":"e2e","DOPPLER_PROJECT":"cli","FOO":"bar","HOME":"123","TEST":"abc"}' +actual="$("$DOPPLER_BINARY" run --mount secrets.env --format json --command "cat \$DOPPLER_CLI_SECRETS_PATH")" +if [[ "$actual" != "$(echo -e "$EXPECTED_SECRETS")" ]]; then + echo "ERROR: mounted secrets file with --format json has invalid contents" + exit 1 +fi + +beforeEach + # verify specified name transformer is used EXPECTED_SECRETS='{"TF_VAR_doppler_config":"e2e","TF_VAR_doppler_environment":"e2e","TF_VAR_doppler_project":"cli","TF_VAR_foo":"bar","TF_VAR_home":"123","TF_VAR_test":"abc"}' actual="$("$DOPPLER_BINARY" run --mount secrets.json --name-transformer tf-var --command "cat \$DOPPLER_CLI_SECRETS_PATH")" @@ -105,7 +115,7 @@ fi beforeEach -# verify --mount-template can be used with --mount-format=template +# verify --mount-template can be used with --mount-format=template (deprecated flag) EXPECTED_SECRETS='e2e' actual="$("$DOPPLER_BINARY" run --mount secrets.json --mount-template /dev/stdin --mount-format template --command "cat \$DOPPLER_CLI_SECRETS_PATH" <<<'{{.DOPPLER_CONFIG}}')" if [[ "$actual" != "$EXPECTED_SECRETS" ]]; then @@ -115,9 +125,19 @@ fi beforeEach -# verify --mount-template cannot be used with --mount-format=json -"$DOPPLER_BINARY" run --mount secrets.json --mount-template /dev/stdin --mount-format json --command "cat \$DOPPLER_CLI_SECRETS_PATH" <<<'{{.DOPPLER_CONFIG}}' && \ - (echo "ERROR: mounted secrets with template was successful with invalid --mount-format" && exit 1) +# verify --mount-template can be used with --format=template (new flag) +EXPECTED_SECRETS='e2e' +actual="$("$DOPPLER_BINARY" run --mount secrets.json --mount-template /dev/stdin --format template --command "cat \$DOPPLER_CLI_SECRETS_PATH" <<<'{{.DOPPLER_CONFIG}}')" +if [[ "$actual" != "$EXPECTED_SECRETS" ]]; then + echo "ERROR: mounted secrets file with --format template has invalid contents" + exit 1 +fi + +beforeEach + +# verify --mount-template cannot be used with --format=json +"$DOPPLER_BINARY" run --mount secrets.json --mount-template /dev/stdin --format json --command "cat \$DOPPLER_CLI_SECRETS_PATH" <<<'{{.DOPPLER_CONFIG}}' && \ + (echo "ERROR: mounted secrets with template was successful with invalid --format" && exit 1) beforeEach