Skip to content

Update backend formatting and add --format flag #530

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
amoses12 wants to merge 1 commit into
base: master
Choose a base branch
from
+176 −44
Open

Update backend formatting and add --format flag #530

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 93 additions & 31 deletions pkg/cmd/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,40 +123,25 @@ doppler run --mount secrets.json -- cat secrets.json`,
utils.HandleError(errors.New("invalid passphrase"))
}

if !enableFallback {
flags := []string{"fallback", "fallback-only", "offline", "fallback-readonly", "no-exit-on-write-failure", "passphrase"}
for _, flag := range flags {
if cmd.Flags().Changed(flag) {
utils.LogWarning(fmt.Sprintf("--%s has no effect when the fallback file is disabled", flag))
}
}
}

fallbackOpts := controllers.FallbackOptions{
Enable: enableFallback,
Path: fallbackPath,
LegacyPath: legacyFallbackPath,
Readonly: fallbackReadonly,
Exclusive: fallbackOnly,
ExclusiveFlag: fallbackFlag,
ExitOnWriteFailure: exitOnWriteFailure,
Passphrase: passphrase,
}

mountPath := cmd.Flag("mount").Value.String()
mountFormatString := cmd.Flag("mount-format").Value.String()
// --format is the primary flag, --mount-format is a deprecated alias
mountFormatString := cmd.Flag("format").Value.String()
if cmd.Flags().Changed("mount-format") && !cmd.Flags().Changed("format") {
// Use --mount-format value if specified and --format was not
mountFormatString = cmd.Flag("mount-format").Value.String()
}
mountTemplate := cmd.Flag("mount-template").Value.String()
maxReads := utils.GetIntFlag(cmd, "mount-max-reads", 32)
// only auto-detect the format if it hasn't been explicitly specified
shouldAutoDetectFormat := !cmd.Flags().Changed("mount-format")
shouldAutoDetectFormat := !cmd.Flags().Changed("format") && !cmd.Flags().Changed("mount-format")
shouldMountFile := mountPath != ""
shouldMountTemplate := mountTemplate != ""

var mountFormat string
if mountFormatVal, ok := models.SecretsMountFormatMap[mountFormatString]; ok {
mountFormat = mountFormatVal
} else {
utils.HandleError(fmt.Errorf("Invalid mount format. Valid formats are %s", models.SecretsMountFormats))
utils.HandleError(fmt.Errorf("Invalid format. Valid formats are %s", models.SecretsMountFormats))
}

if preserveEnv != "false" {
Expand Down Expand Up @@ -197,14 +182,66 @@ doppler run --mount secrets.json -- cat secrets.json`,

if shouldMountTemplate {
if mountFormat != models.TemplateMountFormat {
utils.HandleError(errors.New("--mount-template can only be used with --mount-format=template"))
utils.HandleError(errors.New("--mount-template can only be used with --format=template"))
}
templateBody = controllers.ReadTemplateFile(mountTemplate)
} else if mountFormat == models.TemplateMountFormat {
utils.HandleError(errors.New("--mount-template must be specified when using --mount-format=template"))
utils.HandleError(errors.New("--mount-template must be specified when using --format=template"))
}
}

// Determine the API format for mounting
// Template format requires JSON for client-side rendering
// JSON format uses the standard FetchSecrets path with caching
// Other formats (env, docker, etc.) use backend formatting via http.DownloadSecrets
var mountAPIFormat models.SecretsFormat
if shouldMountFile && mountFormat != models.TemplateMountFormat {
// Find the matching SecretsFormat enum (same pattern as secrets download)
for _, val := range models.SecretsFormatList {
if val.String() == mountFormat {
mountAPIFormat = val
break
}
}
}

// For non-JSON mount formats, disable caching/fallback
if shouldMountFile && mountAPIFormat != models.JSON && mountFormat != models.TemplateMountFormat {
enableFallback = false
enableCache = false
fallbackPath = ""
legacyFallbackPath = ""
metadataPath = ""

flags := []string{"fallback", "fallback-only", "offline", "fallback-readonly", "no-exit-on-write-failure", "passphrase"}
for _, flag := range flags {
if cmd.Flags().Changed(flag) {
utils.LogWarning(fmt.Sprintf("--%s has no effect when mount format is %s", flag, mountFormat))
}
}
}

// Warn about fallback flags that have no effect when fallback is disabled (non-mount case)
if !enableFallback && !shouldMountFile {
flags := []string{"fallback", "fallback-only", "offline", "fallback-readonly", "no-exit-on-write-failure", "passphrase"}
for _, flag := range flags {
if cmd.Flags().Changed(flag) {
utils.LogWarning(fmt.Sprintf("--%s has no effect when the fallback file is disabled", flag))
}
}
}

fallbackOpts := controllers.FallbackOptions{
Enable: enableFallback,
Path: fallbackPath,
LegacyPath: legacyFallbackPath,
Readonly: fallbackReadonly,
Exclusive: fallbackOnly,
ExclusiveFlag: fallbackFlag,
ExitOnWriteFailure: exitOnWriteFailure,
Passphrase: passphrase,
}

mountOptions := controllers.MountOptions{
Enable: shouldMountFile,
Format: mountFormat,
Expand Down Expand Up @@ -252,8 +289,25 @@ doppler run --mount secrets.json -- cat secrets.json`,
}

startProcess := func() {
// ensure we can fetch the new secrets before restarting the process
secrets, fromCache := controllers.FetchSecrets(localConfig, enableCache, fallbackOpts, metadataPath, nameTransformer, dynamicSecretsTTL, format, secretsToInclude)
var secrets map[string]string
var fromCache bool

// For non-JSON, non-template mount formats, use backend formatting
useBackendFormatting := shouldMountFile && mountAPIFormat != models.JSON && mountFormat != models.TemplateMountFormat
if useBackendFormatting {
var apiError http.Error
_, _, formattedBytes, apiError := http.DownloadSecrets(localConfig.APIHost.Value, utils.GetBool(localConfig.VerifyTLS.Value, true), localConfig.Token.Value, localConfig.EnclaveProject.Value, localConfig.EnclaveConfig.Value, mountAPIFormat, nameTransformer, "", dynamicSecretsTTL, secretsToInclude)
if !apiError.IsNil() {
utils.HandleError(apiError.Unwrap(), apiError.Message)
}
mountOptions.FormattedBytes = formattedBytes
secrets = map[string]string{}
fromCache = false
} else {
// For JSON and template formats, use the standard FetchSecrets path with caching
secrets, fromCache = controllers.FetchSecrets(localConfig, enableCache, fallbackOpts, metadataPath, nameTransformer, dynamicSecretsTTL, format, secretsToInclude)
}
Comment on lines +296 to +309
Copy link
Copy Markdown
Member

@emily-curry emily-curry Apr 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

blocking: It is pretty unfortunate that we're introducing a regression in functionality here, where now mounting to an env file (or other formats) can no longer use the fallback file or cache. I was hoping that we'd unwind some of the internals of FetchSecrets, it really should just return a raw byte array, since it accepts format as an argument. All FetchSecrets does with it internally is some validations before writing the fallback file, I'm not convinced those are necessary.

If we're not going to do the work to keep the same level of functionality, we still should not be passing around an empty map and putting the raw bytes somewhere else, that's super hacky. At the very least, secrets here should be a byte array, ValidateSecrets can be moved closer to FetchSecrets, and PrepareSecrets should accept a byte array.

Finally, we should get rid of all of the dead code. SecretsToBytes can be removed entirely. secrets_mount.go can also be removed entirely (though the template option will need to be accommodated elsewhere. we should not introduce a breaking change, but this is not a "real" format, it's json piped through a user-defined local template file).


secretsFetchedAt := time.Now()
if secretsFetchedAt.After(lastSecretsFetch) {
lastSecretsFetch = secretsFetchedAt
Expand All @@ -262,7 +316,10 @@ doppler run --mount secrets.json -- cat secrets.json`,
watchedValuesMayBeStale = false
}

controllers.ValidateSecrets(secrets, secretsToInclude, exitOnMissingIncludedSecrets, mountOptions)
// Validation requires parsed secrets; skip for backend-formatted mount (raw bytes)
if !useBackendFormatting {
controllers.ValidateSecrets(secrets, secretsToInclude, exitOnMissingIncludedSecrets, mountOptions)
}

isRestart := c != nil
// terminate the old process
Expand Down Expand Up @@ -656,13 +713,18 @@ func init() {
runCmd.Flags().Bool("no-liveness-ping", false, "disable the periodic liveness ping")
// secrets mount flags
runCmd.Flags().String("mount", "", "write secrets to an ephemeral file, accessible at DOPPLER_CLI_SECRETS_PATH. when enabled, secrets are NOT injected into the environment")
runCmd.Flags().String("mount-format", "json", fmt.Sprintf("file format to use. if not specified, will be auto-detected from mount name. one of %v", models.SecretsMountFormats))
err = runCmd.RegisterFlagCompletionFunc("mount-format", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return []string{projectTemplateFileName}, cobra.ShellCompDirectiveDefault
runCmd.Flags().String("format", "json", fmt.Sprintf("file format to use. if not specified, will be auto-detected from mount name. one of %v", models.SecretsMountFormats))
err = runCmd.RegisterFlagCompletionFunc("format", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return models.SecretsMountFormats, cobra.ShellCompDirectiveDefault
})
if err != nil {
utils.HandleError(err)
}
// --mount-format is a deprecated alias for --format
runCmd.Flags().String("mount-format", "json", fmt.Sprintf("file format to use. one of %v", models.SecretsMountFormats))
if err := runCmd.Flags().MarkDeprecated("mount-format", "use --format instead"); err != nil {
utils.HandleError(err)
}
runCmd.Flags().String("mount-template", "", "template file to use. secrets will be rendered into this template before mount. see 'doppler secrets substitute' for more info.")
runCmd.Flags().Int("mount-max-reads", 0, "maximum number of times the mounted secrets file can be read (0 for unlimited)")
runCmd.Flags().StringSliceVar(&secretsToInclude, "only-secrets", []string{}, "only include the specified secrets")
Expand Down
24 changes: 16 additions & 8 deletions pkg/controllers/secrets.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,11 +77,12 @@ type FallbackOptions struct {
}

type MountOptions struct {
Enable bool
Format string
Path string
Template string
MaxReads int
Enable bool
Format string
Path string
Template string
MaxReads int
FormattedBytes []byte // Pre-formatted bytes from the backend (for non-JSON formats)
}

func GetSecrets(config models.ScopedOptions) (map[string]models.ComputedSecret, Error) {
Expand Down Expand Up @@ -378,9 +379,16 @@ func PrepareSecrets(dopplerSecrets map[string]string, originalEnv []string, pres
secrets = dopplerSecrets
env = originalEnv

secretsBytes, err := SecretsToBytes(secrets, mountOptions.Format, mountOptions.Template)
if !err.IsNil() {
utils.HandleError(err.Unwrap(), err.Message)
var secretsBytes []byte
// Use pre-formatted bytes from the backend if available (for non-JSON formats)
if mountOptions.FormattedBytes != nil {
secretsBytes = mountOptions.FormattedBytes
} else {
var err Error
secretsBytes, err = SecretsToBytes(secrets, mountOptions.Format, mountOptions.Template)
if !err.IsNil() {
utils.HandleError(err.Unwrap(), err.Message)
}
}
absMountPath, handler, err := MountSecrets(secretsBytes, mountOptions.Path, mountOptions.MaxReads)
if !err.IsNil() {
Expand Down
42 changes: 42 additions & 0 deletions pkg/controllers/secrets_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,3 +166,45 @@ func TestMountSecretsBrokenPipe(t *testing.T) {
assert.NoError(t, readErr, "mount should survive broken pipe")
assert.Equal(t, secrets, content)
}

func TestPrepareSecretsUsesFormattedBytes(t *testing.T) {
if !utils.SupportsNamedPipes {
t.Skip("Named pipes not supported on this platform")
}

// Pre-formatted bytes from the backend (simulating what http.DownloadSecrets returns)
// This tests that PrepareSecrets uses FormattedBytes directly instead of calling SecretsToBytes
formattedBytes := []byte("SECRET=value\\nwith_escaped_newline")

mountPath := filepath.Join(t.TempDir(), "formatted_secrets")
mountOptions := MountOptions{
Enable: true,
Format: "docker",
Path: mountPath,
FormattedBytes: formattedBytes,
MaxReads: 1,
}

// Pass empty secrets map since FormattedBytes should be used instead
env, cleanup := PrepareSecrets(map[string]string{}, []string{}, "false", mountOptions)
if cleanup != nil {
defer cleanup()
}

// Verify DOPPLER_CLI_SECRETS_PATH is set
var secretsPath string
for _, e := range env {
if strings.HasPrefix(e, "DOPPLER_CLI_SECRETS_PATH=") {
secretsPath = strings.TrimPrefix(e, "DOPPLER_CLI_SECRETS_PATH=")
break
}
}
assert.NotEmpty(t, secretsPath, "DOPPLER_CLI_SECRETS_PATH should be set")

time.Sleep(50 * time.Millisecond)

// Read the mounted file and verify it contains the pre-formatted bytes
content, readErr := os.ReadFile(secretsPath)
assert.NoError(t, readErr)
assert.Equal(t, formattedBytes, content, "mounted file should contain pre-formatted bytes from backend")
}
30 changes: 25 additions & 5 deletions tests/e2e/run-mount.sh
100755 → 100644
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ fi

beforeEach

# verify specified format is used
# verify specified format is used (--mount-format, deprecated)
EXPECTED_SECRETS='{"DOPPLER_CONFIG":"e2e","DOPPLER_ENVIRONMENT":"e2e","DOPPLER_PROJECT":"cli","FOO":"bar","HOME":"123","TEST":"abc"}'
actual="$("$DOPPLER_BINARY" run --mount secrets.env --mount-format json --command "cat \$DOPPLER_CLI_SECRETS_PATH")"
if [[ "$actual" != "$(echo -e "$EXPECTED_SECRETS")" ]]; then
Expand All @@ -85,6 +85,16 @@ fi

beforeEach

# verify specified format is used (--format, new flag)
EXPECTED_SECRETS='{"DOPPLER_CONFIG":"e2e","DOPPLER_ENVIRONMENT":"e2e","DOPPLER_PROJECT":"cli","FOO":"bar","HOME":"123","TEST":"abc"}'
actual="$("$DOPPLER_BINARY" run --mount secrets.env --format json --command "cat \$DOPPLER_CLI_SECRETS_PATH")"
if [[ "$actual" != "$(echo -e "$EXPECTED_SECRETS")" ]]; then
echo "ERROR: mounted secrets file with --format json has invalid contents"
exit 1
fi

beforeEach

# verify specified name transformer is used
EXPECTED_SECRETS='{"TF_VAR_doppler_config":"e2e","TF_VAR_doppler_environment":"e2e","TF_VAR_doppler_project":"cli","TF_VAR_foo":"bar","TF_VAR_home":"123","TF_VAR_test":"abc"}'
actual="$("$DOPPLER_BINARY" run --mount secrets.json --name-transformer tf-var --command "cat \$DOPPLER_CLI_SECRETS_PATH")"
Expand All @@ -105,7 +115,7 @@ fi

beforeEach

# verify --mount-template can be used with --mount-format=template
# verify --mount-template can be used with --mount-format=template (deprecated flag)
EXPECTED_SECRETS='e2e'
actual="$("$DOPPLER_BINARY" run --mount secrets.json --mount-template /dev/stdin --mount-format template --command "cat \$DOPPLER_CLI_SECRETS_PATH" <<<'{{.DOPPLER_CONFIG}}')"
if [[ "$actual" != "$EXPECTED_SECRETS" ]]; then
Expand All @@ -115,9 +125,19 @@ fi

beforeEach

# verify --mount-template cannot be used with --mount-format=json
"$DOPPLER_BINARY" run --mount secrets.json --mount-template /dev/stdin --mount-format json --command "cat \$DOPPLER_CLI_SECRETS_PATH" <<<'{{.DOPPLER_CONFIG}}' && \
(echo "ERROR: mounted secrets with template was successful with invalid --mount-format" && exit 1)
# verify --mount-template can be used with --format=template (new flag)
EXPECTED_SECRETS='e2e'
actual="$("$DOPPLER_BINARY" run --mount secrets.json --mount-template /dev/stdin --format template --command "cat \$DOPPLER_CLI_SECRETS_PATH" <<<'{{.DOPPLER_CONFIG}}')"
if [[ "$actual" != "$EXPECTED_SECRETS" ]]; then
echo "ERROR: mounted secrets file with --format template has invalid contents"
exit 1
fi

beforeEach

# verify --mount-template cannot be used with --format=json
"$DOPPLER_BINARY" run --mount secrets.json --mount-template /dev/stdin --format json --command "cat \$DOPPLER_CLI_SECRETS_PATH" <<<'{{.DOPPLER_CONFIG}}' && \
(echo "ERROR: mounted secrets with template was successful with invalid --format" && exit 1)

beforeEach

Expand Down
Loading