Redacting secret values in the configuration summary

src/pkg/cli/compose/config_detector.go

@@ -6,20 +6,20 @@ import (
 	"github.com/DefangLabs/secret-detector/pkg/scanner"
 )
 
-func IsSecret(input string) (bool, error) {
+func IsSecret(key, value string) (bool, []string, error) {
 	// DetectConfig checks if the input string contains any sensitive information
-	detectorTypes, err := detectConfig(input)
+	detectorTypes, err := detectConfig(key + ": " + value)
 	if err != nil {
-		return false, fmt.Errorf("failed to detect config: %w", err)
+		return false, nil, fmt.Errorf("failed to detect config: %w", err)
 	}
 
 	// If no detectors were triggered, return false
 	if len(detectorTypes) == 0 {
-		return false, nil
+		return false, nil, nil
 	}
 
 	// If any detectors were triggered, return true
-	return true, nil
+	return true, detectorTypes, nil
 }
 
 // assume that the input is a key-value pair string
@@ -33,9 +33,9 @@ func detectConfig(input string) (detectorTypes []string, err error) {
 
 	// create a custom scanner config
 	cfg := scanner.NewConfigWithDefaults()
-	cfg.Transformers = []string{"json"}
+	cfg.Transformers = []string{"yaml"}
 	cfg.DetectorConfigs["keyword"] = []string{"3"}
-	cfg.DetectorConfigs["high_entropy_string"] = []string{"4"}
+	cfg.DetectorConfigs["high_entropy_string"] = []string{"3.7"}
 
 	// create a scanner from scanner config
 	scannerClient, err := scanner.NewScannerFromConfig(cfg)

src/pkg/cli/compose/config_detector_test.go

@@ -18,6 +18,7 @@ func TestDetectConfig(t *testing.T) {
 		{"AROA1234567890ABCDEF", []string{"AWS Client ID"}},
 		{"REDIS_URL=rediss://foo:p41fce90d44ac1d891bd21fdbc5dfc1bd7f163e33a6934c30093eaf56c1c23937@ec2-98-85-106-43.compute-1.amazonaws.com:8240", []string{"URL with password"}},
 		{"REDIS_URL=rediss://:p41fce90d44ac1d891bd21fdbc5dfc1bd7f163e33a6934c30093eaf56c1c23937@ec2-98-85-106-43.compute-1.amazonaws.com:8240", []string{"URL with password"}},
+		{"ENCRYPTION_KEY: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", []string{"High entropy string"}},
 	}
 
 	for _, tt := range tests {

src/pkg/cli/compose/validation.go

@@ -204,19 +204,16 @@ func validateService(svccfg *composeTypes.ServiceConfig, project *composeTypes.P
 	// check for compose file environment variables that may be sensitive
 	for key, value := range svccfg.Environment {
 		if value != nil {
-			// format input as a key-value pair string
-			input := key + "=" + *value
-
 			// call detectConfig to check for sensitive information
-			ds, err := detectConfig(input)
+			isSecret, ds, err := IsSecret(key, *value)
 			if err != nil {
 				return fmt.Errorf("service %q: %w", svccfg.Name, err)
 			}
 
 			// show warning if sensitive information is detected
-			if len(ds) > 0 {
+			if isSecret {
 				term.Warnf("service %q: environment %q may contain sensitive information; consider using 'defang config set %s' to securely store this value", svccfg.Name, key, key)
-				term.Debugf("service %q: environment %q may contain detected secrets of type: %q", svccfg.Name, key, ds)
+				term.Debugf("service %q: environment %q may contain detected secrets of type: %v", svccfg.Name, key, ds)
 			}
 		}
 	}

src/pkg/cli/composeUp.go

@@ -87,7 +87,7 @@ func ComposeUp(ctx context.Context, fabric client.FabricClient, provider client.
 	if upload != compose.UploadModeIgnore {
 		// Ignore missing configs in preview mode, because we don't want to fail the preview if some configs are missing.
 		if upload != compose.UploadModeEstimate {
-			if err := PrintConfigSummaryAndValidate(ctx, provider, project); err != nil {
+			if err := PrintRedactedConfigSummaryAndValidate(ctx, provider, project); err != nil {
 				return nil, project, err
 			}
 		}

src/pkg/cli/configResolution.go

@@ -19,7 +19,7 @@ type configOutput struct {
 	Source      Source `json:"source,omitempty"`
 }
 
-const configMaskedValue = "*****"
+const configMaskedValue = "******"
 
 type Source string
 
@@ -33,8 +33,16 @@ func (s Source) String() string {
 	return string(s)
 }
 
+func maskTrailingConfigValue(value string) string {
+	// Mask the value if it looks like a secret and only show the first 4 characters
+	if len(value) <= 4 {
+		return configMaskedValue
+	}
+	return value[:4] + strings.Repeat("*", 2)
+}
+
 // determineConfigSource determines the source of an environment variable
-// and returns the appropriate source type and value to display
+// and returns the appropriate source type and value to display.
 func determineConfigSource(envKey string, envValue *string, defangConfigs map[string]struct{}) (Source, string) {
 	// If the key itself is a defang config, mask it
 	if _, isDefangConfig := defangConfigs[envKey]; isDefangConfig {
@@ -57,7 +65,9 @@ func determineConfigSource(envKey string, envValue *string, defangConfigs map[st
 	return SourceComposeFile, *envValue
 }
 
-func printConfigResolutionSummary(project *types.Project, defangConfig []string) error {
+// printConfigResolutionSummary prints a summary of where each environment variable in the compose file is coming from (compose file, defang config, or interpolation).
+// If redact is true, it will mask values that are from the compose file and look like secrets.
+func printConfigResolutionSummary(project *types.Project, defangConfig []string, redact bool) error {
 	configset := make(map[string]struct{})
 	for _, name := range defangConfig {
 		configset[name] = struct{}{}
@@ -68,6 +78,16 @@ func printConfigResolutionSummary(project *types.Project, defangConfig []string)
 	for serviceName, service := range project.Services {
 		for envKey, envValue := range service.Environment {
 			source, value := determineConfigSource(envKey, envValue, configset)
+			if redact && source == SourceComposeFile {
+				isSecret, _, err := compose.IsSecret(envKey, value)
+				if err != nil {
+					return err
+				}
+
+				if isSecret {
+					value = maskTrailingConfigValue(value)
+				}
+			}
 			projectEnvVars = append(projectEnvVars, configOutput{
 				Service:     serviceName,
 				Environment: envKey,
@@ -97,13 +117,13 @@ func printConfigResolutionSummary(project *types.Project, defangConfig []string)
 	return term.Table(projectEnvVars, "Service", "Environment", "Source", "Value")
 }
 
-func PrintConfigSummaryAndValidate(ctx context.Context, provider client.Provider, project *compose.Project) error {
+func printConfigSummaryAndValidate(ctx context.Context, provider client.Provider, project *compose.Project, redact bool) error {
 	configs, err := provider.ListConfig(ctx, &defangv1.ListConfigsRequest{Project: project.Name})
 	if err != nil {
 		return err
 	}
 
-	err = printConfigResolutionSummary(project, configs.Names)
+	err = printConfigResolutionSummary(project, configs.Names, redact)
 	if err != nil {
 		return err
 	}
@@ -115,3 +135,11 @@ func PrintConfigSummaryAndValidate(ctx context.Context, provider client.Provider
 
 	return nil
 }
+
+func PrintConfigSummaryAndValidate(ctx context.Context, provider client.Provider, project *compose.Project) error {
+	return printConfigSummaryAndValidate(ctx, provider, project, false)
+}
+
+func PrintRedactedConfigSummaryAndValidate(ctx context.Context, provider client.Provider, project *compose.Project) error {
+	return printConfigSummaryAndValidate(ctx, provider, project, true)
+}

src/pkg/cli/configResolution_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func TestPrintConfigResolutionSummary(t *testing.T) {
-	testAllConfigResolutionFiles(t, func(t *testing.T, name, path string) {
+	testAllConfigResolutionFiles(t, "testdata/config-resolution", func(t *testing.T, name, path string) {
 		stdout, _ := term.SetupTestTerm(t)
 
 		loader := compose.NewLoader(compose.WithPath(path))
@@ -36,7 +36,7 @@ func TestPrintConfigResolutionSummary(t *testing.T) {
 			defangConfigs = []string{}
 		}
 
-		err = printConfigResolutionSummary(proj, defangConfigs)
+		err = printConfigResolutionSummary(proj, defangConfigs, false)
 		if err != nil {
 			t.Fatalf("PrintConfigResolutionSummary() error = %v", err)
 		}
@@ -50,11 +50,35 @@ func TestPrintConfigResolutionSummary(t *testing.T) {
 	})
 }
 
-func testAllConfigResolutionFiles(t *testing.T, f func(t *testing.T, name, path string)) {
+func TestPrintRedactedConfigResolutionSummary(t *testing.T) {
+	testAllConfigResolutionFiles(t, "testdata/redact-config", func(t *testing.T, name, path string) {
+		stdout, _ := term.SetupTestTerm(t)
+
+		loader := compose.NewLoader(compose.WithPath(path))
+		proj, err := loader.LoadProject(t.Context())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = printConfigResolutionSummary(proj, nil, true)
+		if err != nil {
+			t.Fatalf("PrintConfigResolutionSummary() error = %v", err)
+		}
+
+		output := stdout.Bytes()
+
+		// Compare the output with the golden file
+		if err := pkg.Compare(output, path+".golden"); err != nil {
+			t.Error(err)
+		}
+	})
+}
+
+func testAllConfigResolutionFiles(t *testing.T, dir string, f func(t *testing.T, name, path string)) {
 	t.Helper()
 
 	composeRegex := regexp.MustCompile(`^(?i)(docker-)?compose.ya?ml$`)
-	err := filepath.WalkDir("testdata/configresolution", func(path string, d os.DirEntry, err error) error {
+	err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
 		if err != nil || d.IsDir() || !composeRegex.MatchString(d.Name()) {
 			return err
 		}

src/pkg/cli/testdata/config-resolution/defang-config-only/compose.yaml.golden

@@ -0,0 +1,6 @@
+ * Service environment variables resolution summary:
+
+SERVICE   ENVIRONMENT  SOURCE  VALUE
+service1  API_TOKEN    Config  ******  
+service1  DB_USER      Config  ******  
+service1  SECRET_KEY   Config  ******  

src/pkg/cli/testdata/configresolution/mixed-sources/compose.yaml.golden → src/pkg/cli/testdata/config-resolution/mixed-sources/compose.yaml.golden

@@ -4,4 +4,4 @@ SERVICE   ENVIRONMENT  SOURCE                 VALUE
 service1  FROM_ENV     Compose                from_ENV                                    
 service1  PLAIN_VAR    Compose                plain_value                                 
 service1  PUBLIC_URL   Config (interpolated)  https://example-${SECRET_KEY}-from_ENV.com  
-service1  SECRET_KEY   Config                 *****                                       
+service1  SECRET_KEY   Config                 ******                                      

src/pkg/cli/testdata/configresolution/multiple-services/compose.yaml.golden → src/pkg/cli/testdata/config-resolution/multiple-services/compose.yaml.golden

@@ -2,8 +2,8 @@
 
 SERVICE   ENVIRONMENT     SOURCE   VALUE
 service1  APP_ENV         Compose  production  
-service1  DATABASE_URL    Config   *****       
+service1  DATABASE_URL    Config   ******      
 service1  LOG_LEVEL       Compose  info        
-service2  REDIS_PASSWORD  Config   *****       
+service2  REDIS_PASSWORD  Config   ******      
 service2  REDIS_PORT      Compose  6379        
 service3  DB_USER         Compose  Defang      

src/pkg/cli/testdata/redact-config/api-keys/compose.yaml

@@ -0,0 +1,9 @@
+services:
+  backend:
+    image: node
+    environment:
+      APP_NAME: my-application
+      API_KEY: api-key=50m34p1k3y
+      SECRET_TOKEN: secret_token=ab12cd34ef56
+      AUTH_PASSWORD: password=hunter12345
+      NORMAL_CONFIG: debug

src/pkg/cli/testdata/redact-config/api-keys/compose.yaml.golden

@@ -0,0 +1,8 @@
+ * Service environment variables resolution summary:
+
+SERVICE  ENVIRONMENT    SOURCE   VALUE
+backend  API_KEY        Compose  api-**          
+backend  APP_NAME       Compose  my-application  
+backend  AUTH_PASSWORD  Compose  pass**          
+backend  NORMAL_CONFIG  Compose  debug           
+backend  SECRET_TOKEN   Compose  secr**          

src/pkg/cli/testdata/redact-config/cloud-credentials/compose.yaml

@@ -0,0 +1,9 @@
+services:
+  deployer:
+    image: alpine
+    environment:
+      GITHUB_TOKEN: ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ1234567890
+      AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE
+      AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+      AWS_CLIENT_ID: AROA1234567890ABCDEF
+      REGION: us-east-1

src/pkg/cli/testdata/redact-config/cloud-credentials/compose.yaml.golden

@@ -0,0 +1,8 @@
+ * Service environment variables resolution summary:
+
+SERVICE   ENVIRONMENT            SOURCE   VALUE
+deployer  AWS_ACCESS_KEY_ID      Compose  AKIA**     
+deployer  AWS_CLIENT_ID          Compose  AROA**     
+deployer  AWS_SECRET_ACCESS_KEY  Compose  wJal**     
+deployer  GITHUB_TOKEN           Compose  ghp_**     
+deployer  REGION                 Compose  us-east-1  

src/pkg/cli/testdata/redact-config/database-urls/compose.yaml

@@ -0,0 +1,10 @@
+services:
+  api:
+    image: node
+    environment:
+      DATABASE_URL: postgres://dbuser:p455w0rd@db.example.com:5432/myapp
+      MONGO_URI: mongodb://admin:s3cretP4ss@mongo.example.com:27017/mydb
+      MYSQL_URL: mysql://root:r00tP4ssw0rd@mysql.example.com:3306/app
+      REDIS_URL: rediss://default:p41fce90d44ac1d891bd21fdbc5dfc1bd7f163e33a6934c30093eaf56c1c23937@redis.example.com:6380
+      DB_HOST: db.example.com
+      DB_PORT: "5432"

src/pkg/cli/testdata/redact-config/database-urls/compose.yaml.golden

@@ -0,0 +1,9 @@
+ * Service environment variables resolution summary:
+
+SERVICE  ENVIRONMENT   SOURCE   VALUE
+api      DATABASE_URL  Compose  post**          
+api      DB_HOST       Compose  db.example.com  
+api      DB_PORT       Compose  5432            
+api      MONGO_URI     Compose  mong**          
+api      MYSQL_URL     Compose  mysq**          
+api      REDIS_URL     Compose  redi**          

src/pkg/cli/testdata/redact-config/empty-values/compose.yaml

@@ -0,0 +1,12 @@
+services:
+  service1:
+    image: nginx
+    environment:
+      EMPTY_VAR:
+      WITH_VALUE: some_value
+  service2:
+    image: nginx
+    environment:
+      - EMPTY_STRING=
+      - EMPTY_VAR
+      - WITH_VALUE=some_value

src/pkg/cli/testdata/redact-config/empty-values/compose.yaml.golden

@@ -0,0 +1,8 @@
+ * Service environment variables resolution summary:
+
+SERVICE   ENVIRONMENT   SOURCE   VALUE
+service1  EMPTY_VAR     Config               
+service1  WITH_VALUE    Compose  some_value  
+service2  EMPTY_STRING  Compose              
+service2  EMPTY_VAR     Config               
+service2  WITH_VALUE    Compose  some_value  

src/pkg/cli/testdata/redact-config/high-entropy-strings/compose.yaml

@@ -0,0 +1,9 @@
+services:
+  app:
+    image: python
+    environment:
+      SESSION_SECRET: VEfk5vO0Q53VkK_uicor
+      ENCRYPTION_KEY: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08
+      SIGNING_KEY: xK9mPqR7sT2vW5yZ8aB3cD6eF
+      APP_ENV: production
+      LOG_LEVEL: info

src/pkg/cli/testdata/redact-config/high-entropy-strings/compose.yaml.golden

@@ -0,0 +1,8 @@
+ * Service environment variables resolution summary:
+
+SERVICE  ENVIRONMENT     SOURCE   VALUE
+app      APP_ENV         Compose  production  
+app      ENCRYPTION_KEY  Compose  9f86**      
+app      LOG_LEVEL       Compose  info        
+app      SESSION_SECRET  Compose  VEfk**      
+app      SIGNING_KEY     Compose  xK9m**      

src/pkg/cli/testdata/redact-config/mixed-secrets-and-plain/compose.yaml

@@ -0,0 +1,16 @@
+services:
+  frontend:
+    image: nginx
+    environment:
+      APP_NAME: my-frontend
+      PORT: "3000"
+  backend:
+    image: node
+    environment:
+      APP_NAME: my-backend
+      PORT: "8080"
+      DATABASE_URL: postgres://admin:p455w0rd@db.example.com:5432/myapp
+      GITHUB_TOKEN: ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ1234567890
+      API_KEY: api-key=50m34p1k3y
+      LOG_LEVEL: info
+      NODE_ENV: production

src/pkg/cli/testdata/redact-config/mixed-secrets-and-plain/compose.yaml.golden

@@ -0,0 +1,12 @@
+ * Service environment variables resolution summary:
+
+SERVICE   ENVIRONMENT   SOURCE   VALUE
+backend   API_KEY       Compose  api-**       
+backend   APP_NAME      Compose  my-backend   
+backend   DATABASE_URL  Compose  post**       
+backend   GITHUB_TOKEN  Compose  ghp_**       
+backend   LOG_LEVEL     Compose  info         
+backend   NODE_ENV      Compose  production   
+backend   PORT          Compose  8080         
+frontend  APP_NAME      Compose  my-frontend  
+frontend  PORT          Compose  3000         

src/pkg/cli/testdata/redact-config/not-secrets/compose.yaml

@@ -0,0 +1,11 @@
+services:
+  web:
+    image: nginx
+    environment:
+      APP_ENV: production
+      LOG_LEVEL: info
+      PORT: "8080"
+      API_URL: https://api.example.com
+      DOCS_PATH: /leaderboard/api/hubs
+      NODE_ENV: production
+      GREETING: not a secret