NGINX Configs to Reference Remote External Files

internal/config/config.go

@@ -157,6 +157,7 @@ func ResolveConfig() (*Config, error) {
 		Features:           viperInstance.GetStringSlice(FeaturesKey),
 		Labels:             resolveLabels(),
 		LibDir:             viperInstance.GetString(LibDirPathKey),
+		ExternalDataSource: resolveExternalDataSource(),
 		SyslogServer:       resolveSyslogServer(),
 	}
 
@@ -475,6 +476,7 @@ func registerFlags() {
 	registerCollectorFlags(fs)
 	registerClientFlags(fs)
 	registerDataPlaneFlags(fs)
+	registerExternalDataSourceFlags(fs)
 
 	fs.SetNormalizeFunc(normalizeFunc)
 
@@ -489,6 +491,24 @@ func registerFlags() {
 	})
 }
 
+func registerExternalDataSourceFlags(fs *flag.FlagSet) {
+	fs.String(
+		ExternalDataSourceProxyUrlKey,
+		DefExternalDataSourceProxyUrl,
+		"Url to the proxy service to fetch the external file.",
+	)
+	fs.StringSlice(
+		ExternalDataSourceAllowDomainsKey,
+		[]string{},
+		"List of allowed domains for external data sources.",
+	)
+	fs.Int64(
+		ExternalDataSourceMaxBytesKey,
+		DefExternalDataSourceMaxBytes,
+		"Maximum size in bytes for external data sources.",
+	)
+}
+
 func registerDataPlaneFlags(fs *flag.FlagSet) {
 	fs.Duration(
 		NginxReloadMonitoringPeriodKey,
@@ -628,6 +648,11 @@ func registerClientFlags(fs *flag.FlagSet) {
 		DefMaxFileSize,
 		"Max file size in bytes.",
 	)
+	fs.Duration(
+		ClientFileDownloadTimeoutKey,
+		DefClientFileDownloadTimeout,
+		"Timeout value in seconds, to downloading file for config apply.",
+	)
 
 	fs.Int(
 		ClientGRPCMaxParallelFileOperationsKey,
@@ -1120,6 +1145,7 @@ func resolveClient() *Client {
 			RandomizationFactor: viperInstance.GetFloat64(ClientBackoffRandomizationFactorKey),
 			Multiplier:          viperInstance.GetFloat64(ClientBackoffMultiplierKey),
 		},
+		FileDownloadTimeout: viperInstance.GetDuration(ClientFileDownloadTimeoutKey),
 	}
 }
 
@@ -1560,3 +1586,26 @@ func areCommandServerProxyTLSSettingsSet() bool {
 		viperInstance.IsSet(CommandServerProxyTLSSkipVerifyKey) ||
 		viperInstance.IsSet(CommandServerProxyTLSServerNameKey)
 }
+
+func resolveExternalDataSource() *ExternalDataSource {
+	proxyURLStruct := ProxyURL{
+		URL: viperInstance.GetString(ExternalDataSourceProxyUrlKey),
+	}
+	externalDataSource := &ExternalDataSource{
+		ProxyURL:       proxyURLStruct,
+		AllowedDomains: viperInstance.GetStringSlice(ExternalDataSourceAllowDomainsKey),
+		MaxBytes:       viperInstance.GetInt64(ExternalDataSourceMaxBytesKey),
+	}
+
+	// Validate domains
+	if len(externalDataSource.AllowedDomains) > 0 {
+		for _, domain := range externalDataSource.AllowedDomains {
+			if strings.ContainsAny(domain, "/\\ ") || domain == "" {
+				slog.Error("domain is not specified in allowed_domains")
+				return nil
+			}
+		}
+	}
+
+	return externalDataSource
+}

internal/config/config_test.go

@@ -1425,6 +1425,13 @@ func createConfig() *Config {
 			config.FeatureCertificates, config.FeatureFileWatcher, config.FeatureMetrics,
 			config.FeatureAPIAction, config.FeatureLogsNap,
 		},
+		ExternalDataSource: &ExternalDataSource{
+			ProxyURL: ProxyURL{
+				URL: "",
+			},
+			AllowedDomains: nil,
+			MaxBytes:       0,
+		},
 	}
 }
 

internal/config/defaults.go

@@ -82,6 +82,8 @@ const (
 	DefBackoffMaxInterval         = 20 * time.Second
 	DefBackoffMaxElapsedTime      = 1 * time.Minute
 
+	DefClientFileDownloadTimeout = 60 * time.Second
+
 	// Watcher defaults
 	DefInstanceWatcherMonitoringFrequency       = 5 * time.Second
 	DefInstanceHealthWatcherMonitoringFrequency = 5 * time.Second
@@ -114,6 +116,9 @@ const (
 
 	// File defaults
 	DefLibDir = "/var/lib/nginx-agent"
+
+	DefExternalDataSourceProxyUrl = ""
+	DefExternalDataSourceMaxBytes = 100 * 1024 * 1024
 )
 
 func DefaultFeatures() []string {

internal/config/flags.go

@@ -25,6 +25,7 @@ const (
 	InstanceHealthWatcherMonitoringFrequencyKey = "watchers_instance_health_watcher_monitoring_frequency"
 	FileWatcherKey                              = "watchers_file_watcher"
 	LibDirPathKey                               = "lib_dir"
+	ExternalDataSourceRootKey                   = "external_data_source"
 )
 
 var (
@@ -47,6 +48,7 @@ var (
 	ClientBackoffMaxElapsedTimeKey      = pre(ClientRootKey) + "backoff_max_elapsed_time"
 	ClientBackoffRandomizationFactorKey = pre(ClientRootKey) + "backoff_randomization_factor"
 	ClientBackoffMultiplierKey          = pre(ClientRootKey) + "backoff_multiplier"
+	ClientFileDownloadTimeoutKey        = pre(ClientRootKey) + "file_download_timeout"
 
 	CollectorConfigPathKey                      = pre(CollectorRootKey) + "config_path"
 	CollectorAdditionalConfigPathsKey           = pre(CollectorRootKey) + "additional_config_paths"
@@ -141,6 +143,11 @@ var (
 
 	FileWatcherMonitoringFrequencyKey = pre(FileWatcherKey) + "monitoring_frequency"
 	NginxExcludeFilesKey              = pre(FileWatcherKey) + "exclude_files"
+
+	ExternalDataSourceProxyKey        = pre(ExternalDataSourceRootKey) + "proxy"
+	ExternalDataSourceProxyUrlKey     = pre(ExternalDataSourceProxyKey) + "url"
+	ExternalDataSourceMaxBytesKey     = pre(ExternalDataSourceRootKey) + "max_bytes"
+	ExternalDataSourceAllowDomainsKey = pre(ExternalDataSourceRootKey) + "allowed_domains"
 )
 
 func pre(prefixes ...string) string {

internal/config/types.go

@@ -36,21 +36,22 @@ func parseServerType(str string) (ServerType, bool) {
 
 type (
 	Config struct {
-		Command            *Command         `yaml:"command"             mapstructure:"command"`
-		AuxiliaryCommand   *Command         `yaml:"auxiliary_command"   mapstructure:"auxiliary_command"`
-		Log                *Log             `yaml:"log"                 mapstructure:"log"`
-		DataPlaneConfig    *DataPlaneConfig `yaml:"data_plane_config"   mapstructure:"data_plane_config"`
-		Client             *Client          `yaml:"client"              mapstructure:"client"`
-		Collector          *Collector       `yaml:"collector"           mapstructure:"collector"`
-		Watchers           *Watchers        `yaml:"watchers"            mapstructure:"watchers"`
-		SyslogServer       *SyslogServer    `yaml:"syslog_server"       mapstructure:"syslog_server"`
-		Labels             map[string]any   `yaml:"labels"              mapstructure:"labels"`
-		Version            string           `yaml:"-"`
-		Path               string           `yaml:"-"`
-		UUID               string           `yaml:"-"`
-		LibDir             string           `yaml:"-"`
-		AllowedDirectories []string         `yaml:"allowed_directories" mapstructure:"allowed_directories"`
-		Features           []string         `yaml:"features"            mapstructure:"features"`
+		Command            *Command            `yaml:"command"              mapstructure:"command"`
+		AuxiliaryCommand   *Command            `yaml:"auxiliary_command"    mapstructure:"auxiliary_command"`
+		Log                *Log                `yaml:"log"                  mapstructure:"log"`
+		DataPlaneConfig    *DataPlaneConfig    `yaml:"data_plane_config"    mapstructure:"data_plane_config"`
+		Client             *Client             `yaml:"client"               mapstructure:"client"`
+		Collector          *Collector          `yaml:"collector"            mapstructure:"collector"`
+		Watchers           *Watchers           `yaml:"watchers"             mapstructure:"watchers"`
+		SyslogServer       *SyslogServer       `yaml:"syslog_server"        mapstructure:"syslog_server"`
+		ExternalDataSource *ExternalDataSource `yaml:"external_data_source" mapstructure:"external_data_source"`
+		Labels             map[string]any      `yaml:"labels"               mapstructure:"labels"`
+		Version            string              `yaml:"-"`
+		Path               string              `yaml:"-"`
+		UUID               string              `yaml:"-"`
+		LibDir             string              `yaml:"-"`
+		AllowedDirectories []string            `yaml:"allowed_directories"  mapstructure:"allowed_directories"`
+		Features           []string            `yaml:"features"             mapstructure:"features"`
 	}
 
 	Log struct {
@@ -74,9 +75,10 @@ type (
 	}
 
 	Client struct {
-		HTTP    *HTTP    `yaml:"http"    mapstructure:"http"`
-		Grpc    *GRPC    `yaml:"grpc"    mapstructure:"grpc"`
-		Backoff *BackOff `yaml:"backoff" mapstructure:"backoff"`
+		HTTP                *HTTP         `yaml:"http"                  mapstructure:"http"`
+		Grpc                *GRPC         `yaml:"grpc"                  mapstructure:"grpc"`
+		Backoff             *BackOff      `yaml:"backoff"               mapstructure:"backoff"`
+		FileDownloadTimeout time.Duration `yaml:"file_download_timeout" mapstructure:"file_download_timeout"`
 	}
 
 	HTTP struct {
@@ -358,6 +360,16 @@ type (
 		Token      string        `yaml:"token,omitempty"       mapstructure:"token"`
 		Timeout    time.Duration `yaml:"timeout"               mapstructure:"timeout"`
 	}
+
+	ProxyURL struct {
+		URL string `yaml:"url" mapstructure:"url"`
+	}
+
+	ExternalDataSource struct {
+		ProxyURL       ProxyURL `yaml:"proxy"           mapstructure:"proxy"`
+		AllowedDomains []string `yaml:"allowed_domains" mapstructure:"allowed_domains"`
+		MaxBytes       int64    `yaml:"max_bytes"       mapstructure:"max_bytes"`
+	}
 )
 
 func (col *Collector) Validate(allowedDirectories []string) error {