From ae7fa147f7c90925a379d790ab0bf1040d805bb2 Mon Sep 17 00:00:00 2001 From: Akshay Chawla Date: Wed, 12 Nov 2025 15:29:57 +0000 Subject: [PATCH 1/8] third party config support --- internal/config/config.go | 43 ++++ internal/config/config_test.go | 7 + internal/config/defaults.go | 3 + internal/config/flags.go | 6 + internal/config/types.go | 39 ++-- internal/file/file_manager_service.go | 303 ++++++++++++++++++++++--- internal/file/file_service_operator.go | 17 ++ internal/model/config.go | 4 + internal/model/file.go | 1 + 9 files changed, 376 insertions(+), 47 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 038b0db78..a90d92af1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -123,6 +123,7 @@ func ResolveConfig() (*Config, error) { Features: viperInstance.GetStringSlice(FeaturesKey), Labels: resolveLabels(), LibDir: viperInstance.GetString(LibDirPathKey), + ExternalDataSource: resolveExternalDataSource(), } defaultCollector(collector, config) @@ -434,6 +435,7 @@ func registerFlags() { registerCollectorFlags(fs) registerClientFlags(fs) registerDataPlaneFlags(fs) + registerExternalDataSourceFlags(fs) fs.SetNormalizeFunc(normalizeFunc) @@ -448,6 +450,24 @@ func registerFlags() { }) } +func registerExternalDataSourceFlags(fs *flag.FlagSet) { + fs.String( + ExternalDataSourceProxyUrlKey, + DefExternalDataSourceProxyUrl, + "Url to the proxy service to fetch the external file.", + ) + fs.StringSlice( + ExternalDataSourceAllowDomainsKey, + []string{}, + "List of allowed domains for external data sources.", + ) + fs.Int64( + ExternalDataSourceMaxBytesKey, + DefExternalDataSourceMaxBytes, + "Maximum size in bytes for external data sources.", + ) +} + func registerDataPlaneFlags(fs *flag.FlagSet) { fs.Duration( NginxReloadMonitoringPeriodKey, @@ -1506,3 +1526,26 @@ func areCommandServerProxyTLSSettingsSet() bool { viperInstance.IsSet(CommandServerProxyTLSSkipVerifyKey) || viperInstance.IsSet(CommandServerProxyTLSServerNameKey) } + +func resolveExternalDataSource() *ExternalDataSource { + proxyURLStruct := ProxyURL{ + URL: viperInstance.GetString(ExternalDataSourceProxyUrlKey), + } + externalDataSource := &ExternalDataSource{ + ProxyURL: proxyURLStruct, + AllowedDomains: viperInstance.GetStringSlice(ExternalDataSourceAllowDomainsKey), + MaxBytes: viperInstance.GetInt64(ExternalDataSourceMaxBytesKey), + } + + // Validate domains + if len(externalDataSource.AllowedDomains) > 0 { + for _, domain := range externalDataSource.AllowedDomains { + if strings.ContainsAny(domain, "/\\ ") || domain == "" { + slog.Error("domain is not specified in allowed_domains") + return nil + } + } + } + + return externalDataSource +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index cdbbcaf47..fd9071656 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1325,6 +1325,13 @@ func createConfig() *Config { config.FeatureCertificates, config.FeatureFileWatcher, config.FeatureMetrics, config.FeatureAPIAction, config.FeatureLogsNap, }, + ExternalDataSource: &ExternalDataSource{ + ProxyURL: ProxyURL{ + URL: "", + }, + AllowedDomains: nil, + MaxBytes: 0, + }, } } diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 615c7bc8b..a9ef34c6d 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -111,6 +111,9 @@ const ( // File defaults DefLibDir = "/var/lib/nginx-agent" + + DefExternalDataSourceProxyUrl = "" + DefExternalDataSourceMaxBytes = 100 * 1024 * 1024 ) func DefaultFeatures() []string { diff --git a/internal/config/flags.go b/internal/config/flags.go index 697c2906e..1bd4b5a10 100644 --- a/internal/config/flags.go +++ b/internal/config/flags.go @@ -25,6 +25,7 @@ const ( InstanceHealthWatcherMonitoringFrequencyKey = "watchers_instance_health_watcher_monitoring_frequency" FileWatcherKey = "watchers_file_watcher" LibDirPathKey = "lib_dir" + ExternalDataSourceRootKey = "external_data_source" ) var ( @@ -138,6 +139,11 @@ var ( FileWatcherMonitoringFrequencyKey = pre(FileWatcherKey) + "monitoring_frequency" NginxExcludeFilesKey = pre(FileWatcherKey) + "exclude_files" + + ExternalDataSourceProxyKey = pre(ExternalDataSourceRootKey) + "proxy" + ExternalDataSourceProxyUrlKey = pre(ExternalDataSourceProxyKey) + "url" + ExternalDataSourceMaxBytesKey = pre(ExternalDataSourceRootKey) + "max_bytes" + ExternalDataSourceAllowDomainsKey = pre(ExternalDataSourceRootKey) + "allowed_domains" ) func pre(prefixes ...string) string { diff --git a/internal/config/types.go b/internal/config/types.go index fe3bc1773..e8dc88a4f 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -36,20 +36,21 @@ func parseServerType(str string) (ServerType, bool) { type ( Config struct { - Command *Command `yaml:"command" mapstructure:"command"` - AuxiliaryCommand *Command `yaml:"auxiliary_command" mapstructure:"auxiliary_command"` - Log *Log `yaml:"log" mapstructure:"log"` - DataPlaneConfig *DataPlaneConfig `yaml:"data_plane_config" mapstructure:"data_plane_config"` - Client *Client `yaml:"client" mapstructure:"client"` - Collector *Collector `yaml:"collector" mapstructure:"collector"` - Watchers *Watchers `yaml:"watchers" mapstructure:"watchers"` - Labels map[string]any `yaml:"labels" mapstructure:"labels"` - Version string `yaml:"-"` - Path string `yaml:"-"` - UUID string `yaml:"-"` - LibDir string `yaml:"-"` - AllowedDirectories []string `yaml:"allowed_directories" mapstructure:"allowed_directories"` - Features []string `yaml:"features" mapstructure:"features"` + Watchers *Watchers `yaml:"watchers" mapstructure:"watchers"` + Labels map[string]any `yaml:"labels" mapstructure:"labels"` + Log *Log `yaml:"log" mapstructure:"log"` + DataPlaneConfig *DataPlaneConfig `yaml:"data_plane_config" mapstructure:"data_plane_config"` + Client *Client `yaml:"client" mapstructure:"client"` + Collector *Collector `yaml:"collector" mapstructure:"collector"` + AuxiliaryCommand *Command `yaml:"auxiliary_command" mapstructure:"auxiliary_command"` + ExternalDataSource *ExternalDataSource `yaml:"external_data_source" mapstructure:"external_data_source"` + Command *Command `yaml:"command" mapstructure:"command"` + Path string `yaml:"-"` + Version string `yaml:"-"` + LibDir string `yaml:"-"` + UUID string `yaml:"-"` + AllowedDirectories []string `yaml:"allowed_directories" mapstructure:"allowed_directories"` + Features []string `yaml:"features" mapstructure:"features"` } Log struct { @@ -352,6 +353,16 @@ type ( Token string `yaml:"token,omitempty" mapstructure:"token"` Timeout time.Duration `yaml:"timeout" mapstructure:"timeout"` } + + ProxyURL struct { + URL string `yaml:"url" mapstructure:"url"` + } + + ExternalDataSource struct { + ProxyURL ProxyURL `yaml:"proxy" mapstructure:"proxy"` + AllowedDomains []string `yaml:"allowed_domains" mapstructure:"allowed_domains"` + MaxBytes int64 `yaml:"max_bytes" mapstructure:"max_bytes"` + } ) func (col *Collector) Validate(allowedDirectories []string) error { diff --git a/internal/file/file_manager_service.go b/internal/file/file_manager_service.go index 6c7ed4afb..5ada5f443 100644 --- a/internal/file/file_manager_service.go +++ b/internal/file/file_manager_service.go @@ -11,11 +11,16 @@ import ( "encoding/json" "errors" "fmt" + "io" "log/slog" + "net/http" + "net/url" "os" "path/filepath" "strconv" + "strings" "sync" + "time" "google.golang.org/grpc" @@ -39,6 +44,13 @@ const ( executePerm = 0o111 ) +const fileDownloadTimeout = 60 * time.Second + +type DownloadHeaders struct { + ETag string + LastModified string +} + type ( fileOperator interface { Write(ctx context.Context, fileContent []byte, fileName, filePermissions string) error @@ -73,6 +85,7 @@ type ( ) error SetIsConnected(isConnected bool) RenameFile(ctx context.Context, hash, fileName, tempDir string) error + RenameExternalFile(ctx context.Context, fileName, tempDir string) error UpdateClient(ctx context.Context, fileServiceClient mpi.FileServiceClient) } @@ -103,26 +116,28 @@ type FileManagerService struct { // map of files and the actions performed on them during config apply fileActions map[string]*model.FileCache // key is file path // map of the files currently on disk, used to determine the file action during config apply - currentFilesOnDisk map[string]*mpi.File // key is file path - previousManifestFiles map[string]*model.ManifestFile - manifestFilePath string - rollbackManifest bool - filesMutex sync.RWMutex + currentFilesOnDisk map[string]*mpi.File // key is file path + previousManifestFiles map[string]*model.ManifestFile + newExternalFileHeaders map[string]DownloadHeaders + manifestFilePath string + rollbackManifest bool + filesMutex sync.RWMutex } func NewFileManagerService(fileServiceClient mpi.FileServiceClient, agentConfig *config.Config, manifestLock *sync.RWMutex, ) *FileManagerService { return &FileManagerService{ - agentConfig: agentConfig, - fileOperator: NewFileOperator(manifestLock), - fileServiceOperator: NewFileServiceOperator(agentConfig, fileServiceClient, manifestLock), - fileActions: make(map[string]*model.FileCache), - currentFilesOnDisk: make(map[string]*mpi.File), - previousManifestFiles: make(map[string]*model.ManifestFile), - rollbackManifest: true, - manifestFilePath: agentConfig.LibDir + "/manifest.json", - manifestLock: manifestLock, + agentConfig: agentConfig, + fileOperator: NewFileOperator(manifestLock), + fileServiceOperator: NewFileServiceOperator(agentConfig, fileServiceClient, manifestLock), + fileActions: make(map[string]*model.FileCache), + currentFilesOnDisk: make(map[string]*mpi.File), + previousManifestFiles: make(map[string]*model.ManifestFile), + newExternalFileHeaders: make(map[string]DownloadHeaders), + rollbackManifest: true, + manifestFilePath: agentConfig.LibDir + "/manifest.json", + manifestLock: manifestLock, } } @@ -232,7 +247,7 @@ func (fms *FileManagerService) Rollback(ctx context.Context, instanceID string) delete(fms.currentFilesOnDisk, fileAction.File.GetFileMeta().GetName()) continue - case model.Delete, model.Update: + case model.Delete, model.Update, model.ExternalFile: content, err := fms.restoreFiles(fileAction) if err != nil { return err @@ -387,13 +402,16 @@ func (fms *FileManagerService) DetermineFileActions( modifiedFile.Action = model.Add fileDiff[fileName] = modifiedFile - continue // if file currently exists and file hash is different, file has been updated // copy contents, set file action } else if ok && modifiedFile.File.GetFileMeta().GetHash() != currentFile.GetFileMeta().GetHash() { modifiedFile.Action = model.Update fileDiff[fileName] = modifiedFile } + if modifiedFile.File.GetExternalDataSource() != nil || currentFile.GetExternalDataSource() != nil { + modifiedFile.Action = model.ExternalFile + fileDiff[fileName] = modifiedFile + } } return fileDiff, nil @@ -571,33 +589,40 @@ func (fms *FileManagerService) executeFileActions(ctx context.Context) (actionEr func (fms *FileManagerService) downloadUpdatedFilesToTempLocation(ctx context.Context) (updateError error) { for _, fileAction := range fms.fileActions { - if fileAction.Action == model.Add || fileAction.Action == model.Update { - tempFilePath := tempFilePath(fileAction.File.GetFileMeta().GetName()) + tempFilePath := tempFilePath(fileAction.File.GetFileMeta().GetName()) + switch fileAction.Action { + case model.ExternalFile: + updateError = fms.handleExternalFileDownload(ctx, fileAction, tempFilePath) + case model.Add, model.Update: slog.DebugContext( ctx, "Downloading file to temp location", "file", tempFilePath, ) + updateError = fms.fileUpdate(ctx, fileAction.File, tempFilePath) + case model.Delete, model.Unchanged: + continue + } - updateErr := fms.fileUpdate(ctx, fileAction.File, tempFilePath) - if updateErr != nil { - updateError = updateErr - break - } + if updateError != nil { + return updateError } } - return updateError + return nil } func (fms *FileManagerService) moveOrDeleteFiles(ctx context.Context, actionError error) error { actionsLoop: for _, fileAction := range fms.fileActions { + var err error + fileMeta := fileAction.File.GetFileMeta() + tempFilePath := tempFilePath(fileAction.File.GetFileMeta().GetName()) switch fileAction.Action { case model.Delete: slog.DebugContext(ctx, "Deleting file", "file", fileAction.File.GetFileMeta().GetName()) - if err := os.Remove(fileAction.File.GetFileMeta().GetName()); err != nil && !os.IsNotExist(err) { + if err = os.Remove(fileAction.File.GetFileMeta().GetName()); err != nil && !os.IsNotExist(err) { actionError = fmt.Errorf("error deleting file: %s error: %w", fileAction.File.GetFileMeta().GetName(), err) @@ -606,17 +631,16 @@ actionsLoop: continue case model.Add, model.Update: - fileMeta := fileAction.File.GetFileMeta() - tempFilePath := tempFilePath(fileAction.File.GetFileMeta().GetName()) - err := fms.fileServiceOperator.RenameFile(ctx, fileMeta.GetHash(), tempFilePath, fileMeta.GetName()) - if err != nil { - actionError = err - - break actionsLoop - } + err = fms.fileServiceOperator.RenameFile(ctx, fileMeta.GetHash(), tempFilePath, fileMeta.GetName()) + case model.ExternalFile: + err = fms.fileServiceOperator.RenameExternalFile(ctx, tempFilePath, fileMeta.GetName()) case model.Unchanged: slog.DebugContext(ctx, "File unchanged") } + if err != nil { + actionError = err + break actionsLoop + } } return actionError @@ -772,3 +796,216 @@ func tempBackupFilePath(fileName string) string { tempFileName := "." + filepath.Base(fileName) + ".agent.backup" return filepath.Join(filepath.Dir(fileName), tempFileName) } + +func (fms *FileManagerService) handleExternalFileDownload(ctx context.Context, fileAction *model.FileCache, + tempFilePath string, +) error { + location := fileAction.File.GetExternalDataSource().GetLocation() + slog.InfoContext(ctx, "Downloading external file from", "location", location) + + var contentToWrite []byte + var downloadErr, updateError error + var headers DownloadHeaders + + contentToWrite, headers, downloadErr = fms.downloadFileContent(ctx, fileAction.File) + + if downloadErr != nil { + updateError = fmt.Errorf("failed to download file %s from %s: %w", + fileAction.File.GetFileMeta().GetName(), location, downloadErr) + + return updateError + } + + if contentToWrite == nil { + slog.DebugContext(ctx, "External file unchanged (304), skipping disk write.", + "file", fileAction.File.GetFileMeta().GetName()) + return nil + } + + fileName := fileAction.File.GetFileMeta().GetName() + fms.newExternalFileHeaders[fileName] = headers + + updateErr := fms.writeContentToTempFile(ctx, contentToWrite, tempFilePath) + + return updateErr +} + +func (fms *FileManagerService) writeContentToTempFile( + ctx context.Context, + content []byte, + path string, +) error { + writeErr := fms.fileOperator.Write( + ctx, + content, + path, + "0600", + ) + + if writeErr != nil { + return fmt.Errorf("failed to write downloaded content to temp file %s: %w", path, writeErr) + } + + return nil +} + +// downloadFileContent performs an HTTP GET request to the given URL and returns the file content as a byte slice. +func (fms *FileManagerService) downloadFileContent( + ctx context.Context, + file *mpi.File, +) (content []byte, headers DownloadHeaders, err error) { + fileName := file.GetFileMeta().GetName() + downloadURL := file.GetExternalDataSource().GetLocation() + externalConfig := fms.agentConfig.ExternalDataSource + + if !isDomainAllowed(downloadURL, externalConfig.AllowedDomains) { + return nil, DownloadHeaders{}, fmt.Errorf("download URL %s is not in the allowed domains list", downloadURL) + } + + httpClient, err := fms.setupHTTPClient(ctx, externalConfig.ProxyURL.URL) + if err != nil { + return nil, DownloadHeaders{}, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) + if err != nil { + return nil, DownloadHeaders{}, fmt.Errorf("failed to create request for %s: %w", downloadURL, err) + } + + if externalConfig.ProxyURL.URL != "" { + fms.addConditionalHeaders(ctx, req, fileName) + } else { + slog.DebugContext(ctx, "No proxy configured; sending plain HTTP request without caching headers.") + } + + resp, err := httpClient.Do(req) + if err != nil { + return nil, DownloadHeaders{}, fmt.Errorf("failed to execute download request for %s: %w", downloadURL, err) + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK: + headers.ETag = resp.Header.Get("ETag") + headers.LastModified = resp.Header.Get("Last-Modified") + case http.StatusNotModified: + slog.InfoContext(ctx, "File content unchanged (304 Not Modified)", "file_name", fileName) + return nil, DownloadHeaders{}, nil + default: + return nil, DownloadHeaders{}, fmt.Errorf("download failed with status code %d", resp.StatusCode) + } + + reader := io.Reader(resp.Body) + if fms.agentConfig.ExternalDataSource.MaxBytes > 0 { + reader = io.LimitReader(resp.Body, fms.agentConfig.ExternalDataSource.MaxBytes) + } + + content, err = io.ReadAll(reader) + if err != nil { + return nil, DownloadHeaders{}, fmt.Errorf("failed to read content from response body: %w", err) + } + + slog.InfoContext(ctx, "Successfully downloaded file content", "file_name", fileName, "size", len(content)) + + return content, headers, nil +} + +func isDomainAllowed(downloadURL string, allowedDomains []string) bool { + u, err := url.Parse(downloadURL) + if err != nil { + slog.Debug("Failed to parse download URL for domain check", "url", downloadURL, "error", err) + return false + } + + hostname := u.Hostname() + if hostname == "" { + return false + } + + for _, pattern := range allowedDomains { + if pattern == "" { + continue + } + + if pattern == hostname { + return true + } + + if isWildcardMatch(hostname, pattern) { + return true + } + } + + return false +} + +func (fms *FileManagerService) setupHTTPClient(ctx context.Context, proxyURLString string) (*http.Client, error) { + var transport *http.Transport + + if proxyURLString != "" { + proxyURL, err := url.Parse(proxyURLString) + if err != nil { + return nil, fmt.Errorf("invalid proxy URL configured: %w", err) + } + slog.DebugContext(ctx, "Configuring HTTP client to use proxy", "proxy_url", proxyURLString) + transport = &http.Transport{ + Proxy: http.ProxyURL(proxyURL), + } + } else { + slog.DebugContext(ctx, "Configuring HTTP client for direct connection (no proxy)") + transport = &http.Transport{ + Proxy: nil, + } + } + + httpClient := &http.Client{ + Transport: transport, + Timeout: fileDownloadTimeout, + } + + return httpClient, nil +} + +func (fms *FileManagerService) addConditionalHeaders(ctx context.Context, req *http.Request, fileName string) { + slog.DebugContext(ctx, "Proxy configured; adding headers to GET request.") + + manifestFiles, _, manifestFileErr := fms.manifestFile() + + if manifestFileErr != nil && !errors.Is(manifestFileErr, os.ErrNotExist) { + slog.WarnContext(ctx, "Error reading manifest file for headers", "error", manifestFileErr) + } + + manifestFile, ok := manifestFiles[fileName] + + if ok && manifestFile != nil && manifestFile.ManifestFileMeta != nil { + fileMeta := manifestFile.ManifestFileMeta + + if fileMeta.ETag != "" { + req.Header.Set("If-None-Match", fileMeta.ETag) + } + if fileMeta.LastModified != "" { + req.Header.Set("If-Modified-Since", fileMeta.LastModified) + } + } else { + slog.DebugContext(ctx, "File not found in manifest or missing metadata; skipping conditional headers.", + "file", fileName) + } +} + +func isWildcardMatch(hostname, pattern string) bool { + if !strings.HasPrefix(pattern, "*.") { + return false + } + + baseDomain := pattern[2:] + if strings.HasSuffix(hostname, baseDomain) { + // Check to ensure it's a true subdomain match (e.g., must have a '.' + // before baseDomain unless it IS the baseDomain) + // This handles cases like preventing 'foo.com' matching '*.oo.com' + if hostname == baseDomain || hostname[len(hostname)-len(baseDomain)-1] == '.' { + return true + } + } + + return false +} diff --git a/internal/file/file_service_operator.go b/internal/file/file_service_operator.go index 19211c600..339c98eec 100644 --- a/internal/file/file_service_operator.go +++ b/internal/file/file_service_operator.go @@ -276,6 +276,23 @@ func (fso *FileServiceOperator) UpdateFile( return fso.sendUpdateFileStream(ctx, fileToUpdate, fso.agentConfig.Client.Grpc.FileChunkSize) } +func (fso *FileServiceOperator) RenameExternalFile( + ctx context.Context, source, desination string, +) error { + slog.DebugContext(ctx, fmt.Sprintf("Moving file %s to %s (no hash validation)", source, desination)) + + if err := os.MkdirAll(filepath.Dir(desination), dirPerm); err != nil { + return fmt.Errorf("failed to create directories for %s: %w", desination, err) + } + + moveErr := os.Rename(source, desination) + if moveErr != nil { + return fmt.Errorf("failed to move file: %w", moveErr) + } + + return nil +} + // renameFile, renames (moves) file from tempDir to new location to update file. func (fso *FileServiceOperator) RenameFile( ctx context.Context, hash, source, desination string, diff --git a/internal/model/config.go b/internal/model/config.go index 78c764ce0..98cc6705c 100644 --- a/internal/model/config.go +++ b/internal/model/config.go @@ -47,6 +47,10 @@ type ManifestFileMeta struct { Referenced bool `json:"referenced"` // File is not managed by the agent Unmanaged bool `json:"unmanaged"` + // ETag of the 3rd Party external file + ETag string `json:"etag"` + // Last modified time of the 3rd Party external file + LastModified string `json:"last_modified"` } type ConfigApplyMessage struct { Error error diff --git a/internal/model/file.go b/internal/model/file.go index fc6c5baca..671a4ff85 100644 --- a/internal/model/file.go +++ b/internal/model/file.go @@ -19,4 +19,5 @@ const ( Update Delete Unchanged + ExternalFile ) From 14f1190b612f12ffb177d17e69371ec91a72d6cc Mon Sep 17 00:00:00 2001 From: Akshay Chawla Date: Wed, 12 Nov 2025 15:43:35 +0000 Subject: [PATCH 2/8] lint fix --- internal/file/file_manager_service.go | 1 + internal/model/config.go | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/file/file_manager_service.go b/internal/file/file_manager_service.go index 5ada5f443..8615b444a 100644 --- a/internal/file/file_manager_service.go +++ b/internal/file/file_manager_service.go @@ -819,6 +819,7 @@ func (fms *FileManagerService) handleExternalFileDownload(ctx context.Context, f if contentToWrite == nil { slog.DebugContext(ctx, "External file unchanged (304), skipping disk write.", "file", fileAction.File.GetFileMeta().GetName()) + return nil } diff --git a/internal/model/config.go b/internal/model/config.go index 98cc6705c..5c5feb11a 100644 --- a/internal/model/config.go +++ b/internal/model/config.go @@ -41,16 +41,16 @@ type ManifestFileMeta struct { Name string `json:"name"` // The hash of the file contents sha256, hex encoded Hash string `json:"hash"` + // ETag of the 3rd Party external file + ETag string `json:"etag"` + // Last modified time of the 3rd Party external file + LastModified string `json:"last_modified"` // The size of the file in bytes Size int64 `json:"size"` // File referenced in the NGINX config Referenced bool `json:"referenced"` // File is not managed by the agent Unmanaged bool `json:"unmanaged"` - // ETag of the 3rd Party external file - ETag string `json:"etag"` - // Last modified time of the 3rd Party external file - LastModified string `json:"last_modified"` } type ConfigApplyMessage struct { Error error From d31b2df6146872881599c0e4d6b64a70bdcbda0b Mon Sep 17 00:00:00 2001 From: Akshay Chawla Date: Thu, 13 Nov 2025 12:12:30 +0000 Subject: [PATCH 3/8] Added a generic file timeout --- internal/config/config.go | 6 ++++++ internal/config/defaults.go | 2 ++ internal/config/flags.go | 1 + internal/config/types.go | 7 ++++--- internal/file/file_manager_service.go | 5 +---- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 80d201180..1ba2c5d1e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -648,6 +648,11 @@ func registerClientFlags(fs *flag.FlagSet) { DefMaxFileSize, "Max file size in bytes.", ) + fs.Duration( + ClientFileDownloadTimeoutKey, + DefClientFileDownloadTimeout, + "Timeout value in seconds, to downloading file for config apply.", + ) } func registerCommandFlags(fs *flag.FlagSet) { @@ -1133,6 +1138,7 @@ func resolveClient() *Client { RandomizationFactor: viperInstance.GetFloat64(ClientBackoffRandomizationFactorKey), Multiplier: viperInstance.GetFloat64(ClientBackoffMultiplierKey), }, + FileDownloadTimeout: viperInstance.GetDuration(ClientFileDownloadTimeoutKey), } } diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 6035ec4fc..545e37a24 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -81,6 +81,8 @@ const ( DefBackoffMaxInterval = 20 * time.Second DefBackoffMaxElapsedTime = 1 * time.Minute + DefClientFileDownloadTimeout = 60 * time.Second + // Watcher defaults DefInstanceWatcherMonitoringFrequency = 5 * time.Second DefInstanceHealthWatcherMonitoringFrequency = 5 * time.Second diff --git a/internal/config/flags.go b/internal/config/flags.go index 21517cb62..953ce7f0d 100644 --- a/internal/config/flags.go +++ b/internal/config/flags.go @@ -47,6 +47,7 @@ var ( ClientBackoffMaxElapsedTimeKey = pre(ClientRootKey) + "backoff_max_elapsed_time" ClientBackoffRandomizationFactorKey = pre(ClientRootKey) + "backoff_randomization_factor" ClientBackoffMultiplierKey = pre(ClientRootKey) + "backoff_multiplier" + ClientFileDownloadTimeoutKey = pre(ClientRootKey) + "file_download_timeout" CollectorConfigPathKey = pre(CollectorRootKey) + "config_path" CollectorAdditionalConfigPathsKey = pre(CollectorRootKey) + "additional_config_paths" diff --git a/internal/config/types.go b/internal/config/types.go index 39488e38d..6aff4cd78 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -75,9 +75,10 @@ type ( } Client struct { - HTTP *HTTP `yaml:"http" mapstructure:"http"` - Grpc *GRPC `yaml:"grpc" mapstructure:"grpc"` - Backoff *BackOff `yaml:"backoff" mapstructure:"backoff"` + HTTP *HTTP `yaml:"http" mapstructure:"http"` + Grpc *GRPC `yaml:"grpc" mapstructure:"grpc"` + Backoff *BackOff `yaml:"backoff" mapstructure:"backoff"` + FileDownloadTimeout time.Duration `yaml:"file_download_timeout" mapstructure:"file_download_timeout"` } HTTP struct { diff --git a/internal/file/file_manager_service.go b/internal/file/file_manager_service.go index 8615b444a..e60c96d2c 100644 --- a/internal/file/file_manager_service.go +++ b/internal/file/file_manager_service.go @@ -20,7 +20,6 @@ import ( "strconv" "strings" "sync" - "time" "google.golang.org/grpc" @@ -44,8 +43,6 @@ const ( executePerm = 0o111 ) -const fileDownloadTimeout = 60 * time.Second - type DownloadHeaders struct { ETag string LastModified string @@ -961,7 +958,7 @@ func (fms *FileManagerService) setupHTTPClient(ctx context.Context, proxyURLStri httpClient := &http.Client{ Transport: transport, - Timeout: fileDownloadTimeout, + Timeout: fms.agentConfig.Client.FileDownloadTimeout, } return httpClient, nil From 501e08f665995ae054708bf3da386650e7d065a1 Mon Sep 17 00:00:00 2001 From: Akshay Chawla Date: Thu, 13 Nov 2025 15:48:00 +0000 Subject: [PATCH 4/8] added generic timeout to create connection --- internal/file/file_service_operator.go | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/internal/file/file_service_operator.go b/internal/file/file_service_operator.go index 339c98eec..8dba2acf7 100644 --- a/internal/file/file_service_operator.go +++ b/internal/file/file_service_operator.go @@ -79,7 +79,10 @@ func (fso *FileServiceOperator) File( defer backoffCancel() getFile := func() (*mpi.GetFileResponse, error) { - return fso.fileServiceClient.GetFile(ctx, &mpi.GetFileRequest{ + grpcCtx, cancel := context.WithTimeout(ctx, fso.agentConfig.Client.FileDownloadTimeout) + defer cancel() + + return fso.fileServiceClient.GetFile(grpcCtx, &mpi.GetFileRequest{ MessageMeta: &mpi.MessageMeta{ MessageId: id.GenerateMessageID(), CorrelationId: logger.CorrelationID(ctx), @@ -225,7 +228,10 @@ func (fso *FileServiceOperator) ChunkedFile( ) error { slog.DebugContext(ctx, "Getting chunked file", "file", file.GetFileMeta().GetName()) - stream, err := fso.fileServiceClient.GetFileStream(ctx, &mpi.GetFileRequest{ + grpcCtx, cancel := context.WithTimeout(ctx, fso.agentConfig.Client.FileDownloadTimeout) + defer cancel() + + stream, err := fso.fileServiceClient.GetFileStream(grpcCtx, &mpi.GetFileRequest{ MessageMeta: &mpi.MessageMeta{ MessageId: id.GenerateMessageID(), CorrelationId: logger.CorrelationID(ctx), @@ -388,12 +394,15 @@ func (fso *FileServiceOperator) sendUpdateFileRequest( return nil, errors.New("CreateConnection rpc has not being called yet") } - response, updateError := fso.fileServiceClient.UpdateFile(ctx, request) + grpcCtx, cancel := context.WithTimeout(ctx, fso.agentConfig.Client.FileDownloadTimeout) + defer cancel() + + response, updateError := fso.fileServiceClient.UpdateFile(grpcCtx, request) validatedError := internalgrpc.ValidateGrpcError(updateError) if validatedError != nil { - slog.ErrorContext(ctx, "Failed to send update file", "error", validatedError) + slog.ErrorContext(grpcCtx, "Failed to send update file", "error", validatedError) return nil, validatedError } @@ -423,7 +432,10 @@ func (fso *FileServiceOperator) sendUpdateFileStream( return errors.New("file chunk size must be greater than zero") } - updateFileStreamClient, err := fso.fileServiceClient.UpdateFileStream(ctx) + grpcCtx, cancel := context.WithTimeout(ctx, fso.agentConfig.Client.FileDownloadTimeout) + defer cancel() + + updateFileStreamClient, err := fso.fileServiceClient.UpdateFileStream(grpcCtx) if err != nil { return err } From b7eaef745ab1c482fdce3d21357a008692fba4ca Mon Sep 17 00:00:00 2001 From: Akshay Chawla Date: Wed, 19 Nov 2025 07:39:21 +0000 Subject: [PATCH 5/8] added uT --- internal/config/config.go | 25 ++++++++---- internal/config/config_test.go | 70 ++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 8 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index c5fcb0900..26c6ed81d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1597,15 +1597,24 @@ func resolveExternalDataSource() *ExternalDataSource { MaxBytes: viperInstance.GetInt64(ExternalDataSourceMaxBytesKey), } - // Validate domains - if len(externalDataSource.AllowedDomains) > 0 { - for _, domain := range externalDataSource.AllowedDomains { - if strings.ContainsAny(domain, "/\\ ") || domain == "" { - slog.Error("domain is not specified in allowed_domains") - return nil - } - } + if err := validateAllowedDomains(externalDataSource.AllowedDomains); err != nil { + return nil } return externalDataSource } + +func validateAllowedDomains(domains []string) error { + if len(domains) == 0 { + return nil + } + + for _, domain := range domains { + if strings.ContainsAny(domain, "/\\ ") || domain == "" { + slog.Error("domain is not specified in allowed_domains") + return errors.New("invalid domain found in allowed_domains") + } + } + + return nil +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 55534cd41..b35818ebe 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1576,3 +1576,73 @@ func TestValidateLabel(t *testing.T) { }) } } + +func TestValidateAllowedDomains(t *testing.T) { + tests := []struct { + name string + domains []string + wantErr bool + }{ + { + name: "Test 1: Success: Empty slice", + domains: []string{}, + wantErr: false, + }, + { + name: "Test 2: Success: Nil slice", + domains: nil, + wantErr: false, + }, + { + name: "Test 3: Success: Valid domains", + domains: []string{"example.com", "api.nginx.com", "sub.domain.io"}, + wantErr: false, + }, + { + name: "Test 4: Failure: Domain contains space", + domains: []string{"valid.com", "bad domain.com"}, + wantErr: true, + }, + { + name: "Test 5: Failure: Empty string domain", + domains: []string{"valid.com", ""}, + wantErr: true, + }, + { + name: "Test 6: Failure: Domain contains forward slash /", + domains: []string{"domain.com/path"}, + wantErr: true, + }, + { + name: "Test 7: Failure: Domain contains backward slash \\", + domains: []string{"domain.com\\path"}, + wantErr: true, + }, + { + name: "Test 8: Failure: Mixed valid and invalid (first is invalid)", + domains: []string{" only.com", "good.com"}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var logBuffer bytes.Buffer + logHandler := slog.NewTextHandler(&logBuffer, &slog.HandlerOptions{Level: slog.LevelError}) + + originalLogger := slog.Default() + slog.SetDefault(slog.New(logHandler)) + defer slog.SetDefault(originalLogger) + + actualErr := validateAllowedDomains(tt.domains) + + if tt.wantErr { + require.Error(t, actualErr, "Expected an error but got nil.") + assert.Contains(t, logBuffer.String(), "domain is not specified in allowed_domains", + "Expected the error log message to be present in the output.") + } else { + assert.NoError(t, actualErr, "Did not expect an error but got one: %v", actualErr) + } + }) + } +} From a83e45b9841b152e7e5fc6e7fd7f46702d5f9de3 Mon Sep 17 00:00:00 2001 From: Akshay Chawla Date: Wed, 19 Nov 2025 09:54:14 +0000 Subject: [PATCH 6/8] Added UT --- internal/file/file_manager_service_test.go | 176 +++++++++++++++++++++ 1 file changed, 176 insertions(+) diff --git a/internal/file/file_manager_service_test.go b/internal/file/file_manager_service_test.go index e42128063..4052a89d5 100644 --- a/internal/file/file_manager_service_test.go +++ b/internal/file/file_manager_service_test.go @@ -10,11 +10,15 @@ import ( "encoding/json" "errors" "fmt" + "net/http" + "net/http/httptest" + "net/url" "os" "path/filepath" "sync" "testing" + "github.com/nginx/agent/v3/internal/config" "github.com/nginx/agent/v3/internal/model" "github.com/nginx/agent/v3/pkg/files" @@ -1173,3 +1177,175 @@ rQHX6DP4w6IwZY8JB8LS }) } } + +func TestFileManagerService_DetermineFileActions_ExternalFile(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + fileName := filepath.Join(tempDir, "external.conf") + + modifiedFiles := map[string]*model.FileCache{ + fileName: { + File: &mpi.File{ + FileMeta: &mpi.FileMeta{ + Name: fileName, + }, + ExternalDataSource: &mpi.ExternalDataSource{Location: "http://example.com/file"}, + }, + }, + } + + fakeFileServiceClient := &v1fakes.FakeFileServiceClient{} + fileManagerService := NewFileManagerService(fakeFileServiceClient, types.AgentConfig(), &sync.RWMutex{}) + fileManagerService.agentConfig.AllowedDirectories = []string{tempDir} + + diff, err := fileManagerService.DetermineFileActions(ctx, map[string]*mpi.File{}, modifiedFiles) + require.NoError(t, err) + + fc, ok := diff[fileName] + require.True(t, ok, "expected file to be present in diff") + assert.Equal(t, model.ExternalFile, fc.Action) +} + +func TestFileManagerService_downloadExternalFiles_Cases(t *testing.T) { + type tc struct { + name string + handler http.HandlerFunc + allowedDomains []string + maxBytes int + expectError bool + expectErrContains string + expectTempFile bool + expectContent []byte + expectHeaderETag string + expectHeaderLastMod string + } + + tests := []tc{ + { + name: "Success", + handler: func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("ETag", "test-etag") + w.Header().Set("Last-Modified", "Mon, 02 Jan 2006 15:04:05 MST") + w.WriteHeader(200) + _, _ = w.Write([]byte("external file content")) + }, + allowedDomains: nil, // will be set per test from ts + maxBytes: 0, + expectError: false, + expectTempFile: true, + expectContent: []byte("external file content"), + expectHeaderETag: "test-etag", + expectHeaderLastMod: "Mon, 02 Jan 2006 15:04:05 MST", + }, + { + name: "NotModified", + handler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotModified) + }, + allowedDomains: nil, + maxBytes: 0, + expectError: false, + expectTempFile: false, + expectContent: nil, + expectHeaderETag: "", + expectHeaderLastMod: "", + }, + { + name: "NotAllowedDomain", + handler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + _, _ = w.Write([]byte("external file content")) + }, + allowedDomains: []string{"not-the-host"}, + maxBytes: 0, + expectError: true, + expectErrContains: "not in the allowed domains", + expectTempFile: false, + }, + { + name: "NotFound", + handler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + }, + allowedDomains: nil, + maxBytes: 0, + expectError: true, + expectErrContains: "status code 404", + expectTempFile: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + fileName := filepath.Join(tempDir, "external.conf") + + ts := httptest.NewServer(http.HandlerFunc(test.handler)) + defer ts.Close() + + u, err := url.Parse(ts.URL) + require.NoError(t, err) + host := u.Hostname() + + fakeFileServiceClient := &v1fakes.FakeFileServiceClient{} + fileManagerService := NewFileManagerService(fakeFileServiceClient, types.AgentConfig(), &sync.RWMutex{}) + + // If the test provided allowedDomains, use it; otherwise allow this test server's host + if test.allowedDomains == nil { + fileManagerService.agentConfig.ExternalDataSource = &config.ExternalDataSource{ + ProxyURL: config.ProxyURL{URL: ""}, + AllowedDomains: []string{host}, + MaxBytes: int64(test.maxBytes), + } + } else { + fileManagerService.agentConfig.ExternalDataSource = &config.ExternalDataSource{ + ProxyURL: config.ProxyURL{URL: ""}, + AllowedDomains: test.allowedDomains, + MaxBytes: int64(test.maxBytes), + } + } + + fileManagerService.fileActions = map[string]*model.FileCache{ + fileName: { + File: &mpi.File{ + FileMeta: &mpi.FileMeta{Name: fileName}, + ExternalDataSource: &mpi.ExternalDataSource{Location: ts.URL}, + }, + Action: model.ExternalFile, + }, + } + + err = fileManagerService.downloadUpdatedFilesToTempLocation(ctx) + + if test.expectError { + require.Error(t, err) + if test.expectErrContains != "" { + assert.Contains(t, err.Error(), test.expectErrContains) + } + // ensure no temp file left + _, statErr := os.Stat(tempFilePath(fileName)) + assert.True(t, os.IsNotExist(statErr)) + return + } + + require.NoError(t, err) + + if test.expectTempFile { + b, readErr := os.ReadFile(tempFilePath(fileName)) + require.NoError(t, readErr) + assert.Equal(t, test.expectContent, b) + + h, ok := fileManagerService.newExternalFileHeaders[fileName] + require.True(t, ok) + assert.Equal(t, test.expectHeaderETag, h.ETag) + assert.Equal(t, test.expectHeaderLastMod, h.LastModified) + + _ = os.Remove(tempFilePath(fileName)) + } else { + _, statErr := os.Stat(tempFilePath(fileName)) + assert.True(t, os.IsNotExist(statErr)) + } + }) + } +} From 529c8393ebecdf83b1f91acb90f65aa3ede39644 Mon Sep 17 00:00:00 2001 From: Akshay Chawla Date: Wed, 19 Nov 2025 10:35:37 +0000 Subject: [PATCH 7/8] fix lint issues --- internal/file/file_manager_service_test.go | 25 ++++++++++++---------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/internal/file/file_manager_service_test.go b/internal/file/file_manager_service_test.go index 4052a89d5..ca14b2f5d 100644 --- a/internal/file/file_manager_service_test.go +++ b/internal/file/file_manager_service_test.go @@ -17,6 +17,7 @@ import ( "path/filepath" "sync" "testing" + "time" "github.com/nginx/agent/v3/internal/config" "github.com/nginx/agent/v3/internal/model" @@ -1198,7 +1199,7 @@ func TestFileManagerService_DetermineFileActions_ExternalFile(t *testing.T) { fileManagerService := NewFileManagerService(fakeFileServiceClient, types.AgentConfig(), &sync.RWMutex{}) fileManagerService.agentConfig.AllowedDirectories = []string{tempDir} - diff, err := fileManagerService.DetermineFileActions(ctx, map[string]*mpi.File{}, modifiedFiles) + diff, err := fileManagerService.DetermineFileActions(ctx, make(map[string]*mpi.File), modifiedFiles) require.NoError(t, err) fc, ok := diff[fileName] @@ -1206,18 +1207,19 @@ func TestFileManagerService_DetermineFileActions_ExternalFile(t *testing.T) { assert.Equal(t, model.ExternalFile, fc.Action) } +//nolint:gocognit,revive,govet // cognitive complexity is 25 func TestFileManagerService_downloadExternalFiles_Cases(t *testing.T) { type tc struct { + allowedDomains []string + expectContent []byte name string + expectHeaderETag string + expectHeaderLastMod string + expectErrContains string handler http.HandlerFunc - allowedDomains []string maxBytes int expectError bool - expectErrContains string expectTempFile bool - expectContent []byte - expectHeaderETag string - expectHeaderLastMod string } tests := []tc{ @@ -1225,8 +1227,8 @@ func TestFileManagerService_downloadExternalFiles_Cases(t *testing.T) { name: "Success", handler: func(w http.ResponseWriter, r *http.Request) { w.Header().Set("ETag", "test-etag") - w.Header().Set("Last-Modified", "Mon, 02 Jan 2006 15:04:05 MST") - w.WriteHeader(200) + w.Header().Set("Last-Modified", time.RFC1123) + w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("external file content")) }, allowedDomains: nil, // will be set per test from ts @@ -1235,7 +1237,7 @@ func TestFileManagerService_downloadExternalFiles_Cases(t *testing.T) { expectTempFile: true, expectContent: []byte("external file content"), expectHeaderETag: "test-etag", - expectHeaderLastMod: "Mon, 02 Jan 2006 15:04:05 MST", + expectHeaderLastMod: time.RFC1123, }, { name: "NotModified", @@ -1253,7 +1255,7 @@ func TestFileManagerService_downloadExternalFiles_Cases(t *testing.T) { { name: "NotAllowedDomain", handler: func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) + w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("external file content")) }, allowedDomains: []string{"not-the-host"}, @@ -1281,7 +1283,7 @@ func TestFileManagerService_downloadExternalFiles_Cases(t *testing.T) { tempDir := t.TempDir() fileName := filepath.Join(tempDir, "external.conf") - ts := httptest.NewServer(http.HandlerFunc(test.handler)) + ts := httptest.NewServer(test.handler) defer ts.Close() u, err := url.Parse(ts.URL) @@ -1326,6 +1328,7 @@ func TestFileManagerService_downloadExternalFiles_Cases(t *testing.T) { // ensure no temp file left _, statErr := os.Stat(tempFilePath(fileName)) assert.True(t, os.IsNotExist(statErr)) + return } From 2a51d2268a29b8c2c52d754c7c31f3992b700c5e Mon Sep 17 00:00:00 2001 From: Akshay Chawla Date: Wed, 19 Nov 2025 12:51:39 +0000 Subject: [PATCH 8/8] Added UT for Proxy URL --- internal/file/file_manager_service_test.go | 85 ++++++++++++++++----- internal/file/file_service_operator_test.go | 83 ++++++++++++++++++++ 2 files changed, 149 insertions(+), 19 deletions(-) diff --git a/internal/file/file_manager_service_test.go b/internal/file/file_manager_service_test.go index ca14b2f5d..9efd7ed97 100644 --- a/internal/file/file_manager_service_test.go +++ b/internal/file/file_manager_service_test.go @@ -1207,7 +1207,7 @@ func TestFileManagerService_DetermineFileActions_ExternalFile(t *testing.T) { assert.Equal(t, model.ExternalFile, fc.Action) } -//nolint:gocognit,revive,govet // cognitive complexity is 25 +//nolint:gocognit,revive,govet // cognitive complexity is 22 func TestFileManagerService_downloadExternalFiles_Cases(t *testing.T) { type tc struct { allowedDomains []string @@ -1224,14 +1224,14 @@ func TestFileManagerService_downloadExternalFiles_Cases(t *testing.T) { tests := []tc{ { - name: "Success", + name: "Test 1: Success", handler: func(w http.ResponseWriter, r *http.Request) { w.Header().Set("ETag", "test-etag") w.Header().Set("Last-Modified", time.RFC1123) w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("external file content")) }, - allowedDomains: nil, // will be set per test from ts + allowedDomains: nil, maxBytes: 0, expectError: false, expectTempFile: true, @@ -1240,7 +1240,7 @@ func TestFileManagerService_downloadExternalFiles_Cases(t *testing.T) { expectHeaderLastMod: time.RFC1123, }, { - name: "NotModified", + name: "Test 2: NotModified", handler: func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotModified) }, @@ -1253,7 +1253,7 @@ func TestFileManagerService_downloadExternalFiles_Cases(t *testing.T) { expectHeaderLastMod: "", }, { - name: "NotAllowedDomain", + name: "Test 3: NotAllowedDomain", handler: func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("external file content")) @@ -1265,7 +1265,7 @@ func TestFileManagerService_downloadExternalFiles_Cases(t *testing.T) { expectTempFile: false, }, { - name: "NotFound", + name: "Test 4: NotFound", handler: func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) }, @@ -1275,6 +1275,32 @@ func TestFileManagerService_downloadExternalFiles_Cases(t *testing.T) { expectErrContains: "status code 404", expectTempFile: false, }, + { + name: "Test 5: ProxyWithConditionalHeaders", + handler: func(w http.ResponseWriter, r *http.Request) { + // verify conditional headers from manifest are added + if r.Header.Get("If-None-Match") != "manifest-test-etag" { + http.Error(w, "missing If-None-Match", http.StatusBadRequest) + return + } + if r.Header.Get("If-Modified-Since") != time.RFC1123 { + http.Error(w, "missing If-Modified-Since", http.StatusBadRequest) + return + } + w.Header().Set("ETag", "resp-etag") + w.Header().Set("Last-Modified", time.RFC1123) + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("external file via proxy")) + }, + allowedDomains: nil, + maxBytes: 0, + expectError: false, + expectTempFile: true, + expectContent: []byte("external file via proxy"), + expectHeaderETag: "resp-etag", + expectHeaderLastMod: time.RFC1123, + expectErrContains: "", + }, } for _, test := range tests { @@ -1293,21 +1319,43 @@ func TestFileManagerService_downloadExternalFiles_Cases(t *testing.T) { fakeFileServiceClient := &v1fakes.FakeFileServiceClient{} fileManagerService := NewFileManagerService(fakeFileServiceClient, types.AgentConfig(), &sync.RWMutex{}) - // If the test provided allowedDomains, use it; otherwise allow this test server's host - if test.allowedDomains == nil { - fileManagerService.agentConfig.ExternalDataSource = &config.ExternalDataSource{ - ProxyURL: config.ProxyURL{URL: ""}, - AllowedDomains: []string{host}, - MaxBytes: int64(test.maxBytes), - } - } else { - fileManagerService.agentConfig.ExternalDataSource = &config.ExternalDataSource{ - ProxyURL: config.ProxyURL{URL: ""}, - AllowedDomains: test.allowedDomains, - MaxBytes: int64(test.maxBytes), + eds := &config.ExternalDataSource{ + ProxyURL: config.ProxyURL{URL: ""}, + AllowedDomains: []string{host}, + MaxBytes: int64(test.maxBytes), + } + + if test.allowedDomains != nil { + eds.AllowedDomains = test.allowedDomains + } + + if test.name == "Test 5: ProxyWithConditionalHeaders" { + manifestFiles := map[string]*model.ManifestFile{ + fileName: { + ManifestFileMeta: &model.ManifestFileMeta{ + Name: fileName, + ETag: "manifest-test-etag", + LastModified: time.RFC1123, + }, + }, } + manifestJSON, mErr := json.MarshalIndent(manifestFiles, "", " ") + require.NoError(t, mErr) + + manifestFile, mErr := os.CreateTemp(tempDir, "manifest.json") + require.NoError(t, mErr) + _, mErr = manifestFile.Write(manifestJSON) + require.NoError(t, mErr) + _ = manifestFile.Close() + + fileManagerService.agentConfig.LibDir = tempDir + fileManagerService.manifestFilePath = manifestFile.Name() + + eds.ProxyURL = config.ProxyURL{URL: ts.URL} } + fileManagerService.agentConfig.ExternalDataSource = eds + fileManagerService.fileActions = map[string]*model.FileCache{ fileName: { File: &mpi.File{ @@ -1325,7 +1373,6 @@ func TestFileManagerService_downloadExternalFiles_Cases(t *testing.T) { if test.expectErrContains != "" { assert.Contains(t, err.Error(), test.expectErrContains) } - // ensure no temp file left _, statErr := os.Stat(tempFilePath(fileName)) assert.True(t, os.IsNotExist(statErr)) diff --git a/internal/file/file_service_operator_test.go b/internal/file/file_service_operator_test.go index f8206e145..f176dbe2a 100644 --- a/internal/file/file_service_operator_test.go +++ b/internal/file/file_service_operator_test.go @@ -187,3 +187,86 @@ func TestFileManagerService_UpdateFile_LargeFile(t *testing.T) { helpers.RemoveFileWithErrorCheck(t, testFile.Name()) } + +func TestFileServiceOperator_RenameExternalFile(t *testing.T) { + tests := []struct { + prepare func(t *testing.T) (src, dst string) + name string + wantErrMsg string + wantErr bool + }{ + { + name: "Test 1: success", + prepare: func(t *testing.T) (string, string) { + t.Helper() + tmp := t.TempDir() + src := filepath.Join(tmp, "src.txt") + dst := filepath.Join(tmp, "subdir", "dest.txt") + content := []byte("hello world") + require.NoError(t, os.WriteFile(src, content, 0o600)) + + return src, dst + }, + wantErr: false, + }, + { + name: "Test 2: mkdirall_fail", + prepare: func(t *testing.T) (string, string) { + t.Helper() + tmp := t.TempDir() + parentFile := filepath.Join(tmp, "not_a_dir") + require.NoError(t, os.WriteFile(parentFile, []byte("block"), 0o600)) + dst := filepath.Join(parentFile, "dest.txt") + src := filepath.Join(tmp, "src.txt") + require.NoError(t, os.WriteFile(src, []byte("content"), 0o600)) + + return src, dst + }, + wantErr: true, + wantErrMsg: "failed to create directories for", + }, + { + name: "Test 3: rename_fail", + prepare: func(t *testing.T) (string, string) { + t.Helper() + tmp := t.TempDir() + src := filepath.Join(tmp, "does_not_exist.txt") + dst := filepath.Join(tmp, "subdir", "dest.txt") + + return src, dst + }, + wantErr: true, + wantErrMsg: "failed to move file", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + fso := NewFileServiceOperator(types.AgentConfig(), nil, &sync.RWMutex{}) + + src, dst := tc.prepare(t) + + err := fso.RenameExternalFile(ctx, src, dst) + if tc.wantErr { + require.Error(t, err) + if tc.wantErrMsg != "" { + require.Contains(t, err.Error(), tc.wantErrMsg) + } + + return + } + + require.NoError(t, err) + + dstContent, readErr := os.ReadFile(dst) + require.NoError(t, readErr) + if tc.name == "success" { + require.Equal(t, []byte("hello world"), dstContent) + } + + _, statErr := os.Stat(src) + require.True(t, os.IsNotExist(statErr)) + }) + } +}