From 1714332a471ea07380c58beb5707e114e1dcad86 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 23 Apr 2026 08:40:34 +0000 Subject: [PATCH 1/2] api: treat in_use_newer like in_use for repository list query Add parseRepositoryListInUseQuery so in_use_mode=in_use_newer maps to the same filter as in_use. Document in_use vs in_use_mode precedence in Swagger. Repository list remains boolean InUse-only in the state store; semver rules apply only to per-repository tag views. Co-authored-by: Stefan Knott --- internal/api/api.go | 22 +++++++++ internal/api/handlers.go | 4 +- .../api/repository_list_in_use_query_test.go | 49 +++++++++++++++++++ internal/statestore/sqlite_repositories.go | 3 ++ 4 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 internal/api/repository_list_in_use_query_test.go diff --git a/internal/api/api.go b/internal/api/api.go index 8c96ae3..559fba8 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -275,6 +275,28 @@ func parseQueryParamBool(r *http.Request, key string) *bool { return &boolValue } +// parseRepositoryListInUseQuery resolves the repository list in-use filter. +// If "in_use" is present it wins; otherwise "in_use_mode" is used (legacy / UI compatibility). +// For repository listing, in_use_newer is equivalent to in_use (semver applies to tag lists only). +func parseRepositoryListInUseQuery(r *http.Request) *bool { + if _, ok := r.URL.Query()["in_use"]; ok { + return parseQueryParamBool(r, "in_use") + } + mode := strings.ToLower(strings.TrimSpace(parseQueryParam(r, "in_use_mode"))) + switch mode { + case "", "all": + return nil + case "in_use", "in-use", "in_use_newer", "in-use-newer": + v := true + return &v + case "not_in_use", "not-in-use", "out": + v := false + return &v + default: + return nil + } +} + // handleRootRedirect redirects / to /swagger/ func (s *APIServer) handleRootRedirect(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/" { diff --git a/internal/api/handlers.go b/internal/api/handlers.go index 3a17c95..2d55812 100644 --- a/internal/api/handlers.go +++ b/internal/api/handlers.go @@ -1092,6 +1092,8 @@ func (s *APIServer) handleGenerateKyvernoPolicy(w http.ResponseWriter, r *http.R // @Param search query string false "Filter by repository name" // @Param max_age query int false "Maximum age of last scan in seconds (e.g., 86400 for last 24 hours)" // @Param sort_by query string false "Sort order: age_desc (default), age_asc, name_asc, name_desc, status_asc, status_desc" Enums(age_desc,age_asc,name_asc,name_desc,status_asc,status_desc) +// @Param in_use query bool false "When set, filter by runtime in-use (true) or not in-use (false). Takes precedence over in_use_mode when both are sent." +// @Param in_use_mode query string false "Legacy filter: all (default), in_use, not_in_use, in_use_newer. For this endpoint in_use_newer is treated like in_use (semver applies to tag lists only). Ignored when in_use is present." Enums(all,in_use,not_in_use,in_use_newer) // @Param limit query int false "Maximum number of results" default(100) // @Param offset query int false "Pagination offset" default(0) // @Success 200 {object} map[string]interface{} "List of repositories with aggregated data" @@ -1106,7 +1108,7 @@ func (s *APIServer) handleListRepositories(w http.ResponseWriter, r *http.Reques } // Parse query parameters - inUse := parseQueryParamBool(r, "in_use") + inUse := parseRepositoryListInUseQuery(r) filter := statestore.RepositoryFilter{ Search: parseQueryParam(r, "search"), PolicyStatus: parseQueryParam(r, "policy_status"), diff --git a/internal/api/repository_list_in_use_query_test.go b/internal/api/repository_list_in_use_query_test.go new file mode 100644 index 0000000..6800005 --- /dev/null +++ b/internal/api/repository_list_in_use_query_test.go @@ -0,0 +1,49 @@ +package api + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestParseRepositoryListInUseQuery(t *testing.T) { + t.Parallel() + cases := []struct { + name string + raw string + wantNil bool + wantTrue bool + }{ + {name: "no_query", raw: "", wantNil: true}, + {name: "mode_all", raw: "?in_use_mode=all", wantNil: true}, + {name: "mode_in_use", raw: "?in_use_mode=in_use", wantTrue: true}, + {name: "mode_in_use_upper", raw: "?in_use_mode=IN_USE", wantTrue: true}, + {name: "mode_in_use_newer", raw: "?in_use_mode=in_use_newer", wantTrue: true}, + {name: "mode_in_use_newer_hyphen", raw: "?in_use_mode=in-use-newer", wantTrue: true}, + {name: "mode_not_in_use", raw: "?in_use_mode=not_in_use", wantTrue: false}, + {name: "mode_out", raw: "?in_use_mode=out", wantTrue: false}, + {name: "in_use_wins_false", raw: "?in_use=false&in_use_mode=in_use", wantTrue: false}, + {name: "in_use_wins_zero", raw: "?in_use=0&in_use_mode=in_use", wantTrue: false}, + {name: "in_use_wins_true", raw: "?in_use=true&in_use_mode=not_in_use", wantTrue: true}, + } + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + req := httptest.NewRequest(http.MethodGet, "/api/v1/repositories"+tc.raw, nil) + got := parseRepositoryListInUseQuery(req) + if tc.wantNil { + if got != nil { + t.Fatalf("expected nil, got %v", *got) + } + return + } + if got == nil { + t.Fatal("expected non-nil") + } + if *got != tc.wantTrue { + t.Fatalf("expected %v, got %v", tc.wantTrue, *got) + } + }) + } +} diff --git a/internal/statestore/sqlite_repositories.go b/internal/statestore/sqlite_repositories.go index f73b067..d6acaeb 100644 --- a/internal/statestore/sqlite_repositories.go +++ b/internal/statestore/sqlite_repositories.go @@ -10,6 +10,9 @@ import ( ) func (s *SQLiteStore) ListRepositories(ctx context.Context, filter RepositoryFilter) (*RepositoriesListResponse, error) { + // Repository list "in use" is a boolean filter from the API. Semver-based "newer than + // min in-use tag" applies only to per-repository tag lists (GetRepository), not here. + // First, get total count countQuery := ` SELECT COUNT(DISTINCT r.id) From f677fec21286cb19c8fcc720f3e47a9ff4f91edd Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 23 Apr 2026 08:51:24 +0000 Subject: [PATCH 2/2] statestore: denormalize repository list into repository_summary Add repository_summary with indexes and an AFTER INSERT trigger on repositories. Backfill on store open when counts drift or rows have updated_at=0. Refresh summary on RecordScan, cluster inventory sync/delete, whitelist changes, artifact digest cleanup, and excess scan cleanup. Rewrite ListRepositories to JOIN repository_summary with SQL filters and LIMIT/OFFSET. Repair uninitialized summary rows lazily on list for test DBs that insert repositories directly. Co-authored-by: Stefan Knott --- internal/statestore/sqlite.go | 31 ++ internal/statestore/sqlite_cleanup.go | 64 +++- internal/statestore/sqlite_repositories.go | 227 ++++--------- .../statestore/sqlite_repository_summary.go | 310 ++++++++++++++++++ .../sqlite_repository_summary_test.go | 78 +++++ internal/statestore/sqlite_runtime.go | 8 + internal/statestore/sqlite_scans_write.go | 4 + .../sqlite_tasks_runtime_unused_whitelist.go | 8 + internal/statestore/sqlite_test.go | 2 +- 9 files changed, 551 insertions(+), 181 deletions(-) create mode 100644 internal/statestore/sqlite_repository_summary.go create mode 100644 internal/statestore/sqlite_repository_summary_test.go diff --git a/internal/statestore/sqlite.go b/internal/statestore/sqlite.go index e14a67c..9f263a5 100644 --- a/internal/statestore/sqlite.go +++ b/internal/statestore/sqlite.go @@ -1,6 +1,7 @@ package statestore import ( + "context" "database/sql" "fmt" "time" @@ -69,6 +70,11 @@ func NewSQLiteStore(dbPath string) (*SQLiteStore, error) { return nil, errors.NewPermanentf("failed to initialize schema: %w", err) } + if err := store.ensureRepositorySummaries(context.Background()); err != nil { + db.Close() + return nil, errors.NewPermanentf("failed to ensure repository summaries: %w", err) + } + return store, nil } @@ -187,6 +193,31 @@ func (s *SQLiteStore) initSchema() error { created_at INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as integer)) ); + CREATE TABLE IF NOT EXISTS repository_summary ( + repository_id INTEGER PRIMARY KEY REFERENCES repositories(id) ON DELETE CASCADE, + artifact_count INTEGER NOT NULL DEFAULT 0, + last_scan_time INTEGER, + max_critical INTEGER NOT NULL DEFAULT 0, + max_high INTEGER NOT NULL DEFAULT 0, + max_medium INTEGER NOT NULL DEFAULT 0, + max_low INTEGER NOT NULL DEFAULT 0, + policy_passed INTEGER NOT NULL DEFAULT 1, + policy_status TEXT NOT NULL DEFAULT 'passed', + runtime_used INTEGER NOT NULL DEFAULT 0, + whitelisted INTEGER NOT NULL DEFAULT 0, + updated_at INTEGER NOT NULL DEFAULT 0 + ); + + CREATE INDEX IF NOT EXISTS idx_repository_summary_last_scan ON repository_summary(last_scan_time); + CREATE INDEX IF NOT EXISTS idx_repository_summary_list ON repository_summary(runtime_used, whitelisted, last_scan_time); + CREATE INDEX IF NOT EXISTS idx_repository_summary_policy ON repository_summary(policy_status, last_scan_time); + + CREATE TRIGGER IF NOT EXISTS tr_repositories_ai_repository_summary + AFTER INSERT ON repositories + BEGIN + INSERT OR IGNORE INTO repository_summary (repository_id) VALUES (NEW.id); + END; + CREATE INDEX IF NOT EXISTS idx_artifacts_repository ON artifacts(repository_id); CREATE INDEX IF NOT EXISTS idx_artifacts_digest ON artifacts(digest); CREATE INDEX IF NOT EXISTS idx_artifacts_next_scan ON artifacts(next_scan_at); diff --git a/internal/statestore/sqlite_cleanup.go b/internal/statestore/sqlite_cleanup.go index a8b26c0..4bb4e11 100644 --- a/internal/statestore/sqlite_cleanup.go +++ b/internal/statestore/sqlite_cleanup.go @@ -10,7 +10,10 @@ import ( ) func (s *SQLiteStore) CleanupArtifactScans(ctx context.Context, digest string) error { - return s.executeCleanup(ctx, func(tx *sql.Tx) error { + var repoIDsToRefresh []int64 + seenRefresh := make(map[int64]struct{}) + + err := s.executeCleanup(ctx, func(tx *sql.Tx) error { // First, get all artifact IDs and repository IDs for this digest rows, err := tx.QueryContext(ctx, ` SELECT a.id, a.repository_id @@ -93,11 +96,25 @@ func (s *SQLiteStore) CleanupArtifactScans(ctx context.Context, digest string) e if err != nil { return errors.NewTransientf("failed to delete empty repository: %w", err) } + } else { + if _, ok := seenRefresh[repositoryID]; !ok { + seenRefresh[repositoryID] = struct{}{} + repoIDsToRefresh = append(repoIDsToRefresh, repositoryID) + } } } return nil }) + if err != nil { + return err + } + for _, id := range repoIDsToRefresh { + if err := s.refreshRepositorySummary(ctx, id); err != nil { + return err + } + } + return nil } // executeCleanup is a helper method for transaction management in cleanup operations @@ -180,7 +197,9 @@ func (s *SQLiteStore) CleanupExcessScans(ctx context.Context, digest string, max return errors.NewPermanentf("maxScansToKeep must be positive, got %d", maxScansToKeep) } - return s.executeCleanup(ctx, func(tx *sql.Tx) error { + repoSeen := make(map[int64]struct{}) + + err := s.executeCleanup(ctx, func(tx *sql.Tx) error { // Get all artifact IDs for this digest rows, err := tx.QueryContext(ctx, ` SELECT id FROM artifacts WHERE digest = ? @@ -209,17 +228,35 @@ func (s *SQLiteStore) CleanupExcessScans(ctx context.Context, digest string, max // Clean up excess scans for each artifact for _, artifactID := range artifactIDs { - if err := s.cleanupExcessScansForArtifact(tx, ctx, artifactID, maxScansToKeep); err != nil { + changed, err := s.cleanupExcessScansForArtifact(tx, ctx, artifactID, maxScansToKeep) + if err != nil { return err } + if changed { + var repoID int64 + if err := tx.QueryRowContext(ctx, `SELECT repository_id FROM artifacts WHERE id = ?`, artifactID).Scan(&repoID); err != nil { + return errors.NewTransientf("failed to read repository_id for artifact after excess scan cleanup: %w", err) + } + repoSeen[repoID] = struct{}{} + } } return nil }) + if err != nil { + return err + } + for id := range repoSeen { + if err := s.refreshRepositorySummary(ctx, id); err != nil { + return err + } + } + return nil } -// cleanupExcessScansForArtifact is a helper to clean up scans for a single artifact -func (s *SQLiteStore) cleanupExcessScansForArtifact(tx *sql.Tx, ctx context.Context, artifactID int64, maxScansToKeep int) error { +// cleanupExcessScansForArtifact is a helper to clean up scans for a single artifact. +// It returns whether any scan rows were deleted (last_scan_id may have changed). +func (s *SQLiteStore) cleanupExcessScansForArtifact(tx *sql.Tx, ctx context.Context, artifactID int64, maxScansToKeep int) (bool, error) { // Get scan IDs to keep (most recent N scans) rows, err := tx.QueryContext(ctx, ` SELECT id FROM scan_records @@ -228,7 +265,7 @@ func (s *SQLiteStore) cleanupExcessScansForArtifact(tx *sql.Tx, ctx context.Cont LIMIT ? `, artifactID, maxScansToKeep) if err != nil { - return errors.NewTransientf("failed to query scans to keep: %w", err) + return false, errors.NewTransientf("failed to query scans to keep: %w", err) } defer rows.Close() @@ -236,18 +273,18 @@ func (s *SQLiteStore) cleanupExcessScansForArtifact(tx *sql.Tx, ctx context.Cont for rows.Next() { var scanID int64 if err := rows.Scan(&scanID); err != nil { - return errors.NewTransientf("failed to scan keep scan ID: %w", err) + return false, errors.NewTransientf("failed to scan keep scan ID: %w", err) } keepScanIDs = append(keepScanIDs, scanID) } if err := rows.Err(); err != nil { - return errors.NewTransientf("error iterating keep scan IDs: %w", err) + return false, errors.NewTransientf("error iterating keep scan IDs: %w", err) } // If we have fewer scans than the limit, nothing to clean up if len(keepScanIDs) < maxScansToKeep { - return nil + return false, nil } // Build placeholders for the IN clause @@ -267,12 +304,12 @@ func (s *SQLiteStore) cleanupExcessScansForArtifact(tx *sql.Tx, ctx context.Cont result, err := tx.ExecContext(ctx, deleteQuery, args...) if err != nil { - return errors.NewTransientf("failed to delete excess scan records: %w", err) + return false, errors.NewTransientf("failed to delete excess scan records: %w", err) } deletedCount, err := result.RowsAffected() if err != nil { - return errors.NewTransientf("failed to get deleted rows count: %w", err) + return false, errors.NewTransientf("failed to get deleted rows count: %w", err) } if deletedCount > 0 { @@ -284,10 +321,11 @@ func (s *SQLiteStore) cleanupExcessScansForArtifact(tx *sql.Tx, ctx context.Cont WHERE id = ? `, keepScanIDs[0], artifactID) // keepScanIDs[0] is the most recent if err != nil { - return errors.NewTransientf("failed to update artifact last_scan_id: %w", err) + return false, errors.NewTransientf("failed to update artifact last_scan_id: %w", err) } } + return true, nil } - return nil + return false, nil } diff --git a/internal/statestore/sqlite_repositories.go b/internal/statestore/sqlite_repositories.go index d6acaeb..c9c029d 100644 --- a/internal/statestore/sqlite_repositories.go +++ b/internal/statestore/sqlite_repositories.go @@ -13,242 +13,135 @@ func (s *SQLiteStore) ListRepositories(ctx context.Context, filter RepositoryFil // Repository list "in use" is a boolean filter from the API. Semver-based "newer than // min in-use tag" applies only to per-repository tag lists (GetRepository), not here. - // First, get total count - countQuery := ` - SELECT COUNT(DISTINCT r.id) - FROM repositories r - LEFT JOIN artifacts a ON r.id = a.repository_id - LEFT JOIN scan_records sr ON a.last_scan_id = sr.id - WHERE 1=1 - ` - countArgs := []interface{}{} + if err := s.repairRepositorySummariesForList(ctx); err != nil { + return nil, err + } + + whereSQL := " WHERE 1=1" + args := make([]interface{}, 0) if filter.Search != "" { - countQuery += " AND r.name LIKE ?" - countArgs = append(countArgs, "%"+filter.Search+"%") + whereSQL += " AND r.name LIKE ?" + args = append(args, "%"+filter.Search+"%") } if filter.MaxAge > 0 { - countQuery += " AND (SELECT MAX(sr2.created_at) FROM scan_records sr2 JOIN artifacts a2 ON sr2.artifact_id = a2.id WHERE a2.repository_id = r.id) >= ?" - countArgs = append(countArgs, time.Now().Unix()-int64(filter.MaxAge)) + whereSQL += " AND rs.last_scan_time IS NOT NULL AND rs.last_scan_time >= ?" + args = append(args, time.Now().Unix()-int64(filter.MaxAge)) + } + + if filter.PolicyStatus != "" { + whereSQL += " AND rs.policy_status = ?" + args = append(args, filter.PolicyStatus) + } + + if filter.InUse != nil { + if *filter.InUse { + whereSQL += " AND (rs.runtime_used = 1 OR rs.whitelisted = 1)" + } else { + whereSQL += " AND (rs.runtime_used = 0 OR rs.whitelisted = 1)" + } } + countQuery := `SELECT COUNT(*) FROM repositories r INNER JOIN repository_summary rs ON r.id = rs.repository_id` + whereSQL var total int - err := s.db.QueryRowContext(ctx, countQuery, countArgs...).Scan(&total) - if err != nil { + if err := s.db.QueryRowContext(ctx, countQuery, args...).Scan(&total); err != nil { return nil, errors.NewTransientf("failed to count repositories: %w", err) } - // Query repositories with aggregated data query := ` - SELECT - r.id, + SELECT r.name, - COUNT(DISTINCT a.id) as artifact_count, - (SELECT MAX(sr2.created_at) FROM scan_records sr2 - JOIN artifacts a2 ON sr2.artifact_id = a2.id - WHERE a2.repository_id = r.id) as last_scan_time, - MAX(sr.critical_vuln_count) as max_critical, - MAX(sr.high_vuln_count) as max_high, - MAX(sr.medium_vuln_count) as max_medium, - MAX(sr.low_vuln_count) as max_low, - CASE WHEN COUNT(CASE WHEN sr.policy_passed = 0 THEN 1 END) > 0 THEN 0 ELSE 1 END as policy_passed, - CASE - WHEN COUNT(CASE WHEN sr.policy_status = 'failed' THEN 1 END) > 0 THEN 'failed' - WHEN COUNT(CASE WHEN sr.policy_status = 'pending' THEN 1 END) > 0 THEN 'pending' - WHEN COUNT(CASE WHEN sr.policy_passed = 0 THEN 1 END) > 0 THEN 'failed' - ELSE 'passed' - END as policy_status, - CASE WHEN EXISTS (SELECT 1 FROM artifacts ai JOIN cluster_images ci ON ci.digest = ai.digest WHERE ai.repository_id = r.id) THEN 1 ELSE 0 END as runtime_used + rs.artifact_count, + rs.last_scan_time, + rs.max_critical, + rs.max_high, + rs.max_medium, + rs.max_low, + rs.policy_passed, + rs.policy_status, + rs.runtime_used, + rs.whitelisted FROM repositories r - LEFT JOIN artifacts a ON r.id = a.repository_id - LEFT JOIN scan_records sr ON a.last_scan_id = sr.id - WHERE 1=1 - ` - args := []interface{}{} + INNER JOIN repository_summary rs ON r.id = rs.repository_id + ` + whereSQL - if filter.Search != "" { - query += " AND r.name LIKE ?" - args = append(args, "%"+filter.Search+"%") - } - - if filter.MaxAge > 0 { - query += " AND (SELECT MAX(sr2.created_at) FROM scan_records sr2 JOIN artifacts a2 ON sr2.artifact_id = a2.id WHERE a2.repository_id = r.id) >= ?" - args = append(args, time.Now().Unix()-int64(filter.MaxAge)) - } - - query += " GROUP BY r.id, r.name" - - // Add sorting switch filter.SortBy { case "name_asc": query += " ORDER BY r.name ASC" case "name_desc": query += " ORDER BY r.name DESC" case "artifacts_asc": - query += " ORDER BY artifact_count ASC, r.name ASC" + query += " ORDER BY rs.artifact_count ASC, r.name ASC" case "artifacts_desc": - query += " ORDER BY artifact_count DESC, r.name ASC" + query += " ORDER BY rs.artifact_count DESC, r.name ASC" case "age_desc", "": - // Default: most recently scanned first - query += " ORDER BY last_scan_time DESC NULLS LAST" + query += " ORDER BY rs.last_scan_time DESC NULLS LAST, r.name ASC" case "age_asc": - // Oldest scanned first - query += " ORDER BY last_scan_time ASC NULLS FIRST" + query += " ORDER BY rs.last_scan_time ASC NULLS FIRST, r.name ASC" case "status_asc": - // Failed first (0), then passed (1) - query += " ORDER BY policy_passed ASC, r.name ASC" + query += " ORDER BY rs.policy_passed ASC, r.name ASC" case "status_desc": - // Passed first (1), then failed (0) - query += " ORDER BY policy_passed DESC, r.name ASC" + query += " ORDER BY rs.policy_passed DESC, r.name ASC" default: query += " ORDER BY r.name ASC" } - needsPostFilter := filter.InUse != nil || filter.PolicyStatus != "" - - if !needsPostFilter && filter.Limit > 0 { + listArgs := append([]interface{}{}, args...) + if filter.Limit > 0 { query += " LIMIT ?" - args = append(args, filter.Limit) + listArgs = append(listArgs, filter.Limit) } - - if !needsPostFilter && filter.Offset > 0 { + if filter.Offset > 0 { query += " OFFSET ?" - args = append(args, filter.Offset) + listArgs = append(listArgs, filter.Offset) } - rows, err := s.db.QueryContext(ctx, query, args...) + rows, err := s.db.QueryContext(ctx, query, listArgs...) if err != nil { return nil, errors.NewTransientf("failed to list repositories: %w", err) } defer rows.Close() var repositories []RepositoryInfo - repoNamesByID := make(map[int64]string) - repoIDs := make([]int64, 0) for rows.Next() { var repo RepositoryInfo - var repoID int64 // repository id (not needed in response, but must scan into something) var lastScanTimeUnix sql.NullInt64 - var maxCritical sql.NullInt64 - var maxHigh sql.NullInt64 - var maxMedium sql.NullInt64 - var maxLow sql.NullInt64 var policyPassed int - var policyStatus string var runtimeUsed int + var whitelisted int - err := rows.Scan( - &repoID, // repository id (not needed in response) + if err := rows.Scan( &repo.Name, &repo.ArtifactCount, &lastScanTimeUnix, - &maxCritical, - &maxHigh, - &maxMedium, - &maxLow, + &repo.VulnerabilityCount.Critical, + &repo.VulnerabilityCount.High, + &repo.VulnerabilityCount.Medium, + &repo.VulnerabilityCount.Low, &policyPassed, - &policyStatus, + &repo.PolicyStatus, &runtimeUsed, - ) - if err != nil { + &whitelisted, + ); err != nil { return nil, errors.NewTransientf("failed to scan repository row: %w", err) } - // Store Unix timestamps directly if lastScanTimeUnix.Valid { repo.LastScanTime = &lastScanTimeUnix.Int64 } - // Aggregate vulnerability counts from most vulnerable artifact - if maxCritical.Valid { - repo.VulnerabilityCount.Critical = int(maxCritical.Int64) - } - if maxHigh.Valid { - repo.VulnerabilityCount.High = int(maxHigh.Int64) - } - if maxMedium.Valid { - repo.VulnerabilityCount.Medium = int(maxMedium.Int64) - } - if maxLow.Valid { - repo.VulnerabilityCount.Low = int(maxLow.Int64) - } - repo.PolicyPassed = policyPassed == 1 - repo.PolicyStatus = policyStatus repo.RuntimeUsed = runtimeUsed == 1 + repo.Whitelisted = whitelisted == 1 repositories = append(repositories, repo) - repoNamesByID[repoID] = repo.Name - repoIDs = append(repoIDs, repoID) } if err := rows.Err(); err != nil { return nil, errors.NewTransientf("error iterating repository rows: %w", err) } - runtimeUsedByRepoID, err := s.repositoryRuntimeUsageByID(ctx, repoNamesByID) - if err != nil { - return nil, err - } - - whitelistSet, err := s.runtimeUnusedRepositoryWhitelistSet(ctx) - if err != nil { - return nil, err - } - - for i, repoID := range repoIDs { - repositories[i].RuntimeUsed = runtimeUsedByRepoID[repoID] - _, repositories[i].Whitelisted = whitelistSet[repositories[i].Name] - } - - if filter.InUse != nil || filter.PolicyStatus != "" { - filtered := repositories - - if filter.PolicyStatus != "" { - result := make([]RepositoryInfo, 0, len(filtered)) - for _, repo := range filtered { - if repo.PolicyStatus == filter.PolicyStatus { - result = append(result, repo) - } - } - filtered = result - } - - if filter.InUse != nil { - result := make([]RepositoryInfo, 0, len(filtered)) - for _, repo := range filtered { - matchesInUse := repo.RuntimeUsed || repo.Whitelisted - // Whitelisted repositories should appear in both in-use and not-in-use - // filtered views so operators can always discover and manage them. - if repo.Whitelisted || matchesInUse == *filter.InUse { - result = append(result, repo) - } - } - filtered = result - } - - total = len(filtered) - - start := filter.Offset - if start < 0 { - start = 0 - } - if start > len(filtered) { - start = len(filtered) - } - - end := len(filtered) - if filter.Limit > 0 { - candidateEnd := start + filter.Limit - if candidateEnd < end { - end = candidateEnd - } - } - - repositories = filtered[start:end] - } - return &RepositoriesListResponse{ Repositories: repositories, Total: total, diff --git a/internal/statestore/sqlite_repository_summary.go b/internal/statestore/sqlite_repository_summary.go new file mode 100644 index 0000000..a4492c5 --- /dev/null +++ b/internal/statestore/sqlite_repository_summary.go @@ -0,0 +1,310 @@ +package statestore + +import ( + "context" + "database/sql" + "log/slog" + "time" + + "github.com/daimoniac/suppline/internal/errors" +) + +// ensureRepositorySummaries inserts missing summary rows and backfills when the table +// is new, incomplete, or never populated (updated_at = 0). +func (s *SQLiteStore) ensureRepositorySummaries(ctx context.Context) error { + if _, err := s.db.ExecContext(ctx, ` + INSERT OR IGNORE INTO repository_summary (repository_id) + SELECT id FROM repositories + `); err != nil { + return errors.NewTransientf("failed to seed repository_summary rows: %w", err) + } + + var repoCount, summaryCount, staleCount, missingJoin int + if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM repositories`).Scan(&repoCount); err != nil { + return errors.NewTransientf("failed to count repositories: %w", err) + } + if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM repository_summary`).Scan(&summaryCount); err != nil { + return errors.NewTransientf("failed to count repository_summary: %w", err) + } + if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM repository_summary WHERE updated_at = 0`).Scan(&staleCount); err != nil { + return errors.NewTransientf("failed to count stale repository_summary rows: %w", err) + } + if err := s.db.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM repositories r + LEFT JOIN repository_summary rs ON rs.repository_id = r.id + WHERE rs.repository_id IS NULL + `).Scan(&missingJoin); err != nil { + return errors.NewTransientf("failed to count repositories missing summary: %w", err) + } + + if missingJoin == 0 && staleCount == 0 && repoCount == summaryCount { + return nil + } + + slog.Info("repository_summary backfill starting", + "repositories", repoCount, + "summaries", summaryCount, + "stale_rows", staleCount, + "missing_summary_rows", missingJoin, + ) + + rows, err := s.db.QueryContext(ctx, `SELECT id FROM repositories ORDER BY id`) + if err != nil { + return errors.NewTransientf("failed to list repositories for summary backfill: %w", err) + } + defer rows.Close() + + var ids []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return errors.NewTransientf("failed to scan repository id: %w", err) + } + ids = append(ids, id) + } + if err := rows.Err(); err != nil { + return errors.NewTransientf("error iterating repositories for summary backfill: %w", err) + } + + for _, id := range ids { + if err := s.refreshRepositorySummary(ctx, id); err != nil { + return err + } + } + + slog.Info("repository_summary backfill complete", "repositories", len(ids)) + return nil +} + +// repairRepositorySummariesForList ensures rows inserted outside RecordScan (e.g. tests) are populated. +func (s *SQLiteStore) repairRepositorySummariesForList(ctx context.Context) error { + if _, err := s.db.ExecContext(ctx, ` + INSERT OR IGNORE INTO repository_summary (repository_id) + SELECT id FROM repositories + `); err != nil { + return errors.NewTransientf("failed to seed repository_summary for list: %w", err) + } + + var stale int + if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM repository_summary WHERE updated_at = 0`).Scan(&stale); err != nil { + return errors.NewTransientf("failed to count uninitialized repository_summary rows: %w", err) + } + if stale == 0 { + return nil + } + + rows, err := s.db.QueryContext(ctx, `SELECT repository_id FROM repository_summary WHERE updated_at = 0`) + if err != nil { + return errors.NewTransientf("failed to query stale repository_summary rows: %w", err) + } + defer rows.Close() + + var ids []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return errors.NewTransientf("failed to scan stale repository_summary id: %w", err) + } + ids = append(ids, id) + } + if err := rows.Err(); err != nil { + return errors.NewTransientf("error iterating stale repository_summary: %w", err) + } + + for _, id := range ids { + if err := s.refreshRepositorySummary(ctx, id); err != nil { + return err + } + } + return nil +} + +// refreshRepositorySummary recomputes denormalized list fields for one repository. +func (s *SQLiteStore) refreshRepositorySummary(ctx context.Context, repositoryID int64) error { + var repoName string + err := s.db.QueryRowContext(ctx, `SELECT name FROM repositories WHERE id = ?`, repositoryID).Scan(&repoName) + if err == sql.ErrNoRows { + return nil + } + if err != nil { + return errors.NewTransientf("failed to load repository name for summary: %w", err) + } + + var artifactCount int + var lastScanTime sql.NullInt64 + var maxCritical, maxHigh, maxMedium, maxLow int64 + var policyPassed int + var policyStatus string + + err = s.db.QueryRowContext(ctx, ` + SELECT + COUNT(DISTINCT a.id), + (SELECT MAX(sr2.created_at) FROM scan_records sr2 + JOIN artifacts a2 ON sr2.artifact_id = a2.id + WHERE a2.repository_id = r.id), + COALESCE(MAX(sr.critical_vuln_count), 0), + COALESCE(MAX(sr.high_vuln_count), 0), + COALESCE(MAX(sr.medium_vuln_count), 0), + COALESCE(MAX(sr.low_vuln_count), 0), + CASE WHEN COUNT(CASE WHEN sr.policy_passed = 0 THEN 1 END) > 0 THEN 0 ELSE 1 END, + CASE + WHEN COUNT(CASE WHEN sr.policy_status = 'failed' THEN 1 END) > 0 THEN 'failed' + WHEN COUNT(CASE WHEN sr.policy_status = 'pending' THEN 1 END) > 0 THEN 'pending' + WHEN COUNT(CASE WHEN sr.policy_passed = 0 THEN 1 END) > 0 THEN 'failed' + ELSE 'passed' + END + FROM repositories r + LEFT JOIN artifacts a ON a.repository_id = r.id + LEFT JOIN scan_records sr ON a.last_scan_id = sr.id + WHERE r.id = ? + GROUP BY r.id + `, repositoryID).Scan( + &artifactCount, + &lastScanTime, + &maxCritical, + &maxHigh, + &maxMedium, + &maxLow, + &policyPassed, + &policyStatus, + ) + if err != nil { + return errors.NewTransientf("failed to aggregate repository summary: %w", err) + } + + runtimeByID, err := s.repositoryRuntimeUsageByID(ctx, map[int64]string{repositoryID: repoName}) + if err != nil { + return err + } + runtimeUsed := 0 + if runtimeByID[repositoryID] { + runtimeUsed = 1 + } + + whitelisted := 0 + if err := s.db.QueryRowContext(ctx, ` + SELECT EXISTS(SELECT 1 FROM runtime_unused_repository_whitelist WHERE repository = ?) + `, repoName).Scan(&whitelisted); err != nil { + return errors.NewTransientf("failed to query whitelist for repository summary: %w", err) + } + + now := time.Now().Unix() + var lastScanArg interface{} + if lastScanTime.Valid { + lastScanArg = lastScanTime.Int64 + } else { + lastScanArg = nil + } + + if _, err := s.db.ExecContext(ctx, ` + INSERT INTO repository_summary ( + repository_id, artifact_count, last_scan_time, + max_critical, max_high, max_medium, max_low, + policy_passed, policy_status, runtime_used, whitelisted, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(repository_id) DO UPDATE SET + artifact_count = excluded.artifact_count, + last_scan_time = excluded.last_scan_time, + max_critical = excluded.max_critical, + max_high = excluded.max_high, + max_medium = excluded.max_medium, + max_low = excluded.max_low, + policy_passed = excluded.policy_passed, + policy_status = excluded.policy_status, + runtime_used = excluded.runtime_used, + whitelisted = excluded.whitelisted, + updated_at = excluded.updated_at + `, + repositoryID, + artifactCount, + lastScanArg, + maxCritical, + maxHigh, + maxMedium, + maxLow, + policyPassed, + policyStatus, + runtimeUsed, + whitelisted, + now, + ); err != nil { + return errors.NewTransientf("failed to upsert repository_summary: %w", err) + } + + return nil +} + +// refreshRuntimeUsedAllRepositories updates runtime_used for every repository (after cluster inventory changes). +func (s *SQLiteStore) refreshRuntimeUsedAllRepositories(ctx context.Context) error { + rows, err := s.db.QueryContext(ctx, `SELECT id, name FROM repositories ORDER BY id`) + if err != nil { + return errors.NewTransientf("failed to list repositories for runtime summary refresh: %w", err) + } + defer rows.Close() + + namesByID := make(map[int64]string) + for rows.Next() { + var id int64 + var name string + if err := rows.Scan(&id, &name); err != nil { + return errors.NewTransientf("failed to scan repository row: %w", err) + } + namesByID[id] = name + } + if err := rows.Err(); err != nil { + return errors.NewTransientf("error iterating repositories: %w", err) + } + + if len(namesByID) == 0 { + return nil + } + + if _, err := s.db.ExecContext(ctx, ` + INSERT OR IGNORE INTO repository_summary (repository_id) SELECT id FROM repositories + `); err != nil { + return errors.NewTransientf("failed to ensure repository_summary rows before runtime refresh: %w", err) + } + + runtimeByID, err := s.repositoryRuntimeUsageByID(ctx, namesByID) + if err != nil { + return err + } + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return errors.NewTransientf("failed to begin transaction for runtime summary refresh: %w", err) + } + defer func() { _ = tx.Rollback() }() + + now := time.Now().Unix() + for id := range namesByID { + runtimeUsed := 0 + if runtimeByID[id] { + runtimeUsed = 1 + } + if _, err := tx.ExecContext(ctx, ` + UPDATE repository_summary SET runtime_used = ?, updated_at = ? WHERE repository_id = ? + `, runtimeUsed, now, id); err != nil { + return errors.NewTransientf("failed to update runtime_used in repository_summary: %w", err) + } + } + + if err := tx.Commit(); err != nil { + return errors.NewTransientf("failed to commit runtime summary refresh: %w", err) + } + + return nil +} + +// refreshRepositorySummaryByName runs refreshRepositorySummary when the repository exists. +func (s *SQLiteStore) refreshRepositorySummaryByName(ctx context.Context, repositoryName string) error { + var repoID int64 + err := s.db.QueryRowContext(ctx, `SELECT id FROM repositories WHERE name = ?`, repositoryName).Scan(&repoID) + if err == sql.ErrNoRows { + return nil + } + if err != nil { + return errors.NewTransientf("failed to resolve repository for summary refresh: %w", err) + } + return s.refreshRepositorySummary(ctx, repoID) +} diff --git a/internal/statestore/sqlite_repository_summary_test.go b/internal/statestore/sqlite_repository_summary_test.go new file mode 100644 index 0000000..e87b2f9 --- /dev/null +++ b/internal/statestore/sqlite_repository_summary_test.go @@ -0,0 +1,78 @@ +package statestore + +import ( + "context" + "database/sql" + "os" + "testing" + "time" +) + +func TestListRepositories_RepositorySummaryMatchesAggregates(t *testing.T) { + ctx := context.Background() + dbPath := "test_repo_summary_list_" + t.Name() + ".db" + defer os.Remove(dbPath) + + store, err := NewSQLiteStore(dbPath) + if err != nil { + t.Fatalf("NewSQLiteStore: %v", err) + } + defer store.Close() + + repoName := "registry.example/ns/app" + now := time.Now().Unix() + if err := store.RecordScan(ctx, &ScanRecord{ + Repository: repoName, + Digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Tag: "1.0.0", + CriticalVulnCount: 2, + HighVulnCount: 1, + MediumVulnCount: 0, + LowVulnCount: 0, + PolicyPassed: true, + PolicyStatus: "passed", + CreatedAt: now, + }); err != nil { + t.Fatalf("RecordScan: %v", err) + } + + var ac int + var lst sql.NullInt64 + var mc, mh, mm, ml int + var pp int + var ps string + err = store.db.QueryRowContext(ctx, ` + SELECT artifact_count, last_scan_time, max_critical, max_high, max_medium, max_low, + policy_passed, policy_status + FROM repository_summary rs + JOIN repositories r ON r.id = rs.repository_id + WHERE r.name = ? + `, repoName).Scan(&ac, &lst, &mc, &mh, &mm, &ml, &pp, &ps) + if err != nil { + t.Fatalf("query summary: %v", err) + } + if ac != 1 || !lst.Valid || lst.Int64 != now { + t.Fatalf("unexpected summary row: count=%d last=%v", ac, lst) + } + if mc != 2 || mh != 1 || mm != 0 || ml != 0 { + t.Fatalf("unexpected vuln max in summary: %d %d %d %d", mc, mh, mm, ml) + } + if pp != 1 || ps != "passed" { + t.Fatalf("unexpected policy in summary: %d %q", pp, ps) + } + + resp, err := store.ListRepositories(ctx, RepositoryFilter{Limit: 10, Offset: 0}) + if err != nil { + t.Fatalf("ListRepositories: %v", err) + } + if resp.Total != 1 || len(resp.Repositories) != 1 { + t.Fatalf("list total=%d repos=%d", resp.Total, len(resp.Repositories)) + } + r := resp.Repositories[0] + if r.Name != repoName || r.ArtifactCount != 1 || r.PolicyStatus != "passed" { + t.Fatalf("unexpected list row: %+v", r) + } + if r.VulnerabilityCount.Critical != 2 || r.VulnerabilityCount.High != 1 { + t.Fatalf("unexpected list vulns: %+v", r.VulnerabilityCount) + } +} diff --git a/internal/statestore/sqlite_runtime.go b/internal/statestore/sqlite_runtime.go index fa64402..77b094f 100644 --- a/internal/statestore/sqlite_runtime.go +++ b/internal/statestore/sqlite_runtime.go @@ -102,6 +102,10 @@ func (s *SQLiteStore) RecordClusterInventory(ctx context.Context, clusterName st return errors.NewTransientf("failed to commit cluster inventory transaction: %w", err) } + if err := s.refreshRuntimeUsedAllRepositories(ctx); err != nil { + return err + } + return nil } @@ -266,6 +270,10 @@ func (s *SQLiteStore) DeleteClusterInventory(ctx context.Context, clusterName st return errors.NewTransientf("failed to delete cluster inventory: %w", err) } + if err := s.refreshRuntimeUsedAllRepositories(ctx); err != nil { + return err + } + return nil } diff --git a/internal/statestore/sqlite_scans_write.go b/internal/statestore/sqlite_scans_write.go index f8896f9..1749afd 100644 --- a/internal/statestore/sqlite_scans_write.go +++ b/internal/statestore/sqlite_scans_write.go @@ -166,6 +166,10 @@ func (s *SQLiteStore) RecordScan(ctx context.Context, record *ScanRecord) error return errors.NewTransientf("failed to commit transaction: %w", err) } + if err := s.refreshRepositorySummary(ctx, repositoryID); err != nil { + return err + } + return nil } diff --git a/internal/statestore/sqlite_tasks_runtime_unused_whitelist.go b/internal/statestore/sqlite_tasks_runtime_unused_whitelist.go index 7d73b99..0a923d0 100644 --- a/internal/statestore/sqlite_tasks_runtime_unused_whitelist.go +++ b/internal/statestore/sqlite_tasks_runtime_unused_whitelist.go @@ -49,6 +49,10 @@ func (s *SQLiteStore) AddRuntimeUnusedRepositoryWhitelist(ctx context.Context, r return errors.NewTransientf("failed to add runtime-unused repository whitelist entry: %w", err) } + if err := s.refreshRepositorySummaryByName(ctx, repository); err != nil { + return err + } + return nil } @@ -65,6 +69,10 @@ func (s *SQLiteStore) RemoveRuntimeUnusedRepositoryWhitelist(ctx context.Context return errors.NewTransientf("failed to remove runtime-unused repository whitelist entry: %w", err) } + if err := s.refreshRepositorySummaryByName(ctx, repository); err != nil { + return err + } + return nil } diff --git a/internal/statestore/sqlite_test.go b/internal/statestore/sqlite_test.go index f3564f1..562083d 100644 --- a/internal/statestore/sqlite_test.go +++ b/internal/statestore/sqlite_test.go @@ -870,7 +870,7 @@ func TestSchemaAndConstraints(t *testing.T) { // Test 1: Verify all tables exist t.Run("All tables are created", func(t *testing.T) { - tables := []string{"repositories", "artifacts", "scan_records", "vulnerabilities"} + tables := []string{"repositories", "artifacts", "scan_records", "vulnerabilities", "repository_summary"} for _, table := range tables { var count int err := store.db.QueryRowContext(ctx, `