Skip to content

Poppy speedup #36

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
cvkramer wants to merge 6 commits into
base: main
Choose a base branch
from
Open

Poppy speedup #36

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e6726d4
Eliminate redundant file reads and cache rebuilding by implementing
c1-charleskramer Jan 13, 2026
1fec62b
Changes Made:
c1-charleskramer Jan 13, 2026
52f14ec
Changes Made:
c1-charleskramer Jan 13, 2026
8ff7e3b
1. Added constant in pkg/connector/syncers.go:
c1-charleskramer Jan 13, 2026
76a8945
Add cache invalidation between syncs
c1-charleskramer Jan 13, 2026
5f97268
cache invalidation unnecessary, handled by baton-sdk
c1-charleskramer Jan 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 86 additions & 0 deletions pkg/connector/cache.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package connector
import (
"context"
"fmt"
"sort"
"strings"
"time"

Expand Down Expand Up @@ -317,3 +318,88 @@ func buildEntitlementCache(ctx context.Context, entitlements []EntitlementData,
l.Info("Built entitlement cache", zap.Int("count", len(cache)))
return cache, nil
}

// buildChildTypesIndex creates an index mapping parent resource IDs to their child resource types.
// This eliminates the need to scan all resources when checking for child types in the List method.
func buildChildTypesIndex(ctx context.Context, resourceData []ResourceData, resourceCache map[string]*v2.Resource) map[string]map[string]struct{} {
l := ctxzap.Extract(ctx)
index := make(map[string]map[string]struct{})

for _, rData := range resourceData {
if rData.ParentResource == "" {
continue
}

// Check if parent exists in the cache
if _, exists := resourceCache[rData.ParentResource]; !exists {
continue
}

childTypeId := strings.ToLower(rData.ResourceType)

// Initialize the set for this parent if it doesn't exist
if index[rData.ParentResource] == nil {
index[rData.ParentResource] = make(map[string]struct{})
}

// Add the child type to the parent's set
index[rData.ParentResource][childTypeId] = struct{}{}
}

l.Debug("Built child types index", zap.Int("parents_with_children", len(index)))
return index
}

// buildSortedResourcesByType creates pre-sorted lists of resources grouped by type.
// This eliminates the need to filter and sort on every pagination request.
func buildSortedResourcesByType(ctx context.Context, resourceCache map[string]*v2.Resource) map[string][]*v2.Resource {
l := ctxzap.Extract(ctx)
index := make(map[string][]*v2.Resource)

// Group resources by type
for _, res := range resourceCache {
typeId := res.Id.ResourceType
index[typeId] = append(index[typeId], res)
}

// Sort each type's list
totalResources := 0
for _, resources := range index {
sort.SliceStable(resources, func(i, j int) bool {
return resources[i].Id.Resource < resources[j].Id.Resource
})
totalResources += len(resources)
}

l.Debug("Built sorted resources by type index",
zap.Int("resource_types", len(index)),
zap.Int("total_resources", totalResources))
return index
}

// buildSortedEntitlementsByResource creates pre-sorted lists of entitlements grouped by resource.
// This eliminates the need to filter and sort on every pagination request.
func buildSortedEntitlementsByResource(ctx context.Context, entitlementCache map[string]*v2.Entitlement) map[string][]*v2.Entitlement {
l := ctxzap.Extract(ctx)
index := make(map[string][]*v2.Entitlement)

// Group entitlements by resource
for _, ent := range entitlementCache {
resourceId := ent.Resource.Id.Resource
index[resourceId] = append(index[resourceId], ent)
}

// Sort each resource's entitlement list
totalEntitlements := 0
for _, entitlements := range index {
sort.SliceStable(entitlements, func(i, j int) bool {
return entitlements[i].Slug < entitlements[j].Slug
})
totalEntitlements += len(entitlements)
}

l.Debug("Built sorted entitlements by resource index",
zap.Int("resources_with_entitlements", len(index)),
zap.Int("total_entitlements", totalEntitlements))
return index
}
90 changes: 81 additions & 9 deletions pkg/connector/connector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,28 +34,100 @@ func (fc *FileConnector) Validate(ctx context.Context) (annotations.Annotations,
return nil, nil
}

// getCachedData returns cached file data and built caches, loading them if not already cached.
// This method is thread-safe and ensures the file is only loaded and processed once per sync.
// The SDK's session cache mechanism ensures fresh data between syncs automatically.
func (fc *FileConnector) getCachedData(ctx context.Context) (*LoadedData, map[string]*v2.ResourceType, map[string]*v2.Resource, map[string]*v2.Entitlement, map[string]map[string]struct{}, error) {
l := ctxzap.Extract(ctx)

// Fast path: check if cache is already populated using read lock
fc.cacheMutex.RLock()
if fc.cachedData != nil {
loadedData := fc.cachedData
resourceTypes := fc.cachedResourceTypes
resources := fc.cachedResources
entitlements := fc.cachedEntitlements
childTypes := fc.cachedChildTypes
fc.cacheMutex.RUnlock()
return loadedData, resourceTypes, resources, entitlements, childTypes, nil
}
fc.cacheMutex.RUnlock()

// Slow path: need to load and build caches, acquire write lock
fc.cacheMutex.Lock()
defer fc.cacheMutex.Unlock()

// Double-check after acquiring write lock (another goroutine may have loaded it)
if fc.cachedData != nil {
return fc.cachedData, fc.cachedResourceTypes, fc.cachedResources, fc.cachedEntitlements, fc.cachedChildTypes, nil
}

l.Info("Loading and caching file data", zap.String("input_file_path", fc.inputFilePath))

// Load file data
loadedData, err := LoadFileData(fc.inputFilePath)
if err != nil {
return nil, nil, nil, nil, nil, fmt.Errorf("failed to load data file: %w", err)
}

// Build all caches
resourceTypesCache, err := buildResourceTypeCache(ctx, loadedData.Resources, loadedData.Users)
if err != nil {
return nil, nil, nil, nil, nil, fmt.Errorf("failed to build resource type cache: %w", err)
}

resourceCache, err := buildResourceCache(ctx, loadedData.Users, loadedData.Resources, resourceTypesCache)
if err != nil {
return nil, nil, nil, nil, nil, fmt.Errorf("failed to build resource cache: %w", err)
}

entitlementCache, err := buildEntitlementCache(ctx, loadedData.Entitlements, resourceCache)
if err != nil {
return nil, nil, nil, nil, nil, fmt.Errorf("failed to build entitlement cache: %w", err)
}

// Build child types index to optimize child type lookups
childTypesIndex := buildChildTypesIndex(ctx, loadedData.Resources, resourceCache)

// Build sorted indexes to eliminate sorting on every pagination request
sortedResourcesByType := buildSortedResourcesByType(ctx, resourceCache)
sortedEntitlementsByResource := buildSortedEntitlementsByResource(ctx, entitlementCache)

// Store in cache
fc.cachedData = loadedData
fc.cachedResourceTypes = resourceTypesCache
fc.cachedResources = resourceCache
fc.cachedEntitlements = entitlementCache
fc.cachedChildTypes = childTypesIndex
fc.cachedSortedResourcesByType = sortedResourcesByType
fc.cachedSortedEntitlementsByRes = sortedEntitlementsByResource

l.Info("Successfully cached file data",
zap.Int("users", len(loadedData.Users)),
zap.Int("resources", len(loadedData.Resources)),
zap.Int("entitlements", len(loadedData.Entitlements)),
zap.Int("grants", len(loadedData.Grants)))

return loadedData, resourceTypesCache, resourceCache, entitlementCache, childTypesIndex, nil
}

// ResourceSyncers returns a list of syncers for the connector.
// function is required by the connectorbuilder.Connector interface.
// It determines resource types from the input file and creates a syncer instance for each type, enabling the SDK to sync them.
// implementation loads minimal data to find resource types, builds the type cache, and creates simple syncers passing only the file path for per-sync loading.
// implementation loads minimal data to find resource types, builds the type cache, and creates simple syncers passing the connector reference.
func (fc *FileConnector) ResourceSyncers(ctx context.Context) []connectorbuilder.ResourceSyncer {
l := ctxzap.Extract(ctx)
l.Info("ResourceSyncers method called", zap.String("input_file_path", fc.inputFilePath))
loadedData, err := LoadFileData(fc.inputFilePath)
if err != nil {
l.Error("Failed to load input data file to determine resource types", zap.Error(err))
return nil
}

resourceTypesCache, err := buildResourceTypeCache(ctx, loadedData.Resources, loadedData.Users)
_, resourceTypesCache, _, _, _, err := fc.getCachedData(ctx)
if err != nil {
l.Error("Failed to build resource type cache", zap.Error(err))
l.Error("Failed to load and cache data", zap.Error(err))
return nil
}

rv := make([]connectorbuilder.ResourceSyncer, 0, len(resourceTypesCache))
for _, rt := range resourceTypesCache {
rv = append(rv, newFileSyncer(rt, fc.inputFilePath))
rv = append(rv, newFileSyncer(rt, fc))
}

l.Info("Created resource syncers", zap.Int("count", len(rv)))
Expand Down
18 changes: 16 additions & 2 deletions pkg/connector/models.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,29 @@ package connector
import (
"context"
"fmt"
"sync"

v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2"
)

// FileConnector struct is the main implementation of the Baton connector for file processing.
// It is required by the connectorbuilder.Connector interface for defining connector behavior.
// It holds the path to the input data file.
// structure provides the context (file path) needed for loading data during sync operations.
// It holds the path to the input data file and caches loaded data to avoid redundant file reads.
// The file is treated as static during a sync operation - cache is built once and reused.
// Instances are created by NewFileConnector.
type FileConnector struct {
inputFilePath string

// Cache fields to avoid redundant file loading and cache rebuilding
cacheMutex sync.RWMutex
cachedData *LoadedData
cachedResourceTypes map[string]*v2.ResourceType
cachedResources map[string]*v2.Resource
cachedEntitlements map[string]*v2.Entitlement
cachedChildTypes map[string]map[string]struct{} // parent resource ID -> set of child resource type IDs
// Pre-sorted indexes to eliminate sorting on every page request
cachedSortedResourcesByType map[string][]*v2.Resource // resource type ID -> sorted list of resources
cachedSortedEntitlementsByRes map[string][]*v2.Entitlement // resource ID -> sorted list of entitlements
}

// LoadedData holds all the data parsed from the input file.
Expand Down
Loading