From 1cdf9c30696c76416e67054e8158e33af63c0896 Mon Sep 17 00:00:00 2001 From: Vigneshraj Sekar Babu Date: Mon, 27 Apr 2026 16:47:20 -0700 Subject: [PATCH 1/2] fix: refresh external secrets before secret use --- .../__tests__/forwardedEnv.test.ts | 7 ++- src/server/lib/agentSession/forwardedEnv.ts | 7 ++- .../__tests__/externalSecret.test.ts | 30 ++++++++- src/server/lib/kubernetes/externalSecret.ts | 33 +++++++++- .../__tests__/secretProcessor.test.ts | 61 +++++++++++++++++++ src/server/services/deploy.ts | 3 +- src/server/services/secretProcessor.ts | 33 ++++++++-- 7 files changed, 163 insertions(+), 11 deletions(-) diff --git a/src/server/lib/agentSession/__tests__/forwardedEnv.test.ts b/src/server/lib/agentSession/__tests__/forwardedEnv.test.ts index 691072b3..39543d79 100644 --- a/src/server/lib/agentSession/__tests__/forwardedEnv.test.ts +++ b/src/server/lib/agentSession/__tests__/forwardedEnv.test.ts @@ -84,6 +84,7 @@ describe('forwardedEnv', () => { processEnvSecrets: jest.fn().mockResolvedValue({ secretRefs: [], expectedKeysPerSecret: {}, + syncTokensPerSecret: {}, warnings: [], }), waitForSecretSync: jest.fn().mockResolvedValue(undefined), @@ -195,6 +196,9 @@ describe('forwardedEnv', () => { expectedKeysPerSecret: { 'agent-env-session-123-aws-secrets': ['PRIVATE_REGISTRY_TOKEN'], }, + syncTokensPerSecret: { + 'agent-env-session-123-aws-secrets': 'sync-123', + }, warnings: [], }); const waitForSecretSync = jest.fn().mockResolvedValue(undefined); @@ -241,7 +245,8 @@ describe('forwardedEnv', () => { expect(waitForSecretSync).toHaveBeenCalledWith( { 'agent-env-session-123-aws-secrets': ['PRIVATE_REGISTRY_TOKEN'] }, 'test-ns', - 30000 + 30000, + { 'agent-env-session-123-aws-secrets': 'sync-123' } ); expect(result.secretProviders).toEqual(['aws']); }); diff --git a/src/server/lib/agentSession/forwardedEnv.ts b/src/server/lib/agentSession/forwardedEnv.ts index 3e2055f0..d9b7d825 100644 --- a/src/server/lib/agentSession/forwardedEnv.ts +++ b/src/server/lib/agentSession/forwardedEnv.ts @@ -157,7 +157,12 @@ export async function resolveForwardedAgentEnv( .map((provider) => provider.secretSyncTimeout) .filter((timeout): timeout is number => timeout !== undefined); const timeout = providerTimeouts.length > 0 ? Math.max(...providerTimeouts) * 1000 : 60000; - await secretProcessor.waitForSecretSync(secretResult.expectedKeysPerSecret, namespace, timeout); + await secretProcessor.waitForSecretSync( + secretResult.expectedKeysPerSecret, + namespace, + timeout, + secretResult.syncTokensPerSecret + ); } return { diff --git a/src/server/lib/kubernetes/__tests__/externalSecret.test.ts b/src/server/lib/kubernetes/__tests__/externalSecret.test.ts index a75851c5..ad1cc7f2 100644 --- a/src/server/lib/kubernetes/__tests__/externalSecret.test.ts +++ b/src/server/lib/kubernetes/__tests__/externalSecret.test.ts @@ -14,7 +14,13 @@ * limitations under the License. */ -import { generateExternalSecretManifest, generateSecretName, groupSecretRefsByProvider } from '../externalSecret'; +import { + EXTERNAL_SECRET_FORCE_SYNC_ANNOTATION, + generateExternalSecretManifest, + generateSecretName, + groupSecretRefsByProvider, + TARGET_SECRET_SYNC_TOKEN_ANNOTATION, +} from '../externalSecret'; import { SecretRefWithEnvKey } from 'server/lib/secretRefs'; describe('externalSecret', () => { @@ -90,6 +96,7 @@ describe('externalSecret', () => { expect(manifest.spec.secretStoreRef.name).toBe('aws-secretsmanager'); expect(manifest.spec.secretStoreRef.kind).toBe('ClusterSecretStore'); expect(manifest.spec.target.name).toBe('api-server-aws-secrets'); + expect(manifest.spec.target.deletionPolicy).toBe('Merge'); expect(manifest.spec.data).toHaveLength(2); }); @@ -167,5 +174,26 @@ describe('externalSecret', () => { expect(manifest.metadata.labels['app.kubernetes.io/managed-by']).toBe('lifecycle'); expect(manifest.metadata.labels['lfc/uuid']).toBe('abc123'); }); + + it('adds sync metadata when a force sync token is provided', () => { + const refs: SecretRefWithEnvKey[] = [{ envKey: 'SECRET', provider: 'aws', path: 'path', key: 'key' }]; + + const manifest = generateExternalSecretManifest({ + name: 'api-server', + namespace: 'lfc-abc123', + provider: 'aws', + secretRefs: refs, + providerConfig, + forceSyncToken: 'sync-123', + }); + + expect(manifest.metadata.annotations).toEqual({ + [EXTERNAL_SECRET_FORCE_SYNC_ANNOTATION]: 'sync-123', + }); + expect(manifest.spec.target.template?.metadata.annotations).toEqual({ + [TARGET_SECRET_SYNC_TOKEN_ANNOTATION]: 'sync-123', + }); + expect(manifest.spec.target.template?.metadata.labels).toEqual(manifest.metadata.labels); + }); }); }); diff --git a/src/server/lib/kubernetes/externalSecret.ts b/src/server/lib/kubernetes/externalSecret.ts index 642f6997..4de4b73e 100644 --- a/src/server/lib/kubernetes/externalSecret.ts +++ b/src/server/lib/kubernetes/externalSecret.ts @@ -22,6 +22,9 @@ import { SecretRefWithEnvKey } from 'server/lib/secretRefs'; import { SecretProviderConfig } from 'server/services/types/globalConfig'; import { buildLifecycleLabels } from 'server/lib/kubernetes/labels'; +export const EXTERNAL_SECRET_FORCE_SYNC_ANNOTATION = 'force-sync'; +export const TARGET_SECRET_SYNC_TOKEN_ANNOTATION = 'lifecycle.dev/secret-sync-token'; + export interface ExternalSecretManifest { apiVersion: string; kind: string; @@ -29,6 +32,7 @@ export interface ExternalSecretManifest { name: string; namespace: string; labels: Record; + annotations?: Record; }; spec: { refreshInterval: string; @@ -38,6 +42,13 @@ export interface ExternalSecretManifest { }; target: { name: string; + deletionPolicy: 'Merge'; + template?: { + metadata: { + labels: Record; + annotations: Record; + }; + }; }; data: Array<{ secretKey: string; @@ -56,6 +67,7 @@ export interface GenerateExternalSecretOptions { secretRefs: SecretRefWithEnvKey[]; providerConfig: SecretProviderConfig; buildUuid?: string; + forceSyncToken?: string; } const MAX_NAME_LENGTH = 63; @@ -87,7 +99,7 @@ export function groupSecretRefsByProvider(refs: SecretRefWithEnvKey[]): Record ({ @@ -23,6 +24,8 @@ jest.mock('server/lib/kubernetes/externalSecret', () => ({ .generateExternalSecretManifest, generateSecretName: jest.requireActual('server/lib/kubernetes/externalSecret').generateSecretName, groupSecretRefsByProvider: jest.requireActual('server/lib/kubernetes/externalSecret').groupSecretRefsByProvider, + TARGET_SECRET_SYNC_TOKEN_ANNOTATION: jest.requireActual('server/lib/kubernetes/externalSecret') + .TARGET_SECRET_SYNC_TOKEN_ANNOTATION, })); jest.mock('server/lib/logger', () => ({ @@ -138,6 +141,44 @@ describe('SecretProcessor', () => { expect(mockReadNamespacedSecret).toHaveBeenCalledTimes(2); }); + it('waits until the secret has the expected sync token', async () => { + const mockReadNamespacedSecret = jest + .fn() + .mockResolvedValueOnce({ + body: { + data: { API_TOKEN: 'b2xk' }, + metadata: { + annotations: { + [TARGET_SECRET_SYNC_TOKEN_ANNOTATION]: 'old-token', + }, + }, + }, + }) + .mockResolvedValueOnce({ + body: { + data: { API_TOKEN: 'bmV3' }, + metadata: { + annotations: { + [TARGET_SECRET_SYNC_TOKEN_ANNOTATION]: 'new-token', + }, + }, + }, + }); + + jest.spyOn(processor as any, 'getK8sClient').mockReturnValue({ + readNamespacedSecret: mockReadNamespacedSecret, + }); + jest.spyOn(processor as any, 'sleep').mockResolvedValue(undefined); + + await expect( + processor.waitForSecretSync({ 'my-secret': ['API_TOKEN'] }, 'test-ns', 5000, { + 'my-secret': 'new-token', + }) + ).resolves.toBeUndefined(); + + expect(mockReadNamespacedSecret).toHaveBeenCalledTimes(2); + }); + it('waits through not found and partial keys until all requested keys land', async () => { const mockReadNamespacedSecret = jest .fn() @@ -286,6 +327,7 @@ describe('SecretProcessor', () => { serviceName: 'api-server', namespace: 'lfc-abc123', buildUuid: 'abc123', + syncToken: 'sync-123', }); expect(applyExternalSecret).toHaveBeenCalledTimes(1); @@ -293,6 +335,21 @@ describe('SecretProcessor', () => { expect.objectContaining({ metadata: expect.objectContaining({ name: 'api-server-aws-secrets', + annotations: { + 'force-sync': 'sync-123', + }, + }), + spec: expect.objectContaining({ + target: expect.objectContaining({ + deletionPolicy: 'Merge', + template: { + metadata: expect.objectContaining({ + annotations: { + [TARGET_SECRET_SYNC_TOKEN_ANNOTATION]: 'sync-123', + }, + }), + }, + }), }), }), 'lfc-abc123' @@ -374,6 +431,10 @@ describe('SecretProcessor', () => { 'api-server-aws-secrets': ['AWS_SECRET'], 'api-server-gcp-secrets': ['GCP_SECRET'], }); + expect(result.syncTokensPerSecret).toEqual({ + 'api-server-aws-secrets': expect.any(String), + 'api-server-gcp-secrets': expect.any(String), + }); }); }); }); diff --git a/src/server/services/deploy.ts b/src/server/services/deploy.ts index e080a061..e27d8e32 100644 --- a/src/server/services/deploy.ts +++ b/src/server/services/deploy.ts @@ -1170,7 +1170,8 @@ export default class DeployService extends BaseService { await secretProcessor.waitForSecretSync( secretResult.expectedKeysPerSecret, deploy.build.namespace, - timeout + timeout, + secretResult.syncTokensPerSecret ); buildSecretNames = secretNames; getLogger().info(`Build: secrets synced count=${buildSecretNames.length}`); diff --git a/src/server/services/secretProcessor.ts b/src/server/services/secretProcessor.ts index a49cc5f4..f3e141b1 100644 --- a/src/server/services/secretProcessor.ts +++ b/src/server/services/secretProcessor.ts @@ -21,9 +21,11 @@ import { generateExternalSecretManifest, groupSecretRefsByProvider, generateSecretName, + TARGET_SECRET_SYNC_TOKEN_ANNOTATION, } from 'server/lib/kubernetes/externalSecret'; import { SecretProvidersConfig } from 'server/services/types/globalConfig'; import { CoreV1Api, KubeConfig } from '@kubernetes/client-node'; +import { v4 as uuid } from 'uuid'; const DEFAULT_SECRET_SYNC_TIMEOUT = 60000; @@ -32,11 +34,13 @@ export interface ProcessEnvSecretsOptions { serviceName: string; namespace: string; buildUuid?: string; + syncToken?: string; } export interface ProcessEnvSecretsResult { secretRefs: SecretRefWithEnvKey[]; expectedKeysPerSecret: Record; + syncTokensPerSecret: Record; warnings: string[]; } @@ -61,7 +65,8 @@ export class SecretProcessor { async waitForSecretSync( expectedKeysPerSecret: Record, namespace: string, - timeoutMs?: number + timeoutMs?: number, + expectedSyncTokensPerSecret: Record = {} ): Promise { const timeout = timeoutMs ?? DEFAULT_SECRET_SYNC_TIMEOUT; const pollInterval = 1000; @@ -72,24 +77,36 @@ export class SecretProcessor { for (const [secretName, expectedKeys] of Object.entries(expectedKeysPerSecret)) { let synced = false; let missingKeys = expectedKeys; + let syncedToken = false; while (!synced) { if (Date.now() - startTime > timeout) { + const tokenMessage = + expectedSyncTokensPerSecret[secretName] && !syncedToken ? ' sync token not observed' : ''; throw new Error( - `Secret sync timeout: ${secretName} missing keys=[${missingKeys.join(', ')}] after ${timeout}ms` + `Secret sync timeout: ${secretName} missing keys=[${missingKeys.join( + ', ' + )}]${tokenMessage} after ${timeout}ms` ); } try { const response = await k8sClient.readNamespacedSecret(secretName, namespace); const data = response.body.data || {}; + const annotations = response.body.metadata?.annotations || {}; + const expectedSyncToken = expectedSyncTokensPerSecret[secretName]; missingKeys = expectedKeys.filter((key) => !Object.prototype.hasOwnProperty.call(data, key)); + syncedToken = !expectedSyncToken || annotations[TARGET_SECRET_SYNC_TOKEN_ANNOTATION] === expectedSyncToken; - if (missingKeys.length === 0) { + if (missingKeys.length === 0 && syncedToken) { getLogger().info(`Secret: synced name=${secretName} namespace=${namespace}`); synced = true; } else { - getLogger().debug(`Secret: waiting for keys name=${secretName} missing=[${missingKeys.join(', ')}]`); + getLogger().debug( + `Secret: waiting for keys name=${secretName} missing=[${missingKeys.join( + ', ' + )}] syncedToken=${syncedToken}` + ); await this.sleep(pollInterval); } } catch (error: any) { @@ -110,6 +127,7 @@ export class SecretProcessor { async processEnvSecrets(options: ProcessEnvSecretsOptions): Promise { const { env, serviceName, namespace, buildUuid } = options; + const syncToken = options.syncToken ?? uuid(); const warnings: string[] = []; const validRefs: SecretRefWithEnvKey[] = []; @@ -131,11 +149,12 @@ export class SecretProcessor { } if (validRefs.length === 0) { - return { secretRefs: [], expectedKeysPerSecret: {}, warnings }; + return { secretRefs: [], expectedKeysPerSecret: {}, syncTokensPerSecret: {}, warnings }; } const grouped = groupSecretRefsByProvider(validRefs); const expectedKeysPerSecret: Record = {}; + const syncTokensPerSecret: Record = {}; for (const [provider, refs] of Object.entries(grouped)) { const providerConfig = this.secretProviders![provider]; @@ -148,11 +167,13 @@ export class SecretProcessor { secretRefs: refs, providerConfig, buildUuid, + forceSyncToken: syncToken, }); try { await applyExternalSecret(manifest, namespace); expectedKeysPerSecret[secretName] = [...new Set(refs.map((ref) => ref.envKey))]; + syncTokensPerSecret[secretName] = syncToken; } catch (error) { const errorMsg = (error as any)?.message || (error as any)?.stderr || String(error); const warning = `Failed to apply ExternalSecret for ${serviceName}: ${errorMsg}`; @@ -160,6 +181,6 @@ export class SecretProcessor { } } - return { secretRefs: validRefs, expectedKeysPerSecret, warnings }; + return { secretRefs: validRefs, expectedKeysPerSecret, syncTokensPerSecret, warnings }; } } From 57f54e25691f24958c3549a0ab64d766188c3913 Mon Sep 17 00:00:00 2001 From: Vigneshraj Sekar Babu Date: Mon, 27 Apr 2026 16:57:58 -0700 Subject: [PATCH 2/2] fix: use lfc sync token annotation --- src/server/lib/kubernetes/externalSecret.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/lib/kubernetes/externalSecret.ts b/src/server/lib/kubernetes/externalSecret.ts index 4de4b73e..b72ac711 100644 --- a/src/server/lib/kubernetes/externalSecret.ts +++ b/src/server/lib/kubernetes/externalSecret.ts @@ -23,7 +23,7 @@ import { SecretProviderConfig } from 'server/services/types/globalConfig'; import { buildLifecycleLabels } from 'server/lib/kubernetes/labels'; export const EXTERNAL_SECRET_FORCE_SYNC_ANNOTATION = 'force-sync'; -export const TARGET_SECRET_SYNC_TOKEN_ANNOTATION = 'lifecycle.dev/secret-sync-token'; +export const TARGET_SECRET_SYNC_TOKEN_ANNOTATION = 'lfc/secret-sync-token'; export interface ExternalSecretManifest { apiVersion: string;