fix: refresh external secrets before secret use

src/server/lib/agentSession/__tests__/forwardedEnv.test.ts

@@ -84,6 +84,7 @@ describe('forwardedEnv', () => {
       processEnvSecrets: jest.fn().mockResolvedValue({
         secretRefs: [],
         expectedKeysPerSecret: {},
+        syncTokensPerSecret: {},
         warnings: [],
       }),
       waitForSecretSync: jest.fn().mockResolvedValue(undefined),
@@ -195,6 +196,9 @@ describe('forwardedEnv', () => {
       expectedKeysPerSecret: {
         'agent-env-session-123-aws-secrets': ['PRIVATE_REGISTRY_TOKEN'],
       },
+      syncTokensPerSecret: {
+        'agent-env-session-123-aws-secrets': 'sync-123',
+      },
       warnings: [],
     });
     const waitForSecretSync = jest.fn().mockResolvedValue(undefined);
@@ -241,7 +245,8 @@ describe('forwardedEnv', () => {
     expect(waitForSecretSync).toHaveBeenCalledWith(
       { 'agent-env-session-123-aws-secrets': ['PRIVATE_REGISTRY_TOKEN'] },
       'test-ns',
-      30000
+      30000,
+      { 'agent-env-session-123-aws-secrets': 'sync-123' }
     );
     expect(result.secretProviders).toEqual(['aws']);
   });

src/server/lib/agentSession/forwardedEnv.ts

@@ -157,7 +157,12 @@ export async function resolveForwardedAgentEnv(
       .map((provider) => provider.secretSyncTimeout)
       .filter((timeout): timeout is number => timeout !== undefined);
     const timeout = providerTimeouts.length > 0 ? Math.max(...providerTimeouts) * 1000 : 60000;
-    await secretProcessor.waitForSecretSync(secretResult.expectedKeysPerSecret, namespace, timeout);
+    await secretProcessor.waitForSecretSync(
+      secretResult.expectedKeysPerSecret,
+      namespace,
+      timeout,
+      secretResult.syncTokensPerSecret
+    );
   }
 
   return {

src/server/lib/kubernetes/__tests__/externalSecret.test.ts

@@ -14,7 +14,13 @@
  * limitations under the License.
  */
 
-import { generateExternalSecretManifest, generateSecretName, groupSecretRefsByProvider } from '../externalSecret';
+import {
+  EXTERNAL_SECRET_FORCE_SYNC_ANNOTATION,
+  generateExternalSecretManifest,
+  generateSecretName,
+  groupSecretRefsByProvider,
+  TARGET_SECRET_SYNC_TOKEN_ANNOTATION,
+} from '../externalSecret';
 import { SecretRefWithEnvKey } from 'server/lib/secretRefs';
 
 describe('externalSecret', () => {
@@ -90,6 +96,7 @@ describe('externalSecret', () => {
       expect(manifest.spec.secretStoreRef.name).toBe('aws-secretsmanager');
       expect(manifest.spec.secretStoreRef.kind).toBe('ClusterSecretStore');
       expect(manifest.spec.target.name).toBe('api-server-aws-secrets');
+      expect(manifest.spec.target.deletionPolicy).toBe('Merge');
       expect(manifest.spec.data).toHaveLength(2);
     });
 
@@ -167,5 +174,26 @@ describe('externalSecret', () => {
       expect(manifest.metadata.labels['app.kubernetes.io/managed-by']).toBe('lifecycle');
       expect(manifest.metadata.labels['lfc/uuid']).toBe('abc123');
     });
+
+    it('adds sync metadata when a force sync token is provided', () => {
+      const refs: SecretRefWithEnvKey[] = [{ envKey: 'SECRET', provider: 'aws', path: 'path', key: 'key' }];
+
+      const manifest = generateExternalSecretManifest({
+        name: 'api-server',
+        namespace: 'lfc-abc123',
+        provider: 'aws',
+        secretRefs: refs,
+        providerConfig,
+        forceSyncToken: 'sync-123',
+      });
+
+      expect(manifest.metadata.annotations).toEqual({
+        [EXTERNAL_SECRET_FORCE_SYNC_ANNOTATION]: 'sync-123',
+      });
+      expect(manifest.spec.target.template?.metadata.annotations).toEqual({
+        [TARGET_SECRET_SYNC_TOKEN_ANNOTATION]: 'sync-123',
+      });
+      expect(manifest.spec.target.template?.metadata.labels).toEqual(manifest.metadata.labels);
+    });
   });
 });

src/server/lib/kubernetes/externalSecret.ts

@@ -22,13 +22,17 @@ import { SecretRefWithEnvKey } from 'server/lib/secretRefs';
 import { SecretProviderConfig } from 'server/services/types/globalConfig';
 import { buildLifecycleLabels } from 'server/lib/kubernetes/labels';
 
+export const EXTERNAL_SECRET_FORCE_SYNC_ANNOTATION = 'force-sync';
+export const TARGET_SECRET_SYNC_TOKEN_ANNOTATION = 'lfc/secret-sync-token';
+
 export interface ExternalSecretManifest {
   apiVersion: string;
   kind: string;
   metadata: {
     name: string;
     namespace: string;
     labels: Record<string, string>;
+    annotations?: Record<string, string>;
   };
   spec: {
     refreshInterval: string;
@@ -38,6 +42,13 @@ export interface ExternalSecretManifest {
     };
     target: {
       name: string;
+      deletionPolicy: 'Merge';
+      template?: {
+        metadata: {
+          labels: Record<string, string>;
+          annotations: Record<string, string>;
+        };
+      };
     };
     data: Array<{
       secretKey: string;
@@ -56,6 +67,7 @@ export interface GenerateExternalSecretOptions {
   secretRefs: SecretRefWithEnvKey[];
   providerConfig: SecretProviderConfig;
   buildUuid?: string;
+  forceSyncToken?: string;
 }
 
 const MAX_NAME_LENGTH = 63;
@@ -87,7 +99,7 @@ export function groupSecretRefsByProvider(refs: SecretRefWithEnvKey[]): Record<s
 }
 
 export function generateExternalSecretManifest(options: GenerateExternalSecretOptions): ExternalSecretManifest {
-  const { name, namespace, provider, secretRefs, providerConfig, buildUuid } = options;
+  const { name, namespace, provider, secretRefs, providerConfig, buildUuid, forceSyncToken } = options;
 
   const secretName = generateSecretName(name, provider);
 
@@ -115,13 +127,30 @@ export function generateExternalSecretManifest(options: GenerateExternalSecretOp
     labels['lfc/uuid'] = buildUuid;
   }
 
+  const annotations = forceSyncToken
+    ? {
+        [EXTERNAL_SECRET_FORCE_SYNC_ANNOTATION]: forceSyncToken,
+      }
+    : undefined;
+  const targetTemplate = forceSyncToken
+    ? {
+        metadata: {
+          labels,
+          annotations: {
+            [TARGET_SECRET_SYNC_TOKEN_ANNOTATION]: forceSyncToken,
+          },
+        },
+      }
+    : undefined;
+
   return {
     apiVersion: 'external-secrets.io/v1',
     kind: 'ExternalSecret',
     metadata: {
       name: secretName,
       namespace,
       labels,
+      ...(annotations ? { annotations } : {}),
     },
     spec: {
       refreshInterval: providerConfig.refreshInterval,
@@ -131,6 +160,8 @@ export function generateExternalSecretManifest(options: GenerateExternalSecretOp
       },
       target: {
         name: secretName,
+        deletionPolicy: 'Merge',
+        ...(targetTemplate ? { template: targetTemplate } : {}),
       },
       data,
     },

src/server/services/__tests__/secretProcessor.test.ts

@@ -15,6 +15,7 @@
  */
 
 import { SecretProcessor } from '../secretProcessor';
+import { TARGET_SECRET_SYNC_TOKEN_ANNOTATION } from 'server/lib/kubernetes/externalSecret';
 import { SecretProvidersConfig } from 'server/services/types/globalConfig';
 
 jest.mock('server/lib/kubernetes/externalSecret', () => ({
@@ -23,6 +24,8 @@ jest.mock('server/lib/kubernetes/externalSecret', () => ({
     .generateExternalSecretManifest,
   generateSecretName: jest.requireActual('server/lib/kubernetes/externalSecret').generateSecretName,
   groupSecretRefsByProvider: jest.requireActual('server/lib/kubernetes/externalSecret').groupSecretRefsByProvider,
+  TARGET_SECRET_SYNC_TOKEN_ANNOTATION: jest.requireActual('server/lib/kubernetes/externalSecret')
+    .TARGET_SECRET_SYNC_TOKEN_ANNOTATION,
 }));
 
 jest.mock('server/lib/logger', () => ({
@@ -138,6 +141,44 @@ describe('SecretProcessor', () => {
       expect(mockReadNamespacedSecret).toHaveBeenCalledTimes(2);
     });
 
+    it('waits until the secret has the expected sync token', async () => {
+      const mockReadNamespacedSecret = jest
+        .fn()
+        .mockResolvedValueOnce({
+          body: {
+            data: { API_TOKEN: 'b2xk' },
+            metadata: {
+              annotations: {
+                [TARGET_SECRET_SYNC_TOKEN_ANNOTATION]: 'old-token',
+              },
+            },
+          },
+        })
+        .mockResolvedValueOnce({
+          body: {
+            data: { API_TOKEN: 'bmV3' },
+            metadata: {
+              annotations: {
+                [TARGET_SECRET_SYNC_TOKEN_ANNOTATION]: 'new-token',
+              },
+            },
+          },
+        });
+
+      jest.spyOn(processor as any, 'getK8sClient').mockReturnValue({
+        readNamespacedSecret: mockReadNamespacedSecret,
+      });
+      jest.spyOn(processor as any, 'sleep').mockResolvedValue(undefined);
+
+      await expect(
+        processor.waitForSecretSync({ 'my-secret': ['API_TOKEN'] }, 'test-ns', 5000, {
+          'my-secret': 'new-token',
+        })
+      ).resolves.toBeUndefined();
+
+      expect(mockReadNamespacedSecret).toHaveBeenCalledTimes(2);
+    });
+
     it('waits through not found and partial keys until all requested keys land', async () => {
       const mockReadNamespacedSecret = jest
         .fn()
@@ -286,13 +327,29 @@ describe('SecretProcessor', () => {
         serviceName: 'api-server',
         namespace: 'lfc-abc123',
         buildUuid: 'abc123',
+        syncToken: 'sync-123',
       });
 
       expect(applyExternalSecret).toHaveBeenCalledTimes(1);
       expect(applyExternalSecret).toHaveBeenCalledWith(
         expect.objectContaining({
           metadata: expect.objectContaining({
             name: 'api-server-aws-secrets',
+            annotations: {
+              'force-sync': 'sync-123',
+            },
+          }),
+          spec: expect.objectContaining({
+            target: expect.objectContaining({
+              deletionPolicy: 'Merge',
+              template: {
+                metadata: expect.objectContaining({
+                  annotations: {
+                    [TARGET_SECRET_SYNC_TOKEN_ANNOTATION]: 'sync-123',
+                  },
+                }),
+              },
+            }),
           }),
         }),
         'lfc-abc123'
@@ -374,6 +431,10 @@ describe('SecretProcessor', () => {
         'api-server-aws-secrets': ['AWS_SECRET'],
         'api-server-gcp-secrets': ['GCP_SECRET'],
       });
+      expect(result.syncTokensPerSecret).toEqual({
+        'api-server-aws-secrets': expect.any(String),
+        'api-server-gcp-secrets': expect.any(String),
+      });
     });
   });
 });

src/server/services/deploy.ts

@@ -1170,7 +1170,8 @@ export default class DeployService extends BaseService {
                   await secretProcessor.waitForSecretSync(
                     secretResult.expectedKeysPerSecret,
                     deploy.build.namespace,
-                    timeout
+                    timeout,
+                    secretResult.syncTokensPerSecret
                   );
                   buildSecretNames = secretNames;
                   getLogger().info(`Build: secrets synced count=${buildSecretNames.length}`);