From 91e1c920f77614850601bc24245cd58139900820 Mon Sep 17 00:00:00 2001 From: namtran6701 Date: Wed, 11 Feb 2026 17:05:12 -0500 Subject: [PATCH 1/3] feat: add cleanup index job configuration to infra --- .github/workflows/dev-provision.yaml | 3 +++ .github/workflows/dev-validate-provision.yaml | 5 +++- .github/workflows/prod-provision.yaml | 5 +++- infra/main.bicep | 24 +++++++++++++++++++ infra/main.parameters.json | 9 +++++++ 5 files changed, 44 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev-provision.yaml b/.github/workflows/dev-provision.yaml index bf6816e7..4493e1ed 100644 --- a/.github/workflows/dev-provision.yaml +++ b/.github/workflows/dev-provision.yaml @@ -84,6 +84,9 @@ jobs: BRAND_ANALYSIS_MODEL: ${{ secrets.BRAND_ANALYSIS_MODEL }} REASONING_EFFORT: ${{ secrets.REASONING_EFFORT }} REPORT_SCHEDULE_CRON: ${{ secrets.REPORT_SCHEDULE_CRON }} + CLEANUP_INDEX_CONTAINER_MAP: ${{ secrets.CLEANUP_INDEX_CONTAINER_MAP }} + CLEANUP_INDEX_DRY_RUN: ${{ secrets.CLEANUP_INDEX_DRY_RUN }} + CLEANUP_INDEX_CRON_JOB: ${{ secrets.CLEANUP_INDEX_CRON_JOB }} # LangSmith Settings LANGCHAIN_API_KEY: ${{ secrets.LANGCHAIN_API_KEY }} diff --git a/.github/workflows/dev-validate-provision.yaml b/.github/workflows/dev-validate-provision.yaml index d714a74a..10685de1 100644 --- a/.github/workflows/dev-validate-provision.yaml +++ b/.github/workflows/dev-validate-provision.yaml @@ -85,6 +85,9 @@ jobs: BRAND_ANALYSIS_MODEL: ${{ secrets.BRAND_ANALYSIS_MODEL }} REASONING_EFFORT: ${{ secrets.REASONING_EFFORT }} REPORT_SCHEDULE_CRON: ${{ secrets.REPORT_SCHEDULE_CRON }} + CLEANUP_INDEX_CONTAINER_MAP: ${{ secrets.CLEANUP_INDEX_CONTAINER_MAP }} + CLEANUP_INDEX_DRY_RUN: ${{ secrets.CLEANUP_INDEX_DRY_RUN }} + CLEANUP_INDEX_CRON_JOB: ${{ secrets.CLEANUP_INDEX_CRON_JOB }} # LangSmith Settings LANGCHAIN_API_KEY: ${{ secrets.LANGCHAIN_API_KEY }} @@ -99,4 +102,4 @@ jobs: USER_DATA_CONTAINER: ${{ secrets.USER_DATA_CONTAINER }} # User Feedback URL - USER_FEEDBACK_URL: ${{ secrets.USER_FEEDBACK_URL }} \ No newline at end of file + USER_FEEDBACK_URL: ${{ secrets.USER_FEEDBACK_URL }} diff --git a/.github/workflows/prod-provision.yaml b/.github/workflows/prod-provision.yaml index b5b3fa4b..30916584 100644 --- a/.github/workflows/prod-provision.yaml +++ b/.github/workflows/prod-provision.yaml @@ -84,6 +84,9 @@ jobs: BRAND_ANALYSIS_MODEL: ${{ secrets.BRAND_ANALYSIS_MODEL }} REASONING_EFFORT: ${{ secrets.REASONING_EFFORT }} REPORT_SCHEDULE_CRON: ${{ secrets.REPORT_SCHEDULE_CRON }} + CLEANUP_INDEX_CONTAINER_MAP: ${{ secrets.CLEANUP_INDEX_CONTAINER_MAP }} + CLEANUP_INDEX_DRY_RUN: ${{ secrets.CLEANUP_INDEX_DRY_RUN }} + CLEANUP_INDEX_CRON_JOB: ${{ secrets.CLEANUP_INDEX_CRON_JOB }} # LangSmith Settings LANGCHAIN_API_KEY: ${{ secrets.LANGCHAIN_API_KEY }} @@ -98,4 +101,4 @@ jobs: USER_DATA_CONTAINER: ${{ secrets.USER_DATA_CONTAINER }} # User Feedback URL - USER_FEEDBACK_URL: ${{ secrets.USER_FEEDBACK_URL }} \ No newline at end of file + USER_FEEDBACK_URL: ${{ secrets.USER_FEEDBACK_URL }} diff --git a/infra/main.bicep b/infra/main.bicep index 8d7f4e36..86a91d39 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -646,6 +646,18 @@ var reasoningEffortReportVar = !empty(reasoningEffortReport) ? reasoningEffortRe param reportScheduleCron string = '' var reportScheduleCronVar = !empty(reportScheduleCron) ? reportScheduleCron : '' +@description('Container to search-index map for the ingestion cleanup job') +param cleanupIndexContainerMap string = '' +var cleanupIndexContainerMapVar = !empty(cleanupIndexContainerMap) ? cleanupIndexContainerMap : '' + +@description('Dry-run flag for the ingestion cleanup job') +param cleanupIndexDryRun string = '' +var cleanupIndexDryRunVar = !empty(cleanupIndexDryRun) ? cleanupIndexDryRun : '' + +@description('Cron schedule for the ingestion cleanup job') +param cleanupIndexCronJob string = '' +var cleanupIndexCronJobVar = !empty(cleanupIndexCronJob) ? cleanupIndexCronJob : '' + // MCP Function app @description('Logging Verbosity') var loggingVerbosity = 'false' @@ -1676,6 +1688,18 @@ module dataIngestion './core/host/functions.bicep' = { name:'COGNITIVE_SERVICES_KEY' value: cognitiveServices.outputs.key } + { + name: 'CLEANUP_INDEX_CONTAINER_MAP' + value: cleanupIndexContainerMapVar + } + { + name: 'CLEANUP_INDEX_DRY_RUN' + value: cleanupIndexDryRunVar + } + { + name: 'CLEANUP_INDEX_CRON_JOB' + value: cleanupIndexCronJobVar + } { name: 'OPENAI_API_KEY' value: mcpOpenAiApiKeyVar diff --git a/infra/main.parameters.json b/infra/main.parameters.json index 305d5960..7dadf38c 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -212,6 +212,15 @@ "reportScheduleCron": { "value": "${REPORT_SCHEDULE_CRON}" }, + "cleanupIndexContainerMap": { + "value": "${CLEANUP_INDEX_CONTAINER_MAP}" + }, + "cleanupIndexDryRun": { + "value": "${CLEANUP_INDEX_DRY_RUN}" + }, + "cleanupIndexCronJob": { + "value": "${CLEANUP_INDEX_CRON_JOB}" + }, "webAppUserFeedbackUrl": { "value": "${USER_FEEDBACK_URL}" } From 5e40c85fd2b1614345d25ec29898743a65de045e Mon Sep 17 00:00:00 2001 From: namtran6701 Date: Wed, 11 Feb 2026 17:19:35 -0500 Subject: [PATCH 2/3] fix escape characters --- .github/workflows/dev-provision.yaml | 2 +- .github/workflows/dev-validate-provision.yaml | 2 +- .github/workflows/prod-provision.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dev-provision.yaml b/.github/workflows/dev-provision.yaml index 4493e1ed..c7937137 100644 --- a/.github/workflows/dev-provision.yaml +++ b/.github/workflows/dev-provision.yaml @@ -84,7 +84,7 @@ jobs: BRAND_ANALYSIS_MODEL: ${{ secrets.BRAND_ANALYSIS_MODEL }} REASONING_EFFORT: ${{ secrets.REASONING_EFFORT }} REPORT_SCHEDULE_CRON: ${{ secrets.REPORT_SCHEDULE_CRON }} - CLEANUP_INDEX_CONTAINER_MAP: ${{ secrets.CLEANUP_INDEX_CONTAINER_MAP }} + CLEANUP_INDEX_CONTAINER_MAP: ${{ replace(secrets.CLEANUP_INDEX_CONTAINER_MAP, '"', '\\"') }} CLEANUP_INDEX_DRY_RUN: ${{ secrets.CLEANUP_INDEX_DRY_RUN }} CLEANUP_INDEX_CRON_JOB: ${{ secrets.CLEANUP_INDEX_CRON_JOB }} diff --git a/.github/workflows/dev-validate-provision.yaml b/.github/workflows/dev-validate-provision.yaml index 10685de1..c1e7e6c5 100644 --- a/.github/workflows/dev-validate-provision.yaml +++ b/.github/workflows/dev-validate-provision.yaml @@ -85,7 +85,7 @@ jobs: BRAND_ANALYSIS_MODEL: ${{ secrets.BRAND_ANALYSIS_MODEL }} REASONING_EFFORT: ${{ secrets.REASONING_EFFORT }} REPORT_SCHEDULE_CRON: ${{ secrets.REPORT_SCHEDULE_CRON }} - CLEANUP_INDEX_CONTAINER_MAP: ${{ secrets.CLEANUP_INDEX_CONTAINER_MAP }} + CLEANUP_INDEX_CONTAINER_MAP: ${{ replace(secrets.CLEANUP_INDEX_CONTAINER_MAP, '"', '\\"') }} CLEANUP_INDEX_DRY_RUN: ${{ secrets.CLEANUP_INDEX_DRY_RUN }} CLEANUP_INDEX_CRON_JOB: ${{ secrets.CLEANUP_INDEX_CRON_JOB }} diff --git a/.github/workflows/prod-provision.yaml b/.github/workflows/prod-provision.yaml index 30916584..6cf33a34 100644 --- a/.github/workflows/prod-provision.yaml +++ b/.github/workflows/prod-provision.yaml @@ -84,7 +84,7 @@ jobs: BRAND_ANALYSIS_MODEL: ${{ secrets.BRAND_ANALYSIS_MODEL }} REASONING_EFFORT: ${{ secrets.REASONING_EFFORT }} REPORT_SCHEDULE_CRON: ${{ secrets.REPORT_SCHEDULE_CRON }} - CLEANUP_INDEX_CONTAINER_MAP: ${{ secrets.CLEANUP_INDEX_CONTAINER_MAP }} + CLEANUP_INDEX_CONTAINER_MAP: ${{ replace(secrets.CLEANUP_INDEX_CONTAINER_MAP, '"', '\\"') }} CLEANUP_INDEX_DRY_RUN: ${{ secrets.CLEANUP_INDEX_DRY_RUN }} CLEANUP_INDEX_CRON_JOB: ${{ secrets.CLEANUP_INDEX_CRON_JOB }} From c134abe0ab80e2759f13215e0cf6c28034a3ddda Mon Sep 17 00:00:00 2001 From: namtran6701 Date: Wed, 11 Feb 2026 17:30:37 -0500 Subject: [PATCH 3/3] remove replace --- .github/workflows/dev-provision.yaml | 2 +- .github/workflows/dev-validate-provision.yaml | 2 +- .github/workflows/prod-provision.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dev-provision.yaml b/.github/workflows/dev-provision.yaml index c7937137..4493e1ed 100644 --- a/.github/workflows/dev-provision.yaml +++ b/.github/workflows/dev-provision.yaml @@ -84,7 +84,7 @@ jobs: BRAND_ANALYSIS_MODEL: ${{ secrets.BRAND_ANALYSIS_MODEL }} REASONING_EFFORT: ${{ secrets.REASONING_EFFORT }} REPORT_SCHEDULE_CRON: ${{ secrets.REPORT_SCHEDULE_CRON }} - CLEANUP_INDEX_CONTAINER_MAP: ${{ replace(secrets.CLEANUP_INDEX_CONTAINER_MAP, '"', '\\"') }} + CLEANUP_INDEX_CONTAINER_MAP: ${{ secrets.CLEANUP_INDEX_CONTAINER_MAP }} CLEANUP_INDEX_DRY_RUN: ${{ secrets.CLEANUP_INDEX_DRY_RUN }} CLEANUP_INDEX_CRON_JOB: ${{ secrets.CLEANUP_INDEX_CRON_JOB }} diff --git a/.github/workflows/dev-validate-provision.yaml b/.github/workflows/dev-validate-provision.yaml index c1e7e6c5..10685de1 100644 --- a/.github/workflows/dev-validate-provision.yaml +++ b/.github/workflows/dev-validate-provision.yaml @@ -85,7 +85,7 @@ jobs: BRAND_ANALYSIS_MODEL: ${{ secrets.BRAND_ANALYSIS_MODEL }} REASONING_EFFORT: ${{ secrets.REASONING_EFFORT }} REPORT_SCHEDULE_CRON: ${{ secrets.REPORT_SCHEDULE_CRON }} - CLEANUP_INDEX_CONTAINER_MAP: ${{ replace(secrets.CLEANUP_INDEX_CONTAINER_MAP, '"', '\\"') }} + CLEANUP_INDEX_CONTAINER_MAP: ${{ secrets.CLEANUP_INDEX_CONTAINER_MAP }} CLEANUP_INDEX_DRY_RUN: ${{ secrets.CLEANUP_INDEX_DRY_RUN }} CLEANUP_INDEX_CRON_JOB: ${{ secrets.CLEANUP_INDEX_CRON_JOB }} diff --git a/.github/workflows/prod-provision.yaml b/.github/workflows/prod-provision.yaml index 6cf33a34..30916584 100644 --- a/.github/workflows/prod-provision.yaml +++ b/.github/workflows/prod-provision.yaml @@ -84,7 +84,7 @@ jobs: BRAND_ANALYSIS_MODEL: ${{ secrets.BRAND_ANALYSIS_MODEL }} REASONING_EFFORT: ${{ secrets.REASONING_EFFORT }} REPORT_SCHEDULE_CRON: ${{ secrets.REPORT_SCHEDULE_CRON }} - CLEANUP_INDEX_CONTAINER_MAP: ${{ replace(secrets.CLEANUP_INDEX_CONTAINER_MAP, '"', '\\"') }} + CLEANUP_INDEX_CONTAINER_MAP: ${{ secrets.CLEANUP_INDEX_CONTAINER_MAP }} CLEANUP_INDEX_DRY_RUN: ${{ secrets.CLEANUP_INDEX_DRY_RUN }} CLEANUP_INDEX_CRON_JOB: ${{ secrets.CLEANUP_INDEX_CRON_JOB }}