feat: ingestion v2 — architecture + Phase 1 streaming + Phase 2 per-scope watermarks + 4 data quality fixes

.gitignore

@@ -56,6 +56,8 @@ npm-debug.log*
 
 # === Claude Code ===
 .claude/settings.local.json
+.claude/scheduled_tasks.lock
+.claude/projects/
 
 # === Factory (read-only reference, not deployed) ===
 # factory/ is tracked but treated as read-only reference docs

pulse/.env.example

@@ -25,15 +25,25 @@ PULSE_DATA_PORT=8000
 # -- Source Connector Tokens ------------------------------------------------
 # GitHub Personal Access Token (repo, read:org scopes)
 GITHUB_TOKEN=
+# GitHub org slug — REQUIRED. Used by discover_repos() — see ingestion-spec §2.3.
+GITHUB_ORG=
 # GitLab Personal Access Token (read_api scope)
 GITLAB_TOKEN=
 # Jira API Token + email
 JIRA_API_TOKEN=
 JIRA_EMAIL=
+# Jira base URL (e.g., https://your-org.atlassian.net)
+JIRA_BASE_URL=
+# JIRA_PROJECTS is intentionally absent. PULSE uses dynamic discovery
+# (ingestion-spec §2.3); the active project list is maintained in
+# `jira_project_catalog` table and resolved by ModeResolver. Do NOT add
+# JIRA_PROJECTS unless you set DYNAMIC_JIRA_DISCOVERY_ENABLED=false (not
+# recommended).
+DYNAMIC_JIRA_DISCOVERY_ENABLED=true
 # Azure DevOps Personal Access Token (Code, Work Items read)
 AZURE_DEVOPS_PAT=
 # Jenkins API credentials (read-only: Overall/Read, Job/Read, Run/Read)
-JENKINS_BASE_URL=https://jenkins.webmotors.com.br
+JENKINS_BASE_URL=
 JENKINS_USERNAME=
 JENKINS_API_TOKEN=
 

pulse/config/connections.yaml

@@ -20,17 +20,13 @@ connections:
     token_env: GITHUB_TOKEN
     base_url: https://api.github.com
     sync_interval_minutes: 15
+    # Per ingestion-spec §2.3 (Discovery-Only): NO explicit list of repos.
+    # The connector calls `discover_repos(active_months=12)` on each cycle
+    # via GraphQL `organization.repositories(orderBy: PUSHED_AT)` filtered
+    # by activity. New repos appear automatically; archived ones drop off
+    # without manual YAML edits.
     scope:
-      repositories:
-        - "webmotors-private/webmotors.next.ui"
-        - "webmotors-private/webmotors.portal.ui"
-        - "webmotors-private/webmotors.buyer.ui"
-        - "webmotors-private/webmotors.buyer.desktop.ui"
-        - "webmotors-private/webmotors.catalogo.next.ui"
-        - "webmotors-private/webmotors.fipe.next.ui"
-        - "webmotors-private/webmotors.pf"
-        - "webmotors-private/eleanor.flutter"
-        - "webmotors-private/webmotors.app.pf.search.bff"
+      active_months: 12
 
   - name: "Webmotors Jenkins"
     source: jenkins
@@ -41,13 +37,11 @@ connections:
     sync_interval_minutes: 15
     scope:
       # Job list is loaded from config/jenkins-job-mapping.json (auto-generated).
-      # Generated 2026-04-14 by READ-ONLY SCM scan of 544 active PRD Jenkins
-      # jobs — each job's lastBuild → remoteUrls resolves the GitHub repo.
-      # Total: 577 PRD jobs across 283 repos.
-      #
-      # To regenerate: run scripts/discover_jenkins_jobs.py (READ-ONLY).
-      # The sync worker reads prd_jobs from the mapping file at startup.
-      jobs_from_mapping: true  # Signals config.py to use jenkins-job-mapping.json
+      # Generated by READ-ONLY SCM scan of active PRD Jenkins jobs — each
+      # job's lastBuild → remoteUrls resolves the GitHub repo. Per
+      # ingestion-spec §3.6, regen via scripts/discover_jenkins_jobs.py
+      # when new repos appear (manual or weekly cron).
+      jobs_from_mapping: true
 
   - name: "Webmotors Jira"
     source: jira
@@ -56,19 +50,15 @@ connections:
     token_env: JIRA_API_TOKEN
     base_url: https://webmotors.atlassian.net
     sync_interval_minutes: 15
+    # Per ingestion-spec §2.3 (Discovery-Only): NO explicit project list.
+    # `ProjectDiscoveryService` lists ALL Jira projects; `SmartPrioritizer`
+    # auto-activates projects with ≥3 PR references. Tenant config in
+    # `tenant_jira_config` controls discovery mode (must be 'smart' for
+    # auto-activation). PII-flagged projects require manual approval.
     scope:
-      projects:
-        # Canais Digitais Web (Kanban)
-        - "DESC"    # PF - Descobrir veículo
-        - "ENO"     # PF - Encontrar oferta
-        - "ANCR"    # PF - Anunciar
-        - "PUSO"    # PF - USO
-        # Canais Digitais App (Kanban)
-        - "APPF"    # PF - Aplicativo
-        # Sprint-based projects
-        - "FID"     # Fidelidade
-        - "CTURBO"  # Consultor Turbo Lab
-        - "PTURB"   # Portal Turbo Lab
+      mode: smart
+      smart_min_pr_references: 3
+      smart_pr_scan_days: 90
 
 # Issue status mapping — Webmotors Jira (Portuguese) → PULSE normalized
 # Primary source; overrides DEFAULT_STATUS_MAPPING in normalizer.py.