feat: Add agent check-in system for health monitoring

db/migrations/check_in_constraints_migration.sql

@@ -0,0 +1,86 @@
+-- Migration: Add CHECK constraints to check-in tables
+-- Purpose: Enforce data integrity constraints identified in code review
+-- Related: TASK-002 Critical Issue #3
+
+-- Add CHECK constraints to agent_check_in_configs table
+ALTER TABLE agent_check_in_configs
+ADD CONSTRAINT chk_timeout_seconds_positive
+CHECK (timeout_seconds >= 0);
+
+ALTER TABLE agent_check_in_configs
+ADD CONSTRAINT chk_retry_attempts_non_negative
+CHECK (retry_attempts >= 0);
+
+ALTER TABLE agent_check_in_configs
+ADD CONSTRAINT chk_retry_delay_seconds_positive
+CHECK (retry_delay_seconds >= 0);
+
+-- Add CHECK constraints to check_in_events table
+ALTER TABLE check_in_events
+ADD CONSTRAINT chk_retry_count_non_negative
+CHECK (retry_count >= 0);
+
+ALTER TABLE check_in_events
+ADD CONSTRAINT chk_latency_non_negative
+CHECK (response_latency_ms IS NULL OR response_latency_ms >= 0);
+
+ALTER TABLE check_in_events
+ADD CONSTRAINT chk_triggered_after_scheduled
+CHECK (triggered_at >= scheduled_at OR triggered_at = '0001-01-01 00:00:00');
+
+-- Add response consistency constraint (ensure response_received matches response_at state)
+ALTER TABLE check_in_events
+ADD CONSTRAINT chk_response_consistency
+CHECK (
+    (response_received = FALSE AND response_at IS NULL) OR
+    (response_received = TRUE AND response_at IS NOT NULL)
+);
+
+-- Note: Cron schedule regex validation is enforced at API layer using robfig/cron parser
+-- PostgreSQL CHECK constraints with regex are limited and can cause performance issues
+-- See handlers_checkin.go lines 104-109 for cron validation via cron.NewParser()
+
+-- Add foreign key constraints for referential integrity and CASCADE cleanup
+ALTER TABLE agent_check_in_configs
+ADD CONSTRAINT fk_agent_checkin_config_agent
+FOREIGN KEY (space_name, agent_name)
+REFERENCES agents(space_name, agent_name)
+ON DELETE CASCADE;
+
+ALTER TABLE check_in_events
+ADD CONSTRAINT fk_checkin_event_config
+FOREIGN KEY (space_name, agent_name)
+REFERENCES agent_check_in_configs(space_name, agent_name)
+ON DELETE CASCADE;
+
+-- Verify indexes exist (these should be auto-created by GORM, but we verify)
+-- Partial indexes:
+-- 1. idx_checkin_enabled on agent_check_in_configs WHERE check_in_enabled = true
+-- 2. idx_pending_checkins on check_in_events WHERE response_received = false AND message_sent = true
+
+-- Composite indexes:
+-- 3. idx_checkin_space_agent on agent_check_in_configs (space_name, agent_name) - unique
+-- 4. idx_event_space_agent on check_in_events (space_name, agent_name)
+-- 5. idx_event_agent_time on check_in_events (agent_name, triggered_at DESC)
+
+-- Regular indexes:
+-- 6. agent_check_in_configs.space_name
+-- 7. check_in_events.space_name
+-- 8. check_in_events.agent_name
+-- 9. check_in_events.message_id
+-- 10. check_in_scheduler_locks.locked_at
+-- 11. check_in_scheduler_locks.expires_at
+
+-- Query to verify all indexes are created:
+-- SELECT tablename, indexname, indexdef FROM pg_indexes
+-- WHERE tablename IN ('agent_check_in_configs', 'check_in_events', 'check_in_scheduler_locks')
+-- ORDER BY tablename, indexname;
+
+-- Query to verify CHECK constraints:
+-- SELECT conname, contype, pg_get_constraintdef(oid)
+-- FROM pg_constraint
+-- WHERE conrelid IN (
+--   'agent_check_in_configs'::regclass,
+--   'check_in_events'::regclass
+-- ) AND contype = 'c'
+-- ORDER BY conrelid::regclass::text, conname;

db/migrations/check_in_constraints_rollback.sql

@@ -0,0 +1,41 @@
+-- Rollback Migration: Remove CHECK constraints and FK constraints from check-in tables
+-- Purpose: Rollback script for check_in_constraints_migration.sql
+-- Related: TASK-002 Critical Issue #3
+
+-- Remove FK constraints first (order matters for dependencies)
+ALTER TABLE check_in_events
+DROP CONSTRAINT IF EXISTS fk_checkin_event_config;
+
+ALTER TABLE agent_check_in_configs
+DROP CONSTRAINT IF EXISTS fk_agent_checkin_config_agent;
+
+-- Remove CHECK constraints from check_in_events table
+ALTER TABLE check_in_events
+DROP CONSTRAINT IF EXISTS chk_response_consistency;
+
+ALTER TABLE check_in_events
+DROP CONSTRAINT IF EXISTS chk_triggered_after_scheduled;
+
+ALTER TABLE check_in_events
+DROP CONSTRAINT IF EXISTS chk_latency_non_negative;
+
+ALTER TABLE check_in_events
+DROP CONSTRAINT IF EXISTS chk_retry_count_non_negative;
+
+-- Remove CHECK constraints from agent_check_in_configs table
+ALTER TABLE agent_check_in_configs
+DROP CONSTRAINT IF EXISTS chk_retry_delay_seconds_positive;
+
+ALTER TABLE agent_check_in_configs
+DROP CONSTRAINT IF EXISTS chk_retry_attempts_non_negative;
+
+ALTER TABLE agent_check_in_configs
+DROP CONSTRAINT IF EXISTS chk_timeout_seconds_positive;
+
+-- Note: This rollback does NOT remove indexes or tables, only the CHECK constraints
+-- To fully rollback the check-in feature, use the full rollback migration below
+
+-- FULL ROLLBACK (USE WITH CAUTION - DESTRUCTIVE):
+-- DROP TABLE IF EXISTS check_in_events CASCADE;
+-- DROP TABLE IF EXISTS agent_check_in_configs CASCADE;
+-- DROP TABLE IF EXISTS check_in_scheduler_locks CASCADE;

db/migrations/verify_check_in_indexes.sql

@@ -0,0 +1,102 @@
+-- Verification Script: Check-in System Database Indexes
+-- Purpose: Verify all required indexes exist for the check-in feature
+-- Related: TASK-002 Critical Issue #3
+
+-- List all indexes on check-in tables (PostgreSQL)
+SELECT
+    tablename,
+    indexname,
+    indexdef
+FROM pg_indexes
+WHERE tablename IN ('agent_check_in_configs', 'check_in_events', 'check_in_scheduler_locks')
+ORDER BY tablename, indexname;
+
+-- Expected indexes (13 total):
+
+-- agent_check_in_configs (4 indexes):
+-- 1. agent_check_in_configs_pkey (PRIMARY KEY on id)
+-- 2. idx_checkin_space_agent (UNIQUE on space_name, agent_name)
+-- 3. idx_checkin_enabled (PARTIAL INDEX WHERE check_in_enabled = true)
+-- 4. idx_agent_check_in_configs_space_name (on space_name)
+
+-- check_in_events (7 indexes):
+-- 5. check_in_events_pkey (PRIMARY KEY on id)
+-- 6. idx_event_space_agent (on space_name, agent_name)
+-- 7. idx_event_agent_time (on agent_name, triggered_at DESC)
+-- 8. idx_pending_checkins (PARTIAL INDEX WHERE response_received = false AND message_sent = true)
+-- 9. idx_check_in_events_space_name (on space_name)
+-- 10. idx_check_in_events_agent_name (on agent_name)
+-- 11. idx_check_in_events_message_id (on message_id)
+
+-- check_in_scheduler_locks (3 indexes):
+-- 12. check_in_scheduler_locks_pkey (PRIMARY KEY on id)
+-- 13. idx_check_in_scheduler_locks_locked_at (on locked_at)
+-- 14. idx_check_in_scheduler_locks_expires_at (on expires_at)
+
+-- Total: 14 indexes (including 3 primary keys)
+-- Partial indexes: 2 (idx_checkin_enabled, idx_pending_checkins)
+-- Unique indexes: 1 (idx_checkin_space_agent)
+
+-- Verify partial indexes specifically:
+SELECT
+    schemaname,
+    tablename,
+    indexname,
+    indexdef
+FROM pg_indexes
+WHERE tablename IN ('agent_check_in_configs', 'check_in_events')
+  AND indexdef LIKE '%WHERE%'
+ORDER BY tablename, indexname;
+
+-- Expected output:
+-- agent_check_in_configs | idx_checkin_enabled | ... WHERE check_in_enabled = true
+-- check_in_events | idx_pending_checkins | ... WHERE response_received = false AND message_sent = true
+
+-- Verify CHECK constraints:
+SELECT
+    conrelid::regclass AS table_name,
+    conname AS constraint_name,
+    pg_get_constraintdef(oid) AS constraint_definition
+FROM pg_constraint
+WHERE conrelid IN (
+    'agent_check_in_configs'::regclass,
+    'check_in_events'::regclass
+)
+AND contype = 'c'
+ORDER BY conrelid::regclass::text, conname;
+
+-- Expected CHECK constraints (7 total):
+-- agent_check_in_configs | chk_timeout_seconds_positive | CHECK (timeout_seconds >= 0)
+-- agent_check_in_configs | chk_retry_attempts_non_negative | CHECK (retry_attempts >= 0)
+-- agent_check_in_configs | chk_retry_delay_seconds_positive | CHECK (retry_delay_seconds >= 0)
+-- check_in_events | chk_retry_count_non_negative | CHECK (retry_count >= 0)
+-- check_in_events | chk_latency_non_negative | CHECK (response_latency_ms IS NULL OR response_latency_ms >= 0)
+-- check_in_events | chk_triggered_after_scheduled | CHECK (triggered_at >= scheduled_at OR triggered_at = '0001-01-01 00:00:00')
+-- check_in_events | chk_response_consistency | CHECK ((response_received = FALSE AND response_at IS NULL) OR (response_received = TRUE AND response_at IS NOT NULL))
+
+-- Note: Cron schedule regex validation is enforced at API layer (handlers_checkin.go:104-109)
+-- using robfig/cron parser. Database CHECK constraints with regex cause performance issues.
+
+-- Verify foreign key constraints:
+SELECT
+    conrelid::regclass AS table_name,
+    conname AS constraint_name,
+    pg_get_constraintdef(oid) AS constraint_definition
+FROM pg_constraint
+WHERE conrelid IN (
+    'agent_check_in_configs'::regclass,
+    'check_in_events'::regclass
+)
+AND contype = 'f'
+ORDER BY conrelid::regclass::text, conname;
+
+-- Expected FK constraints (2 total):
+-- agent_check_in_configs | fk_agent_checkin_config_agent | FOREIGN KEY (space_name, agent_name) REFERENCES agents(space_name, agent_name) ON DELETE CASCADE
+-- check_in_events | fk_checkin_event_config | FOREIGN KEY (space_name, agent_name) REFERENCES agent_check_in_configs(space_name, agent_name) ON DELETE CASCADE
+
+-- For SQLite (used in development):
+-- SQLite stores index info differently, use:
+-- SELECT name, sql FROM sqlite_master WHERE type='index' AND tbl_name IN ('agent_check_in_configs', 'check_in_events', 'check_in_scheduler_locks');
+-- For FK constraints in SQLite:
+-- PRAGMA foreign_key_list('agent_check_in_configs');
+-- PRAGMA foreign_key_list('check_in_events');

go.mod

@@ -10,6 +10,8 @@ require (
 )
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/glebarez/go-sqlite v1.21.2 // indirect
 	github.com/google/jsonschema-go v0.4.2 // indirect
@@ -21,16 +23,24 @@ require (
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/prometheus/client_golang v1.23.2 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.66.1 // indirect
+	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/segmentio/asm v1.1.3 // indirect
 	github.com/segmentio/encoding v0.5.3 // indirect
-	github.com/stretchr/testify v1.9.0 // indirect
+	github.com/stretchr/testify v1.11.1 // indirect
 	github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
+	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	golang.org/x/crypto v0.31.0 // indirect
 	golang.org/x/oauth2 v0.34.0 // indirect
-	golang.org/x/sync v0.10.0 // indirect
+	golang.org/x/sync v0.16.0 // indirect
 	golang.org/x/sys v0.40.0 // indirect
-	golang.org/x/text v0.21.0 // indirect
+	golang.org/x/text v0.28.0 // indirect
+	google.golang.org/protobuf v1.36.8 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	modernc.org/libc v1.22.5 // indirect
 	modernc.org/mathutil v1.5.0 // indirect

go.sum

@@ -1,3 +1,7 @@
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -33,11 +37,23 @@ github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPn
 github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/modelcontextprotocol/go-sdk v1.4.0 h1:u0kr8lbJc1oBcawK7Df+/ajNMpIDFE41OEPxdeTLOn8=
 github.com/modelcontextprotocol/go-sdk v1.4.0/go.mod h1:Nxc2n+n/GdCebUaqCOhTetptS17SXXNu9IfNTaLDi1E=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
+github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
+github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
+github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
+github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
+github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc=
 github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg=
 github.com/segmentio/encoding v0.5.3 h1:OjMgICtcSFuNvQCdwqMCv9Tg7lEOXGwm1J5RPQccx6w=
@@ -47,21 +63,30 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4=
 github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4=
+go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
+go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
 golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
 golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
 golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
+golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
 golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
+golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
 golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
 golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
+google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
+google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=