From 4f63e409f52ded23a73a5bbd5243feb133efbc07 Mon Sep 17 00:00:00 2001 From: Dan Fuller Date: Fri, 15 Aug 2025 15:41:57 -0700 Subject: [PATCH 1/3] feat(crons): Send `detector_id` along with the occurrence, when available This starts sending the `detector_id` to the issue platform, so that we can trigger workflows. --- .../monitors/logic/incident_occurrence.py | 8 +- .../logic/test_incident_occurrence.py | 176 +++++++++++++----- 2 files changed, 138 insertions(+), 46 deletions(-) diff --git a/src/sentry/monitors/logic/incident_occurrence.py b/src/sentry/monitors/logic/incident_occurrence.py index 6d37530e1da453..17499149838075 100644 --- a/src/sentry/monitors/logic/incident_occurrence.py +++ b/src/sentry/monitors/logic/incident_occurrence.py @@ -27,6 +27,7 @@ MonitorEnvironment, MonitorIncident, ) +from sentry.monitors.utils import get_detector_for_monitor from sentry.utils.arroyo_producer import SingletonProducer from sentry.utils.kafka_config import get_kafka_producer_cluster_options, get_topic_definition @@ -144,6 +145,11 @@ def send_incident_occurrence( if last_successful_checkin: last_successful_checkin_timestamp = last_successful_checkin.date_added.isoformat() + detector = get_detector_for_monitor(monitor_env.monitor) + evidence_data = {} + if detector: + evidence_data["detector_id"] = detector.id + occurrence = IssueOccurrence( id=uuid.uuid4().hex, resource_id=None, @@ -170,7 +176,7 @@ def send_incident_occurrence( important=False, ), ], - evidence_data={}, + evidence_data=evidence_data, culprit="", detection_time=current_timestamp, level="error", diff --git a/tests/sentry/monitors/logic/test_incident_occurrence.py b/tests/sentry/monitors/logic/test_incident_occurrence.py index 4610a492447310..9028a595cdd4c0 100644 --- a/tests/sentry/monitors/logic/test_incident_occurrence.py +++ b/tests/sentry/monitors/logic/test_incident_occurrence.py @@ -25,15 +25,13 @@ MonitorStatus, ScheduleType, ) +from sentry.monitors.utils import ensure_cron_detector, get_detector_for_monitor from sentry.testutils.cases import TestCase class IncidentOccurrenceTestCase(TestCase): - @mock.patch("sentry.monitors.logic.incident_occurrence.produce_occurrence_to_kafka") - def test_send_incident_occurrence( - self, mock_produce_occurrence_to_kafka: mock.MagicMock - ) -> None: - monitor = Monitor.objects.create( + def build_occurrence_test_data(self): + self.monitor = Monitor.objects.create( name="test monitor", organization_id=self.organization.id, project_id=self.project.id, @@ -44,51 +42,56 @@ def test_send_incident_occurrence( "checkin_margin": None, }, ) - monitor_environment = MonitorEnvironment.objects.create( - monitor=monitor, + self.monitor_environment = MonitorEnvironment.objects.create( + monitor=self.monitor, environment_id=self.environment.id, status=MonitorStatus.ERROR, ) - successful_checkin = MonitorCheckIn.objects.create( - monitor=monitor, - monitor_environment=monitor_environment, + self.successful_checkin = MonitorCheckIn.objects.create( + monitor=self.monitor, + monitor_environment=self.monitor_environment, project_id=self.project.id, status=CheckInStatus.OK, ) - last_checkin = timezone.now() - trace_id = uuid.uuid4() + self.last_checkin = timezone.now() + self.trace_id = uuid.uuid4() - timeout_checkin = MonitorCheckIn.objects.create( - monitor=monitor, - monitor_environment=monitor_environment, + self.timeout_checkin = MonitorCheckIn.objects.create( + monitor=self.monitor, + monitor_environment=self.monitor_environment, project_id=self.project.id, status=CheckInStatus.TIMEOUT, trace_id=uuid.uuid4(), - date_added=last_checkin - timedelta(minutes=1), + date_added=self.last_checkin - timedelta(minutes=1), ) - failed_checkin = MonitorCheckIn.objects.create( - monitor=monitor, - monitor_environment=monitor_environment, + self.failed_checkin = MonitorCheckIn.objects.create( + monitor=self.monitor, + monitor_environment=self.monitor_environment, project_id=self.project.id, status=CheckInStatus.ERROR, - trace_id=trace_id, - date_added=last_checkin, + trace_id=self.trace_id, + date_added=self.last_checkin, ) - incident = MonitorIncident.objects.create( - monitor=monitor, - monitor_environment=monitor_environment, - starting_checkin=failed_checkin, - starting_timestamp=last_checkin, + self.incident = MonitorIncident.objects.create( + monitor=self.monitor, + monitor_environment=self.monitor_environment, + starting_checkin=self.failed_checkin, + starting_timestamp=self.last_checkin, grouphash="abcd", ) + @mock.patch("sentry.monitors.logic.incident_occurrence.produce_occurrence_to_kafka") + def test_send_incident_occurrence( + self, mock_produce_occurrence_to_kafka: mock.MagicMock + ) -> None: + self.build_occurrence_test_data() send_incident_occurrence( - failed_checkin, - [timeout_checkin, failed_checkin], - incident, - last_checkin, + self.failed_checkin, + [self.timeout_checkin, self.failed_checkin], + self.incident, + self.last_checkin, ) assert mock_produce_occurrence_to_kafka.call_count == 1 @@ -102,8 +105,8 @@ def test_send_incident_occurrence( occurrence, **{ "project_id": self.project.id, - "fingerprint": [incident.grouphash], - "issue_title": f"Monitor failure: {monitor.name}", + "fingerprint": [self.incident.grouphash], + "issue_title": f"Monitor failure: {self.monitor.name}", "subtitle": "Your monitor has reached its failure threshold.", "resource_id": None, "evidence_data": {}, @@ -115,12 +118,95 @@ def test_send_incident_occurrence( }, { "name": "Environment", - "value": monitor_environment.get_environment().name, + "value": self.monitor_environment.get_environment().name, + "important": False, + }, + { + "name": "Last successful check-in", + "value": self.successful_checkin.date_added.isoformat(), + "important": False, + }, + ], + "type": MonitorIncidentType.type_id, + "level": "error", + "culprit": "", + }, + ) == dict(occurrence) + + assert dict( + event, + **{ + "contexts": { + "monitor": { + "status": "error", + "config": self.monitor.config, + "id": str(self.monitor.guid), + "name": self.monitor.name, + "slug": self.monitor.slug, + }, + "trace": { + "trace_id": self.trace_id.hex, + "span_id": None, + }, + }, + "environment": self.monitor_environment.get_environment().name, + "event_id": occurrence["event_id"], + "fingerprint": [self.incident.grouphash], + "platform": "other", + "project_id": self.monitor.project_id, + "sdk": None, + "tags": { + "monitor.id": str(self.monitor.guid), + "monitor.slug": str(self.monitor.slug), + "monitor.incident": str(self.incident.id), + }, + }, + ) == dict(event) + + @mock.patch("sentry.monitors.logic.incident_occurrence.produce_occurrence_to_kafka") + def test_send_incident_occurrence_detector( + self, mock_produce_occurrence_to_kafka: mock.MagicMock + ) -> None: + self.build_occurrence_test_data() + ensure_cron_detector(self.monitor) + send_incident_occurrence( + self.failed_checkin, + [self.timeout_checkin, self.failed_checkin], + self.incident, + self.last_checkin, + ) + + assert mock_produce_occurrence_to_kafka.call_count == 1 + kwargs = mock_produce_occurrence_to_kafka.call_args.kwargs + + occurrence = kwargs["occurrence"] + event = kwargs["event_data"] + occurrence = occurrence.to_dict() + + detector = get_detector_for_monitor(self.monitor) + assert dict( + occurrence, + **{ + "project_id": self.project.id, + "fingerprint": [self.incident.grouphash], + "issue_title": f"Monitor failure: {self.monitor.name}", + "subtitle": "Your monitor has reached its failure threshold.", + "resource_id": None, + "evidence_data": {"detector_id": detector.id}, + "evidence_display": [ + { + "name": "Failure reason", + "value": "1 timeout and 1 error check-ins detected", + "important": True, + }, + { + "name": "Environment", + "value": self.monitor_environment.get_environment().name, "important": False, }, { "name": "Last successful check-in", - "value": successful_checkin.date_added.isoformat(), + "value": self.successful_checkin.date_added.isoformat(), "important": False, }, ], @@ -136,26 +222,26 @@ def test_send_incident_occurrence( "contexts": { "monitor": { "status": "error", - "config": monitor.config, - "id": str(monitor.guid), - "name": monitor.name, - "slug": monitor.slug, + "config": self.monitor.config, + "id": str(self.monitor.guid), + "name": self.monitor.name, + "slug": self.monitor.slug, }, "trace": { - "trace_id": trace_id.hex, + "trace_id": self.trace_id.hex, "span_id": None, }, }, - "environment": monitor_environment.get_environment().name, + "environment": self.monitor_environment.get_environment().name, "event_id": occurrence["event_id"], - "fingerprint": [incident.grouphash], + "fingerprint": [self.incident.grouphash], "platform": "other", - "project_id": monitor.project_id, + "project_id": self.monitor.project_id, "sdk": None, "tags": { - "monitor.id": str(monitor.guid), - "monitor.slug": str(monitor.slug), - "monitor.incident": str(incident.id), + "monitor.id": str(self.monitor.guid), + "monitor.slug": str(self.monitor.slug), + "monitor.incident": str(self.incident.id), }, }, ) == dict(event) From bfe7b744852d8568035c06b09844f3f6d81d4046 Mon Sep 17 00:00:00 2001 From: Dan Fuller Date: Mon, 25 Aug 2025 15:18:13 -0700 Subject: [PATCH 2/3] tests --- src/sentry/monitors/utils.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/sentry/monitors/utils.py b/src/sentry/monitors/utils.py index 957041a67cfb72..3fefacd89cfa49 100644 --- a/src/sentry/monitors/utils.py +++ b/src/sentry/monitors/utils.py @@ -9,6 +9,7 @@ from sentry import audit_log from sentry.api.serializers.rest_framework.rule import RuleSerializer from sentry.db.models import BoundedPositiveIntegerField +from sentry.db.postgres.transactions import in_test_hide_transaction_boundary from sentry.models.group import Group from sentry.models.project import Project from sentry.models.rule import Rule, RuleActivity, RuleActivityType, RuleSource @@ -416,10 +417,11 @@ def ensure_cron_detector(monitor: Monitor): def get_detector_for_monitor(monitor: Monitor) -> Detector | None: try: - return Detector.objects.get( - datasource__type=DATA_SOURCE_CRON_MONITOR, - datasource__source_id=str(monitor.id), - datasource__organization_id=monitor.organization_id, - ) + with in_test_hide_transaction_boundary(): + return Detector.objects.get( + datasource__type=DATA_SOURCE_CRON_MONITOR, + datasource__source_id=str(monitor.id), + datasource__organization_id=monitor.organization_id, + ) except Detector.DoesNotExist: return None From f7d2c44a0f0bbc1a49d2c398e6f4ca79ec4816ed Mon Sep 17 00:00:00 2001 From: Dan Fuller Date: Mon, 25 Aug 2025 15:50:27 -0700 Subject: [PATCH 3/3] mypy --- tests/sentry/monitors/logic/test_incident_occurrence.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/sentry/monitors/logic/test_incident_occurrence.py b/tests/sentry/monitors/logic/test_incident_occurrence.py index 9028a595cdd4c0..152bb41a523d9d 100644 --- a/tests/sentry/monitors/logic/test_incident_occurrence.py +++ b/tests/sentry/monitors/logic/test_incident_occurrence.py @@ -184,6 +184,7 @@ def test_send_incident_occurrence_detector( occurrence = occurrence.to_dict() detector = get_detector_for_monitor(self.monitor) + assert detector assert dict( occurrence, **{