From 644da3e4714810ee016ac47856d750603509e84b Mon Sep 17 00:00:00 2001 From: Dennis Toepker Date: Mon, 9 Mar 2026 10:54:31 -0700 Subject: [PATCH 1/9] Revert "[3.5] Remove PPL alerting feature. (#2017)" This reverts commit c39cacaf3cfe8aef9ba81f033b2e10187347543f. --- alerting/build.gradle | 27 + .../org/opensearch/alerting/AlertingPlugin.kt | 75 +- .../opensearch/alerting/AlertingV2Utils.kt | 219 +++++ .../org/opensearch/alerting/MonitorRunner.kt | 116 +-- .../alerting/MonitorRunnerExecutionContext.kt | 2 + .../alerting/MonitorRunnerService.kt | 150 ++- .../opensearch/alerting/MonitorV2Runner.kt | 29 + .../alerting/PPLSQLMonitorRunner.kt | 679 ++++++++++++++ .../org/opensearch/alerting/PPLUtils.kt | 305 ++++++ .../opensearch/alerting/WorkflowService.kt | 8 +- .../actionv2/DeleteMonitorV2Action.kt | 15 + .../actionv2/DeleteMonitorV2Request.kt | 39 + .../actionv2/DeleteMonitorV2Response.kt | 43 + .../actionv2/ExecuteMonitorV2Action.kt | 15 + .../actionv2/ExecuteMonitorV2Request.kt | 71 ++ .../actionv2/ExecuteMonitorV2Response.kt | 38 + .../alerting/actionv2/GetAlertsV2Action.kt | 15 + .../alerting/actionv2/GetAlertsV2Request.kt | 47 + .../alerting/actionv2/GetAlertsV2Response.kt | 52 ++ .../alerting/actionv2/GetMonitorV2Action.kt | 15 + .../alerting/actionv2/GetMonitorV2Request.kt | 52 ++ .../alerting/actionv2/GetMonitorV2Response.kt | 80 ++ .../alerting/actionv2/IndexMonitorV2Action.kt | 15 + .../actionv2/IndexMonitorV2Request.kt | 69 ++ .../actionv2/IndexMonitorV2Response.kt | 73 ++ .../actionv2/SearchMonitorV2Action.kt | 16 + .../actionv2/SearchMonitorV2Request.kt | 37 + .../alerting/alertsv2/AlertV2Indices.kt | 418 +++++++++ .../alerting/alertsv2/AlertV2Mover.kt | 477 ++++++++++ .../opensearch/alerting/modelv2/AlertV2.kt | 257 +++++ .../opensearch/alerting/modelv2/MonitorV2.kt | 157 ++++ .../alerting/modelv2/MonitorV2RunResult.kt | 49 + .../alerting/modelv2/PPLSQLMonitor.kt | 410 ++++++++ .../modelv2/PPLSQLMonitorRunResult.kt | 74 ++ .../alerting/modelv2/PPLSQLTrigger.kt | 413 +++++++++ .../modelv2/PPLSQLTriggerRunResult.kt | 60 ++ .../opensearch/alerting/modelv2/TriggerV2.kt | 70 ++ .../alerting/modelv2/TriggerV2RunResult.kt | 25 + .../RestDeleteMonitorV2Action.kt | 60 ++ .../RestExecuteMonitorV2Action.kt | 82 ++ .../resthandlerv2/RestGetAlertsV2Action.kt | 72 ++ .../resthandlerv2/RestGetMonitorV2Action.kt | 73 ++ .../resthandlerv2/RestIndexMonitorV2Action.kt | 86 ++ .../RestSearchMonitorV2Action.kt | 128 +++ .../script/PPLTriggerExecutionContext.kt | 27 + .../script/TriggerV2ExecutionContext.kt | 21 + .../alerting/service/DeleteMonitorService.kt | 21 +- .../alerting/settings/AlertingSettings.kt | 109 +++ .../transport/TransportDeleteMonitorAction.kt | 2 +- .../TransportDeleteWorkflowAction.kt | 18 +- .../TransportExecuteWorkflowAction.kt | 8 +- .../transport/TransportGetAlertsAction.kt | 6 +- .../transport/TransportIndexMonitorAction.kt | 11 +- .../transport/TransportIndexWorkflowAction.kt | 16 +- .../transport/TransportSearchMonitorAction.kt | 1 - .../TransportDeleteMonitorV2Action.kt | 148 +++ .../TransportExecuteMonitorV2Action.kt | 229 +++++ .../transportv2/TransportGetAlertsV2Action.kt | 211 +++++ .../TransportGetMonitorV2Action.kt | 172 ++++ .../TransportIndexMonitorV2Action.kt | 868 +++++++++++++++++ .../TransportSearchMonitorV2Action.kt | 122 +++ .../opensearch/alerting/util/IndexUtils.kt | 27 + .../alerting/workflow/WorkflowRunner.kt | 118 +-- .../alerting/alertsv2/alert_v2_mapping.json | 118 +++ .../org/opensearch/alerting/AccessRoles.kt | 2 + .../alerting/AlertingRestTestCase.kt | 543 ++++++++--- .../alerting/PPLSQLMonitorRunnerIT.kt | 496 ++++++++++ .../org/opensearch/alerting/TestHelpers.kt | 281 +++++- .../actionv2/DeleteMonitorV2RequestTests.kt | 29 + .../actionv2/DeleteMonitorV2ResponseTests.kt | 28 + .../actionv2/ExecuteMonitorV2RequestTests.kt | 38 + .../actionv2/ExecuteMonitorV2ResponseTests.kt | 57 ++ .../actionv2/GetAlertsV2RequestTests.kt | 49 + .../actionv2/GetAlertsV2ResponseTests.kt | 93 ++ .../actionv2/GetMonitorV2RequestTests.kt | 31 + .../actionv2/GetMonitorV2ResponseTests.kt | 36 + .../actionv2/IndexMonitorV2RequestTests.kt | 42 + .../actionv2/IndexMonitorV2ResponseTests.kt | 36 + .../actionv2/SearchMonitorV2RequestTests.kt | 34 + .../alerting/alerts/AlertIndicesIT.kt | 4 +- .../alerting/alertsv2/AlertV2IndicesIT.kt | 450 +++++++++ .../alerting/modelv2/AlertV2Tests.kt | 59 ++ .../alerting/modelv2/MonitorV2Tests.kt | 172 ++++ .../alerting/modelv2/RunResultV2Tests.kt | 61 ++ .../alerting/modelv2/TriggerV2Tests.kt | 254 +++++ .../alerting/resthandler/MonitorRestApiIT.kt | 17 +- .../resthandler/MonitorV2RestApiIT.kt | 524 +++++++++++ .../resthandler/SecureMonitorV2RestApiIT.kt | 875 ++++++++++++++++++ core/build.gradle | 51 +- .../core/action/node/ScheduledJobStats.kt | 7 +- .../action/node/ScheduledJobsStatsRequest.kt | 9 +- .../node/ScheduledJobsStatsTransportAction.kt | 3 +- .../alerting/core/ppl/PPLPluginInterface.kt | 65 ++ .../RestScheduledJobStatsHandler.kt | 61 +- .../core/resthandler/StatsRequestUtils.kt | 63 ++ .../alerting/core/schedule/JobScheduler.kt | 21 +- .../core/settings/AlertingV2Settings.kt | 22 + .../alerting/core/util/XContentExtensions.kt | 13 + .../opensearchapi/OpenSearchExtensions.kt | 15 + .../resources/mappings/scheduled-jobs.json | 197 +++- ...ensearch-alerting.release-notes-3.5.0.0.md | 7 - 101 files changed, 11716 insertions(+), 495 deletions(-) create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/AlertingV2Utils.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/MonitorV2Runner.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/PPLSQLMonitorRunner.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Action.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Request.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Response.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Action.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Request.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Response.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Action.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Request.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Response.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Action.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Request.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Response.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Action.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Request.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Response.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2Action.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2Request.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Indices.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Mover.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/modelv2/AlertV2.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2RunResult.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitor.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitorRunResult.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTrigger.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTriggerRunResult.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2RunResult.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestDeleteMonitorV2Action.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestExecuteMonitorV2Action.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetAlertsV2Action.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetMonitorV2Action.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestIndexMonitorV2Action.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestSearchMonitorV2Action.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/script/PPLTriggerExecutionContext.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/script/TriggerV2ExecutionContext.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportDeleteMonitorV2Action.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportExecuteMonitorV2Action.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetAlertsV2Action.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetMonitorV2Action.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportIndexMonitorV2Action.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportSearchMonitorV2Action.kt create mode 100644 alerting/src/main/resources/org/opensearch/alerting/alertsv2/alert_v2_mapping.json create mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/PPLSQLMonitorRunnerIT.kt create mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2RequestTests.kt create mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2ResponseTests.kt create mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2RequestTests.kt create mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2ResponseTests.kt create mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2RequestTests.kt create mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2ResponseTests.kt create mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2RequestTests.kt create mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2ResponseTests.kt create mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2RequestTests.kt create mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2ResponseTests.kt create mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2RequestTests.kt create mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/alertsv2/AlertV2IndicesIT.kt create mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/modelv2/AlertV2Tests.kt create mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/modelv2/MonitorV2Tests.kt create mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/modelv2/RunResultV2Tests.kt create mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/modelv2/TriggerV2Tests.kt create mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorV2RestApiIT.kt create mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureMonitorV2RestApiIT.kt create mode 100644 core/src/main/kotlin/org/opensearch/alerting/core/ppl/PPLPluginInterface.kt create mode 100644 core/src/main/kotlin/org/opensearch/alerting/core/resthandler/StatsRequestUtils.kt create mode 100644 core/src/main/kotlin/org/opensearch/alerting/core/settings/AlertingV2Settings.kt create mode 100644 core/src/main/kotlin/org/opensearch/alerting/core/util/XContentExtensions.kt delete mode 100644 release-notes/opensearch-alerting.release-notes-3.5.0.0.md diff --git a/alerting/build.gradle b/alerting/build.gradle index 5a53f45d7..74c2c8d4a 100644 --- a/alerting/build.gradle +++ b/alerting/build.gradle @@ -156,6 +156,8 @@ dependencies { // Needed for integ tests zipArchive group: 'org.opensearch.plugin', name:'opensearch-notifications-core', version: "${opensearch_build}" zipArchive group: 'org.opensearch.plugin', name:'notifications', version: "${opensearch_build}" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-sql-plugin', version: "${opensearch_build}" // Needed for security tests if (securityEnabled) { @@ -173,7 +175,10 @@ dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}" implementation "org.jetbrains:annotations:13.0" + // SQL/PPL plugin dependencies are included in alerting-core api project(":alerting-core") + implementation 'org.json:json:20240303' + implementation "com.github.seancfoley:ipaddress:5.4.1" implementation project(path: ":alerting-spi", configuration: 'shadow') @@ -253,6 +258,28 @@ testClusters.integTest { } })) + plugin(provider({ + new RegularFile() { + @Override + File getAsFile() { + return configurations.zipArchive.asFileTree.matching { + include '**/opensearch-job-scheduler*' + }.singleFile + } + } + })) + + plugin(provider({ + new RegularFile() { + @Override + File getAsFile() { + return configurations.zipArchive.asFileTree.matching { + include '**/opensearch-sql-plugin*' + }.singleFile + } + } + })) + if (securityEnabled) { plugin(provider({ new RegularFile() { diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index c7415e061..d80d7b114 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -14,8 +14,17 @@ import org.opensearch.alerting.action.GetEmailGroupAction import org.opensearch.alerting.action.GetRemoteIndexesAction import org.opensearch.alerting.action.SearchEmailAccountAction import org.opensearch.alerting.action.SearchEmailGroupAction +import org.opensearch.alerting.actionv2.DeleteMonitorV2Action +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Action +import org.opensearch.alerting.actionv2.GetAlertsV2Action +import org.opensearch.alerting.actionv2.GetMonitorV2Action +import org.opensearch.alerting.actionv2.IndexMonitorV2Action +import org.opensearch.alerting.actionv2.SearchMonitorV2Action import org.opensearch.alerting.alerts.AlertIndices import org.opensearch.alerting.alerts.AlertIndices.Companion.ALL_ALERT_INDEX_PATTERN +import org.opensearch.alerting.alertsv2.AlertV2Indices +import org.opensearch.alerting.alertsv2.AlertV2Indices.Companion.ALL_ALERT_V2_INDEX_PATTERN +import org.opensearch.alerting.alertsv2.AlertV2Mover import org.opensearch.alerting.comments.CommentsIndices import org.opensearch.alerting.comments.CommentsIndices.Companion.ALL_COMMENTS_INDEX_PATTERN import org.opensearch.alerting.core.JobSweeper @@ -25,8 +34,10 @@ import org.opensearch.alerting.core.action.node.ScheduledJobsStatsTransportActio import org.opensearch.alerting.core.lock.LockService import org.opensearch.alerting.core.resthandler.RestScheduledJobStatsHandler import org.opensearch.alerting.core.schedule.JobScheduler +import org.opensearch.alerting.core.settings.AlertingV2Settings import org.opensearch.alerting.core.settings.LegacyOpenDistroScheduledJobSettings import org.opensearch.alerting.core.settings.ScheduledJobSettings +import org.opensearch.alerting.modelv2.MonitorV2 import org.opensearch.alerting.remote.monitors.RemoteMonitorRegistry import org.opensearch.alerting.resthandler.RestAcknowledgeAlertAction import org.opensearch.alerting.resthandler.RestAcknowledgeChainedAlertAction @@ -51,6 +62,12 @@ import org.opensearch.alerting.resthandler.RestSearchAlertingCommentAction import org.opensearch.alerting.resthandler.RestSearchEmailAccountAction import org.opensearch.alerting.resthandler.RestSearchEmailGroupAction import org.opensearch.alerting.resthandler.RestSearchMonitorAction +import org.opensearch.alerting.resthandlerv2.RestDeleteMonitorV2Action +import org.opensearch.alerting.resthandlerv2.RestExecuteMonitorV2Action +import org.opensearch.alerting.resthandlerv2.RestGetAlertsV2Action +import org.opensearch.alerting.resthandlerv2.RestGetMonitorV2Action +import org.opensearch.alerting.resthandlerv2.RestIndexMonitorV2Action +import org.opensearch.alerting.resthandlerv2.RestSearchMonitorV2Action import org.opensearch.alerting.script.TriggerScript import org.opensearch.alerting.service.DeleteMonitorService import org.opensearch.alerting.settings.AlertingSettings @@ -88,6 +105,12 @@ import org.opensearch.alerting.transport.TransportSearchAlertingCommentAction import org.opensearch.alerting.transport.TransportSearchEmailAccountAction import org.opensearch.alerting.transport.TransportSearchEmailGroupAction import org.opensearch.alerting.transport.TransportSearchMonitorAction +import org.opensearch.alerting.transportv2.TransportDeleteMonitorV2Action +import org.opensearch.alerting.transportv2.TransportExecuteMonitorV2Action +import org.opensearch.alerting.transportv2.TransportGetAlertsV2Action +import org.opensearch.alerting.transportv2.TransportGetMonitorV2Action +import org.opensearch.alerting.transportv2.TransportIndexMonitorV2Action +import org.opensearch.alerting.transportv2.TransportSearchMonitorV2Action import org.opensearch.alerting.util.DocLevelMonitorQueries import org.opensearch.alerting.util.destinationmigration.DestinationMigrationCoordinator import org.opensearch.cluster.metadata.IndexNameExpressionResolver @@ -170,6 +193,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R @JvmField val UI_METADATA_EXCLUDE = arrayOf("monitor.${Monitor.UI_METADATA_FIELD}") @JvmField val TENANT_ID_HEADER = "x-tenant-id" @JvmField val MONITOR_BASE_URI = "/_plugins/_alerting/monitors" + @JvmField val MONITOR_V2_BASE_URI = "/_plugins/_alerting/v2/monitors" @JvmField val WORKFLOW_BASE_URI = "/_plugins/_alerting/workflows" @JvmField val REMOTE_BASE_URI = "/_plugins/_alerting/remote" @JvmField val DESTINATION_BASE_URI = "/_plugins/_alerting/destinations" @@ -182,7 +206,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R @JvmField val FINDING_BASE_URI = "/_plugins/_alerting/findings" @JvmField val COMMENTS_BASE_URI = "/_plugins/_alerting/comments" - @JvmField val ALERTING_JOB_TYPES = listOf("monitor", "workflow") + @JvmField val ALERTING_JOB_TYPES = listOf("monitor", "workflow", "monitor_v2") } lateinit var runner: MonitorRunnerService @@ -193,8 +217,10 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R lateinit var docLevelMonitorQueries: DocLevelMonitorQueries lateinit var threadPool: ThreadPool lateinit var alertIndices: AlertIndices + lateinit var alertV2Indices: AlertV2Indices lateinit var clusterService: ClusterService lateinit var destinationMigrationCoordinator: DestinationMigrationCoordinator + lateinit var alertV2Mover: AlertV2Mover var monitorTypeToMonitorRunners: MutableMap = mutableMapOf() override fun getRestHandlers( @@ -207,6 +233,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R nodesInCluster: Supplier ): List { return listOf( + // Alerting V1 RestGetMonitorAction(), RestDeleteMonitorAction(), RestIndexMonitorAction(), @@ -231,11 +258,20 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R RestIndexAlertingCommentAction(), RestSearchAlertingCommentAction(), RestDeleteAlertingCommentAction(), + + // Alerting V2 + RestIndexMonitorV2Action(), + RestExecuteMonitorV2Action(), + RestDeleteMonitorV2Action(), + RestGetMonitorV2Action(), + RestSearchMonitorV2Action(settings, clusterService), + RestGetAlertsV2Action() ) } override fun getActions(): List> { return listOf( + // Alerting V1 ActionPlugin.ActionHandler(ScheduledJobsStatsAction.INSTANCE, ScheduledJobsStatsTransportAction::class.java), ActionPlugin.ActionHandler(AlertingActions.INDEX_MONITOR_ACTION_TYPE, TransportIndexMonitorAction::class.java), ActionPlugin.ActionHandler(AlertingActions.GET_MONITOR_ACTION_TYPE, TransportGetMonitorAction::class.java), @@ -262,13 +298,22 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R ActionPlugin.ActionHandler(AlertingActions.DELETE_COMMENT_ACTION_TYPE, TransportDeleteAlertingCommentAction::class.java), ActionPlugin.ActionHandler(ExecuteWorkflowAction.INSTANCE, TransportExecuteWorkflowAction::class.java), ActionPlugin.ActionHandler(GetRemoteIndexesAction.INSTANCE, TransportGetRemoteIndexesAction::class.java), - ActionPlugin.ActionHandler(DocLevelMonitorFanOutAction.INSTANCE, TransportDocLevelMonitorFanOutAction::class.java) + ActionPlugin.ActionHandler(DocLevelMonitorFanOutAction.INSTANCE, TransportDocLevelMonitorFanOutAction::class.java), + + // Alerting V2 + ActionPlugin.ActionHandler(IndexMonitorV2Action.INSTANCE, TransportIndexMonitorV2Action::class.java), + ActionPlugin.ActionHandler(GetMonitorV2Action.INSTANCE, TransportGetMonitorV2Action::class.java), + ActionPlugin.ActionHandler(SearchMonitorV2Action.INSTANCE, TransportSearchMonitorV2Action::class.java), + ActionPlugin.ActionHandler(DeleteMonitorV2Action.INSTANCE, TransportDeleteMonitorV2Action::class.java), + ActionPlugin.ActionHandler(ExecuteMonitorV2Action.INSTANCE, TransportExecuteMonitorV2Action::class.java), + ActionPlugin.ActionHandler(GetAlertsV2Action.INSTANCE, TransportGetAlertsV2Action::class.java) ) } override fun getNamedXContent(): List { return listOf( Monitor.XCONTENT_REGISTRY, + MonitorV2.XCONTENT_REGISTRY, SearchInput.XCONTENT_REGISTRY, DocLevelMonitorInput.XCONTENT_REGISTRY, QueryLevelTrigger.XCONTENT_REGISTRY, @@ -298,6 +343,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R val settings = environment.settings() val lockService = LockService(client, clusterService) alertIndices = AlertIndices(settings, client, threadPool, clusterService) + alertV2Indices = AlertV2Indices(settings, client, threadPool, clusterService) val sdkClient: SdkClient = SdkClientFactory.createSdkClient( client, @@ -323,6 +369,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R .registerSettings(settings) .registerThreadPool(threadPool) .registerAlertIndices(alertIndices) + .registerAlertV2Indices(alertV2Indices) .registerInputService( InputService( client, @@ -349,6 +396,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R scheduler = JobScheduler(threadPool, runner) sweeper = JobSweeper(environment.settings(), client, clusterService, threadPool, xContentRegistry, scheduler, ALERTING_JOB_TYPES) destinationMigrationCoordinator = DestinationMigrationCoordinator(client, clusterService, threadPool, scheduledJobIndices) + alertV2Mover = AlertV2Mover(environment.settings(), client, threadPool, clusterService, xContentRegistry) this.threadPool = threadPool this.clusterService = clusterService @@ -377,6 +425,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R commentsIndices, docLevelMonitorQueries, destinationMigrationCoordinator, + alertV2Mover, lockService, alertService, triggerService, @@ -468,7 +517,24 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R AlertingSettings.REMOTE_METADATA_ENDPOINT, AlertingSettings.REMOTE_METADATA_REGION, AlertingSettings.REMOTE_METADATA_SERVICE_NAME, - AlertingSettings.MULTI_TENANT_TRIGGER_EVAL_ENABLED + AlertingSettings.MULTI_TENANT_TRIGGER_EVAL_ENABLED, + AlertingSettings.ALERT_V2_HISTORY_ENABLED, + AlertingSettings.ALERT_V2_HISTORY_ROLLOVER_PERIOD, + AlertingSettings.ALERT_V2_HISTORY_INDEX_MAX_AGE, + AlertingSettings.ALERT_V2_HISTORY_MAX_DOCS, + AlertingSettings.ALERT_V2_HISTORY_RETENTION_PERIOD, + AlertingSettings.ALERT_V2_MONITOR_EXECUTION_MAX_DURATION, + AlertingSettings.ALERTING_V2_MAX_MONITORS, + AlertingSettings.ALERTING_V2_MAX_THROTTLE_DURATION, + AlertingSettings.ALERTING_V2_MAX_EXPIRE_DURATION, + AlertingSettings.ALERTING_V2_MAX_LOOK_BACK_WINDOW, + AlertingSettings.ALERTING_V2_MAX_QUERY_LENGTH, + AlertingSettings.ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS, + AlertingSettings.ALERT_V2_QUERY_RESULTS_MAX_SIZE, + AlertingSettings.ALERT_V2_PER_RESULT_TRIGGER_MAX_ALERTS, + AlertingSettings.NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH, + AlertingSettings.NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH, + AlertingV2Settings.ALERTING_V2_ENABLED, ) } @@ -486,7 +552,8 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R return listOf( SystemIndexDescriptor(ALL_ALERT_INDEX_PATTERN, "Alerting Plugin system index pattern"), SystemIndexDescriptor(SCHEDULED_JOBS_INDEX, "Alerting Plugin Configuration index"), - SystemIndexDescriptor(ALL_COMMENTS_INDEX_PATTERN, "Alerting Comments system index pattern") + SystemIndexDescriptor(ALL_COMMENTS_INDEX_PATTERN, "Alerting Comments system index pattern"), + SystemIndexDescriptor(ALL_ALERT_V2_INDEX_PATTERN, "Alerting V2 Alerts index pattern") ) } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingV2Utils.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingV2Utils.kt new file mode 100644 index 000000000..8a3936eb6 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingV2Utils.kt @@ -0,0 +1,219 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting + +import org.apache.lucene.search.TotalHits +import org.apache.lucene.search.TotalHits.Relation +import org.opensearch.OpenSearchSecurityException +import org.opensearch.action.search.SearchResponse +import org.opensearch.action.search.ShardSearchFailure +import org.opensearch.alerting.AlertingPlugin.Companion.MONITOR_BASE_URI +import org.opensearch.alerting.AlertingPlugin.Companion.MONITOR_V2_BASE_URI +import org.opensearch.alerting.action.GetDestinationsAction +import org.opensearch.alerting.action.GetDestinationsRequest +import org.opensearch.alerting.action.GetDestinationsResponse +import org.opensearch.alerting.model.destination.Destination +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.opensearchapi.suspendUntil +import org.opensearch.alerting.util.destinationmigration.NotificationActionConfigs +import org.opensearch.alerting.util.destinationmigration.NotificationApiUtils.Companion.getNotificationConfigInfo +import org.opensearch.alerting.util.destinationmigration.getTitle +import org.opensearch.alerting.util.destinationmigration.publishLegacyNotification +import org.opensearch.alerting.util.destinationmigration.sendNotification +import org.opensearch.alerting.util.isAllowed +import org.opensearch.alerting.util.isTestAction +import org.opensearch.commons.alerting.model.Monitor +import org.opensearch.commons.alerting.model.ScheduledJob +import org.opensearch.commons.alerting.model.Table +import org.opensearch.commons.alerting.model.Workflow +import org.opensearch.commons.alerting.model.action.Action +import org.opensearch.commons.notifications.model.NotificationConfigInfo +import org.opensearch.index.IndexNotFoundException +import org.opensearch.search.SearchHits +import org.opensearch.search.aggregations.InternalAggregations +import org.opensearch.search.internal.InternalSearchResponse +import org.opensearch.search.profile.SearchProfileShardResults +import org.opensearch.search.suggest.Suggest +import org.opensearch.transport.RemoteTransportException +import org.opensearch.transport.client.node.NodeClient +import java.util.Collections + +object AlertingV2Utils { + // Validates that the given scheduled job is a Monitor + // returns the exception to pass into actionListener.onFailure if not. + fun validateMonitorV1(scheduledJob: ScheduledJob): Exception? { + if (scheduledJob is MonitorV2) { + return IllegalStateException( + "The ID given corresponds to an Alerting V2 Monitor, but a V1 Monitor was expected. " + + "If you wish to operate on a V2 Monitor (e.g. PPL Monitor), please use " + + "the Alerting V2 APIs with endpoint prefix: $MONITOR_V2_BASE_URI." + ) + } else if (scheduledJob !is Monitor && scheduledJob !is Workflow) { + return IllegalStateException( + "The ID given corresponds to a scheduled job of unknown type: ${scheduledJob.javaClass.name}. " + + "Please validate the ID and ensure it corresponds to a valid Monitor." + ) + } + return null + } + + // Validates that the given scheduled job is a MonitorV2 + // returns the exception to pass into actionListener.onFailure if not. + fun validateMonitorV2(scheduledJob: ScheduledJob): Exception? { + if (scheduledJob is Monitor || scheduledJob is Workflow) { + return IllegalStateException( + "The ID given corresponds to an Alerting V1 Monitor, but a V2 Monitor was expected. " + + "If you wish to operate on a V1 Monitor (e.g. Per Query, Per Document, etc), please use " + + "the Alerting V1 APIs with endpoint prefix: $MONITOR_BASE_URI." + ) + } else if (scheduledJob !is MonitorV2) { + return IllegalStateException( + "The ID given corresponds to a scheduled job of unknown type: ${scheduledJob.javaClass.name}. " + + "Please validate the ID and ensure it corresponds to a valid Monitor." + ) + } + return null + } + + // Checks if the exception is caused by an IndexNotFoundException (directly or nested). + // Used in Get and Search monitor functionalities to determine whether a "no results" + // response should be returned + fun isIndexNotFoundException(e: Exception): Boolean { + if (e is IndexNotFoundException) { + return true + } + if (e is RemoteTransportException) { + val cause = e.cause + if (cause is IndexNotFoundException) { + return true + } + } + return false + } + + // Used in Get and Search monitor functionalities to return a "no results" response + fun getEmptySearchResponse(): SearchResponse { + val internalSearchResponse = InternalSearchResponse( + SearchHits(emptyArray(), TotalHits(0L, Relation.EQUAL_TO), 0.0f), + InternalAggregations.from(Collections.emptyList()), + Suggest(Collections.emptyList()), + SearchProfileShardResults(Collections.emptyMap()), + false, + false, + 0 + ) + + return SearchResponse( + internalSearchResponse, + "", + 0, + 0, + 0, + 0, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ) + } + + suspend fun getConfigAndSendNotification( + action: Action, + monitorCtx: MonitorRunnerExecutionContext, + subject: String?, + message: String + ): String { + val config = getConfigForNotificationAction(action, monitorCtx) + if (config.destination == null && config.channel == null) { + throw IllegalStateException("Unable to find a Notification Channel or Destination config with id [${action.destinationId}]") + } + + // Adding a check on TEST_ACTION Destination type here to avoid supporting it as a LegacyBaseMessage type + // just for Alerting integration tests + if (config.destination?.isTestAction() == true) { + return "test action" + } + + if (config.destination?.isAllowed(monitorCtx.allowList) == false) { + throw IllegalStateException( + "Monitor contains a Destination type that is not allowed: ${config.destination.type}" + ) + } + + var actionResponseContent = "" + actionResponseContent = config.channel + ?.sendNotification( + monitorCtx.client!!, + config.channel.getTitle(subject), + message + ) ?: actionResponseContent + + actionResponseContent = config.destination + ?.buildLegacyBaseMessage(subject, message, monitorCtx.destinationContextFactory!!.getDestinationContext(config.destination)) + ?.publishLegacyNotification(monitorCtx.client!!) + ?: actionResponseContent + + return actionResponseContent + } + + /** + * The "destination" ID referenced in a Monitor Action could either be a Notification config or a Destination config + * depending on whether the background migration process has already migrated it from a Destination to a Notification config. + * + * To cover both of these cases, the Notification config will take precedence and if it is not found, the Destination will be retrieved. + */ + private suspend fun getConfigForNotificationAction( + action: Action, + monitorCtx: MonitorRunnerExecutionContext + ): NotificationActionConfigs { + var destination: Destination? = null + var notificationPermissionException: Exception? = null + + var channel: NotificationConfigInfo? = null + try { + channel = getNotificationConfigInfo(monitorCtx.client as NodeClient, action.destinationId) + } catch (e: OpenSearchSecurityException) { + notificationPermissionException = e + } + + // If the channel was not found, try to retrieve the Destination + if (channel == null) { + destination = try { + val table = Table( + "asc", + "destination.name.keyword", + null, + 1, + 0, + null + ) + val getDestinationsRequest = GetDestinationsRequest( + action.destinationId, + 0L, + null, + table, + "ALL" + ) + + val getDestinationsResponse: GetDestinationsResponse = monitorCtx.client!!.suspendUntil { + monitorCtx.client!!.execute(GetDestinationsAction.INSTANCE, getDestinationsRequest, it) + } + getDestinationsResponse.destinations.firstOrNull() + } catch (e: IllegalStateException) { + // Catching the exception thrown when the Destination was not found so the NotificationActionConfigs object can be returned + null + } catch (e: OpenSearchSecurityException) { + if (notificationPermissionException != null) + throw notificationPermissionException + else + throw e + } + + if (destination == null && notificationPermissionException != null) + throw notificationPermissionException + } + + return NotificationActionConfigs(destination, channel) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunner.kt index 49c11500b..04aaac098 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunner.kt @@ -5,35 +5,20 @@ package org.opensearch.alerting -import org.opensearch.OpenSearchSecurityException -import org.opensearch.alerting.action.GetDestinationsAction -import org.opensearch.alerting.action.GetDestinationsRequest -import org.opensearch.alerting.action.GetDestinationsResponse -import org.opensearch.alerting.model.destination.Destination +import org.opensearch.alerting.AlertingV2Utils.getConfigAndSendNotification import org.opensearch.alerting.opensearchapi.InjectorContextElement -import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.opensearchapi.withClosableContext import org.opensearch.alerting.script.QueryLevelTriggerExecutionContext import org.opensearch.alerting.script.TriggerExecutionContext -import org.opensearch.alerting.util.destinationmigration.NotificationActionConfigs -import org.opensearch.alerting.util.destinationmigration.NotificationApiUtils.Companion.getNotificationConfigInfo -import org.opensearch.alerting.util.destinationmigration.getTitle -import org.opensearch.alerting.util.destinationmigration.publishLegacyNotification -import org.opensearch.alerting.util.destinationmigration.sendNotification -import org.opensearch.alerting.util.isAllowed -import org.opensearch.alerting.util.isTestAction import org.opensearch.alerting.util.use import org.opensearch.commons.ConfigConstants import org.opensearch.commons.alerting.model.ActionRunResult import org.opensearch.commons.alerting.model.Monitor import org.opensearch.commons.alerting.model.MonitorRunResult -import org.opensearch.commons.alerting.model.Table import org.opensearch.commons.alerting.model.WorkflowRunContext import org.opensearch.commons.alerting.model.action.Action -import org.opensearch.commons.notifications.model.NotificationConfigInfo import org.opensearch.core.common.Strings import org.opensearch.transport.TransportService -import org.opensearch.transport.client.node.NodeClient import java.time.Instant abstract class MonitorRunner { @@ -97,103 +82,4 @@ abstract class MonitorRunner { ActionRunResult(action.id, action.name, mapOf(), false, MonitorRunnerService.currentTime(), e) } } - - protected suspend fun getConfigAndSendNotification( - action: Action, - monitorCtx: MonitorRunnerExecutionContext, - subject: String?, - message: String - ): String { - val config = getConfigForNotificationAction(action, monitorCtx) - if (config.destination == null && config.channel == null) { - throw IllegalStateException("Unable to find a Notification Channel or Destination config with id [${action.destinationId}]") - } - - // Adding a check on TEST_ACTION Destination type here to avoid supporting it as a LegacyBaseMessage type - // just for Alerting integration tests - if (config.destination?.isTestAction() == true) { - return "test action" - } - - if (config.destination?.isAllowed(monitorCtx.allowList) == false) { - throw IllegalStateException( - "Monitor contains a Destination type that is not allowed: ${config.destination.type}" - ) - } - - var actionResponseContent = "" - actionResponseContent = config.channel - ?.sendNotification( - monitorCtx.client!!, - config.channel.getTitle(subject), - message - ) ?: actionResponseContent - - actionResponseContent = config.destination - ?.buildLegacyBaseMessage(subject, message, monitorCtx.destinationContextFactory!!.getDestinationContext(config.destination)) - ?.publishLegacyNotification(monitorCtx.client!!) - ?: actionResponseContent - - return actionResponseContent - } - - /** - * The "destination" ID referenced in a Monitor Action could either be a Notification config or a Destination config - * depending on whether the background migration process has already migrated it from a Destination to a Notification config. - * - * To cover both of these cases, the Notification config will take precedence and if it is not found, the Destination will be retrieved. - */ - private suspend fun getConfigForNotificationAction( - action: Action, - monitorCtx: MonitorRunnerExecutionContext - ): NotificationActionConfigs { - var destination: Destination? = null - var notificationPermissionException: Exception? = null - - var channel: NotificationConfigInfo? = null - try { - channel = getNotificationConfigInfo(monitorCtx.client as NodeClient, action.destinationId) - } catch (e: OpenSearchSecurityException) { - notificationPermissionException = e - } - - // If the channel was not found, try to retrieve the Destination - if (channel == null) { - destination = try { - val table = Table( - "asc", - "destination.name.keyword", - null, - 1, - 0, - null - ) - val getDestinationsRequest = GetDestinationsRequest( - action.destinationId, - 0L, - null, - table, - "ALL" - ) - - val getDestinationsResponse: GetDestinationsResponse = monitorCtx.client!!.suspendUntil { - monitorCtx.client!!.execute(GetDestinationsAction.INSTANCE, getDestinationsRequest, it) - } - getDestinationsResponse.destinations.firstOrNull() - } catch (e: IllegalStateException) { - // Catching the exception thrown when the Destination was not found so the NotificationActionConfigs object can be returned - null - } catch (e: OpenSearchSecurityException) { - if (notificationPermissionException != null) - throw notificationPermissionException - else - throw e - } - - if (destination == null && notificationPermissionException != null) - throw notificationPermissionException - } - - return NotificationActionConfigs(destination, channel) - } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerExecutionContext.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerExecutionContext.kt index ba12100ba..38659e835 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerExecutionContext.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerExecutionContext.kt @@ -7,6 +7,7 @@ package org.opensearch.alerting import org.opensearch.action.bulk.BackoffPolicy import org.opensearch.alerting.alerts.AlertIndices +import org.opensearch.alerting.alertsv2.AlertV2Indices import org.opensearch.alerting.core.lock.LockService import org.opensearch.alerting.model.destination.DestinationContextFactory import org.opensearch.alerting.remote.monitors.RemoteMonitorRegistry @@ -35,6 +36,7 @@ data class MonitorRunnerExecutionContext( var settings: Settings? = null, var threadPool: ThreadPool? = null, var alertIndices: AlertIndices? = null, + var alertV2Indices: AlertV2Indices? = null, var inputService: InputService? = null, var triggerService: TriggerService? = null, var alertService: AlertService? = null, diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt index 8e4c44760..10e277957 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt @@ -21,18 +21,28 @@ import org.opensearch.alerting.action.ExecuteMonitorResponse import org.opensearch.alerting.action.ExecuteWorkflowAction import org.opensearch.alerting.action.ExecuteWorkflowRequest import org.opensearch.alerting.action.ExecuteWorkflowResponse +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Action +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Request +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Response import org.opensearch.alerting.alerts.AlertIndices import org.opensearch.alerting.alerts.AlertMover.Companion.moveAlerts +import org.opensearch.alerting.alertsv2.AlertV2Indices +import org.opensearch.alerting.alertsv2.AlertV2Mover.Companion.moveAlertV2s import org.opensearch.alerting.core.JobRunner import org.opensearch.alerting.core.ScheduledJobIndices import org.opensearch.alerting.core.lock.LockModel import org.opensearch.alerting.core.lock.LockService import org.opensearch.alerting.model.destination.DestinationContextFactory +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.modelv2.MonitorV2RunResult +import org.opensearch.alerting.modelv2.PPLSQLMonitor +import org.opensearch.alerting.modelv2.PPLSQLMonitor.Companion.PPL_SQL_MONITOR_TYPE import org.opensearch.alerting.opensearchapi.retry import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.remote.monitors.RemoteDocumentLevelMonitorRunner import org.opensearch.alerting.remote.monitors.RemoteMonitorRegistry import org.opensearch.alerting.script.TriggerExecutionContext +import org.opensearch.alerting.script.TriggerV2ExecutionContext import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_BACKOFF_COUNT import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_BACKOFF_MILLIS @@ -137,6 +147,11 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon return this } + fun registerAlertV2Indices(alertV2Indices: AlertV2Indices): MonitorRunnerService { + this.monitorCtx.alertV2Indices = alertV2Indices + return this + } + fun registerInputService(inputService: InputService): MonitorRunnerService { this.monitorCtx.inputService = inputService return this @@ -321,6 +336,18 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon logger.error("Failed to move active alerts for monitor [${job.id}].", e) } } + } else if (job is MonitorV2) { + launch { + try { + monitorCtx.moveAlertsRetryPolicy!!.retry(logger) { + if (monitorCtx.alertV2Indices!!.isAlertV2Initialized()) { + moveAlertV2s(job.id, job, monitorCtx) + } + } + } catch (e: Exception) { + logger.error("Failed to move active alertV2s for monitorV2 [${job.id}].", e) + } + } } else { throw IllegalArgumentException("Invalid job type") } @@ -344,6 +371,15 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon } catch (e: Exception) { logger.error("Failed to move active alerts for monitor [$jobId].", e) } + try { + monitorCtx.moveAlertsRetryPolicy!!.retry(logger) { + if (monitorCtx.alertV2Indices!!.isAlertV2Initialized()) { + moveAlertV2s(jobId, null, monitorCtx) + } + } + } catch (e: Exception) { + logger.error("Failed to move active alertV2s for monitorV2 [$jobId].", e) + } } } @@ -413,6 +449,44 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon } } } + is MonitorV2 -> { + if (job !is PPLSQLMonitor) { + throw IllegalStateException("Invalid MonitorV2 type: ${job.javaClass.name}") + } + + launch { + var monitorLock: LockModel? = null + try { + monitorLock = monitorCtx.client!!.suspendUntil { + monitorCtx.lockService!!.acquireLock(job, it) + } ?: return@launch + logger.debug("lock ${monitorLock!!.lockId} acquired") + logger.debug( + "PERF_DEBUG: executing $PPL_SQL_MONITOR_TYPE ${job.id} on node " + + monitorCtx.clusterService!!.state().nodes().localNode.id + ) + val executeMonitorV2Request = ExecuteMonitorV2Request( + false, + false, + job.id, // only need to pass in MonitorV2 ID + null, // no need to pass in MonitorV2 object itself + TimeValue(periodEnd.toEpochMilli()) + ) + monitorCtx.client!!.suspendUntil { + monitorCtx.client!!.execute( + ExecuteMonitorV2Action.INSTANCE, + executeMonitorV2Request, + it + ) + } + } catch (e: Exception) { + logger.error("MonitorV2 run failed for monitor with id ${job.id}", e) + } finally { + monitorCtx.client!!.suspendUntil { monitorCtx.lockService!!.release(monitorLock, it) } + logger.debug("lock ${monitorLock?.lockId} released") + } + } + } else -> { throw IllegalArgumentException("Invalid job type") } @@ -438,20 +512,7 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon ): MonitorRunResult<*> { // Updating the scheduled job index at the start of monitor execution runs for when there is an upgrade the the schema mapping // has not been updated. - if (!IndexUtils.scheduledJobIndexUpdated && monitorCtx.clusterService != null && monitorCtx.client != null) { - IndexUtils.updateIndexMapping( - ScheduledJob.SCHEDULED_JOBS_INDEX, - ScheduledJobIndices.scheduledJobMappings(), monitorCtx.clusterService!!.state(), monitorCtx.client!!.admin().indices(), - object : ActionListener { - override fun onResponse(response: AcknowledgedResponse) { - } - - override fun onFailure(t: Exception) { - logger.error("Failed to update config index schema", t) - } - } - ) - } + updateAlertingConfigIndexSchema() if (job is Workflow) { logger.info("Executing scheduled workflow - id: ${job.id}, periodStart: $periodStart, periodEnd: $periodEnd, dryrun: $dryrun") @@ -544,6 +605,44 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon } } + // after the above JobRunner interface override runJob calls ExecuteMonitorV2 API, + // the ExecuteMonitorV2 transport action calls this function to call the PPLSQLMonitorRunner, + // where the core PPL/SQL Monitor execution logic resides + suspend fun runJobV2( + monitorV2: MonitorV2, + periodEnd: Instant, + dryrun: Boolean, + manual: Boolean, + transportService: TransportService, + ): MonitorV2RunResult<*> { + updateAlertingConfigIndexSchema() + + val executionId = "${monitorV2.id}_${LocalDateTime.now(ZoneOffset.UTC)}_${UUID.randomUUID()}" + val monitorV2Type = when (monitorV2) { + is PPLSQLMonitor -> PPL_SQL_MONITOR_TYPE + else -> throw IllegalStateException("Unexpected MonitorV2 type: ${monitorV2.javaClass.name}") + } + + logger.info( + "Executing scheduled monitor v2 - id: ${monitorV2.id}, type: $monitorV2Type, " + + "periodEnd: $periodEnd, dryrun: $dryrun, manual: $manual, executionId: $executionId" + ) + + // for now, always call PPLSQLMonitorRunner since only PPL Monitors are initially supported + // to introduce new MonitorV2 type, create its MonitorRunner, and if/else branch + // to the corresponding MonitorRunners based on type. For now, default to PPLSQLMonitorRunner + val runResult = PPLSQLMonitorRunner.runMonitorV2( + monitorV2, + monitorCtx, + periodEnd, + dryrun, + manual, + executionId = executionId, + transportService = transportService, + ) + return runResult + } + // TODO: See if we can move below methods (or few of these) to a common utils internal fun getRolesForMonitor(monitor: Monitor): List { /* @@ -587,4 +686,27 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon .newInstance(template.params + mapOf("ctx" to ctx.asTemplateArg())) .execute() } + + internal fun compileTemplateV2(template: Script, ctx: TriggerV2ExecutionContext): String { + return monitorCtx.scriptService!!.compile(template, TemplateScript.CONTEXT) + .newInstance(template.params + mapOf("ctx" to ctx.asTemplateArg())) + .execute() + } + + private fun updateAlertingConfigIndexSchema() { + if (!IndexUtils.scheduledJobIndexUpdated && monitorCtx.clusterService != null && monitorCtx.client != null) { + IndexUtils.updateIndexMapping( + ScheduledJob.SCHEDULED_JOBS_INDEX, + ScheduledJobIndices.scheduledJobMappings(), monitorCtx.clusterService!!.state(), monitorCtx.client!!.admin().indices(), + object : ActionListener { + override fun onResponse(response: AcknowledgedResponse) { + } + + override fun onFailure(t: Exception) { + logger.error("Failed to update config index schema", t) + } + } + ) + } + } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorV2Runner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorV2Runner.kt new file mode 100644 index 000000000..ccf933148 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorV2Runner.kt @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting + +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.modelv2.MonitorV2RunResult +import org.opensearch.transport.TransportService +import java.time.Instant + +/** + * Interface for monitor V2 runners. All monitor v2 runner classes that house + * a specific v2 monitor type's execution logic must implement this interface. + * + * @opensearch.experimental + */ +interface MonitorV2Runner { + suspend fun runMonitorV2( + monitorV2: MonitorV2, + monitorCtx: MonitorRunnerExecutionContext, // MonitorV2 reads from same context as Monitor does + periodEnd: Instant, + dryRun: Boolean, + manual: Boolean, + executionId: String, + transportService: TransportService + ): MonitorV2RunResult<*> +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/PPLSQLMonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/PPLSQLMonitorRunner.kt new file mode 100644 index 000000000..d183389ff --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/PPLSQLMonitorRunner.kt @@ -0,0 +1,679 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting + +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.withTimeout +import org.apache.logging.log4j.LogManager +import org.json.JSONArray +import org.json.JSONObject +import org.opensearch.ExceptionsHelper +import org.opensearch.action.DocWriteRequest +import org.opensearch.action.bulk.BackoffPolicy +import org.opensearch.action.bulk.BulkRequest +import org.opensearch.action.bulk.BulkResponse +import org.opensearch.action.index.IndexRequest +import org.opensearch.action.support.WriteRequest +import org.opensearch.alerting.AlertingV2Utils.getConfigAndSendNotification +import org.opensearch.alerting.PPLUtils.appendCustomCondition +import org.opensearch.alerting.PPLUtils.appendDataRowsLimit +import org.opensearch.alerting.PPLUtils.capPPLQueryResultsSize +import org.opensearch.alerting.PPLUtils.executePplQuery +import org.opensearch.alerting.PPLUtils.findEvalResultVar +import org.opensearch.alerting.PPLUtils.findEvalResultVarIdxInSchema +import org.opensearch.alerting.alertsv2.AlertV2Indices +import org.opensearch.alerting.modelv2.AlertV2 +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.modelv2.MonitorV2RunResult +import org.opensearch.alerting.modelv2.PPLSQLMonitor +import org.opensearch.alerting.modelv2.PPLSQLMonitorRunResult +import org.opensearch.alerting.modelv2.PPLSQLTrigger +import org.opensearch.alerting.modelv2.PPLSQLTrigger.ConditionType +import org.opensearch.alerting.modelv2.PPLSQLTrigger.NumResultsCondition +import org.opensearch.alerting.modelv2.PPLSQLTrigger.TriggerMode +import org.opensearch.alerting.modelv2.PPLSQLTriggerRunResult +import org.opensearch.alerting.modelv2.TriggerV2.Severity +import org.opensearch.alerting.opensearchapi.InjectorContextElement +import org.opensearch.alerting.opensearchapi.retry +import org.opensearch.alerting.opensearchapi.suspendUntil +import org.opensearch.alerting.opensearchapi.withClosableContext +import org.opensearch.alerting.script.PPLTriggerExecutionContext +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.common.xcontent.XContentFactory +import org.opensearch.commons.alerting.alerts.AlertError +import org.opensearch.commons.alerting.model.Alert +import org.opensearch.commons.alerting.model.ScheduledJob.Companion.SCHEDULED_JOBS_INDEX +import org.opensearch.commons.alerting.model.action.Action +import org.opensearch.commons.alerting.model.userErrorMessage +import org.opensearch.core.common.Strings +import org.opensearch.core.rest.RestStatus +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.index.VersionType +import org.opensearch.transport.TransportService +import org.opensearch.transport.client.node.NodeClient +import java.time.Duration +import java.time.Instant +import java.time.ZoneOffset.UTC +import java.time.format.DateTimeFormatter +import java.time.temporal.ChronoUnit +import java.util.Locale +import kotlin.math.min +import kotlin.time.measureTimedValue + +/** + * This class contains the core logic for running a PPLSQLMonitor. + * The logic for checking throttles, executing the PPL Query, evaluating + * the results against the trigger condition, generating alerts, sending + * notifications, and updating the monitor document with last triggered + * time, are all here. + * + * @opensearch.experimental + */ + +object PPLSQLMonitorRunner : MonitorV2Runner { + private val logger = LogManager.getLogger(javaClass) + + override suspend fun runMonitorV2( + monitorV2: MonitorV2, + monitorCtx: MonitorRunnerExecutionContext, // MonitorV2 reads from same context as Monitor + periodEnd: Instant, + dryRun: Boolean, + manual: Boolean, + executionId: String, + transportService: TransportService, + ): MonitorV2RunResult<*> { + if (monitorV2 !is PPLSQLMonitor) { + throw IllegalStateException("Unexpected monitor type: ${monitorV2.javaClass.name}") + } + + if (monitorV2.id == MonitorV2.NO_ID) { + throw IllegalStateException("Received PPL Monitor to execute that unexpectedly has no ID") + } + + logger.debug("Running PPL Monitor: ${monitorV2.id}. Thread: ${Thread.currentThread().name}") + + // time the monitor execution run for informational logging + val monitorRunStart = Instant.now() + + val pplSqlMonitor = monitorV2 + val nodeClient = monitorCtx.client as NodeClient + + // create some objects that will be used later + val triggerResults = mutableMapOf() + val pplSqlQueryResults = mutableMapOf>() + + // set the current execution time + // use threadpool time for cross node consistency + val timeOfCurrentExecution = Instant.ofEpochMilli(MonitorRunnerService.monitorCtx.threadPool!!.absoluteTimeInMillis()) + + // check for and create the active alerts and alert history indices + // so we have indices to write alerts to + try { + monitorCtx.alertV2Indices!!.createOrUpdateAlertV2Index() + monitorCtx.alertV2Indices!!.createOrUpdateInitialAlertV2HistoryIndex() + } catch (e: Exception) { + val id = if (pplSqlMonitor.id.trim().isEmpty()) "_na_" else pplSqlMonitor.id + logger.error("Error loading alerts for monitorV2: $id", e) + return PPLSQLMonitorRunResult(pplSqlMonitor.name, e, mapOf(), mapOf()) + } + + val timeFilteredQuery = if (pplSqlMonitor.lookBackWindow != null) { + logger.debug("look back window specified for PPL Monitor: ${monitorV2.id}, injecting look back window time filter") + // if lookback window is specified, inject a top level lookback window time filter + // into the PPL query + val lookBackWindow = pplSqlMonitor.lookBackWindow!! + val lookbackPeriodStart = periodEnd.minus(lookBackWindow, ChronoUnit.MINUTES) + val timeFilteredQuery = addTimeFilter(pplSqlMonitor.query, lookbackPeriodStart, periodEnd, pplSqlMonitor.timestampField!!) + logger.debug("time filtered query: $timeFilteredQuery") + timeFilteredQuery + } else { + logger.debug("look back window not specified for PPL Monitor: ${monitorV2.id}, proceeding with original base query") + // otherwise, don't inject any time filter whatsoever + // unless the query itself has user-specified time filters, this query + // will return all applicable data in the cluster + pplSqlMonitor.query + } + + val monitorExecutionDuration = monitorCtx + .clusterService!! + .clusterSettings + .get(AlertingSettings.ALERT_V2_MONITOR_EXECUTION_MAX_DURATION) + + // for storing any exception that may or may not happen + // while executing monitor + var exception: Exception? = null + + // run each trigger + try { + withTimeout(monitorExecutionDuration.millis) { + runTriggers( + pplSqlMonitor, + timeFilteredQuery, + timeOfCurrentExecution, + manual, + dryRun, + triggerResults, + pplSqlQueryResults, + executionId, + monitorCtx, + nodeClient, + transportService + ) + } + } catch (e: TimeoutCancellationException) { + // generate an alert that the monitor's triggers took + // too long to run. this error alert is generated + // even if some triggers managed to run successfully within + // the above time frame and generate their own alerts + monitorCtx.retryPolicy?.let { + saveAlertsV2( + generateErrorAlert(null, pplSqlMonitor, e, executionId, timeOfCurrentExecution), + pplSqlMonitor, + it, + nodeClient + ) + } + + exception = e + } + + // for throttle checking purposes, reindex the PPL Monitor into the alerting-config index + // with updated last triggered times for each of its triggers + if (triggerResults.any { it.value.triggered }) { + updateMonitorWithLastTriggeredTimes(pplSqlMonitor, nodeClient) + } + + val monitorRunEnd = Instant.now() + + val monitorRunTime = Duration.between(monitorRunStart, monitorRunEnd) + + logger.info("monitor ${pplSqlMonitor.id} execution $executionId run time: $monitorRunTime") + + return PPLSQLMonitorRunResult( + pplSqlMonitor.name, + exception, + triggerResults, + pplSqlQueryResults + ) + } + + suspend fun runTriggers( + pplSqlMonitor: PPLSQLMonitor, + timeFilteredQuery: String, + timeOfCurrentExecution: Instant, + manual: Boolean, + dryRun: Boolean, + triggerResults: MutableMap, + pplSqlQueryResults: MutableMap>, + executionId: String, + monitorCtx: MonitorRunnerExecutionContext, + nodeClient: NodeClient, + transportService: TransportService + ) { + for (pplSqlTrigger in pplSqlMonitor.triggers) { + try { + // check for throttle and skip execution + // before even running the trigger itself + val throttled = checkForThrottle(pplSqlTrigger, timeOfCurrentExecution, manual) + if (throttled) { + logger.info("throttling trigger ${pplSqlTrigger.id} from monitor ${pplSqlMonitor.id}") + + // automatically return that this trigger is untriggered + triggerResults[pplSqlTrigger.id] = PPLSQLTriggerRunResult(pplSqlTrigger.name, false, null) + + continue + } + logger.debug("throttle check passed, executing trigger ${pplSqlTrigger.id} from monitor ${pplSqlMonitor.id}") + + logger.debug("checking if custom condition is used and appending to base query") + // if trigger uses custom condition, append the custom condition to query, otherwise simply proceed + val queryToExecute = if (pplSqlTrigger.conditionType == ConditionType.NUMBER_OF_RESULTS) { // number of results trigger + timeFilteredQuery + } else { // custom condition trigger + appendCustomCondition(timeFilteredQuery, pplSqlTrigger.customCondition!!) + } + + // limit the number of PPL query result data rows returned + val dataRowsLimit = monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS) + val limitedQueryToExecute = appendDataRowsLimit(queryToExecute, dataRowsLimit) + + // TODO: after getting ppl query results, see if the number of results + // retrieved equals the max allowed number of query results. this implies + // query results might have been excluded, in which case a warning message + // in the alert and notification must be added that results were excluded + // and an alert that should have been generated might not have been + + logger.debug("executing the PPL query of monitor: ${pplSqlMonitor.id}") + // execute the PPL query + val (queryResponseJson, timeTaken) = measureTimedValue { + withClosableContext( + InjectorContextElement( + pplSqlMonitor.id, + monitorCtx.settings!!, + monitorCtx.threadPool!!.threadContext, + pplSqlMonitor.user?.roles, + pplSqlMonitor.user + ) + ) { + executePplQuery( + limitedQueryToExecute, + monitorCtx.clusterService!!.state().nodes.localNode, + transportService + ) + } + } + logger.debug("query results for trigger ${pplSqlTrigger.id}: $queryResponseJson") + logger.debug("time taken to execute query against sql/ppl plugin: $timeTaken") + + // store the query results for Execute Monitor API response + // unlike the query results stored in alerts and notifications, which must be size capped + // (because they will be stored in the OpenSearch cluster or sent as notification) and must be based + // on only the query results that met the trigger condition (because alerts should generate + // on query results that met trigger condition, not those that didn't), the pplQueryResults + // here will be returned as part of the Execute Monitor API response. This will return the original, + // untouched set of query results, and whether this causes size exceed errors is deferred + // to HTTP's response size limits + pplSqlQueryResults[pplSqlTrigger.id] = queryResponseJson.toMap() + + // determine if the trigger condition has been met + val triggered = if (pplSqlTrigger.conditionType == ConditionType.NUMBER_OF_RESULTS) { // number of results trigger + evaluateNumResultsTrigger(queryResponseJson, pplSqlTrigger.numResultsCondition!!, pplSqlTrigger.numResultsValue!!) + } else { // custom condition trigger + evaluateCustomTrigger(queryResponseJson, pplSqlTrigger.customCondition!!) + } + + logger.debug("PPLTrigger ${pplSqlTrigger.name} with ID ${pplSqlTrigger.id} triggered: $triggered") + + // store the trigger execution results for Execute Monitor API response + triggerResults[pplSqlTrigger.id] = PPLSQLTriggerRunResult(pplSqlTrigger.name, triggered, null) + + if (triggered) { + logger.debug("generating alerts for PPLTrigger ${pplSqlTrigger.name} with ID ${pplSqlTrigger.id}") + // retrieve some limits from settings + val maxQueryResultsSize = + monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.ALERT_V2_QUERY_RESULTS_MAX_SIZE) + val maxAlerts = + monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.ALERT_V2_PER_RESULT_TRIGGER_MAX_ALERTS) + + // if trigger is on result set mode, this list will have exactly 1 element + // if trigger is on per result mode, this list will have as many elements as the query results had + // trigger condition-meeting rows, up to the max number of alerts a per result trigger can generate + val preparedQueryResults = splitUpQueryResults(pplSqlTrigger, queryResponseJson, maxQueryResultsSize, maxAlerts) + + // generate alerts based on trigger mode + // if this trigger is on result_set mode, this list contains exactly 1 alert + // if this trigger is on per_result mode, this list has as many alerts as there are + // trigger condition-meeting query results + val thisTriggersGeneratedAlerts = generateAlerts( + pplSqlTrigger, + pplSqlMonitor, + preparedQueryResults, + executionId, + timeOfCurrentExecution + ) + + // for future throttle checks, update the trigger's last execution time + // in the monitor object stored in memory + pplSqlTrigger.lastTriggeredTime = timeOfCurrentExecution + + // send alert notifications + for (action in pplSqlTrigger.actions) { + for (queryResult in preparedQueryResults) { + val pplTriggerExecutionContext = PPLTriggerExecutionContext( + pplSqlMonitor, + null, + pplSqlTrigger, + queryResult + ) + + runAction( + action, + pplTriggerExecutionContext, + monitorCtx, + pplSqlMonitor, + dryRun + ) + } + } + + // write the alerts to the alerts index + monitorCtx.retryPolicy?.let { + saveAlertsV2(thisTriggersGeneratedAlerts, pplSqlMonitor, it, nodeClient) + } + + logger.debug("PPL Trigger ${pplSqlTrigger.id} executed successfully") + } + } catch (e: Exception) { + logger.error( + "failed to run PPL Trigger ${pplSqlTrigger.name} (id: ${pplSqlTrigger.id} " + + "from PPL Monitor ${pplSqlMonitor.name} (id: ${pplSqlMonitor.id}", + e + ) + + // generate an alert with an error message + monitorCtx.retryPolicy?.let { + saveAlertsV2( + generateErrorAlert(pplSqlTrigger, pplSqlMonitor, e, executionId, timeOfCurrentExecution), + pplSqlMonitor, + it, + nodeClient + ) + } + } + } + } + + // returns true if the pplTrigger should be throttled + private fun checkForThrottle(pplTrigger: PPLSQLTrigger, timeOfCurrentExecution: Instant, manual: Boolean): Boolean { + // manual calls from the user to execute a monitor should never be throttled + if (manual) { + return false + } + + // the interval between throttledTimeBound and now is the throttle window + // i.e. any PPLTrigger whose last trigger time is in this window must be throttled + val throttleTimeBound = pplTrigger.throttleDuration?.let { + timeOfCurrentExecution.minus(pplTrigger.throttleDuration, ChronoUnit.MINUTES) + } + + // the trigger must be throttled if... + return pplTrigger.throttleDuration != null && // throttling is enabled on the PPLTrigger + pplTrigger.lastTriggeredTime != null && // and it has triggered before at least once + pplTrigger.lastTriggeredTime!!.isAfter(throttleTimeBound!!) // and it's not yet out of its throttle window + } + + // adds monitor schedule-based time filter + // query: the raw PPL Monitor query + // lookbackPeriodStart: the lower bound of the query interval based on monitor schedule and look back window + // periodEnd: the upper bound of the initially computed query interval based on monitor schedule + // timestampField: the timestamp field that will be used to time bound the query results + private fun addTimeFilter(query: String, lookbackPeriodStart: Instant, periodEnd: Instant, timestampField: String): String { + // PPL plugin only accepts timestamp strings in this format + val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss", Locale.ROOT).withZone(UTC) + + val periodStartPplTimestamp = formatter.format(lookbackPeriodStart) + val periodEndPplTimeStamp = formatter.format(periodEnd) + + val timeFilterAppend = "| where $timestampField > TIMESTAMP('$periodStartPplTimestamp') and " + + "$timestampField < TIMESTAMP('$periodEndPplTimeStamp')" + val timeFilterReplace = "$timeFilterAppend |" + + val timeFilteredQuery: String = if (query.contains("|")) { + // if Monitor query contains piped statements, inject the time filter + // as the first piped statement (i.e. before more complex statements + // like aggregations can take effect later in the query) + query.replaceFirst("|", timeFilterReplace) + } else { + // otherwise the query contains no piped statements and is simply a + // `search source=` statement, simply append time filter at the end + query + timeFilterAppend + } + + return timeFilteredQuery + } + + private fun evaluateNumResultsTrigger( + pplQueryResponse: JSONObject, + numResultsCondition: NumResultsCondition, + numResultsValue: Long + ): Boolean { + val numResults = pplQueryResponse.getLong("total") + return when (numResultsCondition) { + NumResultsCondition.GREATER_THAN -> numResults > numResultsValue + NumResultsCondition.GREATER_THAN_EQUAL -> numResults >= numResultsValue + NumResultsCondition.LESS_THAN -> numResults < numResultsValue + NumResultsCondition.LESS_THAN_EQUAL -> numResults <= numResultsValue + NumResultsCondition.EQUAL -> numResults == numResultsValue + NumResultsCondition.NOT_EQUAL -> numResults != numResultsValue + } + } + + private fun evaluateCustomTrigger(pplQueryResponse: JSONObject, customCondition: String): Boolean { + // find the name of the eval result variable defined in custom condition + val evalResultVarName = findEvalResultVar(customCondition) + + // find the index eval statement result variable in the PPL query response schema + val evalResultVarIdx = findEvalResultVarIdxInSchema(pplQueryResponse, evalResultVarName) + + val dataRowList = pplQueryResponse.getJSONArray("datarows") + for (i in 0 until dataRowList.length()) { + val dataRow = dataRowList.getJSONArray(i) + val evalResult = dataRow.getBoolean(evalResultVarIdx) + if (evalResult) { + return true + } + } + + return false + } + + // prepares the query results to be passed into alerts and notifications based on trigger mode + // if result set, alert and notification simply stores all query results. + // if per result, each alert and notification stores a single row of the query results. + // this function then ensures that only a capped number of results are returned to generate alerts + // and notifications based on. it also caps the size of the query results themselves. + private fun splitUpQueryResults( + pplTrigger: PPLSQLTrigger, + pplQueryResults: JSONObject, + maxQueryResultsSize: Long, + maxAlerts: Int + ): List { + // case: result set + // return the results as a single set of all the results + if (pplTrigger.mode == TriggerMode.RESULT_SET) { + val sizeCappedRelevantQueryResultRows = capPPLQueryResultsSize(pplQueryResults, maxQueryResultsSize) + return listOf(sizeCappedRelevantQueryResultRows) + } + + // case: per result + // prepare to generate an alert for each relevant query result row, + // up to the maxAlerts limit + val individualRows = mutableListOf() + if (pplTrigger.conditionType == ConditionType.NUMBER_OF_RESULTS) { + // nested case: number_of_results + val numAlertsToGenerate = min(maxAlerts, pplQueryResults.getInt("total")) + for (i in 0 until numAlertsToGenerate) { + addRowToList(individualRows, pplQueryResults, i, maxQueryResultsSize) + } + } else { + // nested case: custom + val evalResultVarName = findEvalResultVar(pplTrigger.customCondition!!) + val evalResultVarIdx = findEvalResultVarIdxInSchema(pplQueryResults, evalResultVarName) + val dataRowList = pplQueryResults.getJSONArray("datarows") + for (i in 0 until dataRowList.length()) { + val dataRow = dataRowList.getJSONArray(i) + val evalResult = dataRow.getBoolean(evalResultVarIdx) + if (evalResult) { + addRowToList(individualRows, pplQueryResults, i, maxQueryResultsSize) + } + if (individualRows.size >= maxAlerts) { + break + } + } + } + + logger.debug("individualRows: $individualRows") + + return individualRows + } + + private fun addRowToList( + individualRows: MutableList, + pplQueryResults: JSONObject, + i: Int, + maxQueryResultsSize: Long + ) { + val individualRow = JSONObject() + individualRow.put("total", 1) // set the size explicitly to 1 for consistency + individualRow.put("size", 1) + individualRow.put("schema", JSONArray(pplQueryResults.getJSONArray("schema").toList())) + individualRow.put( + "datarows", + JSONArray().put( + JSONArray(pplQueryResults.getJSONArray("datarows").getJSONArray(i).toList()) + ) + ) + val sizeCappedIndividualRow = capPPLQueryResultsSize(individualRow, maxQueryResultsSize) + individualRows.add(sizeCappedIndividualRow) + } + + private fun generateAlerts( + pplSqlTrigger: PPLSQLTrigger, + pplSqlMonitor: PPLSQLMonitor, + preparedQueryResults: List, + executionId: String, + timeOfCurrentExecution: Instant + ): List { + val alertV2s = mutableListOf() + for (queryResult in preparedQueryResults) { + val alertV2 = AlertV2( + monitorId = pplSqlMonitor.id, + monitorName = pplSqlMonitor.name, + monitorVersion = pplSqlMonitor.version, + monitorUser = pplSqlMonitor.user, + triggerId = pplSqlTrigger.id, + triggerName = pplSqlTrigger.name, + query = pplSqlMonitor.query, + queryResults = queryResult.toMap(), + triggeredTime = timeOfCurrentExecution, + severity = pplSqlTrigger.severity, + executionId = executionId + ) + alertV2s.add(alertV2) + } + + return alertV2s.toList() // return as immutable list + } + + private fun generateErrorAlert( + pplSqlTrigger: PPLSQLTrigger?, + pplSqlMonitor: PPLSQLMonitor, + exception: Exception, + executionId: String, + timeOfCurrentExecution: Instant + ): List { + val errorMessage = "Failed to run PPL Monitor ${pplSqlMonitor.id}, PPL Trigger ${pplSqlTrigger?.id}: " + + exception.userErrorMessage() + val obfuscatedErrorMessage = AlertError.obfuscateIPAddresses(errorMessage) + + val alertV2 = AlertV2( + monitorId = pplSqlMonitor.id, + monitorName = pplSqlMonitor.name, + monitorVersion = pplSqlMonitor.version, + monitorUser = pplSqlMonitor.user, + triggerId = pplSqlTrigger?.id ?: "", + triggerName = pplSqlTrigger?.name ?: "", + query = pplSqlMonitor.query, + queryResults = mapOf(), + triggeredTime = timeOfCurrentExecution, + errorMessage = obfuscatedErrorMessage, + severity = Severity.ERROR, + executionId = executionId + ) + + return listOf(alertV2) + } + + private suspend fun saveAlertsV2( + alerts: List, + pplSqlMonitor: PPLSQLMonitor, + retryPolicy: BackoffPolicy, + client: NodeClient + ) { + logger.debug("received alerts: $alerts") + + var requestsToRetry = alerts.flatMap { alert -> + listOf>( + IndexRequest(AlertV2Indices.ALERT_V2_INDEX) + .routing(pplSqlMonitor.id) // set routing ID to PPL Monitor ID + .source(alert.toXContentWithUser(XContentFactory.jsonBuilder())) + .id(if (alert.id != Alert.NO_ID) alert.id else null) + ) + } + + if (requestsToRetry.isEmpty()) return + // Retry Bulk requests if there was any 429 response + retryPolicy.retry(logger, listOf(RestStatus.TOO_MANY_REQUESTS)) { + val bulkRequest = BulkRequest().add(requestsToRetry).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + val bulkResponse: BulkResponse = client.suspendUntil { client.bulk(bulkRequest, it) } + val failedResponses = (bulkResponse.items ?: arrayOf()).filter { it.isFailed } + failedResponses.forEach { + logger.debug("write alerts failed responses: ${it.failureMessage}") + } + requestsToRetry = failedResponses.filter { it.status() == RestStatus.TOO_MANY_REQUESTS } + .map { bulkRequest.requests()[it.itemId] as IndexRequest } + + if (requestsToRetry.isNotEmpty()) { + val retryCause = failedResponses.first { it.status() == RestStatus.TOO_MANY_REQUESTS }.failure.cause + throw ExceptionsHelper.convertToOpenSearchException(retryCause) + } + } + } + + // during monitor execution, the ppl sql monitor object stored in memory had its triggers updated + // with their last trigger times. this function simply indexes those updated triggers into the + // alerting-config index + private suspend fun updateMonitorWithLastTriggeredTimes(pplSqlMonitor: PPLSQLMonitor, client: NodeClient) { + val indexRequest = IndexRequest(SCHEDULED_JOBS_INDEX) + .id(pplSqlMonitor.id) + .source( + pplSqlMonitor.toXContentWithUser( + XContentFactory.jsonBuilder(), + ToXContent.MapParams( + mapOf("with_type" to "true") + ) + ) + ) + .routing(pplSqlMonitor.id) + .version(pplSqlMonitor.version) + .versionType(VersionType.EXTERNAL_GTE) + + val indexResponse = client.suspendUntil { index(indexRequest, it) } + + logger.debug("PPLSQLMonitor update with last execution times index response: ${indexResponse.result}") + } + + suspend fun runAction( + action: Action, + triggerCtx: PPLTriggerExecutionContext, + monitorCtx: MonitorRunnerExecutionContext, + pplSqlMonitor: PPLSQLMonitor, + dryrun: Boolean + ) { + // this function can throw an exception, which is caught by the try + // catch in runMonitor() to generate an error alert + + val notifSubject = if (action.subjectTemplate != null) + MonitorRunnerService.compileTemplateV2(action.subjectTemplate!!, triggerCtx) + else "" + + var notifMessage = MonitorRunnerService.compileTemplateV2(action.messageTemplate, triggerCtx) + if (Strings.isNullOrEmpty(notifMessage)) { + throw IllegalStateException("Message content missing in the Destination with id: ${action.destinationId}") + } + + if (!dryrun) { + monitorCtx.client!!.threadPool().threadContext.stashContext().use { + withClosableContext( + InjectorContextElement( + pplSqlMonitor.id, + monitorCtx.settings!!, + monitorCtx.threadPool!!.threadContext, + pplSqlMonitor.user?.roles, + pplSqlMonitor.user + ) + ) { + getConfigAndSendNotification( + action, + monitorCtx, + notifSubject, + notifMessage + ) + } + } + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt b/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt new file mode 100644 index 000000000..0bb4babc8 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt @@ -0,0 +1,305 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting + +import org.json.JSONArray +import org.json.JSONObject +import org.opensearch.alerting.core.ppl.PPLPluginInterface +import org.opensearch.alerting.opensearchapi.suspendUntil +import org.opensearch.cluster.node.DiscoveryNode +import org.opensearch.sql.plugin.transport.TransportPPLQueryRequest +import org.opensearch.transport.TransportService + +object PPLUtils { + + // TODO: these are in-house PPL query parsers, find a PPL plugin dependency that does this for us + /* Regular Expressions */ + // captures the name of the result variable in a PPL monitor's custom condition + // e.g. custom condition: `eval apple = avg_latency > 100` + // captures: "apple" + private val evalResultVarRegex = """\beval\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*=""".toRegex() + + // captures the list of indices and index patterns that a given PPL query searches + // e.g. PPL query: search source = index_1,index_pattern*,index_3 | where responseCode = 500 | head 10 + // captures: index_1,index_pattern*,index_3 + private val indicesListRegex = + """(?i)source(?:\s*)=(?:\s*)((?:`[^`]+`|[-\w.*'+]+(?:\*)?)(?:\s*,\s*(?:`[^`]+`|[-\w.*'+]+\*?))*)\s*\|*""".toRegex() + + /** + * Appends a user-defined custom condition to a PPL query. + * + * This method is used exclusively for custom condition triggers. It concatenates + * the custom condition to the base PPL query using the pipe operator (|), allowing the condition + * to evaluate each query result data row. + * + * @param query The base PPL query string (e.g., "source=logs | where status=error") + * @param customCondition The custom trigger condition to append (e.g., "eval result = avg > 3") + * @return The combined PPL query with the custom condition appended + * + * @example + * ``` + * val baseQuery = "source=logs | stats max(price) as max_price by region" + * val condition = "eval result = max_price > 300" + * val result = appendCustomCondition(baseQuery, condition) + * // Returns: "source=logs | stats max(price) as max_price by region | eval result = max_price > 300" + * ``` + * + * @note This method does not validate the syntax of either the query or custom condition. + * It is assumed that upstream workflows have already validated the base query, + * and that downstream workflows will validate the constructed query + */ + fun appendCustomCondition(query: String, customCondition: String): String { + return "$query | $customCondition" + } + + /** + * Appends a limit on the number of documents/data rows to retrieve from a PPL query. + * + * This method uses the PPL `head` command to restrict the number of rows returned by + * the query. This is used to prevent memory issues and improving performance when + * only a subset of results is needed for alert evaluation. + * + * @param query The base PPL query string + * @param maxDataRows The maximum number of data rows to retrieve + * @return The PPL query with a head limit appended (e.g., "source=logs | head 1000") + * + * @example + * ``` + * val query = "source=logs | where status=error" + * val limitedQuery = appendDataRowsLimit(query, 100) + * // Returns: "source=logs | where status=error | head 100" + * ``` + */ + fun appendDataRowsLimit(query: String, maxDataRows: Long): String { + return "$query | head $maxDataRows" + } + + /** + * Executes a PPL query and returns the response as a parsable JSONObject. + * + * This method calls the PPL Plugin's Execute API via the transport layer to execute the provided query + * and parses the response into a structured JSON format suitable for trigger evaluation + * + * @param query The PPL query string to execute + * @param client The NodeClient used to communicate with the PPL plugin + * @return A JSONObject containing the query execution results + * + * @throws Exception if the query execution fails or the response cannot be parsed as JSON + * + * @note The response format follows the PPL plugin's Execute API response structure with + * "schema", "datarows", "total", and "size" fields. + */ + suspend fun executePplQuery( + query: String, + localNode: DiscoveryNode, + transportService: TransportService + ): JSONObject { + // call PPL plugin to execute query + val transportPplQueryRequest = TransportPPLQueryRequest( + query, + JSONObject(mapOf("query" to query)), + null // null path falls back to a default path internal to SQL/PPL Plugin + ) + + val transportPplQueryResponse = PPLPluginInterface.suspendUntil { + this.executeQuery( + transportService, + localNode, + transportPplQueryRequest, + it + ) + } + + val queryResponseJson = JSONObject(transportPplQueryResponse.result) + + return queryResponseJson + } + + /** + * Searches a custom condition eval statement for the name of the eval result variable. + * + * Parses a PPL eval expression to extract the variable name being assigned. The eval + * statement must follow the format: `eval = `. This variable + * name is needed to reference the evaluation result in subsequent trigger condition checks. + * + * @param customCondition The PPL custom condition string containing an eval statement (e.g. eval result = avg > 3) + * @return The name of the eval result variable + * @throws IllegalArgumentException if no valid eval statement is found or the syntax is invalid + * + * @example + * ``` + * val condition = "eval error_rate = errors / total" + * val varName = findEvalResultVar(condition) + * // Returns: "error_rate" + * ``` + * + * @note A precheck of the base query + custom condition is assumed to have been done already. + * The function thus expects the PPL keyword "eval" followed by whitespace. Without the + * whitespace (e.g., "evalresult"), the PPL plugin would have thrown a syntax error + * during upstream validations + * @note Variable names must follow standard identifier rules: start with a letter or underscore, + * followed by letters, digits, or underscores (matching `[a-zA-Z_][a-zA-Z0-9_]*`). + * + * TODO: Replace this in-house parser with a PPL plugin dependency that provides proper + * query parsing functionality. + */ + fun findEvalResultVar(customCondition: String): String { + // TODO: these are in-house PPL query parsers, find a PPL plugin dependency that does this for us + val evalResultVar = evalResultVarRegex.find(customCondition)?.groupValues?.get(1) + ?: throw IllegalArgumentException("Given custom condition is invalid, could not find eval result variable") + return evalResultVar + } + + /** + * Finds the index of the eval result variable in the PPL query response schema. + * + * Searches through the schema array in the PPL query response to locate the column + * corresponding to the eval result variable. This index is used to extract the + * eval result values from the datarows in the query response. + * + * @param customConditionQueryResponse The JSONObject containing the PPL query response + * with "schema" and "datarows" fields + * @param evalResultVarName The name of the eval result variable to locate in the schema + * @return The zero-based index of the eval result variable in the schema array + * @throws IllegalStateException if the eval result variable is not found in the schema + * + * @note The eval result variable should always be present in the schema if the query + * executed successfully. If not found, this indicates an unexpected state. + * @note The query response schema is assumed to follow PPL plugin Execute API response schema + */ + fun findEvalResultVarIdxInSchema(customConditionQueryResponse: JSONObject, evalResultVarName: String): Int { + // find the index eval statement result variable in the PPL query response schema + val schemaList = customConditionQueryResponse.getJSONArray("schema") + var evalResultVarIdx = -1 + for (i in 0 until schemaList.length()) { + val schemaObj = schemaList.getJSONObject(i) + val columnName = schemaObj.getString("name") + + if (columnName == evalResultVarName) { + evalResultVarIdx = i + break + } + } + + // eval statement result variable should always be found + if (evalResultVarIdx == -1) { + throw IllegalStateException( + "Expected to find eval statement results variable \"$evalResultVarName\" in results " + + "of PPL query with custom condition, but did not." + ) + } + + return evalResultVarIdx + } + + /** + * Extracts the list of indices from a PPL query's source statement. + * + * Parses the PPL `source=` clause to identify which indices, index patterns, or index + * aliases are being queried. This information is primarily used for permission checks. + * Supports comma-separated lists of indices and wildcard patterns. + * + * @param pplQuery The complete PPL query string containing a source statement + * @return A list of index names, patterns, or aliases (e.g., ["logs-*", "metrics-2024"]) + * @throws IllegalStateException if no valid source statement is found, even after + * the query has been validated by the SQL/PPL plugin + * + * @example + * ``` + * val query = "source=logs-* | where level='ERROR'" + * val indices = getIndicesFromPplQuery(query) + * // Returns: ["logs-*"] + * + * val multiQuery = "source=logs-*, metrics-2024, .kibana | stats count()" + * val multiIndices = getIndicesFromPplQuery(multiQuery) + * // Returns: ["logs-*", "metrics-2024", ".kibana"] + * ``` + * + * @note Supports concrete indices, wildcard patterns (*), dot-prefixed system indices, + * and index aliases + * @note PPL queries contain exactly one source statement, so only the first match is used + * @note The regex pattern handles optional whitespace around `=` and commas + * + */ + fun getIndicesFromPplQuery(pplQuery: String): List { + // use find() instead of findAll() because a PPL query only ever has one source statement + // the only capture group specified in the regex captures the comma separated string of indices/index patterns + val indices = indicesListRegex.find(pplQuery)?.groupValues?.get(1)?.split(",")?.map { it.trim() } + ?: throw IllegalStateException( + "Could not find indices that PPL Monitor query searches even " + + "after validating the query through SQL/PPL plugin." + ) + + // remove any backticks that might have been read in + val unBackTickedIndices = mutableListOf() + indices.forEach { + if (it.startsWith("`") && it.endsWith("`")) { + unBackTickedIndices.add(it.substring(1, it.length - 1)) + } else { + unBackTickedIndices.add(it) + } + } + + return unBackTickedIndices.toList() + } + + /** + * Caps the size of PPL query results to prevent memory issues and oversized alert payloads. + * + * Checks if the serialized query results exceed a specified size limit. If the results + * are within the limit, they are returned unchanged. If they exceed the limit, the datarows + * are replaced with an informational message while preserving the schema and metadata fields. + * This ensures alerts can still be created even when query results are too large. + * + * @param pplQueryResults The PPL query response JSONObject + * @param maxSize The maximum allowed size in bytes (estimated by serialized string length) + * @return The original results if under the limit, or a modified version with datarows replaced by a message + * + * @example + * ``` + * val queryResults = executePplQuery(query, client) + * val cappedResults = capPPLQueryResultsSize(queryResults, maxSize = 5000L) + * + * // If results were too large, datarows will contain: + * // [["The PPL Query results were too large and thus excluded"]] + * // But schema, total, and size fields are preserved + * ``` + * + * @note Size is estimated using `toString().length`, which approximates byte size but may + * not be exact for multi-byte characters + * @note The PPL query results structure includes: + * - `schema`: Array of objects storing data types for each column + * - `datarows`: Array of arrays containing the actual query result rows + * - `total`: Total number of result rows + * - `size`: Same as `total` (redundant field in PPL response) + */ + fun capPPLQueryResultsSize(pplQueryResults: JSONObject, maxSize: Long): JSONObject { + // estimate byte size with serialized string length + // if query results size are already under the limit, do nothing + // and return the query results as is + val pplQueryResultsSize = pplQueryResults.toString().length + if (pplQueryResultsSize <= maxSize) { + return pplQueryResults + } + + // if the query results exceed the limit, we need to replace the query results + // with a message that says the results were too large, but still retain the other + // ppl query response fields like schema, total, and size + val limitExceedMessageQueryResults = JSONObject() + + val schema = JSONArray(pplQueryResults.getJSONArray("schema").toList()) + val datarows = JSONArray().put(JSONArray(listOf("The PPL Query results were too large and thus excluded"))) + val total = pplQueryResults.getInt("total") + val size = pplQueryResults.getInt("size") + + limitExceedMessageQueryResults.put("schema", schema) + limitExceedMessageQueryResults.put("datarows", datarows) + limitExceedMessageQueryResults.put("total", total) + limitExceedMessageQueryResults.put("size", size) + + return limitExceedMessageQueryResults + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/WorkflowService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/WorkflowService.kt index 15f1e192e..83f8b56e0 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/WorkflowService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/WorkflowService.kt @@ -11,6 +11,7 @@ import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse import org.opensearch.action.search.SearchRequest import org.opensearch.action.search.SearchResponse +import org.opensearch.alerting.AlertingV2Utils.validateMonitorV1 import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.common.xcontent.LoggingDeprecationHandler import org.opensearch.common.xcontent.XContentType @@ -132,7 +133,12 @@ class WorkflowService( xContentRegistry, LoggingDeprecationHandler.INSTANCE, hit.sourceAsString ).use { hitsParser -> - val monitor = ScheduledJob.parse(hitsParser, hit.id, hit.version) as Monitor + val scheduledJob = ScheduledJob.parse(hitsParser, hit.id, hit.version) + validateMonitorV1(scheduledJob)?.let { + throw OpenSearchException(it) + } + + val monitor = scheduledJob as Monitor monitors.add(monitor) } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Action.kt new file mode 100644 index 000000000..b182d87d4 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Action.kt @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionType + +class DeleteMonitorV2Action private constructor() : ActionType(NAME, ::DeleteMonitorV2Response) { + companion object { + val INSTANCE = DeleteMonitorV2Action() + const val NAME = "cluster:admin/opensearch/alerting/v2/monitor/delete" + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Request.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Request.kt new file mode 100644 index 000000000..7024842ac --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Request.kt @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionRequest +import org.opensearch.action.ActionRequestValidationException +import org.opensearch.action.support.WriteRequest +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import java.io.IOException + +class DeleteMonitorV2Request : ActionRequest { + val monitorV2Id: String + val refreshPolicy: WriteRequest.RefreshPolicy + + constructor(monitorV2Id: String, refreshPolicy: WriteRequest.RefreshPolicy) : super() { + this.monitorV2Id = monitorV2Id + this.refreshPolicy = refreshPolicy + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + monitorV2Id = sin.readString(), + refreshPolicy = WriteRequest.RefreshPolicy.readFrom(sin) + ) + + override fun validate(): ActionRequestValidationException? { + return null + } + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeString(monitorV2Id) + refreshPolicy.writeTo(out) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Response.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Response.kt new file mode 100644 index 000000000..b4850b662 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Response.kt @@ -0,0 +1,43 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.commons.alerting.util.IndexUtils +import org.opensearch.commons.notifications.action.BaseResponse +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.XContentBuilder + +class DeleteMonitorV2Response : BaseResponse { + var id: String + var version: Long + + constructor( + id: String, + version: Long + ) : super() { + this.id = id + this.version = version + } + + constructor(sin: StreamInput) : this( + sin.readString(), // id + sin.readLong() // version + ) + + override fun writeTo(out: StreamOutput) { + out.writeString(id) + out.writeLong(version) + } + + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + return builder.startObject() + .field(IndexUtils._ID, id) + .field(IndexUtils._VERSION, version) + .endObject() + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Action.kt new file mode 100644 index 000000000..f0de80e8d --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Action.kt @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionType + +class ExecuteMonitorV2Action private constructor() : ActionType(NAME, ::ExecuteMonitorV2Response) { + companion object { + val INSTANCE = ExecuteMonitorV2Action() + const val NAME = "cluster:admin/opensearch/alerting/v2/monitor/execute" + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Request.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Request.kt new file mode 100644 index 000000000..99a01667c --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Request.kt @@ -0,0 +1,71 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionRequest +import org.opensearch.action.ActionRequestValidationException +import org.opensearch.action.ValidateActions +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.common.unit.TimeValue +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import java.io.IOException + +class ExecuteMonitorV2Request : ActionRequest { + val dryrun: Boolean + val manual: Boolean + val monitorV2Id: String? // exactly one of monitorId or monitor must be non-null + val monitorV2: MonitorV2? + val requestEnd: TimeValue + + constructor( + dryrun: Boolean, + manual: Boolean, // if execute was called by user or by scheduled job + monitorV2Id: String?, + monitorV2: MonitorV2?, + requestEnd: TimeValue + ) : super() { + this.dryrun = dryrun + this.manual = manual + this.monitorV2Id = monitorV2Id + this.monitorV2 = monitorV2 + this.requestEnd = requestEnd + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + sin.readBoolean(), // dryrun + sin.readBoolean(), // manual + sin.readOptionalString(), // monitorV2Id + if (sin.readBoolean()) { + MonitorV2.readFrom(sin) // monitorV2 + } else { + null + }, + sin.readTimeValue() // requestEnd + ) + + override fun validate(): ActionRequestValidationException? = + if (monitorV2 == null && monitorV2Id == null) { + ValidateActions.addValidationError("Neither a monitor ID nor monitor object was supplied", null) + } else { + null + } + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeBoolean(dryrun) + out.writeBoolean(manual) + out.writeOptionalString(monitorV2Id) + if (monitorV2 != null) { + out.writeBoolean(true) + MonitorV2.writeTo(out, monitorV2) + } else { + out.writeBoolean(false) + } + out.writeTimeValue(requestEnd) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Response.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Response.kt new file mode 100644 index 000000000..7d6eb8b1f --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Response.kt @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.alerting.modelv2.MonitorV2RunResult +import org.opensearch.core.action.ActionResponse +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.ToXContentObject +import org.opensearch.core.xcontent.XContentBuilder +import java.io.IOException + +class ExecuteMonitorV2Response : ActionResponse, ToXContentObject { + val monitorV2RunResult: MonitorV2RunResult<*> + + constructor(monitorV2RunResult: MonitorV2RunResult<*>) : super() { + this.monitorV2RunResult = monitorV2RunResult + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + MonitorV2RunResult.readFrom(sin) // monitorRunResult + ) + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + MonitorV2RunResult.writeTo(out, monitorV2RunResult) + } + + @Throws(IOException::class) + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + return monitorV2RunResult.toXContent(builder, ToXContent.EMPTY_PARAMS) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Action.kt new file mode 100644 index 000000000..e656d6a71 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Action.kt @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionType + +class GetAlertsV2Action private constructor() : ActionType(NAME, ::GetAlertsV2Response) { + companion object { + val INSTANCE = GetAlertsV2Action() + const val NAME = "cluster:admin/opensearch/alerting/v2/alerts/get" + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Request.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Request.kt new file mode 100644 index 000000000..008057aa4 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Request.kt @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionRequest +import org.opensearch.action.ActionRequestValidationException +import org.opensearch.commons.alerting.model.Table +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import java.io.IOException + +class GetAlertsV2Request : ActionRequest { + val table: Table + val severityLevel: String + val monitorV2Ids: List? + + constructor( + table: Table, + severityLevel: String, + monitorV2Ids: List? = null, + ) : super() { + this.table = table + this.severityLevel = severityLevel + this.monitorV2Ids = monitorV2Ids + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + table = Table.readFrom(sin), + severityLevel = sin.readString(), + monitorV2Ids = sin.readOptionalStringList(), + ) + + override fun validate(): ActionRequestValidationException? { + return null + } + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + table.writeTo(out) + out.writeString(severityLevel) + out.writeOptionalStringCollection(monitorV2Ids) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Response.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Response.kt new file mode 100644 index 000000000..0de492496 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Response.kt @@ -0,0 +1,52 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.alerting.modelv2.AlertV2 +import org.opensearch.commons.notifications.action.BaseResponse +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.XContentBuilder +import java.io.IOException +import java.util.Collections + +class GetAlertsV2Response : BaseResponse { + val alertV2s: List + + // totalAlertV2s is not the same as the size of alertV2s because there can be 30 alerts from the request, but + // the request only asked for 5 alerts, so totalAlertV2s will be 30, but alertV2s will only contain 5 alerts + val totalAlertV2s: Int? + + constructor( + alertV2s: List, + totalAlertV2s: Int? + ) : super() { + this.alertV2s = alertV2s + this.totalAlertV2s = totalAlertV2s + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + alertV2s = Collections.unmodifiableList(sin.readList(::AlertV2)), + totalAlertV2s = sin.readOptionalInt() + ) + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeCollection(alertV2s) + out.writeOptionalInt(totalAlertV2s) + } + + @Throws(IOException::class) + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + builder.startObject() + .field("alerts_v2", alertV2s) + .field("total_alerts_v2", totalAlertV2s) + + return builder.endObject() + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Action.kt new file mode 100644 index 000000000..58b6a1cb9 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Action.kt @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionType + +class GetMonitorV2Action private constructor() : ActionType(NAME, ::GetMonitorV2Response) { + companion object { + val INSTANCE = GetMonitorV2Action() + const val NAME = "cluster:admin/opensearch/alerting/v2/monitor/get" + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Request.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Request.kt new file mode 100644 index 000000000..7f051ff1e --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Request.kt @@ -0,0 +1,52 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionRequest +import org.opensearch.action.ActionRequestValidationException +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.search.fetch.subphase.FetchSourceContext +import java.io.IOException + +class GetMonitorV2Request : ActionRequest { + val monitorV2Id: String + val version: Long + val srcContext: FetchSourceContext? + + constructor( + monitorV2Id: String, + version: Long, + srcContext: FetchSourceContext? + ) : super() { + this.monitorV2Id = monitorV2Id + this.version = version + this.srcContext = srcContext + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + sin.readString(), // monitorV2Id + sin.readLong(), // version + if (sin.readBoolean()) { + FetchSourceContext(sin) // srcContext + } else { + null + } + ) + + override fun validate(): ActionRequestValidationException? { + return null + } + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeString(monitorV2Id) + out.writeLong(version) + out.writeBoolean(srcContext != null) + srcContext?.writeTo(out) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Response.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Response.kt new file mode 100644 index 000000000..5b6df334f --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Response.kt @@ -0,0 +1,80 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.commons.alerting.util.IndexUtils.Companion._ID +import org.opensearch.commons.alerting.util.IndexUtils.Companion._PRIMARY_TERM +import org.opensearch.commons.alerting.util.IndexUtils.Companion._SEQ_NO +import org.opensearch.commons.alerting.util.IndexUtils.Companion._VERSION +import org.opensearch.commons.notifications.action.BaseResponse +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.XContentBuilder +import java.io.IOException + +class GetMonitorV2Response : BaseResponse { + var id: String + var version: Long + var seqNo: Long + var primaryTerm: Long + var monitorV2: MonitorV2? + + constructor( + id: String, + version: Long, + seqNo: Long, + primaryTerm: Long, + monitorV2: MonitorV2? + ) : super() { + this.id = id + this.version = version + this.seqNo = seqNo + this.primaryTerm = primaryTerm + this.monitorV2 = monitorV2 + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + id = sin.readString(), // id + version = sin.readLong(), // version + seqNo = sin.readLong(), // seqNo + primaryTerm = sin.readLong(), // primaryTerm + monitorV2 = if (sin.readBoolean()) { + MonitorV2.readFrom(sin) // monitorV2 + } else { + null + } + ) + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeString(id) + out.writeLong(version) + out.writeLong(seqNo) + out.writeLong(primaryTerm) + if (monitorV2 != null) { + out.writeBoolean(true) + MonitorV2.writeTo(out, monitorV2!!) + } else { + out.writeBoolean(false) + } + } + + @Throws(IOException::class) + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + builder.startObject() + .field(_ID, id) + .field(_VERSION, version) + .field(_SEQ_NO, seqNo) + .field(_PRIMARY_TERM, primaryTerm) + if (monitorV2 != null) { + builder.field("monitorV2", monitorV2) + } + return builder.endObject() + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Action.kt new file mode 100644 index 000000000..aab23b631 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Action.kt @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionType + +class IndexMonitorV2Action private constructor() : ActionType(NAME, ::IndexMonitorV2Response) { + companion object { + val INSTANCE = IndexMonitorV2Action() + const val NAME = "cluster:admin/opensearch/alerting/v2/monitor/write" + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Request.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Request.kt new file mode 100644 index 000000000..105408d07 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Request.kt @@ -0,0 +1,69 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionRequest +import org.opensearch.action.ActionRequestValidationException +import org.opensearch.action.support.WriteRequest +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.rest.RestRequest +import java.io.IOException + +class IndexMonitorV2Request : ActionRequest { + val monitorId: String + val seqNo: Long + val primaryTerm: Long + val refreshPolicy: WriteRequest.RefreshPolicy + val method: RestRequest.Method + var monitorV2: MonitorV2 + val rbacRoles: List? + + constructor( + monitorId: String, + seqNo: Long, + primaryTerm: Long, + refreshPolicy: WriteRequest.RefreshPolicy, + method: RestRequest.Method, + monitorV2: MonitorV2, + rbacRoles: List? = null + ) : super() { + this.monitorId = monitorId + this.seqNo = seqNo + this.primaryTerm = primaryTerm + this.refreshPolicy = refreshPolicy + this.method = method + this.monitorV2 = monitorV2 + this.rbacRoles = rbacRoles + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + monitorId = sin.readString(), + seqNo = sin.readLong(), + primaryTerm = sin.readLong(), + refreshPolicy = WriteRequest.RefreshPolicy.readFrom(sin), + method = sin.readEnum(RestRequest.Method::class.java), + monitorV2 = MonitorV2.readFrom(sin), + rbacRoles = sin.readOptionalStringList() + ) + + override fun validate(): ActionRequestValidationException? { + return null + } + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeString(monitorId) + out.writeLong(seqNo) + out.writeLong(primaryTerm) + refreshPolicy.writeTo(out) + out.writeEnum(method) + MonitorV2.writeTo(out, monitorV2) + out.writeOptionalStringCollection(rbacRoles) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Response.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Response.kt new file mode 100644 index 000000000..99d076334 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Response.kt @@ -0,0 +1,73 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.commons.alerting.util.IndexUtils.Companion._ID +import org.opensearch.commons.alerting.util.IndexUtils.Companion._PRIMARY_TERM +import org.opensearch.commons.alerting.util.IndexUtils.Companion._SEQ_NO +import org.opensearch.commons.alerting.util.IndexUtils.Companion._VERSION +import org.opensearch.commons.notifications.action.BaseResponse +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.XContentBuilder +import java.io.IOException + +class IndexMonitorV2Response : BaseResponse { + var id: String + var version: Long + var seqNo: Long + var primaryTerm: Long + var monitorV2: MonitorV2 + + constructor( + id: String, + version: Long, + seqNo: Long, + primaryTerm: Long, + monitorV2: MonitorV2 + ) : super() { + this.id = id + this.version = version + this.seqNo = seqNo + this.primaryTerm = primaryTerm + this.monitorV2 = monitorV2 + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + sin.readString(), // id + sin.readLong(), // version + sin.readLong(), // seqNo + sin.readLong(), // primaryTerm + MonitorV2.readFrom(sin) // monitorV2 + ) + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeString(id) + out.writeLong(version) + out.writeLong(seqNo) + out.writeLong(primaryTerm) + MonitorV2.writeTo(out, monitorV2) + } + + @Throws(IOException::class) + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + return builder.startObject() + .field(_ID, id) + .field(_VERSION, version) + .field(_SEQ_NO, seqNo) + .field(_PRIMARY_TERM, primaryTerm) + .field(MONITOR_V2_FIELD, monitorV2) + .endObject() + } + + companion object { + const val MONITOR_V2_FIELD = "monitor_v2" + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2Action.kt new file mode 100644 index 000000000..bc4d9e572 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2Action.kt @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionType +import org.opensearch.action.search.SearchResponse + +class SearchMonitorV2Action private constructor() : ActionType(NAME, ::SearchResponse) { + companion object { + val INSTANCE = SearchMonitorV2Action() + const val NAME = "cluster:admin/opensearch/alerting/v2/monitor/search" + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2Request.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2Request.kt new file mode 100644 index 000000000..c16b62fd8 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2Request.kt @@ -0,0 +1,37 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionRequest +import org.opensearch.action.ActionRequestValidationException +import org.opensearch.action.search.SearchRequest +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import java.io.IOException + +class SearchMonitorV2Request : ActionRequest { + val searchRequest: SearchRequest + + constructor( + searchRequest: SearchRequest + ) : super() { + this.searchRequest = searchRequest + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + searchRequest = SearchRequest(sin) + ) + + override fun validate(): ActionRequestValidationException? { + return null + } + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + searchRequest.writeTo(out) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Indices.kt b/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Indices.kt new file mode 100644 index 000000000..75257fc25 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Indices.kt @@ -0,0 +1,418 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.alertsv2 + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import org.apache.logging.log4j.LogManager +import org.opensearch.ExceptionsHelper +import org.opensearch.ResourceAlreadyExistsException +import org.opensearch.action.admin.cluster.state.ClusterStateRequest +import org.opensearch.action.admin.cluster.state.ClusterStateResponse +import org.opensearch.action.admin.indices.alias.Alias +import org.opensearch.action.admin.indices.create.CreateIndexRequest +import org.opensearch.action.admin.indices.create.CreateIndexResponse +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse +import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest +import org.opensearch.action.admin.indices.rollover.RolloverRequest +import org.opensearch.action.admin.indices.rollover.RolloverResponse +import org.opensearch.action.support.IndicesOptions +import org.opensearch.action.support.clustermanager.AcknowledgedResponse +import org.opensearch.alerting.opensearchapi.suspendUntil +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_V2_HISTORY_ENABLED +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_V2_HISTORY_INDEX_MAX_AGE +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_V2_HISTORY_MAX_DOCS +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_V2_HISTORY_RETENTION_PERIOD +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_V2_HISTORY_ROLLOVER_PERIOD +import org.opensearch.alerting.settings.AlertingSettings.Companion.REQUEST_TIMEOUT +import org.opensearch.alerting.util.IndexUtils +import org.opensearch.cluster.ClusterChangedEvent +import org.opensearch.cluster.ClusterStateListener +import org.opensearch.cluster.metadata.IndexMetadata +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.settings.Settings +import org.opensearch.common.unit.TimeValue +import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.alerting.util.AlertingException +import org.opensearch.core.action.ActionListener +import org.opensearch.threadpool.Scheduler.Cancellable +import org.opensearch.threadpool.ThreadPool +import org.opensearch.transport.client.Client +import java.time.Instant + +private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) +private val logger = LogManager.getLogger(AlertV2Indices::class.java) + +/** + * This class handles the rollover and management of v2 alerts history indices + * + * @opensearch.experimental + */ +class AlertV2Indices( + settings: Settings, + private val client: Client, + private val threadPool: ThreadPool, + private val clusterService: ClusterService +) : ClusterStateListener { + + init { + clusterService.addListener(this) + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERT_V2_HISTORY_ENABLED) { alertV2HistoryEnabled = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERT_V2_HISTORY_MAX_DOCS) { alertV2HistoryMaxDocs = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERT_V2_HISTORY_INDEX_MAX_AGE) { alertV2HistoryMaxAge = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERT_V2_HISTORY_ROLLOVER_PERIOD) { + alertV2HistoryRolloverPeriod = it + rescheduleAlertRollover() + } + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERT_V2_HISTORY_RETENTION_PERIOD) { + alertV2HistoryRetentionPeriod = it + } + clusterService.clusterSettings.addSettingsUpdateConsumer(REQUEST_TIMEOUT) { requestTimeout = it } + } + + companion object { + + /** The in progress alert history index. */ + const val ALERT_V2_INDEX = ".opensearch-alerting-v2-alerts" + + /** The alias of the index in which to write alert history */ + const val ALERT_V2_HISTORY_WRITE_INDEX = ".opensearch-alerting-v2-alert-history-write" + + /** The index name pattern referring to all alert history indices */ + const val ALERT_V2_HISTORY_ALL = ".opensearch-alerting-v2-alert-history*" + + /** The index name pattern to create alert history indices */ + const val ALERT_V2_HISTORY_INDEX_PATTERN = "<.opensearch-alerting-v2-alert-history-{now/d}-1>" + + /** The index name pattern to query all alerts, history and current alerts. */ + const val ALL_ALERT_V2_INDEX_PATTERN = ".opensearch-alerting-v2-alert*" + + @JvmStatic + fun alertV2Mapping() = + AlertV2Indices::class.java.getResource("alert_v2_mapping.json").readText() + } + + @Volatile private var alertV2HistoryEnabled = ALERT_V2_HISTORY_ENABLED.get(settings) + + @Volatile private var alertV2HistoryMaxDocs = ALERT_V2_HISTORY_MAX_DOCS.get(settings) + + @Volatile private var alertV2HistoryMaxAge = ALERT_V2_HISTORY_INDEX_MAX_AGE.get(settings) + + @Volatile private var alertV2HistoryRolloverPeriod = ALERT_V2_HISTORY_ROLLOVER_PERIOD.get(settings) + + @Volatile private var alertV2HistoryRetentionPeriod = ALERT_V2_HISTORY_RETENTION_PERIOD.get(settings) + + @Volatile private var requestTimeout = REQUEST_TIMEOUT.get(settings) + + @Volatile private var isClusterManager = false + + // for JobsMonitor to report + var lastRolloverTime: TimeValue? = null + + private var alertV2HistoryIndexInitialized: Boolean = false + + private var alertV2IndexInitialized: Boolean = false + + private var scheduledAlertV2Rollover: Cancellable? = null + + fun onClusterManager() { + try { + // try to rollover immediately as we might be restarting the cluster + rolloverAlertV2HistoryIndex() + + // schedule the next rollover for approx MAX_AGE later + scheduledAlertV2Rollover = threadPool + .scheduleWithFixedDelay({ rolloverAndDeleteAlertV2HistoryIndices() }, alertV2HistoryRolloverPeriod, executorName()) + } catch (e: Exception) { + logger.error("Error rolling over alerts v2 history index.", e) + } + } + + fun offClusterManager() { + scheduledAlertV2Rollover?.cancel() + } + + private fun executorName(): String { + return ThreadPool.Names.MANAGEMENT + } + + override fun clusterChanged(event: ClusterChangedEvent) { + // Instead of using a LocalNodeClusterManagerListener to track clustermanager changes, this service will + // track them here to avoid conditions where clustermanager listener events run after other + // listeners that depend on what happened in the clustermanager listener + if (this.isClusterManager != event.localNodeClusterManager()) { + this.isClusterManager = event.localNodeClusterManager() + if (this.isClusterManager) { + onClusterManager() + } else { + offClusterManager() + } + } + + // if the indexes have been deleted they need to be reinitialized + alertV2IndexInitialized = event.state().routingTable().hasIndex(ALERT_V2_INDEX) + alertV2HistoryIndexInitialized = event.state().metadata().hasAlias(ALERT_V2_HISTORY_WRITE_INDEX) + } + + private fun rescheduleAlertRollover() { + if (clusterService.state().nodes.isLocalNodeElectedClusterManager) { + scheduledAlertV2Rollover?.cancel() + scheduledAlertV2Rollover = threadPool + .scheduleWithFixedDelay({ rolloverAndDeleteAlertV2HistoryIndices() }, alertV2HistoryRolloverPeriod, executorName()) + } + } + + suspend fun createOrUpdateAlertV2Index() { + if (!alertV2IndexInitialized) { + alertV2IndexInitialized = createIndex(ALERT_V2_INDEX, alertV2Mapping()) + if (alertV2IndexInitialized) IndexUtils.alertIndexUpdated() + } else { + if (!IndexUtils.alertIndexUpdated) updateIndexMapping(ALERT_V2_INDEX, alertV2Mapping()) + } + alertV2IndexInitialized + } + + suspend fun createOrUpdateInitialAlertV2HistoryIndex() { + if (!alertV2HistoryIndexInitialized) { + alertV2HistoryIndexInitialized = createIndex(ALERT_V2_HISTORY_INDEX_PATTERN, alertV2Mapping(), ALERT_V2_HISTORY_WRITE_INDEX) + if (alertV2HistoryIndexInitialized) + IndexUtils.lastUpdatedAlertV2HistoryIndex = IndexUtils.getIndexNameWithAlias( + clusterService.state(), + ALERT_V2_HISTORY_WRITE_INDEX + ) + } else { + updateIndexMapping(ALERT_V2_HISTORY_WRITE_INDEX, alertV2Mapping(), true) + } + alertV2HistoryIndexInitialized + } + + fun isAlertV2Initialized(): Boolean { + return alertV2IndexInitialized && alertV2HistoryIndexInitialized + } + + private fun rolloverAndDeleteAlertV2HistoryIndices() { + if (alertV2HistoryEnabled) rolloverAlertV2HistoryIndex() + deleteOldIndices("History", ALERT_V2_HISTORY_ALL) + } + + private suspend fun createIndex(index: String, schemaMapping: String, alias: String? = null): Boolean { + // This should be a fast check of local cluster state. Should be exceedingly rare that the local cluster + // state does not contain the index and multiple nodes concurrently try to create the index. + // If it does happen that error is handled we catch the ResourceAlreadyExistsException + val existsResponse: IndicesExistsResponse = client.admin().indices().suspendUntil { + exists(IndicesExistsRequest(index).local(true), it) + } + if (existsResponse.isExists) return true + + logger.debug("index: [$index] schema mappings: [$schemaMapping]") + val request = CreateIndexRequest(index) + .mapping(schemaMapping) + .settings(Settings.builder().put("index.hidden", true).build()) + + if (alias != null) request.alias(Alias(alias)) + return try { + val createIndexResponse: CreateIndexResponse = client.admin().indices().suspendUntil { create(request, it) } + createIndexResponse.isAcknowledged + } catch (t: Exception) { + if (ExceptionsHelper.unwrapCause(t) is ResourceAlreadyExistsException) { + true + } else { + throw AlertingException.wrap(t) + } + } + } + + private suspend fun updateIndexMapping(index: String, mapping: String, alias: Boolean = false) { + val clusterState = clusterService.state() + var targetIndex = index + if (alias) { + targetIndex = IndexUtils.getIndexNameWithAlias(clusterState, index) + } + + if (targetIndex == IndexUtils.lastUpdatedAlertV2HistoryIndex) { + return + } + + val putMappingRequest: PutMappingRequest = PutMappingRequest(targetIndex) + .source(mapping, XContentType.JSON) + val updateResponse: AcknowledgedResponse = client.admin().indices().suspendUntil { putMapping(putMappingRequest, it) } + if (updateResponse.isAcknowledged) { + logger.info("Index mapping of $targetIndex is updated") + setIndexUpdateFlag(index, targetIndex) + } else { + logger.info("Failed to update index mapping of $targetIndex") + } + } + + private fun setIndexUpdateFlag(index: String, targetIndex: String) { + when (index) { + ALERT_V2_INDEX -> IndexUtils.alertV2IndexUpdated() + ALERT_V2_HISTORY_WRITE_INDEX -> IndexUtils.lastUpdatedAlertV2HistoryIndex = targetIndex + } + } + + private fun rolloverIndex( + initialized: Boolean, + index: String, + pattern: String, + map: String, + docsCondition: Long, + ageCondition: TimeValue, + writeIndex: String + ) { + if (!initialized) { + return + } + + // We have to pass null for newIndexName in order to get Elastic to increment the index count. + val request = RolloverRequest(index, null) + request.createIndexRequest.index(pattern) + .mapping(map) + .settings(Settings.builder().put("index.hidden", true).build()) + request.addMaxIndexDocsCondition(docsCondition) + request.addMaxIndexAgeCondition(ageCondition) + client.admin().indices().rolloverIndex( + request, + object : ActionListener { + override fun onResponse(response: RolloverResponse) { + if (!response.isRolledOver) { + logger.info("$writeIndex not rolled over. Conditions were: ${response.conditionStatus}") + } else { + lastRolloverTime = TimeValue.timeValueMillis(threadPool.absoluteTimeInMillis()) + } + } + override fun onFailure(e: Exception) { + logger.error("$writeIndex not roll over failed.") + } + } + ) + } + + private fun rolloverAlertV2HistoryIndex() { + rolloverIndex( + alertV2HistoryIndexInitialized, + ALERT_V2_HISTORY_WRITE_INDEX, + ALERT_V2_HISTORY_INDEX_PATTERN, + alertV2Mapping(), + alertV2HistoryMaxDocs, + alertV2HistoryMaxAge, + ALERT_V2_HISTORY_WRITE_INDEX + ) + } + + private fun deleteOldIndices(tag: String, indices: String) { + val clusterStateRequest = ClusterStateRequest() + .clear() + .indices(indices) + .metadata(true) + .local(true) + .indicesOptions(IndicesOptions.strictExpand()) + client.admin().cluster().state( + clusterStateRequest, + object : ActionListener { + override fun onResponse(clusterStateResponse: ClusterStateResponse) { + if (clusterStateResponse.state.metadata.indices.isNotEmpty()) { + scope.launch { + val indicesToDelete = getIndicesToDelete(clusterStateResponse) + logger.info("Deleting old $tag indices viz $indicesToDelete") + deleteAllOldHistoryIndices(indicesToDelete) + } + } else { + logger.info("No Old $tag Indices to delete") + } + } + override fun onFailure(e: Exception) { + logger.error("Error fetching cluster state") + } + } + ) + } + + private fun getIndicesToDelete(clusterStateResponse: ClusterStateResponse): List { + val indicesToDelete = mutableListOf() + for (entry in clusterStateResponse.state.metadata.indices) { + val indexMetaData = entry.value + getHistoryIndexToDelete( + indexMetaData, + alertV2HistoryRetentionPeriod.millis, + ALERT_V2_HISTORY_WRITE_INDEX, + alertV2HistoryEnabled + )?.let { indicesToDelete.add(it) } + } + return indicesToDelete + } + + private fun getHistoryIndexToDelete( + indexMetadata: IndexMetadata, + retentionPeriodMillis: Long, + writeIndex: String, + historyEnabled: Boolean + ): String? { + val creationTime = indexMetadata.creationDate + if ((Instant.now().toEpochMilli() - creationTime) > retentionPeriodMillis) { + val alias = indexMetadata.aliases.entries.firstOrNull { writeIndex == it.value.alias } + if (alias != null) { + if (historyEnabled) { + // If the index has the write alias and history is enabled, don't delete the index + return null + } else if (writeIndex == ALERT_V2_HISTORY_WRITE_INDEX) { + // Otherwise reset alertHistoryIndexInitialized since index will be deleted + alertV2HistoryIndexInitialized = false + } + } + + return indexMetadata.index.name + } + return null + } + + private fun deleteAllOldHistoryIndices(indicesToDelete: List) { + if (indicesToDelete.isNotEmpty()) { + val deleteIndexRequest = DeleteIndexRequest(*indicesToDelete.toTypedArray()) + client.admin().indices().delete( + deleteIndexRequest, + object : ActionListener { + override fun onResponse(deleteIndicesResponse: AcknowledgedResponse) { + if (!deleteIndicesResponse.isAcknowledged) { + logger.error( + "Could not delete one or more Alerting V2 history indices: $indicesToDelete. Retrying one by one." + ) + deleteOldHistoryIndex(indicesToDelete) + } + } + override fun onFailure(e: Exception) { + logger.error("Delete for Alerting V2 History Indices $indicesToDelete Failed. Retrying one by one.") + deleteOldHistoryIndex(indicesToDelete) + } + } + ) + } + } + + private fun deleteOldHistoryIndex(indicesToDelete: List) { + for (index in indicesToDelete) { + val singleDeleteRequest = DeleteIndexRequest(*indicesToDelete.toTypedArray()) + client.admin().indices().delete( + singleDeleteRequest, + object : ActionListener { + override fun onResponse(acknowledgedResponse: AcknowledgedResponse?) { + if (acknowledgedResponse != null) { + if (!acknowledgedResponse.isAcknowledged) { + logger.error("Could not delete one or more Alerting V2 history indices: $index") + } + } + } + override fun onFailure(e: Exception) { + logger.error("Exception ${e.message} while deleting the index $index") + } + } + ) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Mover.kt b/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Mover.kt new file mode 100644 index 000000000..bf5aa9fad --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Mover.kt @@ -0,0 +1,477 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.alertsv2 + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import org.apache.logging.log4j.LogManager +import org.opensearch.action.bulk.BulkRequest +import org.opensearch.action.bulk.BulkResponse +import org.opensearch.action.delete.DeleteRequest +import org.opensearch.action.index.IndexRequest +import org.opensearch.action.search.SearchRequest +import org.opensearch.action.search.SearchResponse +import org.opensearch.alerting.MonitorRunnerExecutionContext +import org.opensearch.alerting.alertsv2.AlertV2Indices.Companion.ALERT_V2_HISTORY_WRITE_INDEX +import org.opensearch.alerting.alertsv2.AlertV2Indices.Companion.ALERT_V2_INDEX +import org.opensearch.alerting.modelv2.AlertV2 +import org.opensearch.alerting.modelv2.AlertV2.Companion.TRIGGERED_TIME_FIELD +import org.opensearch.alerting.modelv2.AlertV2.Companion.TRIGGER_V2_ID_FIELD +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.modelv2.MonitorV2.Companion.MONITOR_V2_TYPE +import org.opensearch.alerting.modelv2.MonitorV2.Companion.TRIGGERS_FIELD +import org.opensearch.alerting.modelv2.PPLSQLMonitor.Companion.PPL_SQL_MONITOR_TYPE +import org.opensearch.alerting.modelv2.TriggerV2.Companion.EXPIRE_FIELD +import org.opensearch.alerting.modelv2.TriggerV2.Companion.ID_FIELD +import org.opensearch.alerting.opensearchapi.suspendUntil +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_V2_HISTORY_ENABLED +import org.opensearch.alerting.util.MAX_SEARCH_SIZE +import org.opensearch.cluster.ClusterChangedEvent +import org.opensearch.cluster.ClusterStateListener +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.settings.Settings +import org.opensearch.common.unit.TimeValue +import org.opensearch.common.xcontent.LoggingDeprecationHandler +import org.opensearch.common.xcontent.XContentFactory +import org.opensearch.common.xcontent.XContentHelper +import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.alerting.model.ScheduledJob.Companion.SCHEDULED_JOBS_INDEX +import org.opensearch.core.common.bytes.BytesReference +import org.opensearch.core.rest.RestStatus +import org.opensearch.core.xcontent.NamedXContentRegistry +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.XContentParser +import org.opensearch.index.VersionType +import org.opensearch.index.query.QueryBuilders +import org.opensearch.search.builder.SearchSourceBuilder +import org.opensearch.threadpool.Scheduler +import org.opensearch.threadpool.ThreadPool +import org.opensearch.transport.client.Client +import java.time.Instant +import java.util.concurrent.TimeUnit + +private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) +private val logger = LogManager.getLogger(AlertV2Mover::class.java) + +/** + * This class handles sweeping the active v2 alerts index for expired alerts, and + * either moving them to v2 alerts history index (if alert v2 history enabled) or + * permanently deleting them (if alert v2 history disabled). It also contains the + * logic for moving alerts in response to a monitor update or deletion. + * + * @opensearch.experimental + */ +class AlertV2Mover( + settings: Settings, + private val client: Client, + private val threadPool: ThreadPool, + private val clusterService: ClusterService, + private val xContentRegistry: NamedXContentRegistry, +) : ClusterStateListener { + init { + clusterService.addListener(this) + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERT_V2_HISTORY_ENABLED) { alertV2HistoryEnabled = it } + } + + @Volatile private var isClusterManager = false + + private var alertV2IndexInitialized = false + + private var alertV2HistoryIndexInitialized = false + + private var alertV2HistoryEnabled = ALERT_V2_HISTORY_ENABLED.get(settings) + + private var scheduledAlertsV2CheckAndExpire: Scheduler.Cancellable? = null + + private val executorName = ThreadPool.Names.MANAGEMENT + + private val checkForExpirationInterval = TimeValue(1L, TimeUnit.MINUTES) + + override fun clusterChanged(event: ClusterChangedEvent) { + if (this.isClusterManager != event.localNodeClusterManager()) { + this.isClusterManager = event.localNodeClusterManager() + if (this.isClusterManager) { + onManager() + } else { + offManager() + } + } + + alertV2IndexInitialized = event.state().routingTable().hasIndex(ALERT_V2_INDEX) + alertV2HistoryIndexInitialized = event.state().metadata().hasAlias(ALERT_V2_HISTORY_WRITE_INDEX) + } + + fun onManager() { + try { + // try to sweep current AlertV2s for expiration immediately as we might be restarting the cluster + moveOrDeleteAlertV2s() + // schedule expiration checks and expirations to happen repeatedly at some interval + scheduledAlertsV2CheckAndExpire = threadPool + .scheduleWithFixedDelay({ moveOrDeleteAlertV2s() }, checkForExpirationInterval, executorName) + } catch (e: Exception) { + // This should be run on cluster startup + logger.error( + "Error sweeping AlertV2s for expiration. This cannot be done until clustermanager node is restarted.", + e + ) + } + } + + fun offManager() { + scheduledAlertsV2CheckAndExpire?.cancel() + } + + // if alertV2 history is enabled, move expired alerts to alertV2 history indices + // if alertV2 history is disabled, permanently delete expired alerts + private fun moveOrDeleteAlertV2s() { + if (!areAlertV2IndicesPresent()) { + return + } + + scope.launch { + val expiredAlerts = searchForExpiredAlerts() + + var copyResponse: BulkResponse? = null + val deleteResponse: BulkResponse? + if (!alertV2HistoryEnabled) { + deleteResponse = deleteExpiredAlerts(expiredAlerts) + } else { + copyResponse = copyExpiredAlerts(expiredAlerts) + deleteResponse = deleteExpiredAlertsThatWereCopied(copyResponse, expiredAlerts) + } + checkForFailures(copyResponse) + checkForFailures(deleteResponse) + } + } + + private suspend fun searchForExpiredAlerts(): List { + logger.debug("beginning search for expired alerts") + /* first collect all triggers and their expire durations */ + // when searching the alerting-config index, only trigger IDs and their expire durations are needed + val monitorV2sSearchQuery = SearchSourceBuilder.searchSource() + .query(QueryBuilders.existsQuery(MONITOR_V2_TYPE)) + .fetchSource( + arrayOf( + "$MONITOR_V2_TYPE.$PPL_SQL_MONITOR_TYPE.$TRIGGERS_FIELD.$ID_FIELD", + "$MONITOR_V2_TYPE.$PPL_SQL_MONITOR_TYPE.$TRIGGERS_FIELD.$EXPIRE_FIELD" + ), + null + ) + .size(MAX_SEARCH_SIZE) + .version(true) + val monitorV2sRequest = SearchRequest(SCHEDULED_JOBS_INDEX) + .source(monitorV2sSearchQuery) + val searchMonitorV2sResponse: SearchResponse = client.suspendUntil { search(monitorV2sRequest, it) } + + logger.debug("searching triggers for their expire durations") + // construct a map that stores each trigger's expiration time + // TODO: create XContent parser specifically for responses to the above search to avoid casting + val triggerToExpireDuration = mutableMapOf() + searchMonitorV2sResponse.hits.forEach { hit -> + val monitorV2Obj = hit.sourceAsMap[MONITOR_V2_TYPE] as Map + val pplMonitorObj = monitorV2Obj[PPL_SQL_MONITOR_TYPE] as Map + val triggers = pplMonitorObj[TRIGGERS_FIELD] as List> + for (trigger in triggers) { + val triggerId = trigger[ID_FIELD] as String + val expireDuration = (trigger[EXPIRE_FIELD] as Int).toLong() + logger.debug("triggerId: $triggerId") + logger.debug("triggerExpires: $expireDuration") + triggerToExpireDuration[triggerId] = expireDuration + } + } + + logger.debug("trigger to expire duration map: $triggerToExpireDuration") + + /* now collect all expired alerts */ + logger.debug("searching active alerts index for expired alerts") + + val now = Instant.now().toEpochMilli() + + val expiredAlertsBoolQuery = QueryBuilders.boolQuery() + + // collect, in an overarching should clause, each trigger and its expiration time. + // any alert that matches both the trigger ID and the expiration time check should + // be returned by the search query + triggerToExpireDuration.forEach { (triggerId, expireDuration) -> + val expireDurationMillis = expireDuration * 60 * 1000 + val maxValidTime = now - expireDurationMillis + + expiredAlertsBoolQuery.should( + QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery(TRIGGER_V2_ID_FIELD, triggerId)) + .must(QueryBuilders.rangeQuery(TRIGGERED_TIME_FIELD).lte(maxValidTime)) + ) + } + + // add orphaned alerts to should clause as well (i.e. alerts whose trigger IDs cannot + // be found in the list of currently existent triggers), since orphaned alerts should be expired. + // note: this is a redundancy with MonitorRunnerService's + // postIndex and postDelete, which handles moving alerts in response + // to a monitor update or delete event. this cleanly handles the case + // that even with those measures in place, an alert that came from a + // now nonexistent trigger was somehow found + expiredAlertsBoolQuery.should( + QueryBuilders.boolQuery() + .mustNot(QueryBuilders.termsQuery(TRIGGER_V2_ID_FIELD, triggerToExpireDuration.keys.toList())) + ) + + // Explicitly specify that at least one should clause must match + expiredAlertsBoolQuery.minimumShouldMatch(1) + + // search for the expired alerts + val expiredAlertsSearchQuery = SearchSourceBuilder.searchSource() + .query(expiredAlertsBoolQuery) + .size(MAX_SEARCH_SIZE) + .version(true) + val expiredAlertsRequest = SearchRequest(ALERT_V2_INDEX) + .source(expiredAlertsSearchQuery) + val expiredAlertsResponse: SearchResponse = client.suspendUntil { search(expiredAlertsRequest, it) } + + // parse the search results into full alert docs, as they will need to be + // indexed into alert history indices + val expiredAlertV2s = mutableListOf() + expiredAlertsResponse.hits.forEach { hit -> + expiredAlertV2s.add( + AlertV2.parse(alertV2ContentParser(hit.sourceRef), hit.id, hit.version) + ) + } + + logger.debug("expired alerts: $expiredAlertV2s") + + return expiredAlertV2s + } + + private suspend fun deleteExpiredAlerts(expiredAlerts: List): BulkResponse? { + logger.debug("beginning to hard delete expired alerts permanently") + // If no expired alerts are found, simply return + if (expiredAlerts.isEmpty()) { + return null + } + + val deleteRequests = expiredAlerts.map { + DeleteRequest(ALERT_V2_INDEX, it.id) + .routing(it.monitorId) + .version(it.version) + .versionType(VersionType.EXTERNAL_GTE) + } + + val deleteRequest = BulkRequest().add(deleteRequests) + val deleteResponse: BulkResponse = client.suspendUntil { bulk(deleteRequest, it) } + + return deleteResponse + } + + private suspend fun copyExpiredAlerts(expiredAlerts: List): BulkResponse? { + logger.debug("beginning to copy expired alerts to history write index") + // If no expired alerts are found, simply return + if (expiredAlerts.isEmpty()) { + return null + } + + val indexRequests = expiredAlerts.map { + IndexRequest(ALERT_V2_HISTORY_WRITE_INDEX) + .routing(it.monitorId) + .source(it.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .version(it.version) + .versionType(VersionType.EXTERNAL_GTE) + .id(it.id) + } + + val copyRequest = BulkRequest().add(indexRequests) + val copyResponse: BulkResponse = client.suspendUntil { bulk(copyRequest, it) } + + return copyResponse + } + + private suspend fun deleteExpiredAlertsThatWereCopied(copyResponse: BulkResponse?, expiredAlerts: List): BulkResponse? { + logger.debug("beginning to delete expired alerts that were copied to history write index") + // if there were no expired alerts to copy, skip deleting anything + if (copyResponse == null) { + return null + } + + // pre-index the alerts so retrieving their + // monitor IDs for routing is easier + val alertsById: Map = expiredAlerts.associateBy { it.id } + + val deleteRequests = copyResponse.items.filterNot { it.isFailed }.map { + DeleteRequest(ALERT_V2_INDEX, it.id) + .routing(alertsById[it.id]!!.monitorId) + .version(it.version) + .versionType(VersionType.EXTERNAL_GTE) + } + val deleteRequest = BulkRequest().add(deleteRequests) + val deleteResponse: BulkResponse = client.suspendUntil { bulk(deleteRequest, it) } + + return deleteResponse + } + + private fun checkForFailures(bulkResponse: BulkResponse?) { + bulkResponse?.let { + if (bulkResponse.hasFailures()) { + val retryCause = bulkResponse.items.filter { it.isFailed } + .firstOrNull { it.status() == RestStatus.TOO_MANY_REQUESTS } + ?.failure?.cause + logger.error( + "Failed to move or delete alert v2s: ${bulkResponse.buildFailureMessage()}", + retryCause + ) + } + } + } + + private fun alertV2ContentParser(bytesReference: BytesReference): XContentParser { + return XContentHelper.createParser( + NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, + bytesReference, XContentType.JSON + ) + } + + private fun areAlertV2IndicesPresent(): Boolean { + return alertV2IndexInitialized && alertV2HistoryIndexInitialized + } + + companion object { + // this method is used by MonitorRunnerService's postIndex and postDelete + // functions to move (in the case of alert v2 history enabled) or delete + // (in the case of alert v2 history disabled) the alerts generated by + // a monitor in response to the event that the monitor gets updated + // or deleted + suspend fun moveAlertV2s(monitorV2Id: String, monitorV2: MonitorV2?, monitorCtx: MonitorRunnerExecutionContext) { + logger.debug("beginning to move alerts for postIndex or postDelete of monitor: $monitorV2Id") + val client = monitorCtx.client!! + + // first collect all alerts that came from this updated or deleted monitor + val boolQuery = QueryBuilders.boolQuery() + .filter(QueryBuilders.termQuery(AlertV2.MONITOR_V2_ID_FIELD, monitorV2Id)) + + /* + this monitorV2 != null case happens when this function is called by postIndex. if the monitor is updated, + we don't want to expire alerts that were generated by triggers that still exist + in the updated monitor, so filter those out. only expire alerts from triggers in + this monitor that may no longer exist in the updated version of the monitor. + edge case: user can edit the trigger itself while explicitly keeping the ID the same, + which means alerts generated by that trigger will (incorrectly) not be filtered out by this logic + even though it was edited. to mitigate this, recall that callers of the update monitor API + must supply the full MonitorV2 object of the updated monitor config. this is important + because it means they don't have to reference the triggers by ID when updating the triggers, + they simply declare a whole new monitor with whatever new triggers they want it to have, and when doing this, + likely won't explicitly pass in trigger IDs for their updated triggers that exactly match + the IDs of the old triggers. this means Alerting will generate a new ID for the updated triggers by default, + meaning this logic will pick up those updated triggers and correctly move/delete the alerts + */ + if (monitorV2 != null) { + boolQuery.mustNot(QueryBuilders.termsQuery(TRIGGER_V2_ID_FIELD, monitorV2.triggers.map { it.id })) + } + + val alertsSearchQuery = SearchSourceBuilder.searchSource() + .query(boolQuery) + .size(MAX_SEARCH_SIZE) + .version(true) + val activeAlertsRequest = SearchRequest(ALERT_V2_INDEX) + .source(alertsSearchQuery) + val searchAlertsResponse: SearchResponse = client.suspendUntil { search(activeAlertsRequest, it) } + + // If no alerts are found, simply return + if (searchAlertsResponse.hits.totalHits?.value == 0L) return + + val activeAlerts = mutableListOf() + searchAlertsResponse.hits.forEach { hit -> + activeAlerts.add( + AlertV2.parse( + XContentHelper.createParser( + NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, + hit.sourceRef, XContentType.JSON + ), + hit.id, + hit.version + ) + ) + } + + // pre-index the alerts so retrieving their + // monitor IDs for routing is easier + val alertsById: Map = activeAlerts.associateBy { it.id } + + val alertV2HistoryEnabled = monitorCtx.clusterService!!.clusterSettings.get(ALERT_V2_HISTORY_ENABLED) + + // if alert v2 history is enabled, migrate the relevant alerts + // to the alert v2 history index pattern instead of hard deleting them + var copyResponse: BulkResponse? = null + if (alertV2HistoryEnabled) { + logger.debug("alert v2 history enabled, copying alerts to history write index") + val indexRequests = searchAlertsResponse.hits.map { hit -> + val xcp = XContentHelper.createParser( + NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, + hit.sourceRef, XContentType.JSON + ) + + IndexRequest(ALERT_V2_HISTORY_WRITE_INDEX) + .routing(monitorV2Id) + .source( + AlertV2.parse(xcp, hit.id, hit.version) + .toXContentWithUser(XContentFactory.jsonBuilder()) + ) + .version(hit.version) + .versionType(VersionType.EXTERNAL_GTE) + .id(hit.id) + } + val copyRequest = BulkRequest().add(indexRequests) + copyResponse = client.suspendUntil { bulk(copyRequest, it) } + + if (copyResponse!!.hasFailures()) { + val retryCause = copyResponse.items.filter { it.isFailed } + .firstOrNull { it.status() == RestStatus.TOO_MANY_REQUESTS } + ?.failure?.cause + throw RuntimeException( + "Failed to copy alertV2s for [$monitorV2Id, ${monitorV2?.triggers?.map { it.id }}]: " + + copyResponse.buildFailureMessage(), + retryCause + ) + } + } + + logger.debug("deleting alerts related to monitor: $monitorV2Id") + + // prepare deletion request + val deleteRequests = if (alertV2HistoryEnabled) { + // if alerts were to be migrated, delete only the ones + // that were successfully copied over + copyResponse!!.items.filterNot { it.isFailed }.map { + DeleteRequest(ALERT_V2_INDEX, it.id) + .routing(alertsById[it.id]!!.monitorId) + .version(it.version) + .versionType(VersionType.EXTERNAL_GTE) + } + } else { + // otherwise just directly get the original + // set of alerts + searchAlertsResponse.hits.map { hit -> + DeleteRequest(ALERT_V2_INDEX, hit.id) + .routing(alertsById[hit.id]!!.monitorId) + .version(hit.version) + .versionType(VersionType.EXTERNAL_GTE) + } + } + + // execute delete request + val deleteRequest = BulkRequest().add(deleteRequests) + val deleteResponse: BulkResponse = client.suspendUntil { bulk(deleteRequest, it) } + + if (deleteResponse.hasFailures()) { + val retryCause = deleteResponse.items.filter { it.isFailed } + .firstOrNull { it.status() == RestStatus.TOO_MANY_REQUESTS } + ?.failure?.cause + throw RuntimeException( + "Failed to delete alertV2s for [$monitorV2Id, ${monitorV2?.triggers?.map { it.id }}]: " + + deleteResponse.buildFailureMessage(), + retryCause + ) + } + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/AlertV2.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/AlertV2.kt new file mode 100644 index 000000000..af188f180 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/AlertV2.kt @@ -0,0 +1,257 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.modelv2 + +import org.opensearch.alerting.core.util.nonOptionalTimeField +import org.opensearch.alerting.modelv2.TriggerV2.Severity +import org.opensearch.common.lucene.uid.Versions +import org.opensearch.commons.alerting.util.instant +import org.opensearch.commons.alerting.util.optionalUserField +import org.opensearch.commons.authuser.User +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.common.io.stream.Writeable +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.XContentBuilder +import org.opensearch.core.xcontent.XContentParser +import org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken +import java.io.IOException +import java.time.Instant + +/** + * Alert generated by Alerting V2 + * An alert is created when a Trigger's trigger conditions are met. + * + * @property id Alert ID. Defaults to [NO_ID]. + * @property version Version number of the Alert. Defaults to [NO_VERSION]. + * @property schemaVersion Version of the alerting-alerts index schema when this Alert was indexed. Defaults to [NO_SCHEMA_VERSION]. + * @property monitorId ID of the Monitor that generated this Alert. + * @property monitorName Name of the Monitor that generated this Alert. + * @property monitorVersion Version of the Monitor at the time it generated this Alert. + * @property triggerId ID of the Trigger in the Monitor that generated this alert. + * @property triggerName Name of the trigger in the Monitor that generated this alert. + * @property queryResults Results from the Monitor's query that caused the Trigger to fire. + * @property triggeredTime Timestamp for when the Alert was generated. + * @property errorMessage Optional error message if there were issues during Trigger execution. + * Null indicates no errors occurred. + * @property severity Severity level of the alert (e.g., "HIGH", "MEDIUM", "LOW"). + * @property executionId Optional ID for the Monitor execution that generated this Alert. + * + * @see MonitorV2 For the monitor that generates alerts + * @see TriggerV2 For the trigger conditions that create alerts + * + * Lifecycle: + * 1. AlertV2 is generated when a TriggerV2's condition is met. The TriggerV2 fires and forgets the AlertV2. + * 2. AlertV2 is stored in the alerts index. AlertV2s are stateless. (e.g. they are never ACTIVE or COMPLETED) + * 3. AlertV2 is soft deleted after its expire duration (determined by its trigger), and archived in an alert history index + * 4. Based on the alert v2 history retention period, the AlertV2 is permanently deleted + * + * @opensearch.experimental + */ +data class AlertV2( + val id: String = NO_ID, + val version: Long = NO_VERSION, + val schemaVersion: Int = NO_SCHEMA_VERSION, + val monitorId: String, + val monitorName: String, + val monitorVersion: Long, + val monitorUser: User?, + val triggerId: String, + val triggerName: String, + val query: String, + val queryResults: Map, + val triggeredTime: Instant, + val errorMessage: String? = null, + val severity: Severity, + val executionId: String? = null +) : Writeable, ToXContent { + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + id = sin.readString(), + version = sin.readLong(), + schemaVersion = sin.readInt(), + monitorId = sin.readString(), + monitorName = sin.readString(), + monitorVersion = sin.readLong(), + monitorUser = if (sin.readBoolean()) { + User(sin) + } else { + null + }, + triggerId = sin.readString(), + triggerName = sin.readString(), + query = sin.readString(), + queryResults = sin.readMap(), + triggeredTime = sin.readInstant(), + errorMessage = sin.readOptionalString(), + severity = sin.readEnum(Severity::class.java), + executionId = sin.readOptionalString() + ) + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeString(id) + out.writeLong(version) + out.writeInt(schemaVersion) + out.writeString(monitorId) + out.writeString(monitorName) + out.writeLong(monitorVersion) + out.writeBoolean(monitorUser != null) + monitorUser?.writeTo(out) + out.writeString(triggerId) + out.writeString(triggerName) + out.writeString(query) + out.writeMap(queryResults) + out.writeInstant(triggeredTime) + out.writeOptionalString(errorMessage) + out.writeEnum(severity) + out.writeOptionalString(executionId) + } + + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + return createXContentBuilder(builder, false) + } + + fun toXContentWithUser(builder: XContentBuilder): XContentBuilder { + return createXContentBuilder(builder, true) + } + + private fun createXContentBuilder(builder: XContentBuilder, withUser: Boolean): XContentBuilder { + builder.startObject() + .field(ALERT_V2_ID_FIELD, id) + .field(ALERT_V2_VERSION_FIELD, version) + .field(MONITOR_V2_ID_FIELD, monitorId) + .field(SCHEMA_VERSION_FIELD, schemaVersion) + .field(MONITOR_V2_VERSION_FIELD, monitorVersion) + .field(MONITOR_V2_NAME_FIELD, monitorName) + .field(EXECUTION_ID_FIELD, executionId) + .field(TRIGGER_V2_ID_FIELD, triggerId) + .field(TRIGGER_V2_NAME_FIELD, triggerName) + .field(QUERY_FIELD, query) + .field(QUERY_RESULTS_FIELD, queryResults) + .field(ERROR_MESSAGE_FIELD, errorMessage) + .field(SEVERITY_FIELD, severity.value) + .nonOptionalTimeField(TRIGGERED_TIME_FIELD, triggeredTime) + + if (withUser) { + builder.optionalUserField(MONITOR_V2_USER_FIELD, monitorUser) + } + + builder.endObject() + + return builder + } + + fun asTemplateArg(): Map { + return mapOf( + ALERT_V2_ID_FIELD to id, + ALERT_V2_VERSION_FIELD to version, + ERROR_MESSAGE_FIELD to errorMessage, + EXECUTION_ID_FIELD to executionId, + SEVERITY_FIELD to severity.value + ) + } + + companion object { + const val ALERT_V2_ID_FIELD = "id" + const val ALERT_V2_VERSION_FIELD = "version" + const val MONITOR_V2_ID_FIELD = "monitor_v2_id" + const val MONITOR_V2_VERSION_FIELD = "monitor_v2_version" + const val MONITOR_V2_NAME_FIELD = "monitor_v2_name" + const val MONITOR_V2_USER_FIELD = "monitor_v2_user" + const val TRIGGER_V2_ID_FIELD = "trigger_v2_id" + const val TRIGGER_V2_NAME_FIELD = "trigger_v2_name" + const val TRIGGERED_TIME_FIELD = "triggered_time" + const val QUERY_FIELD = "query" + const val QUERY_RESULTS_FIELD = "query_results" + const val ERROR_MESSAGE_FIELD = "error_message" + const val EXECUTION_ID_FIELD = "execution_id" + const val SEVERITY_FIELD = "severity" + const val SCHEMA_VERSION_FIELD = "schema_version" + + const val NO_ID = "" + const val NO_VERSION = Versions.NOT_FOUND + const val NO_SCHEMA_VERSION = 0 + + @JvmStatic + @JvmOverloads + @Throws(IOException::class) + fun parse(xcp: XContentParser, id: String = NO_ID, version: Long = NO_VERSION): AlertV2 { + var schemaVersion = NO_SCHEMA_VERSION + lateinit var monitorId: String + lateinit var monitorName: String + var monitorVersion: Long = Versions.NOT_FOUND + var monitorUser: User? = null + lateinit var triggerId: String + lateinit var triggerName: String + lateinit var query: String + var queryResults: Map = mapOf() + lateinit var severity: Severity + var triggeredTime: Instant? = null + var errorMessage: String? = null + var executionId: String? = null + + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp) + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + val fieldName = xcp.currentName() + xcp.nextToken() + + when (fieldName) { + MONITOR_V2_ID_FIELD -> monitorId = xcp.text() + SCHEMA_VERSION_FIELD -> schemaVersion = xcp.intValue() + MONITOR_V2_NAME_FIELD -> monitorName = xcp.text() + MONITOR_V2_VERSION_FIELD -> monitorVersion = xcp.longValue() + MONITOR_V2_USER_FIELD -> + monitorUser = if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + null + } else { + User.parse(xcp) + } + TRIGGER_V2_ID_FIELD -> triggerId = xcp.text() + TRIGGER_V2_NAME_FIELD -> triggerName = xcp.text() + QUERY_FIELD -> query = xcp.text() + QUERY_RESULTS_FIELD -> queryResults = xcp.map() + TRIGGERED_TIME_FIELD -> triggeredTime = xcp.instant() + ERROR_MESSAGE_FIELD -> errorMessage = xcp.textOrNull() + EXECUTION_ID_FIELD -> executionId = xcp.textOrNull() + TriggerV2.SEVERITY_FIELD -> { + val input = xcp.text() + val enumMatchResult = Severity.enumFromString(input) + ?: throw IllegalArgumentException( + "Invalid value for ${TriggerV2.SEVERITY_FIELD}: $input. " + + "Supported values are ${Severity.entries.map { it.value }}" + ) + severity = enumMatchResult + } + } + } + + return AlertV2( + id = id, + version = version, + schemaVersion = schemaVersion, + monitorId = requireNotNull(monitorId), + monitorName = requireNotNull(monitorName), + monitorVersion = monitorVersion, + monitorUser = monitorUser, + triggerId = requireNotNull(triggerId), + triggerName = requireNotNull(triggerName), + query = requireNotNull(query), + queryResults = requireNotNull(queryResults), + triggeredTime = requireNotNull(triggeredTime), + errorMessage = errorMessage, + severity = severity, + executionId = executionId + ) + } + + @JvmStatic + @Throws(IOException::class) + fun readFrom(sin: StreamInput): AlertV2 { + return AlertV2(sin) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2.kt new file mode 100644 index 000000000..d7b1ef16c --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2.kt @@ -0,0 +1,157 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.modelv2 + +import org.opensearch.alerting.modelv2.PPLSQLMonitor.Companion.PPL_SQL_MONITOR_TYPE +import org.opensearch.common.CheckedFunction +import org.opensearch.commons.alerting.model.Schedule +import org.opensearch.commons.alerting.model.ScheduledJob +import org.opensearch.commons.authuser.User +import org.opensearch.core.ParseField +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.xcontent.NamedXContentRegistry +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.XContentBuilder +import org.opensearch.core.xcontent.XContentParser +import org.opensearch.core.xcontent.XContentParserUtils +import java.io.IOException +import java.time.Instant + +/** + * Monitor V2 interface. All v2 monitors of different types must implement this interface. + * + * @opensearch.experimental + */ +interface MonitorV2 : ScheduledJob { + override val id: String + override val version: Long + override val name: String + override val enabled: Boolean + override val schedule: Schedule + override val lastUpdateTime: Instant // required for scheduled job maintenance + override val enabledTime: Instant? // required for scheduled job maintenance + val description: String? + val user: User? + val triggers: List + val schemaVersion: Int // for updating monitors + val lookBackWindow: Long? // how far back to look when querying data during monitor execution + val timestampField: String? // field that will be used to inject lookback window time filter + + fun asTemplateArg(): Map + + fun toXContentWithUser(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder + + fun makeCopy( + id: String = this.id, + version: Long = this.version, + name: String = this.name, + enabled: Boolean = this.enabled, + schedule: Schedule = this.schedule, + lastUpdateTime: Instant = this.lastUpdateTime, + enabledTime: Instant? = this.enabledTime, + description: String? = this.description, + user: User? = this.user, + // no support for overriding triggers in interface-level makeCopy(), + // triggers can be copied at instance-level data class copy() + schemaVersion: Int = this.schemaVersion, + lookBackWindow: Long? = this.lookBackWindow, + timestampField: String? = this.timestampField + ): MonitorV2 + + enum class MonitorV2Type(val value: String) { + PPL_SQL_MONITOR(PPL_SQL_MONITOR_TYPE); + + override fun toString(): String { + return value + } + + companion object { + fun enumFromString(value: String): MonitorV2Type? { + return MonitorV2Type.entries.find { it.value == value } + } + } + } + + companion object { + // scheduled job field names + const val MONITOR_V2_TYPE = "monitor_v2" // scheduled job type is MonitorV2 + + // field names + const val NAME_FIELD = "name" + const val ENABLED_FIELD = "enabled" + const val SCHEDULE_FIELD = "schedule" + const val LAST_UPDATE_TIME_FIELD = "last_update_time" + const val ENABLED_TIME_FIELD = "enabled_time" + const val DESCRIPTION_FIELD = "description" + const val USER_FIELD = "user" + const val TRIGGERS_FIELD = "triggers" + const val SCHEMA_VERSION_FIELD = "schema_version" + const val LOOK_BACK_WINDOW_FIELD = "look_back_window_minutes" + const val TIMESTAMP_FIELD = "timestamp_field" + + // default values + const val NO_ID = "" + const val NO_VERSION = 1L + + // hard, nonadjustable limits + const val MONITOR_V2_MAX_TRIGGERS = 10 + const val MONITOR_V2_MIN_LOOK_BACK_WINDOW = 1L // 1 minute + const val ALERTING_V2_MAX_NAME_LENGTH = 30 // max length of any name for monitors, triggers, notif actions, etc + const val UUID_LENGTH = 20 // the length of a UUID generated by UUIDs.base64UUID() + const val DESCRIPTION_MAX_LENGTH = 2000 + + val XCONTENT_REGISTRY = NamedXContentRegistry.Entry( + ScheduledJob::class.java, + ParseField(MONITOR_V2_TYPE), + CheckedFunction { parse(it) } + ) + + @JvmStatic + @Throws(IOException::class) + fun parse(xcp: XContentParser, id: String = NO_ID, version: Long = NO_VERSION): MonitorV2 { + /* parse outer object for monitorV2 type, then delegate to correct monitorV2 parser */ + + XContentParserUtils.ensureExpectedToken( // outer monitor object start + XContentParser.Token.START_OBJECT, + xcp.currentToken(), + xcp + ) + + // monitor type field name + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, xcp.nextToken(), xcp) + val monitorTypeText = xcp.currentName() + val monitorType = MonitorV2Type.enumFromString(monitorTypeText) + ?: throw IllegalStateException( + "when parsing MonitorV2, received invalid monitor type: $monitorTypeText. " + + "Please ensure monitor object is wrapped in an outer $PPL_SQL_MONITOR_TYPE object" + ) + + // inner monitor object start + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp) + + return when (monitorType) { + MonitorV2Type.PPL_SQL_MONITOR -> PPLSQLMonitor.parse(xcp, id, version) + } + } + + fun readFrom(sin: StreamInput): MonitorV2 { + return when (val monitorType = sin.readEnum(MonitorV2Type::class.java)) { + MonitorV2Type.PPL_SQL_MONITOR -> PPLSQLMonitor(sin) + else -> throw IllegalStateException("Unexpected input \"$monitorType\" when reading MonitorV2") + } + } + + fun writeTo(out: StreamOutput, monitorV2: MonitorV2) { + when (monitorV2) { + is PPLSQLMonitor -> { + out.writeEnum(MonitorV2Type.PPL_SQL_MONITOR) + monitorV2.writeTo(out) + } + } + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2RunResult.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2RunResult.kt new file mode 100644 index 000000000..28d91b297 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2RunResult.kt @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.modelv2 + +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.common.io.stream.Writeable +import org.opensearch.core.xcontent.ToXContent + +/** + * Monitor V2 run result interface. All classes that store the results + * of a monitor v2 run must implement this interface + * + * @opensearch.experimental + */ +interface MonitorV2RunResult : Writeable, ToXContent { + val monitorName: String + val error: Exception? + val triggerResults: Map + + enum class MonitorV2RunResultType { + PPL_SQL_MONITOR_RUN_RESULT; + } + + companion object { + const val ERROR_FIELD = "error" + const val TRIGGER_RESULTS_FIELD = "trigger_results" + + fun readFrom(sin: StreamInput): MonitorV2RunResult<*> { + val monitorRunResultType = sin.readEnum(MonitorV2RunResultType::class.java) + return when (monitorRunResultType) { + MonitorV2RunResultType.PPL_SQL_MONITOR_RUN_RESULT -> PPLSQLMonitorRunResult(sin) + else -> throw IllegalStateException("Unexpected input [$monitorRunResultType] when reading MonitorV2RunResult") + } + } + + fun writeTo(out: StreamOutput, monitorV2RunResult: MonitorV2RunResult<*>) { + when (monitorV2RunResult) { + is PPLSQLMonitorRunResult -> { + out.writeEnum(MonitorV2RunResultType.PPL_SQL_MONITOR_RUN_RESULT) + monitorV2RunResult.writeTo(out) + } + } + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitor.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitor.kt new file mode 100644 index 000000000..8b39069ae --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitor.kt @@ -0,0 +1,410 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.modelv2 + +import org.opensearch.alerting.core.util.nonOptionalTimeField +import org.opensearch.alerting.modelv2.MonitorV2.Companion.ALERTING_V2_MAX_NAME_LENGTH +import org.opensearch.alerting.modelv2.MonitorV2.Companion.DESCRIPTION_FIELD +import org.opensearch.alerting.modelv2.MonitorV2.Companion.DESCRIPTION_MAX_LENGTH +import org.opensearch.alerting.modelv2.MonitorV2.Companion.ENABLED_FIELD +import org.opensearch.alerting.modelv2.MonitorV2.Companion.ENABLED_TIME_FIELD +import org.opensearch.alerting.modelv2.MonitorV2.Companion.LAST_UPDATE_TIME_FIELD +import org.opensearch.alerting.modelv2.MonitorV2.Companion.LOOK_BACK_WINDOW_FIELD +import org.opensearch.alerting.modelv2.MonitorV2.Companion.MONITOR_V2_MAX_TRIGGERS +import org.opensearch.alerting.modelv2.MonitorV2.Companion.MONITOR_V2_MIN_LOOK_BACK_WINDOW +import org.opensearch.alerting.modelv2.MonitorV2.Companion.NAME_FIELD +import org.opensearch.alerting.modelv2.MonitorV2.Companion.NO_ID +import org.opensearch.alerting.modelv2.MonitorV2.Companion.NO_VERSION +import org.opensearch.alerting.modelv2.MonitorV2.Companion.SCHEDULE_FIELD +import org.opensearch.alerting.modelv2.MonitorV2.Companion.SCHEMA_VERSION_FIELD +import org.opensearch.alerting.modelv2.MonitorV2.Companion.TIMESTAMP_FIELD +import org.opensearch.alerting.modelv2.MonitorV2.Companion.TRIGGERS_FIELD +import org.opensearch.alerting.modelv2.MonitorV2.Companion.USER_FIELD +import org.opensearch.commons.alerting.model.CronSchedule +import org.opensearch.commons.alerting.model.Schedule +import org.opensearch.commons.alerting.util.AlertingException +import org.opensearch.commons.alerting.util.IndexUtils +import org.opensearch.commons.alerting.util.instant +import org.opensearch.commons.alerting.util.optionalTimeField +import org.opensearch.commons.alerting.util.optionalUserField +import org.opensearch.commons.authuser.User +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.XContentBuilder +import org.opensearch.core.xcontent.XContentParser +import org.opensearch.core.xcontent.XContentParserUtils +import java.io.IOException +import java.time.Instant + +/** + * PPL/SQL Monitor for OpenSearch Alerting V2 + * + * @property id Monitor ID. Defaults to [NO_ID]. + * @property version Version number of the monitor. Defaults to [NO_VERSION]. + * @property name Display name of the monitor. + * @property enabled Boolean flag indicating whether the monitor is currently on or off. + * @property schedule Defines when and how often the monitor should run. Can be a CRON or interval schedule. + * @property lookBackWindow How far back each Monitor execution's query should look back when searching data. + * @property lastUpdateTime Timestamp of the last update to this monitor. + * @property enabledTime Timestamp when the monitor was last enabled. Null if never enabled. + * @property description Optional Monitor description. + * @property triggers List of [PPLTrigger]s associated with this monitor. + * @property schemaVersion Version of the alerting-config index schema used when this Monitor was indexed. Defaults to [NO_SCHEMA_VERSION]. + * @property queryLanguage The query language used. Defaults to [QueryLanguage.PPL]. + * @property query The query string to be executed by this monitor. + * + * @opensearch.experimental + */ +data class PPLSQLMonitor( + override val id: String = NO_ID, + override val version: Long = NO_VERSION, + override val name: String, + override val enabled: Boolean, + override val schedule: Schedule, + override val lookBackWindow: Long?, + override val timestampField: String?, + override val lastUpdateTime: Instant, + override val enabledTime: Instant?, + override val description: String?, + override val user: User?, + override val triggers: List, + override val schemaVersion: Int = IndexUtils.NO_SCHEMA_VERSION, + val queryLanguage: QueryLanguage = QueryLanguage.PPL, // default to PPL, SQL not currently supported + val query: String +) : MonitorV2 { + + // specify scheduled job type + override val type = MonitorV2.MONITOR_V2_TYPE + + override fun fromDocument(id: String, version: Long): PPLSQLMonitor = copy(id = id, version = version) + + init { + // SQL monitors are not yet supported + if (this.queryLanguage == QueryLanguage.SQL) { + throw IllegalArgumentException("SQL queries are not supported. Please use a PPL query.") + } + + require(this.name.length <= ALERTING_V2_MAX_NAME_LENGTH) { + "Monitor name too long, length must be less than $ALERTING_V2_MAX_NAME_LENGTH." + } + + if (this.lookBackWindow != null) { + requireNotNull(this.timestampField) { "If look back window is specified, timestamp field must not be null." } + } else { + require(this.timestampField == null) { "If look back window is not specified, timestamp field must not be specified." } + } + + require(this.triggers.isNotEmpty()) { "Monitor must include at least 1 trigger." } + require(this.triggers.size <= MONITOR_V2_MAX_TRIGGERS) { "Monitors can only have $MONITOR_V2_MAX_TRIGGERS triggers." } + + lookBackWindow?.let { + require(this.lookBackWindow >= MONITOR_V2_MIN_LOOK_BACK_WINDOW) { + "Monitors look back windows must be at least $MONITOR_V2_MIN_LOOK_BACK_WINDOW minute." + } + } + + this.description?.let { + require(this.description.length <= DESCRIPTION_MAX_LENGTH) { "Description must be under $DESCRIPTION_MAX_LENGTH characters." } + } + + // for checking trigger ID uniqueness + val triggerIds = mutableSetOf() + this.triggers.forEach { trigger -> + require(triggerIds.add(trigger.id)) { "Duplicate trigger id: ${trigger.id}. Trigger ids must be unique." } + } + + if (this.enabled) { + requireNotNull(this.enabledTime) + } else { + require(this.enabledTime == null) + } + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + id = sin.readString(), + version = sin.readLong(), + name = sin.readString(), + enabled = sin.readBoolean(), + schedule = Schedule.readFrom(sin), + lookBackWindow = sin.readOptionalLong(), + timestampField = sin.readOptionalString(), + lastUpdateTime = sin.readInstant(), + enabledTime = sin.readOptionalInstant(), + description = sin.readOptionalString(), + user = if (sin.readBoolean()) { + User(sin) + } else { + null + }, + triggers = sin.readList(PPLSQLTrigger.Companion::readFrom), + schemaVersion = sin.readInt(), + queryLanguage = sin.readEnum(QueryLanguage::class.java), + query = sin.readString() + ) + + override fun toXContentWithUser(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + return createXContentBuilder(builder, params, true) + } + + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + return createXContentBuilder(builder, params, false) + } + + private fun createXContentBuilder(builder: XContentBuilder, params: ToXContent.Params, withUser: Boolean): XContentBuilder { + builder.startObject() // overall start object + + // if this is being written as ScheduledJob, add extra object layer and add ScheduledJob + // related metadata, default to false + if (params.paramAsBoolean("with_type", false)) { + builder.startObject(MonitorV2.MONITOR_V2_TYPE) + } + + // wrap PPLSQLMonitor in outer object named after its monitor type + // required for MonitorV2 XContentParser to first encounter this, + // read in monitor type, then delegate to correct parse() function + builder.startObject(PPL_SQL_MONITOR_TYPE) // monitor type start object + + builder.field(NAME_FIELD, name) + builder.field(SCHEDULE_FIELD, schedule) + builder.field(LOOK_BACK_WINDOW_FIELD, lookBackWindow) + builder.field(TIMESTAMP_FIELD, timestampField) + builder.field(ENABLED_FIELD, enabled) + builder.nonOptionalTimeField(LAST_UPDATE_TIME_FIELD, lastUpdateTime) + builder.optionalTimeField(ENABLED_TIME_FIELD, enabledTime) + builder.field(DESCRIPTION_FIELD, description) + builder.field(TRIGGERS_FIELD, triggers.toTypedArray()) + builder.field(SCHEMA_VERSION_FIELD, schemaVersion) + builder.field(QUERY_LANGUAGE_FIELD, queryLanguage.value) + builder.field(QUERY_FIELD, query) + + if (withUser) { + builder.optionalUserField(USER_FIELD, user) + } + + builder.endObject() // monitor type end object + + // if ScheduledJob metadata was added, end the extra object layer that was created + if (params.paramAsBoolean("with_type", false)) { + builder.endObject() + } + + builder.endObject() // overall end object + + return builder + } + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeString(id) + out.writeLong(version) + out.writeString(name) + out.writeBoolean(enabled) + + if (schedule is CronSchedule) { + out.writeEnum(Schedule.TYPE.CRON) + } else { + out.writeEnum(Schedule.TYPE.INTERVAL) + } + schedule.writeTo(out) + + out.writeOptionalLong(lookBackWindow) + out.writeOptionalString(timestampField) + out.writeInstant(lastUpdateTime) + out.writeOptionalInstant(enabledTime) + out.writeOptionalString(description) + + out.writeBoolean(user != null) + user?.writeTo(out) + + out.writeVInt(triggers.size) + triggers.forEach { it.writeTo(out) } + out.writeInt(schemaVersion) + out.writeEnum(queryLanguage) + out.writeString(query) + } + + override fun asTemplateArg(): Map { + return mapOf( + IndexUtils._ID to id, + IndexUtils._VERSION to version, + NAME_FIELD to name, + ENABLED_FIELD to enabled, + SCHEDULE_FIELD to schedule, + LOOK_BACK_WINDOW_FIELD to lookBackWindow, + LAST_UPDATE_TIME_FIELD to lastUpdateTime.toEpochMilli(), + ENABLED_TIME_FIELD to enabledTime?.toEpochMilli(), + QUERY_FIELD to query + ) + } + + override fun makeCopy( + id: String, + version: Long, + name: String, + enabled: Boolean, + schedule: Schedule, + lastUpdateTime: Instant, + enabledTime: Instant?, + description: String?, + user: User?, + schemaVersion: Int, + lookBackWindow: Long?, + timestampField: String? + ): PPLSQLMonitor { + return copy( + id = id, + version = version, + name = name, + enabled = enabled, + schedule = schedule, + lastUpdateTime = lastUpdateTime, + enabledTime = enabledTime, + description = description, + user = user, + schemaVersion = schemaVersion, + lookBackWindow = lookBackWindow, + timestampField = timestampField + ) + } + + enum class QueryLanguage(val value: String) { + PPL(PPL_QUERY_LANGUAGE), + SQL(SQL_QUERY_LANGUAGE); + + companion object { + fun enumFromString(value: String): QueryLanguage? = QueryLanguage.entries.firstOrNull { it.value == value } + } + } + + companion object { + // monitor type name + const val PPL_SQL_MONITOR_TYPE = "ppl_monitor" + + // query languages + const val PPL_QUERY_LANGUAGE = "ppl" + const val SQL_QUERY_LANGUAGE = "sql" + + // field names + const val QUERY_LANGUAGE_FIELD = "query_language" + const val QUERY_FIELD = "query" + + @JvmStatic + @JvmOverloads + @Throws(IOException::class) + fun parse(xcp: XContentParser, id: String = NO_ID, version: Long = NO_VERSION): PPLSQLMonitor { + var name: String? = null + var enabled = true + var schedule: Schedule? = null + var lookBackWindow: Long? = null + var timestampField: String? = null + var lastUpdateTime: Instant? = null + var enabledTime: Instant? = null + var description: String? = null + var user: User? = null + val triggers: MutableList = mutableListOf() + var schemaVersion = IndexUtils.NO_SCHEMA_VERSION + var queryLanguage: QueryLanguage = QueryLanguage.PPL // default to PPL + var query: String? = null + + /* parse */ + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + val fieldName = xcp.currentName() + xcp.nextToken() + + when (fieldName) { + NAME_FIELD -> name = xcp.text() + ENABLED_FIELD -> enabled = xcp.booleanValue() + SCHEDULE_FIELD -> schedule = Schedule.parse(xcp) + LOOK_BACK_WINDOW_FIELD -> { + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { + lookBackWindow = xcp.longValue() + } + } + TIMESTAMP_FIELD -> { + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { + timestampField = xcp.text() + } + } + LAST_UPDATE_TIME_FIELD -> lastUpdateTime = xcp.instant() + ENABLED_TIME_FIELD -> enabledTime = xcp.instant() + DESCRIPTION_FIELD -> { + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { + description = xcp.text() + } + } + USER_FIELD -> { + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { + user = User.parse(xcp) + } + } + TRIGGERS_FIELD -> { + XContentParserUtils.ensureExpectedToken( + XContentParser.Token.START_ARRAY, + xcp.currentToken(), + xcp + ) + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + triggers.add(PPLSQLTrigger.parseInner(xcp)) + } + } + SCHEMA_VERSION_FIELD -> schemaVersion = xcp.intValue() + QUERY_LANGUAGE_FIELD -> { + val input = xcp.text() + val enumMatchResult = QueryLanguage.enumFromString(input) + ?: throw AlertingException.wrap( + IllegalArgumentException( + "Invalid value for $QUERY_LANGUAGE_FIELD: $input. " + + "Supported values are ${QueryLanguage.entries.map { it.value }}" + ) + ) + queryLanguage = enumMatchResult + } + QUERY_FIELD -> query = xcp.text() + else -> throw IllegalArgumentException("Unexpected field when parsing PPL/SQL Monitor: $fieldName") + } + } + + /* validations */ + + // if enabled, set time of MonitorV2 creation/update is set as enable time + if (enabled && enabledTime == null) { + enabledTime = Instant.now() + } else if (!enabled) { + enabledTime = null + } + + lastUpdateTime = lastUpdateTime ?: Instant.now() + + // check for required fields + requireNotNull(name) { "Monitor name is null" } + requireNotNull(schedule) { "Schedule is null" } + requireNotNull(query) { "Query is null" } + requireNotNull(lastUpdateTime) { "Last update time is null" } + + /* return PPLSQLMonitor */ + return PPLSQLMonitor( + id, + version, + name, + enabled, + schedule, + lookBackWindow, + timestampField, + lastUpdateTime, + enabledTime, + description, + user, + triggers, + schemaVersion, + queryLanguage, + query + ) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitorRunResult.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitorRunResult.kt new file mode 100644 index 000000000..12a34c560 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitorRunResult.kt @@ -0,0 +1,74 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.modelv2 + +import org.opensearch.alerting.modelv2.AlertV2.Companion.MONITOR_V2_NAME_FIELD +import org.opensearch.alerting.modelv2.MonitorV2RunResult.Companion.ERROR_FIELD +import org.opensearch.alerting.modelv2.MonitorV2RunResult.Companion.TRIGGER_RESULTS_FIELD +import org.opensearch.commons.utils.STRING_READER +import org.opensearch.commons.utils.STRING_WRITER +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.common.io.stream.Writeable +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.XContentBuilder +import java.io.IOException + +/** + * A class that stores the run results of a PPL/SQL Monitor + * + * @opensearch.experimental + */ +data class PPLSQLMonitorRunResult( + override val monitorName: String, + override val error: Exception?, + override val triggerResults: Map, + val pplQueryResults: Map> // key: trigger id, value: query results +) : MonitorV2RunResult { + + @Throws(IOException::class) + @Suppress("UNCHECKED_CAST") + constructor(sin: StreamInput) : this( + sin.readString(), // monitorName + sin.readException(), // error + sin.readMap(STRING_READER, runResultReader()) as Map, // triggerResults + sin.readMap() as Map> // pplQueryResults + ) + + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + builder.startObject() + builder.field(MONITOR_V2_NAME_FIELD, monitorName) + builder.field(ERROR_FIELD, error?.message) + builder.field(TRIGGER_RESULTS_FIELD, triggerResults) + builder.field(PPL_QUERY_RESULTS_FIELD, pplQueryResults) + builder.endObject() + return builder + } + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeString(monitorName) + out.writeException(error) + out.writeMap(triggerResults, STRING_WRITER, runResultWriter()) + out.writeMap(pplQueryResults) + } + + companion object { + const val PPL_QUERY_RESULTS_FIELD = "ppl_query_results" + + private fun runResultReader(): Writeable.Reader { + return Writeable.Reader { + PPLSQLTriggerRunResult.readFrom(it) + } + } + + private fun runResultWriter(): Writeable.Writer { + return Writeable.Writer { streamOutput: StreamOutput, runResult: PPLSQLTriggerRunResult -> + runResult.writeTo(streamOutput) + } + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTrigger.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTrigger.kt new file mode 100644 index 000000000..b902d39a2 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTrigger.kt @@ -0,0 +1,413 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.modelv2 + +import org.opensearch.alerting.modelv2.MonitorV2.Companion.ALERTING_V2_MAX_NAME_LENGTH +import org.opensearch.alerting.modelv2.MonitorV2.Companion.UUID_LENGTH +import org.opensearch.alerting.modelv2.TriggerV2.Companion.ACTIONS_FIELD +import org.opensearch.alerting.modelv2.TriggerV2.Companion.DEFAULT_EXPIRE_DURATION +import org.opensearch.alerting.modelv2.TriggerV2.Companion.EXPIRE_FIELD +import org.opensearch.alerting.modelv2.TriggerV2.Companion.ID_FIELD +import org.opensearch.alerting.modelv2.TriggerV2.Companion.LAST_TRIGGERED_FIELD +import org.opensearch.alerting.modelv2.TriggerV2.Companion.MONITOR_V2_MIN_EXPIRE_DURATION_MINUTES +import org.opensearch.alerting.modelv2.TriggerV2.Companion.MONITOR_V2_MIN_THROTTLE_DURATION_MINUTES +import org.opensearch.alerting.modelv2.TriggerV2.Companion.NAME_FIELD +import org.opensearch.alerting.modelv2.TriggerV2.Companion.NOTIFICATIONS_ID_MAX_LENGTH +import org.opensearch.alerting.modelv2.TriggerV2.Companion.SEVERITY_FIELD +import org.opensearch.alerting.modelv2.TriggerV2.Companion.THROTTLE_FIELD +import org.opensearch.alerting.modelv2.TriggerV2.Severity +import org.opensearch.common.CheckedFunction +import org.opensearch.common.UUIDs +import org.opensearch.commons.alerting.model.action.Action +import org.opensearch.commons.alerting.util.instant +import org.opensearch.commons.alerting.util.optionalTimeField +import org.opensearch.core.ParseField +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.xcontent.NamedXContentRegistry +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.XContentBuilder +import org.opensearch.core.xcontent.XContentParser +import org.opensearch.core.xcontent.XContentParserUtils +import java.io.IOException +import java.time.Instant + +/** + * The PPL/SQL Trigger for PPL/SQL Monitors + * + * There are two types of PPLTrigger conditions: NUMBER_OF_RESULTS and CUSTOM + * NUMBER_OF_RESULTS: triggers based on whether the number of query results returned by the PPLSQLMonitor + * query meets some threshold + * CUSTOM: triggers based on a custom condition that user specifies (a single ppl eval statement) + * + * PPLTriggers can run on two modes: RESULT_SET and PER_RESULT + * RESULT_SET: exactly one Alert is generated when the Trigger condition is met + * PER_RESULT: one Alert is generated per trigger condition-meeting query result row + * + * @property id Trigger ID, defaults to a base64 UUID. + * @property name Display name of the Trigger. + * @property severity The severity level of the Trigger. + * @property throttleDuration Optional duration (in minutes) for which alerts from this Trigger should be throttled/suppressed. + * Null indicates no throttling. + * @property expireDuration Duration (in minutes) after which alerts from this Trigger should be deleted permanently. + * @property lastTriggeredTime The last time this Trigger generated an Alert. Null if Trigger hasn't generated an Alert yet. + * @property actions List of notification-sending actions to run when the Trigger condition is met. + * @property mode Specifies whether the trigger evaluates the entire result set or each result individually. + * Can be either [TriggerMode.RESULT_SET] or [TriggerMode.PER_RESULT]. + * @property conditionType The type of condition to evaluate. + * Can be either [ConditionType.NUMBER_OF_RESULTS] or [ConditionType.CUSTOM]. + * @property numResultsCondition The comparison operator for NUMBER_OF_RESULTS conditions. Required if using NUMBER_OF_RESULTS conditions, + * required to be null otherwise. + * @property numResultsValue The threshold value for NUMBER_OF_RESULTS conditions. Required if using NUMBER_OF_RESULTS conditions, + * required to be null otherwise. + * @property customCondition A custom condition expression. Required if using CUSTOM conditions, + * required to be null otherwise. + * + * @opensearch.experimental + */ +data class PPLSQLTrigger( + override val id: String = UUIDs.base64UUID(), + override val name: String, + override val severity: Severity, + override val throttleDuration: Long?, + override val expireDuration: Long = DEFAULT_EXPIRE_DURATION, + override var lastTriggeredTime: Instant?, + override val actions: List, + val mode: TriggerMode, // RESULT_SET or PER_RESULT + val conditionType: ConditionType, // NUMBER_OF_RESULTS or CUSTOM + val numResultsCondition: NumResultsCondition?, + val numResultsValue: Long?, + val customCondition: String? +) : TriggerV2 { + + init { + requireNotNull(this.name) { "Trigger name must be included." } + requireNotNull(this.severity) { "Trigger severity must be included." } + requireNotNull(this.mode) { "Trigger mode must be included." } + requireNotNull(this.conditionType) { "Trigger condition type must be included." } + + require(this.id.length <= UUID_LENGTH) { + "Trigger ID too long, length must be less than $UUID_LENGTH." + } + + require(this.name.length <= ALERTING_V2_MAX_NAME_LENGTH) { + "Trigger name too long, length must be less than $ALERTING_V2_MAX_NAME_LENGTH." + } + + require(this.expireDuration >= MONITOR_V2_MIN_EXPIRE_DURATION_MINUTES) { + "expire duration cannot be less than $MONITOR_V2_MIN_EXPIRE_DURATION_MINUTES, was $expireDuration." + } + + this.throttleDuration?.let { + require(it >= MONITOR_V2_MIN_THROTTLE_DURATION_MINUTES) { + "Throttle duration cannot be less than $MONITOR_V2_MIN_THROTTLE_DURATION_MINUTES, was $throttleDuration." + } + } + + this.actions.forEach { + require(it.name.length <= ALERTING_V2_MAX_NAME_LENGTH) { + "Name of action with ID ${it.id} too long, length must be less than $ALERTING_V2_MAX_NAME_LENGTH." + } + require(it.destinationId.length <= NOTIFICATIONS_ID_MAX_LENGTH) { + "Channel ID of action with ID ${it.id} too long, length must be less than $NOTIFICATIONS_ID_MAX_LENGTH." + } + require(it.destinationId.isNotEmpty()) { + "Channel ID should not be empty." + } + require(it.destinationId.matches(validCharsRegex)) { + "Channel ID should only have alphanumeric characters, dashes, and underscores." + } + } + + when (this.conditionType) { + ConditionType.NUMBER_OF_RESULTS -> { + requireNotNull(this.numResultsCondition) { + "if trigger condition is of type ${ConditionType.NUMBER_OF_RESULTS.value}, " + + "$NUM_RESULTS_CONDITION_FIELD must be included." + } + requireNotNull(this.numResultsValue) { + "if trigger condition is of type ${ConditionType.NUMBER_OF_RESULTS.value}, " + + "$NUM_RESULTS_VALUE_FIELD must be included." + } + require(this.customCondition == null) { + "if trigger condition is of type ${ConditionType.NUMBER_OF_RESULTS.value}, " + + "$CUSTOM_CONDITION_FIELD must not be included." + } + } + ConditionType.CUSTOM -> { + requireNotNull(this.customCondition) { + "if trigger condition is of type ${ConditionType.CUSTOM.value}, " + + "$CUSTOM_CONDITION_FIELD must be included." + } + require(this.numResultsCondition == null) { + "if trigger condition is of type ${ConditionType.CUSTOM.value}, " + + "$NUM_RESULTS_CONDITION_FIELD must not be included." + } + require(this.numResultsValue == null) { + "if trigger condition is of type ${ConditionType.CUSTOM.value}, " + + "$NUM_RESULTS_VALUE_FIELD must not be included." + } + } + } + + if (conditionType == ConditionType.NUMBER_OF_RESULTS) { + require(this.numResultsValue!! >= 0L) { "Number of results to check for cannot be negative." } + } + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + sin.readString(), // id + sin.readString(), // name + sin.readEnum(Severity::class.java), // severity + sin.readOptionalLong(), // throttleDuration + sin.readLong(), // expireDuration + sin.readOptionalInstant(), // lastTriggeredTime + sin.readList(::Action), // actions + sin.readEnum(TriggerMode::class.java), // trigger mode + sin.readEnum(ConditionType::class.java), // condition type + if (sin.readBoolean()) sin.readEnum(NumResultsCondition::class.java) else null, // num results condition + sin.readOptionalLong(), // num results value + sin.readOptionalString() // custom condition + ) + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeString(id) + out.writeString(name) + out.writeEnum(severity) + out.writeOptionalLong(throttleDuration) + out.writeLong(expireDuration) + out.writeOptionalInstant(lastTriggeredTime) + out.writeCollection(actions) + out.writeEnum(mode) + out.writeEnum(conditionType) + + out.writeBoolean(numResultsCondition != null) + numResultsCondition?.let { out.writeEnum(numResultsCondition) } + + out.writeOptionalLong(numResultsValue) + out.writeOptionalString(customCondition) + } + + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params?): XContentBuilder { + builder.startObject() + builder.field(ID_FIELD, id) + builder.field(NAME_FIELD, name) + builder.field(SEVERITY_FIELD, severity.value) + throttleDuration?.let { builder.field(THROTTLE_FIELD, throttleDuration) } + builder.field(EXPIRE_FIELD, expireDuration) + builder.optionalTimeField(LAST_TRIGGERED_FIELD, lastTriggeredTime) + builder.field(ACTIONS_FIELD, actions.toTypedArray()) + builder.field(MODE_FIELD, mode.value) + builder.field(CONDITION_TYPE_FIELD, conditionType.value) + numResultsCondition?.let { builder.field(NUM_RESULTS_CONDITION_FIELD, numResultsCondition.value) } + numResultsValue?.let { builder.field(NUM_RESULTS_VALUE_FIELD, numResultsValue) } + customCondition?.let { builder.field(CUSTOM_CONDITION_FIELD, customCondition) } + builder.endObject() + return builder + } + + fun asTemplateArg(): Map { + return mapOf( + ID_FIELD to id, + NAME_FIELD to name, + SEVERITY_FIELD to severity.value, + THROTTLE_FIELD to throttleDuration, + EXPIRE_FIELD to expireDuration, + ACTIONS_FIELD to actions.map { it.asTemplateArg() }, + MODE_FIELD to mode.value, + CONDITION_TYPE_FIELD to conditionType.value, + NUM_RESULTS_CONDITION_FIELD to numResultsCondition?.value, + NUM_RESULTS_VALUE_FIELD to numResultsValue, + CUSTOM_CONDITION_FIELD to customCondition + ) + } + + enum class TriggerMode(val value: String) { + RESULT_SET("result_set"), + PER_RESULT("per_result"); + + companion object { + fun enumFromString(value: String): TriggerMode? = entries.firstOrNull { it.value == value } + } + } + + enum class ConditionType(val value: String) { + NUMBER_OF_RESULTS("number_of_results"), + CUSTOM("custom"); + + companion object { + fun enumFromString(value: String): ConditionType? = entries.firstOrNull { it.value == value } + } + } + + enum class NumResultsCondition(val value: String) { + GREATER_THAN(">"), + GREATER_THAN_EQUAL(">="), + LESS_THAN("<"), + LESS_THAN_EQUAL("<="), + EQUAL("=="), + NOT_EQUAL("!="); + + companion object { + fun enumFromString(value: String): NumResultsCondition? = entries.firstOrNull { it.value == value } + } + } + + companion object { + // trigger wrapper object field name + const val PPL_SQL_TRIGGER_FIELD = "ppl_trigger" + + // field names + const val MODE_FIELD = "mode" + const val CONDITION_TYPE_FIELD = "type" + const val NUM_RESULTS_CONDITION_FIELD = "num_results_condition" + const val NUM_RESULTS_VALUE_FIELD = "num_results_value" + const val CUSTOM_CONDITION_FIELD = "custom_condition" + + // regular expression for validating that a string contains + // only valid chars (letters, numbers, -, _) + private val validCharsRegex = """^[a-zA-Z0-9_-]+$""".toRegex() + + val XCONTENT_REGISTRY = NamedXContentRegistry.Entry( + TriggerV2::class.java, + ParseField(PPL_SQL_TRIGGER_FIELD), + CheckedFunction { parseInner(it) } + ) + + @JvmStatic + @Throws(IOException::class) + fun parseInner(xcp: XContentParser): PPLSQLTrigger { + var id = UUIDs.base64UUID() // assign a default triggerId if one is not specified + var name: String? = null + var severity: Severity? = null + var throttleDuration: Long? = null + var expireDuration: Long = DEFAULT_EXPIRE_DURATION + var lastTriggeredTime: Instant? = null + val actions: MutableList = mutableListOf() + var mode: TriggerMode? = null + var conditionType: ConditionType? = null + var numResultsCondition: NumResultsCondition? = null + var numResultsValue: Long? = null + var customCondition: String? = null + + /* parse */ + XContentParserUtils.ensureExpectedToken( // outer trigger object start + XContentParser.Token.START_OBJECT, + xcp.currentToken(), xcp + ) + + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + val fieldName = xcp.currentName() + xcp.nextToken() + + when (fieldName) { + ID_FIELD -> id = xcp.text() + NAME_FIELD -> name = xcp.text() + SEVERITY_FIELD -> { + val input = xcp.text() + val enumMatchResult = Severity.enumFromString(input) + ?: throw IllegalArgumentException( + "Invalid value for $SEVERITY_FIELD: $input. " + + "Supported values are ${Severity.entries.map { it.value }}" + ) + severity = enumMatchResult + } + MODE_FIELD -> { + val input = xcp.text() + val enumMatchResult = TriggerMode.enumFromString(input) + ?: throw IllegalArgumentException( + "Invalid value for $MODE_FIELD: $input. " + + "Supported values are ${TriggerMode.entries.map { it.value }}" + ) + mode = enumMatchResult + } + CONDITION_TYPE_FIELD -> { + val input = xcp.text() + val enumMatchResult = ConditionType.enumFromString(input) + ?: throw IllegalArgumentException( + "Invalid value for $CONDITION_TYPE_FIELD: $input. " + + "Supported values are ${ConditionType.entries.map { it.value }}" + ) + conditionType = enumMatchResult + } + NUM_RESULTS_CONDITION_FIELD -> { + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { + val input = xcp.text() + val enumMatchResult = NumResultsCondition.enumFromString(input) + ?: throw IllegalArgumentException( + "Invalid value for $NUM_RESULTS_CONDITION_FIELD: $input. " + + "Supported values are ${NumResultsCondition.entries.map { it.value }}" + ) + numResultsCondition = enumMatchResult + } + } + NUM_RESULTS_VALUE_FIELD -> { + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { + numResultsValue = xcp.longValue() + } + } + CUSTOM_CONDITION_FIELD -> { + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { + customCondition = xcp.text() + } + } + THROTTLE_FIELD -> { + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { + throttleDuration = xcp.longValue() + } + } + EXPIRE_FIELD -> { + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { + expireDuration = xcp.longValue() + } + } + LAST_TRIGGERED_FIELD -> lastTriggeredTime = xcp.instant() + ACTIONS_FIELD -> { + XContentParserUtils.ensureExpectedToken( + XContentParser.Token.START_ARRAY, + xcp.currentToken(), + xcp + ) + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + actions.add(Action.parse(xcp)) + } + } + else -> throw IllegalArgumentException("Unexpected field when parsing PPL Trigger: $fieldName") + } + } + + /* validations */ + requireNotNull(name) { "Trigger name must be included" } + requireNotNull(severity) { "Trigger severity must be included" } + requireNotNull(mode) { "Trigger mode must be included" } + requireNotNull(conditionType) { "Trigger condition type must be included" } + + // 3. prepare and return PPLTrigger object + return PPLSQLTrigger( + id, + name, + severity, + throttleDuration, + expireDuration, + lastTriggeredTime, + actions, + mode, + conditionType, + numResultsCondition, + numResultsValue, + customCondition + ) + } + + @JvmStatic + @Throws(IOException::class) + fun readFrom(sin: StreamInput): PPLSQLTrigger { + return PPLSQLTrigger(sin) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTriggerRunResult.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTriggerRunResult.kt new file mode 100644 index 000000000..0c505d9e6 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTriggerRunResult.kt @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.modelv2 + +import org.opensearch.alerting.modelv2.TriggerV2RunResult.Companion.ERROR_FIELD +import org.opensearch.alerting.modelv2.TriggerV2RunResult.Companion.NAME_FIELD +import org.opensearch.alerting.modelv2.TriggerV2RunResult.Companion.TRIGGERED_FIELD +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.XContentBuilder +import java.io.IOException + +/** + * A class that stores the run results of an individual + * PPL/SQL trigger within a PPL/SQL monitor + * + * @opensearch.experimental + */ +data class PPLSQLTriggerRunResult( + override var triggerName: String, + override var triggered: Boolean, + override var error: Exception?, +) : TriggerV2RunResult { + + @Throws(IOException::class) + @Suppress("UNCHECKED_CAST") + constructor(sin: StreamInput) : this( + triggerName = sin.readString(), + triggered = sin.readBoolean(), + error = sin.readException() + ) + + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + builder.startObject() + builder.field(NAME_FIELD, triggerName) + builder.field(TRIGGERED_FIELD, triggered) + builder.field(ERROR_FIELD, error?.message) + builder.endObject() + return builder + } + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeString(triggerName) + out.writeBoolean(triggered) + out.writeException(error) + } + + companion object { + @JvmStatic + @Throws(IOException::class) + fun readFrom(sin: StreamInput): PPLSQLTriggerRunResult { + return PPLSQLTriggerRunResult(sin) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2.kt new file mode 100644 index 000000000..a4fe90e14 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2.kt @@ -0,0 +1,70 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.modelv2 + +import org.opensearch.alerting.modelv2.PPLSQLTrigger.Companion.PPL_SQL_TRIGGER_FIELD +import org.opensearch.commons.alerting.model.action.Action +import org.opensearch.commons.notifications.model.BaseModel +import java.time.Instant + +/** + * Trigger V2 interface. All triggers of different v2 monitor + * types must implement this interface + * + * @opensearch.experimental + */ +interface TriggerV2 : BaseModel { + + val id: String + val name: String + val severity: Severity + val throttleDuration: Long? + val expireDuration: Long + var lastTriggeredTime: Instant? + val actions: List + + enum class TriggerV2Type(val value: String) { + PPL_TRIGGER(PPL_SQL_TRIGGER_FIELD); + + override fun toString(): String { + return value + } + } + + enum class Severity(val value: String) { + INFO("info"), + ERROR("error"), + LOW("low"), + MEDIUM("medium"), + HIGH("high"), + CRITICAL("critical"); + + companion object { + fun enumFromString(value: String): Severity? { + return entries.find { it.value == value } + } + } + } + + companion object { + // field names + const val ID_FIELD = "id" + const val NAME_FIELD = "name" + const val SEVERITY_FIELD = "severity" + const val THROTTLE_FIELD = "throttle_minutes" + const val LAST_TRIGGERED_FIELD = "last_triggered_time" + const val EXPIRE_FIELD = "expires_minutes" + const val ACTIONS_FIELD = "actions" + + // hard, nonadjustable limits + const val MONITOR_V2_MIN_THROTTLE_DURATION_MINUTES = 1L // one minute min duration to match scheduled job interval granularity + const val MONITOR_V2_MIN_EXPIRE_DURATION_MINUTES = 1L // one minute min duration to match scheduled job interval granularity + const val NOTIFICATIONS_ID_MAX_LENGTH = 512 // length limit for notifications channel custom ID at channel creation time + + // default fallback values of fields if none are passed in + const val DEFAULT_EXPIRE_DURATION = (7 * 24 * 60).toLong() // 7 days in minutes + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2RunResult.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2RunResult.kt new file mode 100644 index 000000000..8e3069972 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2RunResult.kt @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.modelv2 + +import org.opensearch.core.common.io.stream.Writeable +import org.opensearch.core.xcontent.ToXContent + +/** + * Trigger V2 Run Result interface. All classes that store the run results + * of an individual v2 trigger must implement this interface + */ +interface TriggerV2RunResult : Writeable, ToXContent { + val triggerName: String + val triggered: Boolean + val error: Exception? + + companion object { + const val NAME_FIELD = "name" + const val TRIGGERED_FIELD = "triggered" + const val ERROR_FIELD = "error" + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestDeleteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestDeleteMonitorV2Action.kt new file mode 100644 index 000000000..9756dd8b0 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestDeleteMonitorV2Action.kt @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.resthandlerv2 + +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import org.opensearch.action.support.WriteRequest.RefreshPolicy +import org.opensearch.alerting.AlertingPlugin +import org.opensearch.alerting.actionv2.DeleteMonitorV2Action +import org.opensearch.alerting.actionv2.DeleteMonitorV2Request +import org.opensearch.alerting.util.REFRESH +import org.opensearch.rest.BaseRestHandler +import org.opensearch.rest.RestHandler.Route +import org.opensearch.rest.RestRequest +import org.opensearch.rest.RestRequest.Method.DELETE +import org.opensearch.rest.action.RestToXContentListener +import org.opensearch.transport.client.node.NodeClient +import java.io.IOException + +private val log: Logger = LogManager.getLogger(RestDeleteMonitorV2Action::class.java) + +/** + * This class consists of the REST handler to delete V2 monitors. + * When a monitor is deleted, all alerts are moved to the alert history index if alerting v2 history is enabled, + * or permanently deleted if alerting v2 history is disabled. + * If this process fails the monitor is not deleted. + * + * @opensearch.experimental + */ +class RestDeleteMonitorV2Action : BaseRestHandler() { + + override fun getName(): String { + return "delete_monitor_v2_action" + } + + override fun routes(): List { + return mutableListOf( + Route( + DELETE, + "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitor_id}" + ) + ) + } + + @Throws(IOException::class) + override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { + val monitorV2Id = request.param("monitor_id") + log.info("${request.method()} ${AlertingPlugin.MONITOR_V2_BASE_URI}/$monitorV2Id") + + val refreshPolicy = RefreshPolicy.parse(request.param(REFRESH, RefreshPolicy.IMMEDIATE.value)) + val deleteMonitorV2Request = DeleteMonitorV2Request(monitorV2Id, refreshPolicy) + + return RestChannelConsumer { channel -> + client.execute(DeleteMonitorV2Action.INSTANCE, deleteMonitorV2Request, RestToXContentListener(channel)) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestExecuteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestExecuteMonitorV2Action.kt new file mode 100644 index 000000000..4c706e747 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestExecuteMonitorV2Action.kt @@ -0,0 +1,82 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.resthandlerv2 + +import org.apache.logging.log4j.LogManager +import org.opensearch.alerting.AlertingPlugin +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Action +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Request +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.common.unit.TimeValue +import org.opensearch.commons.alerting.util.AlertingException +import org.opensearch.core.xcontent.XContentParser.Token.START_OBJECT +import org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.rest.BaseRestHandler +import org.opensearch.rest.RestHandler.Route +import org.opensearch.rest.RestRequest +import org.opensearch.rest.RestRequest.Method.POST +import org.opensearch.rest.action.RestToXContentListener +import org.opensearch.transport.client.node.NodeClient +import java.time.Instant + +private val log = LogManager.getLogger(RestExecuteMonitorV2Action::class.java) + +/** + * This class consists of the REST handler to execute V2 monitors manually. + * In addition to monitors running on their scheduled jobs, this API allows users + * to execute the monitor themselves to generate alerts and send notifications accordingly + * + * @opensearch.experimental + */ +class RestExecuteMonitorV2Action : BaseRestHandler() { + + override fun getName(): String = "execute_monitor_v2_action" + + override fun routes(): List { + return listOf( + Route( + POST, + "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitor_id}/_execute" + ), + Route( + POST, + "${AlertingPlugin.MONITOR_V2_BASE_URI}/_execute" + ) + ) + } + + override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { + log.debug("${request.method()} ${AlertingPlugin.MONITOR_V2_BASE_URI}/_execute") + + return RestChannelConsumer { channel -> + val dryrun = request.paramAsBoolean("dryrun", false) + val requestEnd = request.paramAsTime("period_end", TimeValue(Instant.now().toEpochMilli())) + + if (request.hasParam("monitor_id")) { + val monitorV2Id = request.param("monitor_id") + val execMonitorV2Request = ExecuteMonitorV2Request(dryrun, true, monitorV2Id, null, requestEnd) + client.execute(ExecuteMonitorV2Action.INSTANCE, execMonitorV2Request, RestToXContentListener(channel)) + } else { + val xcp = request.contentParser() + ensureExpectedToken(START_OBJECT, xcp.nextToken(), xcp) + + val monitorV2: MonitorV2 + try { + monitorV2 = MonitorV2.parse(xcp) + } catch (e: Exception) { + throw AlertingException.wrap(e) + } + + val execMonitorV2Request = ExecuteMonitorV2Request(dryrun, true, null, monitorV2, requestEnd) + client.execute(ExecuteMonitorV2Action.INSTANCE, execMonitorV2Request, RestToXContentListener(channel)) + } + } + } + + override fun responseParams(): Set { + return setOf("dryrun", "period_end", "monitor_id") + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetAlertsV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetAlertsV2Action.kt new file mode 100644 index 000000000..560d243bc --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetAlertsV2Action.kt @@ -0,0 +1,72 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.resthandlerv2 + +import org.apache.logging.log4j.LogManager +import org.opensearch.alerting.AlertingPlugin +import org.opensearch.alerting.actionv2.GetAlertsV2Action +import org.opensearch.alerting.actionv2.GetAlertsV2Request +import org.opensearch.commons.alerting.model.Table +import org.opensearch.rest.BaseRestHandler +import org.opensearch.rest.RestHandler.Route +import org.opensearch.rest.RestRequest +import org.opensearch.rest.RestRequest.Method.GET +import org.opensearch.rest.action.RestToXContentListener +import org.opensearch.transport.client.node.NodeClient + +/** + * This class consists of the REST handler to retrieve V2 alerts. + * + * @opensearch.experimental + */ +class RestGetAlertsV2Action : BaseRestHandler() { + + private val log = LogManager.getLogger(RestGetAlertsV2Action::class.java) + + override fun getName(): String { + return "get_alerts_v2_action" + } + + override fun routes(): List { + return listOf( + Route( + GET, + "${AlertingPlugin.MONITOR_V2_BASE_URI}/alerts" + ) + ) + } + + override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { + log.debug("${request.method()} ${AlertingPlugin.MONITOR_V2_BASE_URI}/alerts") + + val sortString = request.param("sortString", "monitor_v2_name.keyword") + val sortOrder = request.param("sortOrder", "asc") + val missing: String? = request.param("missing") + val size = request.paramAsInt("size", 20) + val startIndex = request.paramAsInt("startIndex", 0) + val searchString = request.param("searchString", "") + val severityLevel = request.param("severityLevel", "ALL") + val monitorId: String? = request.param("monitorId") + val table = Table( + sortOrder, + sortString, + missing, + size, + startIndex, + searchString + ) + + val getAlertsV2Request = GetAlertsV2Request( + table, + severityLevel, + monitorId?.let { listOf(monitorId) } + ) + return RestChannelConsumer { + channel -> + client.execute(GetAlertsV2Action.INSTANCE, getAlertsV2Request, RestToXContentListener(channel)) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetMonitorV2Action.kt new file mode 100644 index 000000000..5c471156c --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetMonitorV2Action.kt @@ -0,0 +1,73 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.resthandlerv2 + +import org.apache.logging.log4j.LogManager +import org.opensearch.alerting.AlertingPlugin +import org.opensearch.alerting.actionv2.GetMonitorV2Action +import org.opensearch.alerting.actionv2.GetMonitorV2Request +import org.opensearch.alerting.modelv2.MonitorV2.Companion.UUID_LENGTH +import org.opensearch.alerting.util.context +import org.opensearch.rest.BaseRestHandler +import org.opensearch.rest.RestHandler.Route +import org.opensearch.rest.RestRequest +import org.opensearch.rest.RestRequest.Method.GET +import org.opensearch.rest.RestRequest.Method.HEAD +import org.opensearch.rest.action.RestActions +import org.opensearch.rest.action.RestToXContentListener +import org.opensearch.search.fetch.subphase.FetchSourceContext +import org.opensearch.transport.client.node.NodeClient + +private val log = LogManager.getLogger(RestGetMonitorV2Action::class.java) + +/** + * This class consists of the REST handler to retrieve a V2 monitor by its ID. + * + * @opensearch.experimental + */ +class RestGetMonitorV2Action : BaseRestHandler() { + + override fun getName(): String { + return "get_monitor_v2_action" + } + + override fun routes(): List { + return listOf( + Route( + GET, + "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitor_id}" + ), + Route( + HEAD, + "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitor_id}" + ) + ) + } + + override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { + log.debug("${request.method()} ${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitor_id}") + + val monitorV2Id = request.param("monitor_id") + if (monitorV2Id == null || monitorV2Id.isEmpty()) { + throw IllegalArgumentException("No MonitorV2 ID provided") + } + + if (monitorV2Id.length != UUID_LENGTH) { + throw IllegalArgumentException("MonitorV2 ID provided does not have correct length") + } + + var srcContext = context(request) + if (request.method() == HEAD) { + srcContext = FetchSourceContext.DO_NOT_FETCH_SOURCE + } + + val getMonitorV2Request = GetMonitorV2Request(monitorV2Id, RestActions.parseVersion(request), srcContext) + return RestChannelConsumer { + channel -> + client.execute(GetMonitorV2Action.INSTANCE, getMonitorV2Request, RestToXContentListener(channel)) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestIndexMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestIndexMonitorV2Action.kt new file mode 100644 index 000000000..8ff03e791 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestIndexMonitorV2Action.kt @@ -0,0 +1,86 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.resthandlerv2 + +import org.apache.logging.log4j.LogManager +import org.opensearch.action.support.WriteRequest +import org.opensearch.alerting.AlertingPlugin +import org.opensearch.alerting.actionv2.IndexMonitorV2Action +import org.opensearch.alerting.actionv2.IndexMonitorV2Request +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.util.IF_PRIMARY_TERM +import org.opensearch.alerting.util.IF_SEQ_NO +import org.opensearch.alerting.util.REFRESH +import org.opensearch.commons.alerting.util.AlertingException +import org.opensearch.core.xcontent.XContentParser.Token +import org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.index.seqno.SequenceNumbers +import org.opensearch.rest.BaseRestHandler +import org.opensearch.rest.RestHandler.Route +import org.opensearch.rest.RestRequest +import org.opensearch.rest.RestRequest.Method.POST +import org.opensearch.rest.RestRequest.Method.PUT +import org.opensearch.rest.action.RestToXContentListener +import org.opensearch.transport.client.node.NodeClient +import java.io.IOException + +private val log = LogManager.getLogger(RestIndexMonitorV2Action::class.java) + +/** + * Rest handlers to create and update V2 monitors + * + * @opensearch.experimental + */ +class RestIndexMonitorV2Action : BaseRestHandler() { + override fun getName(): String { + return "index_monitor_v2_action" + } + + override fun routes(): List { + return listOf( + Route( + POST, + AlertingPlugin.MONITOR_V2_BASE_URI + ), + Route( + PUT, + "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitor_id}" + ) + ) + } + + @Throws(IOException::class) + override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { + log.debug("${request.method()} ${request.path()}") + + val xcp = request.contentParser() + ensureExpectedToken(Token.START_OBJECT, xcp.nextToken(), xcp) + + val monitorV2: MonitorV2 + val rbacRoles: List? + try { + monitorV2 = MonitorV2.parse(xcp) + rbacRoles = request.contentParser().map()["rbac_roles"] as List? + } catch (e: Exception) { + throw AlertingException.wrap(IllegalArgumentException(e.localizedMessage)) + } + + val id = request.param("monitor_id", MonitorV2.NO_ID) + val seqNo = request.paramAsLong(IF_SEQ_NO, SequenceNumbers.UNASSIGNED_SEQ_NO) + val primaryTerm = request.paramAsLong(IF_PRIMARY_TERM, SequenceNumbers.UNASSIGNED_PRIMARY_TERM) + val refreshPolicy = if (request.hasParam(REFRESH)) { + WriteRequest.RefreshPolicy.parse(request.param(REFRESH)) + } else { + WriteRequest.RefreshPolicy.IMMEDIATE + } + + val indexMonitorV2Request = IndexMonitorV2Request(id, seqNo, primaryTerm, refreshPolicy, request.method(), monitorV2, rbacRoles) + + return RestChannelConsumer { channel -> + client.execute(IndexMonitorV2Action.INSTANCE, indexMonitorV2Request, RestToXContentListener(channel)) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestSearchMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestSearchMonitorV2Action.kt new file mode 100644 index 000000000..a179078f8 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestSearchMonitorV2Action.kt @@ -0,0 +1,128 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.resthandlerv2 + +import org.apache.logging.log4j.LogManager +import org.opensearch.action.search.SearchRequest +import org.opensearch.action.search.SearchResponse +import org.opensearch.alerting.AlertingPlugin +import org.opensearch.alerting.actionv2.SearchMonitorV2Action +import org.opensearch.alerting.actionv2.SearchMonitorV2Request +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.alerting.util.context +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.settings.Settings +import org.opensearch.common.xcontent.LoggingDeprecationHandler +import org.opensearch.common.xcontent.XContentFactory.jsonBuilder +import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.alerting.model.ScheduledJob +import org.opensearch.commons.alerting.model.ScheduledJob.Companion.SCHEDULED_JOBS_INDEX +import org.opensearch.core.common.bytes.BytesReference +import org.opensearch.core.rest.RestStatus +import org.opensearch.core.xcontent.ToXContent.EMPTY_PARAMS +import org.opensearch.rest.BaseRestHandler +import org.opensearch.rest.BytesRestResponse +import org.opensearch.rest.RestChannel +import org.opensearch.rest.RestHandler.Route +import org.opensearch.rest.RestRequest +import org.opensearch.rest.RestRequest.Method.GET +import org.opensearch.rest.RestRequest.Method.POST +import org.opensearch.rest.RestResponse +import org.opensearch.rest.action.RestResponseListener +import org.opensearch.search.builder.SearchSourceBuilder +import org.opensearch.transport.client.node.NodeClient +import java.io.IOException + +private val log = LogManager.getLogger(RestSearchMonitorV2Action::class.java) + +/** + * This class consists of the REST handler to search for v2 monitors with some OpenSearch search query. + * + * @opensearch.experimental + */ +class RestSearchMonitorV2Action( + val settings: Settings, + clusterService: ClusterService, +) : BaseRestHandler() { + + @Volatile private var filterBy = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) + + init { + clusterService.clusterSettings.addSettingsUpdateConsumer(AlertingSettings.FILTER_BY_BACKEND_ROLES) { filterBy = it } + } + + override fun getName(): String { + return "search_monitor_v2_action" + } + + override fun routes(): List { + return listOf( + Route( + POST, + "${AlertingPlugin.MONITOR_V2_BASE_URI}/_search" + ), + Route( + GET, + "${AlertingPlugin.MONITOR_V2_BASE_URI}/_search" + ) + ) + } + + @Throws(IOException::class) + override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { + log.debug("${request.method()} ${AlertingPlugin.MONITOR_V2_BASE_URI}/_search") + + val searchSourceBuilder = SearchSourceBuilder() + searchSourceBuilder.parseXContent(request.contentOrSourceParamParser()) + searchSourceBuilder.fetchSource(context(request)) + + val searchRequest = SearchRequest() + .source(searchSourceBuilder) + .indices(SCHEDULED_JOBS_INDEX) + + val searchMonitorV2Request = SearchMonitorV2Request(searchRequest) + return RestChannelConsumer { channel -> + client.execute(SearchMonitorV2Action.INSTANCE, searchMonitorV2Request, searchMonitorResponse(channel)) + } + } + + // once the search response is received, rewrite the search hits to remove the extra "monitor_v2" JSON object wrapper + // that is used as ScheduledJob metadata + private fun searchMonitorResponse(channel: RestChannel): RestResponseListener { + return object : RestResponseListener(channel) { + @Throws(Exception::class) + override fun buildResponse(response: SearchResponse): RestResponse { + if (response.isTimedOut) { + return BytesRestResponse(RestStatus.REQUEST_TIMEOUT, response.toString()) + } + + try { + for (hit in response.hits) { + XContentType.JSON.xContent().createParser( + channel.request().xContentRegistry, + LoggingDeprecationHandler.INSTANCE, hit.sourceAsString + ).use { hitsParser -> + // when reconstructing XContent, intentionally leave out + // user field in response for security reasons by + // calling ScheduledJob.toXContent instead of + // a MonitorV2's toXContentWithUser + val monitorV2 = ScheduledJob.parse(hitsParser, hit.id, hit.version) + val xcb = monitorV2.toXContent(jsonBuilder(), EMPTY_PARAMS) + + // rewrite the search hit as just the MonitorV2 source, + // without the extra "monitor_v2" JSON object wrapper + hit.sourceRef(BytesReference.bytes(xcb)) + } + } + } catch (e: Exception) { + // Swallow exception and return response as is + log.error("The monitor_v2 parsing failed. Will return response as is.") + } + return BytesRestResponse(RestStatus.OK, response.toXContent(channel.newBuilder(), EMPTY_PARAMS)) + } + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/script/PPLTriggerExecutionContext.kt b/alerting/src/main/kotlin/org/opensearch/alerting/script/PPLTriggerExecutionContext.kt new file mode 100644 index 000000000..1b95da951 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/script/PPLTriggerExecutionContext.kt @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.script + +import org.json.JSONObject +import org.opensearch.alerting.modelv2.PPLSQLMonitor +import org.opensearch.alerting.modelv2.PPLSQLMonitorRunResult.Companion.PPL_QUERY_RESULTS_FIELD +import org.opensearch.alerting.modelv2.PPLSQLTrigger +import org.opensearch.alerting.modelv2.PPLSQLTrigger.Companion.PPL_SQL_TRIGGER_FIELD + +data class PPLTriggerExecutionContext( + override val monitorV2: PPLSQLMonitor, + override val error: Exception? = null, + val pplTrigger: PPLSQLTrigger, + var pplQueryResults: JSONObject // can be a full set of PPL query results, or an individual result row +) : TriggerV2ExecutionContext(monitorV2, error) { + + override fun asTemplateArg(): Map { + val templateArg = super.asTemplateArg().toMutableMap() + templateArg[PPL_SQL_TRIGGER_FIELD] = pplTrigger.asTemplateArg() + templateArg[PPL_QUERY_RESULTS_FIELD] = pplQueryResults.toMap() + return templateArg.toMap() + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/script/TriggerV2ExecutionContext.kt b/alerting/src/main/kotlin/org/opensearch/alerting/script/TriggerV2ExecutionContext.kt new file mode 100644 index 000000000..97384845c --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/script/TriggerV2ExecutionContext.kt @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.script + +import org.opensearch.alerting.modelv2.MonitorV2 + +abstract class TriggerV2ExecutionContext( + open val monitorV2: MonitorV2, + open val error: Exception? = null +) { + + open fun asTemplateArg(): Map { + return mapOf( + "monitorV2" to monitorV2.asTemplateArg(), + "error" to error + ) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/service/DeleteMonitorService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/service/DeleteMonitorService.kt index fbc655543..c1a56a4c6 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/service/DeleteMonitorService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/service/DeleteMonitorService.kt @@ -22,6 +22,7 @@ import org.opensearch.action.support.IndicesOptions import org.opensearch.action.support.WriteRequest.RefreshPolicy import org.opensearch.action.support.clustermanager.AcknowledgedResponse import org.opensearch.alerting.MonitorMetadataService +import org.opensearch.alerting.actionv2.DeleteMonitorV2Response import org.opensearch.alerting.core.lock.LockModel import org.opensearch.alerting.core.lock.LockService import org.opensearch.alerting.opensearchapi.suspendUntil @@ -74,6 +75,19 @@ object DeleteMonitorService : return DeleteMonitorResponse(deleteResponse.id, deleteResponse.version) } + /** + * Deletes the monitorV2, which does not come with other metadata and queries + * like doc level monitors + * @param monitorV2Id monitorV2 ID to be deleted + * @param refreshPolicy + */ + suspend fun deleteMonitorV2(monitorV2Id: String, refreshPolicy: RefreshPolicy): DeleteMonitorV2Response { + val deleteResponse = deleteMonitor(monitorV2Id, refreshPolicy) + deleteLock(monitorV2Id) + return DeleteMonitorV2Response(deleteResponse.id, deleteResponse.version) + } + + // both Alerting v1 and v2 workflows flow through this function private suspend fun deleteMonitor(monitorId: String, refreshPolicy: RefreshPolicy): DeleteResponse { val deleteMonitorRequest = DeleteRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, monitorId) .setRefreshPolicy(refreshPolicy) @@ -167,7 +181,12 @@ object DeleteMonitorService : } private suspend fun deleteLock(monitor: Monitor) { - client.suspendUntil { lockService.deleteLock(LockModel.generateLockId(monitor.id), it) } + deleteLock(monitor.id) + } + + // both Alerting v1 and v2 workflows flow through this function + private suspend fun deleteLock(monitorId: String) { + client.suspendUntil { lockService.deleteLock(LockModel.generateLockId(monitorId), it) } } /** diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt index ba4dfc159..9fdbb1fec 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt @@ -309,6 +309,115 @@ class AlertingSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ) + val ALERT_V2_HISTORY_ENABLED = Setting.boolSetting( + "plugins.alerting.v2.alert_history_enabled", + true, + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val ALERT_V2_HISTORY_ROLLOVER_PERIOD = Setting.positiveTimeSetting( + "plugins.alerting.v2.alert_history_rollover_period", + TimeValue(12, TimeUnit.HOURS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val ALERT_V2_HISTORY_INDEX_MAX_AGE = Setting.positiveTimeSetting( + "plugins.alerting.v2.alert_history_max_age", + TimeValue(30, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val ALERT_V2_HISTORY_MAX_DOCS = Setting.longSetting( + "plugins.alerting.v2.alert_history_max_docs", + 1000L, 0L, + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val ALERT_V2_HISTORY_RETENTION_PERIOD = Setting.positiveTimeSetting( + "plugins.alerting.v2.alert_history_retention_period", + TimeValue(60, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val ALERT_V2_MONITOR_EXECUTION_MAX_DURATION = Setting.positiveTimeSetting( + "plugins.alerting.v2.alert_monitor_execution_max_duration", + TimeValue(4, TimeUnit.MINUTES), + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val ALERTING_V2_MAX_MONITORS = Setting.intSetting( + "plugins.alerting.v2.monitor.max_monitors", + 1000, + 1, + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val ALERTING_V2_MAX_THROTTLE_DURATION = Setting.longSetting( + "plugins.alerting.v2.monitor.max_throttle_duration", + 7200L, // 5 days, 7200 minutes + 2L, + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val ALERTING_V2_MAX_EXPIRE_DURATION = Setting.longSetting( + "plugins.alerting.v2.monitor.max_expire_duration", + 43200L, // 30 days, 43200 minutes + 2L, + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val ALERTING_V2_MAX_LOOK_BACK_WINDOW = Setting.longSetting( + "plugins.alerting.v2.monitor.max_look_back_window", + 10080L, // 7 days, 10080 minutes + 2L, + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val ALERTING_V2_MAX_QUERY_LENGTH = Setting.longSetting( + "plugins.alerting.v2.monitor.max_query_length", + 2000L, + 0L, + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + // max data rows to retrieve when executing PPL query against + // SQL/PPL plugin during monitor execution + val ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS = Setting.longSetting( + "plugins.alerting.v2.query_results_max_datarows", + 10000L, + 1L, + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + // max size of query results to store in alerts and notifications + val ALERT_V2_QUERY_RESULTS_MAX_SIZE = Setting.longSetting( + "plugins.alerting.v2.query_results_max_size", + 3000L, + 0L, + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val ALERT_V2_PER_RESULT_TRIGGER_MAX_ALERTS = Setting.intSetting( + "plugins.alerting.v2.per_result_trigger_max_alerts", + 10, + 1, + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH = Setting.intSetting( + "plugins.alerting.v2.notification_subject_source_max_length", + 1000, + 100, + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH = Setting.intSetting( + "plugins.alerting.v2.notification_message_source_max_length", + 3000, + 1000, + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + val NOTIFICATION_CONTEXT_RESULTS_ALLOWED_ROLES: Setting> = Setting.listSetting( "plugins.alerting.notification_context_results_allowed_roles", listOf(), diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteMonitorAction.kt index 97b513230..b23c8ba05 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteMonitorAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteMonitorAction.kt @@ -88,7 +88,7 @@ class TransportDeleteMonitorAction @Inject constructor( ) { suspend fun resolveUserAndStart(refreshPolicy: RefreshPolicy) { try { - val monitor = getMonitor() + val monitor = getMonitor() ?: return // null means there was an issue retrieving the Monitor val canDelete = user == null || !doFilterForUser(user) || checkUserPermissionsWithResource(user, monitor.user, actionListener, "monitor", monitorId) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteWorkflowAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteWorkflowAction.kt index bf0d44eab..60e018e50 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteWorkflowAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteWorkflowAction.kt @@ -23,6 +23,7 @@ import org.opensearch.action.search.SearchResponse import org.opensearch.action.support.ActionFilters import org.opensearch.action.support.HandledTransportAction import org.opensearch.action.support.WriteRequest.RefreshPolicy +import org.opensearch.alerting.AlertingV2Utils.validateMonitorV1 import org.opensearch.alerting.core.lock.LockModel import org.opensearch.alerting.core.lock.LockService import org.opensearch.alerting.opensearchapi.addFilter @@ -299,7 +300,13 @@ class TransportDeleteWorkflowAction @Inject constructor( xContentRegistry, LoggingDeprecationHandler.INSTANCE, hit.sourceAsString ).use { hitsParser -> - val monitor = ScheduledJob.parse(hitsParser, hit.id, hit.version) as Monitor + val scheduledJob = ScheduledJob.parse(hitsParser, hit.id, hit.version) + + validateMonitorV1(scheduledJob)?.let { + throw OpenSearchException(it) + } + + val monitor = scheduledJob as Monitor deletableMonitors.add(monitor) } } @@ -327,12 +334,17 @@ class TransportDeleteWorkflowAction @Inject constructor( ) } - private fun parseWorkflow(getResponse: GetResponse): Workflow { + private fun parseWorkflow(getResponse: GetResponse): Workflow? { val xcp = XContentHelper.createParser( xContentRegistry, LoggingDeprecationHandler.INSTANCE, getResponse.sourceAsBytesRef, XContentType.JSON ) - return ScheduledJob.parse(xcp, getResponse.id, getResponse.version) as Workflow + val scheduledJob = ScheduledJob.parse(xcp, getResponse.id, getResponse.version) + validateMonitorV1(scheduledJob)?.let { + actionListener.onFailure(AlertingException.wrap(it)) + return null + } + return scheduledJob as Workflow } private suspend fun deleteWorkflow(deleteRequest: DeleteRequest): DeleteResponse { diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteWorkflowAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteWorkflowAction.kt index 8b3272e74..aa3c5af1d 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteWorkflowAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteWorkflowAction.kt @@ -14,6 +14,7 @@ import org.opensearch.action.get.GetRequest import org.opensearch.action.get.GetResponse import org.opensearch.action.support.ActionFilters import org.opensearch.action.support.HandledTransportAction +import org.opensearch.alerting.AlertingV2Utils.validateMonitorV1 import org.opensearch.alerting.MonitorRunnerService import org.opensearch.alerting.action.ExecuteWorkflowAction import org.opensearch.alerting.action.ExecuteWorkflowRequest @@ -119,7 +120,12 @@ class TransportExecuteWorkflowAction @Inject constructor( xContentRegistry, LoggingDeprecationHandler.INSTANCE, response.sourceAsBytesRef, XContentType.JSON ).use { xcp -> - val workflow = ScheduledJob.parse(xcp, response.id, response.version) as Workflow + val scheduledJob = ScheduledJob.parse(xcp, response.id, response.version) + validateMonitorV1(scheduledJob)?.let { + actionListener.onFailure(AlertingException.wrap(it)) + return + } + val workflow = scheduledJob as Workflow executeWorkflow(workflow) } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetAlertsAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetAlertsAction.kt index a936ecedb..b8e6d31b7 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetAlertsAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetAlertsAction.kt @@ -218,7 +218,11 @@ class TransportGetAlertsAction @Inject constructor( xContentRegistry, LoggingDeprecationHandler.INSTANCE, getResponse.sourceAsBytesRef, XContentType.JSON ) - return ScheduledJob.parse(xcp, getResponse.id, getResponse.version) as Monitor + val scheduledJob = ScheduledJob.parse(xcp, getResponse.id, getResponse.version) + validateMonitorV1(scheduledJob)?.let { + throw it + } + return scheduledJob as Monitor } catch (t: Exception) { log.error("Failure in fetching monitor ${getAlertsRequest.monitorId} to resolve alert index in get alerts action", t) return null diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt index cab10849e..a7015c263 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt @@ -27,6 +27,7 @@ import org.opensearch.action.support.HandledTransportAction import org.opensearch.action.support.WriteRequest.RefreshPolicy import org.opensearch.action.support.clustermanager.AcknowledgedResponse import org.opensearch.alerting.AlertingPlugin +import org.opensearch.alerting.AlertingV2Utils.validateMonitorV1 import org.opensearch.alerting.MonitorMetadataService import org.opensearch.alerting.core.ScheduledJobIndices import org.opensearch.alerting.opensearchapi.suspendUntil @@ -636,7 +637,15 @@ class TransportIndexMonitorAction @Inject constructor( xContentRegistry, LoggingDeprecationHandler.INSTANCE, getResponse.sourceAsBytesRef, XContentType.JSON ) - val monitor = ScheduledJob.parse(xcp, getResponse.id, getResponse.version) as Monitor + val scheduledJob = ScheduledJob.parse(xcp, getResponse.id, getResponse.version) + + validateMonitorV1(scheduledJob)?.let { + actionListener.onFailure(AlertingException.wrap(it)) + return + } + + val monitor = scheduledJob as Monitor + onGetResponse(monitor) } catch (t: Exception) { actionListener.onFailure(AlertingException.wrap(t)) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexWorkflowAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexWorkflowAction.kt index 17dc67d92..6101c66e3 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexWorkflowAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexWorkflowAction.kt @@ -27,6 +27,7 @@ import org.opensearch.action.search.SearchResponse import org.opensearch.action.support.ActionFilters import org.opensearch.action.support.HandledTransportAction import org.opensearch.action.support.clustermanager.AcknowledgedResponse +import org.opensearch.alerting.AlertingV2Utils.validateMonitorV1 import org.opensearch.alerting.MonitorMetadataService import org.opensearch.alerting.MonitorRunnerService.monitorCtx import org.opensearch.alerting.WorkflowMetadataService @@ -458,7 +459,12 @@ class TransportIndexWorkflowAction @Inject constructor( xContentRegistry, LoggingDeprecationHandler.INSTANCE, getResponse.sourceAsBytesRef, XContentType.JSON ) - val workflow = ScheduledJob.parse(xcp, getResponse.id, getResponse.version) as Workflow + val scheduledJob = ScheduledJob.parse(xcp, getResponse.id, getResponse.version) + validateMonitorV1(scheduledJob)?.let { + actionListener.onFailure(AlertingException.wrap(it)) + return + } + val workflow = scheduledJob as Workflow onGetResponse(workflow) } catch (t: Exception) { actionListener.onFailure(AlertingException.wrap(t)) @@ -470,7 +476,7 @@ class TransportIndexWorkflowAction @Inject constructor( user, currentWorkflow.user, actionListener, - "workfklow", + "workflow", request.workflowId ) ) { @@ -731,7 +737,11 @@ class TransportIndexWorkflowAction @Inject constructor( xContentRegistry, LoggingDeprecationHandler.INSTANCE, hit.sourceAsString ).use { hitsParser -> - val monitor = ScheduledJob.parse(hitsParser, hit.id, hit.version) as Monitor + val scheduledJob = ScheduledJob.parse(hitsParser, hit.id, hit.version) + validateMonitorV1(scheduledJob)?.let { + throw OpenSearchException(it) + } + val monitor = scheduledJob as Monitor monitors.add(monitor) } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportSearchMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportSearchMonitorAction.kt index 771133628..8896bff78 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportSearchMonitorAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportSearchMonitorAction.kt @@ -11,7 +11,6 @@ import org.apache.lucene.search.TotalHits.Relation import org.opensearch.action.ActionRequest import org.opensearch.action.search.SearchRequest import org.opensearch.action.search.SearchResponse -import org.opensearch.action.search.SearchResponse.Clusters import org.opensearch.action.search.ShardSearchFailure import org.opensearch.action.support.ActionFilters import org.opensearch.action.support.HandledTransportAction diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportDeleteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportDeleteMonitorV2Action.kt new file mode 100644 index 000000000..b35381474 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportDeleteMonitorV2Action.kt @@ -0,0 +1,148 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.transportv2 + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import org.apache.logging.log4j.LogManager +import org.opensearch.OpenSearchStatusException +import org.opensearch.action.get.GetRequest +import org.opensearch.action.get.GetResponse +import org.opensearch.action.support.ActionFilters +import org.opensearch.action.support.HandledTransportAction +import org.opensearch.alerting.AlertingV2Utils +import org.opensearch.alerting.actionv2.DeleteMonitorV2Action +import org.opensearch.alerting.actionv2.DeleteMonitorV2Request +import org.opensearch.alerting.actionv2.DeleteMonitorV2Response +import org.opensearch.alerting.core.settings.AlertingV2Settings.Companion.ALERTING_V2_ENABLED +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.opensearchapi.suspendUntil +import org.opensearch.alerting.service.DeleteMonitorService +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.alerting.transport.SecureTransportAction +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.inject.Inject +import org.opensearch.common.settings.Settings +import org.opensearch.common.xcontent.LoggingDeprecationHandler +import org.opensearch.common.xcontent.XContentHelper +import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.alerting.model.ScheduledJob +import org.opensearch.commons.alerting.util.AlertingException +import org.opensearch.core.action.ActionListener +import org.opensearch.core.rest.RestStatus +import org.opensearch.core.xcontent.NamedXContentRegistry +import org.opensearch.tasks.Task +import org.opensearch.transport.TransportService +import org.opensearch.transport.client.Client + +private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) +private val log = LogManager.getLogger(TransportDeleteMonitorV2Action::class.java) + +/** + * Transport action that contains the core logic for deleting monitor V2s. + * + * @opensearch.experimental + */ +class TransportDeleteMonitorV2Action @Inject constructor( + transportService: TransportService, + val client: Client, + actionFilters: ActionFilters, + val clusterService: ClusterService, + settings: Settings, + val xContentRegistry: NamedXContentRegistry +) : HandledTransportAction( + DeleteMonitorV2Action.NAME, transportService, actionFilters, ::DeleteMonitorV2Request +), + SecureTransportAction { + + @Volatile private var alertingV2Enabled = ALERTING_V2_ENABLED.get(settings) + + @Volatile override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) + + init { + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_ENABLED) { alertingV2Enabled = it } + listenFilterBySettingChange(clusterService) + } + + override fun doExecute(task: Task, request: DeleteMonitorV2Request, actionListener: ActionListener) { + if (!alertingV2Enabled) { + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "Alerting V2 is currently disabled, please enable it with the " + + "cluster setting: ${ALERTING_V2_ENABLED.key}.", + RestStatus.FORBIDDEN + ), + ) + ) + return + } + + val user = readUserFromThreadContext(client) + + if (!validateUserBackendRoles(user, actionListener)) { + return + } + + scope.launch { + try { + val monitorV2 = getMonitorV2(request.monitorV2Id, actionListener) ?: return@launch + + val canDelete = user == null || !doFilterForUser(user) || + checkUserPermissionsWithResource(user, monitorV2!!.user, actionListener, "monitor_v2", request.monitorV2Id) + + if (canDelete) { + val deleteResponse = + DeleteMonitorService.deleteMonitorV2(request.monitorV2Id, request.refreshPolicy) + actionListener.onResponse(deleteResponse) + } else { + actionListener.onFailure( + AlertingException( + "Not allowed to delete this Monitor V2", + RestStatus.FORBIDDEN, + IllegalStateException() + ) + ) + } + } catch (e: Exception) { + actionListener.onFailure(e) + } + + // scheduled AlertV2Mover will sweep the alerts and find that this monitor no longer exists, + // and expire this monitor's alerts accordingly + } + } + + private suspend fun getMonitorV2(monitorV2Id: String, actionListener: ActionListener): MonitorV2? { + val getRequest = GetRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, monitorV2Id) + + val getResponse: GetResponse = client.suspendUntil { get(getRequest, it) } + if (!getResponse.isExists) { + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException("Monitor V2 with $monitorV2Id is not found", RestStatus.NOT_FOUND) + ) + ) + return null + } + + val xcp = XContentHelper.createParser( + xContentRegistry, LoggingDeprecationHandler.INSTANCE, + getResponse.sourceAsBytesRef, XContentType.JSON + ) + val scheduledJob = ScheduledJob.parse(xcp, getResponse.id, getResponse.version) + + AlertingV2Utils.validateMonitorV2(scheduledJob)?.let { + actionListener.onFailure(AlertingException.wrap(it)) + return null + } + + val monitorV2 = scheduledJob as MonitorV2 + + return monitorV2 + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportExecuteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportExecuteMonitorV2Action.kt new file mode 100644 index 000000000..05055afef --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportExecuteMonitorV2Action.kt @@ -0,0 +1,229 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.transportv2 + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import org.apache.logging.log4j.LogManager +import org.opensearch.OpenSearchStatusException +import org.opensearch.action.get.GetRequest +import org.opensearch.action.get.GetResponse +import org.opensearch.action.support.ActionFilters +import org.opensearch.action.support.HandledTransportAction +import org.opensearch.alerting.AlertingV2Utils.validateMonitorV2 +import org.opensearch.alerting.MonitorRunnerService +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Action +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Request +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Response +import org.opensearch.alerting.core.settings.AlertingV2Settings.Companion.ALERTING_V2_ENABLED +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.modelv2.PPLSQLMonitor +import org.opensearch.alerting.modelv2.PPLSQLMonitor.Companion.PPL_SQL_MONITOR_TYPE +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.alerting.transport.SecureTransportAction +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.inject.Inject +import org.opensearch.common.settings.Settings +import org.opensearch.common.xcontent.LoggingDeprecationHandler +import org.opensearch.common.xcontent.XContentHelper +import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.ConfigConstants +import org.opensearch.commons.alerting.model.ScheduledJob +import org.opensearch.commons.alerting.util.AlertingException +import org.opensearch.commons.authuser.User +import org.opensearch.core.action.ActionListener +import org.opensearch.core.rest.RestStatus +import org.opensearch.core.xcontent.NamedXContentRegistry +import org.opensearch.tasks.Task +import org.opensearch.transport.TransportService +import org.opensearch.transport.client.Client +import java.time.Instant + +private val log = LogManager.getLogger(TransportExecuteMonitorV2Action::class.java) + +/** + * Transport action for executing monitor V2s by calling the respective monitor V2 runners. + * + * @opensearch.experimental + */ +class TransportExecuteMonitorV2Action @Inject constructor( + private val transportService: TransportService, + private val client: Client, + private val clusterService: ClusterService, + private val runner: MonitorRunnerService, + actionFilters: ActionFilters, + val xContentRegistry: NamedXContentRegistry, + private val settings: Settings +) : HandledTransportAction( + ExecuteMonitorV2Action.NAME, transportService, actionFilters, ::ExecuteMonitorV2Request +), + SecureTransportAction { + + @Volatile private var alertingV2Enabled = ALERTING_V2_ENABLED.get(settings) + + @Volatile override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) + + init { + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_ENABLED) { alertingV2Enabled = it } + listenFilterBySettingChange(clusterService) + } + + override fun doExecute( + task: Task, + execMonitorV2Request: ExecuteMonitorV2Request, + actionListener: ActionListener + ) { + if (!alertingV2Enabled) { + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "Alerting V2 is currently disabled, please enable it with the " + + "cluster setting: ${ALERTING_V2_ENABLED.key}", + RestStatus.FORBIDDEN + ), + ) + ) + return + } + + val userStr = client.threadPool().threadContext.getTransient(ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT) + log.debug("User and roles string from thread context: $userStr") + val user: User? = User.parse(userStr) + + client.threadPool().threadContext.stashContext().use { + /* first define a function that will be used later to run MonitorV2s */ + val executeMonitorV2 = fun (monitorV2: MonitorV2) { + runner.launch { + // get execution end, this will be used to compute the execution interval + // via look back window (if one is supplied) + val periodEnd = Instant.ofEpochMilli(execMonitorV2Request.requestEnd.millis) + + // call the MonitorRunnerService to execute the MonitorV2 + try { + val monitorV2Type = when (monitorV2) { + is PPLSQLMonitor -> PPL_SQL_MONITOR_TYPE + else -> throw IllegalStateException("Unexpected MonitorV2 type: ${monitorV2.javaClass.name}") + } + log.info( + "Executing MonitorV2 from API - id: ${monitorV2.id}, type: $monitorV2Type, " + + "periodEnd: $periodEnd, manual: ${execMonitorV2Request.manual}" + ) + val monitorV2RunResult = runner.runJobV2( + monitorV2, + periodEnd, + execMonitorV2Request.dryrun, + execMonitorV2Request.manual, + transportService + ) + withContext(Dispatchers.IO) { + actionListener.onResponse(ExecuteMonitorV2Response(monitorV2RunResult)) + } + } catch (e: Exception) { + log.error("Unexpected error running monitor", e) + withContext(Dispatchers.IO) { + actionListener.onFailure(AlertingException.wrap(e)) + } + } + } + } + + /* now execute the MonitorV2 */ + + // if both monitor_v2 id and object were passed in, ignore object and proceed with id + if (execMonitorV2Request.monitorV2Id != null && execMonitorV2Request.monitorV2 != null) { + log.info( + "Both a monitor_v2 id and monitor_v2 object were passed in to ExecuteMonitorV2" + + "request. Proceeding to execute by monitor_v2 ID and ignoring monitor_v2 object." + ) + } + + if (execMonitorV2Request.monitorV2Id != null) { // execute with monitor ID case + // search the alerting-config index for the MonitorV2 with this ID + val getMonitorV2Request = GetRequest(ScheduledJob.SCHEDULED_JOBS_INDEX).id(execMonitorV2Request.monitorV2Id) + client.get( + getMonitorV2Request, + object : ActionListener { + override fun onResponse(getMonitorV2Response: GetResponse) { + if (!getMonitorV2Response.isExists) { + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "Can't find monitorV2 with id: ${getMonitorV2Response.id} to execute", + RestStatus.NOT_FOUND + ) + ) + ) + return + } + + if (getMonitorV2Response.isSourceEmpty) { + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "Found monitorV2 with id: ${getMonitorV2Response.id} but it was empty", + RestStatus.NO_CONTENT + ) + ) + ) + return + } + + val xcp = XContentHelper.createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, + getMonitorV2Response.sourceAsBytesRef, + XContentType.JSON + ) + + val scheduledJob = ScheduledJob.parse(xcp, getMonitorV2Response.id, getMonitorV2Response.version) + + validateMonitorV2(scheduledJob)?.let { + actionListener.onFailure(AlertingException.wrap(it)) + return + } + + val monitorV2 = scheduledJob as MonitorV2 + + // security is enabled and filterby is enabled + // only run this check on manual executions, + // automatic scheduled job executions should + // bypass this check and proceed to execution + if (execMonitorV2Request.manual && + !checkUserPermissionsWithResource( + user, + monitorV2.user, + actionListener, + "monitor", + execMonitorV2Request.monitorV2Id + ) + ) { + return + } + + try { + executeMonitorV2(monitorV2) + } catch (e: Exception) { + actionListener.onFailure(AlertingException.wrap(e)) + } + } + + override fun onFailure(t: Exception) { + actionListener.onFailure(AlertingException.wrap(t)) + } + } + ) + } else { // execute with monitor object case + try { + val monitorV2 = execMonitorV2Request.monitorV2!!.makeCopy(user = user) + executeMonitorV2(monitorV2) + } catch (e: Exception) { + actionListener.onFailure(AlertingException.wrap(e)) + } + } + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetAlertsV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetAlertsV2Action.kt new file mode 100644 index 000000000..4a0cec89d --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetAlertsV2Action.kt @@ -0,0 +1,211 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.transportv2 + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import org.apache.logging.log4j.LogManager +import org.opensearch.OpenSearchStatusException +import org.opensearch.action.search.SearchRequest +import org.opensearch.action.search.SearchResponse +import org.opensearch.action.support.ActionFilters +import org.opensearch.action.support.HandledTransportAction +import org.opensearch.alerting.actionv2.GetAlertsV2Action +import org.opensearch.alerting.actionv2.GetAlertsV2Request +import org.opensearch.alerting.actionv2.GetAlertsV2Response +import org.opensearch.alerting.alertsv2.AlertV2Indices +import org.opensearch.alerting.core.settings.AlertingV2Settings.Companion.ALERTING_V2_ENABLED +import org.opensearch.alerting.modelv2.AlertV2 +import org.opensearch.alerting.modelv2.AlertV2.Companion.MONITOR_V2_ID_FIELD +import org.opensearch.alerting.modelv2.AlertV2.Companion.MONITOR_V2_NAME_FIELD +import org.opensearch.alerting.modelv2.AlertV2.Companion.MONITOR_V2_USER_FIELD +import org.opensearch.alerting.modelv2.AlertV2.Companion.SEVERITY_FIELD +import org.opensearch.alerting.modelv2.AlertV2.Companion.TRIGGER_V2_NAME_FIELD +import org.opensearch.alerting.opensearchapi.addFilter +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.alerting.transport.SecureTransportAction +import org.opensearch.alerting.util.use +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.inject.Inject +import org.opensearch.common.settings.Settings +import org.opensearch.common.xcontent.LoggingDeprecationHandler +import org.opensearch.common.xcontent.XContentHelper +import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.alerting.util.AlertingException +import org.opensearch.commons.authuser.User +import org.opensearch.commons.authuser.User.BACKEND_ROLES_FIELD +import org.opensearch.core.action.ActionListener +import org.opensearch.core.common.io.stream.NamedWriteableRegistry +import org.opensearch.core.rest.RestStatus +import org.opensearch.core.xcontent.NamedXContentRegistry +import org.opensearch.index.query.Operator +import org.opensearch.index.query.QueryBuilders +import org.opensearch.search.builder.SearchSourceBuilder +import org.opensearch.search.sort.SortBuilders +import org.opensearch.search.sort.SortOrder +import org.opensearch.tasks.Task +import org.opensearch.transport.TransportService +import org.opensearch.transport.client.Client +import java.io.IOException + +private val log = LogManager.getLogger(TransportGetAlertsV2Action::class.java) +private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) + +/** + * Transport action that contains the core logic for retrieving v2 alerts. + * + * @opensearch.experimental + */ +class TransportGetAlertsV2Action @Inject constructor( + transportService: TransportService, + val client: Client, + clusterService: ClusterService, + actionFilters: ActionFilters, + val settings: Settings, + val xContentRegistry: NamedXContentRegistry, + val namedWriteableRegistry: NamedWriteableRegistry +) : HandledTransportAction( + GetAlertsV2Action.NAME, + transportService, + actionFilters, + ::GetAlertsV2Request +), + SecureTransportAction { + + @Volatile private var alertingV2Enabled = ALERTING_V2_ENABLED.get(settings) + + @Volatile + override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) + + init { + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_ENABLED) { alertingV2Enabled = it } + listenFilterBySettingChange(clusterService) + } + + override fun doExecute( + task: Task, + getAlertsV2Request: GetAlertsV2Request, + actionListener: ActionListener, + ) { + if (!alertingV2Enabled) { + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "Alerting V2 is currently disabled, please enable it with the " + + "cluster setting: ${ALERTING_V2_ENABLED.key}", + RestStatus.FORBIDDEN + ), + ) + ) + return + } + + val user = readUserFromThreadContext(client) + + val tableProp = getAlertsV2Request.table + val sortBuilder = SortBuilders + .fieldSort(tableProp.sortString) + .order(SortOrder.fromString(tableProp.sortOrder)) + if (!tableProp.missing.isNullOrBlank()) { + sortBuilder.missing(tableProp.missing) + } + + val queryBuilder = QueryBuilders.boolQuery() + + if (getAlertsV2Request.severityLevel != "ALL") { + queryBuilder.filter(QueryBuilders.termQuery(SEVERITY_FIELD, getAlertsV2Request.severityLevel)) + } + + if (!getAlertsV2Request.monitorV2Ids.isNullOrEmpty()) { + queryBuilder.filter(QueryBuilders.termsQuery(MONITOR_V2_ID_FIELD, getAlertsV2Request.monitorV2Ids)) + } + + if (!tableProp.searchString.isNullOrBlank()) { + queryBuilder + .must( + QueryBuilders + .queryStringQuery(tableProp.searchString) + .defaultOperator(Operator.AND) + .field(MONITOR_V2_NAME_FIELD) + .field(TRIGGER_V2_NAME_FIELD) + ) + } + val searchSourceBuilder = SearchSourceBuilder() + .version(true) + .seqNoAndPrimaryTerm(true) + .query(queryBuilder) + .sort(sortBuilder) + .size(tableProp.size) + .from(tableProp.startIndex) + + client.threadPool().threadContext.stashContext().use { + scope.launch { + try { + getAlerts(AlertV2Indices.ALERT_V2_INDEX, searchSourceBuilder, actionListener, user) + } catch (t: Exception) { + log.error("Failed to get alerts", t) + if (t is AlertingException) { + actionListener.onFailure(t) + } else { + actionListener.onFailure(AlertingException.wrap(t)) + } + } + } + } + } + + fun getAlerts( + alertIndex: String, + searchSourceBuilder: SearchSourceBuilder, + actionListener: ActionListener, + user: User? + ) { + try { + // if user is null, security plugin is disabled or user is super-admin + // if doFilterForUser() is false, security is enabled but filterby is disabled + if (user != null && doFilterForUser(user)) { + // if security is enabled and filterby is enabled, add search filter + log.info("Filtering result by: ${user.backendRoles}") + addFilter(user, searchSourceBuilder, "$MONITOR_V2_USER_FIELD.$BACKEND_ROLES_FIELD.keyword") + } + + search(alertIndex, searchSourceBuilder, actionListener) + } catch (ex: IOException) { + actionListener.onFailure(AlertingException.wrap(ex)) + } + } + + fun search(alertIndex: String, searchSourceBuilder: SearchSourceBuilder, actionListener: ActionListener) { + val searchRequest = SearchRequest() + .indices(alertIndex) + .source(searchSourceBuilder) + + client.search( + searchRequest, + object : ActionListener { + override fun onResponse(response: SearchResponse) { + val totalAlertCount = response.hits.totalHits?.value?.toInt() + val alerts = response.hits.map { hit -> + val xcp = XContentHelper.createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, + hit.sourceRef, + XContentType.JSON + ) + val alertV2 = AlertV2.parse(xcp, hit.id, hit.version) + alertV2 + } + actionListener.onResponse(GetAlertsV2Response(alerts, totalAlertCount)) + } + + override fun onFailure(t: Exception) { + actionListener.onFailure(t) + } + } + ) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetMonitorV2Action.kt new file mode 100644 index 000000000..bc10421c7 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetMonitorV2Action.kt @@ -0,0 +1,172 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.transportv2 + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import org.apache.logging.log4j.LogManager +import org.opensearch.OpenSearchStatusException +import org.opensearch.action.get.GetRequest +import org.opensearch.action.get.GetResponse +import org.opensearch.action.support.ActionFilters +import org.opensearch.action.support.HandledTransportAction +import org.opensearch.alerting.AlertingV2Utils.isIndexNotFoundException +import org.opensearch.alerting.AlertingV2Utils.validateMonitorV2 +import org.opensearch.alerting.actionv2.GetMonitorV2Action +import org.opensearch.alerting.actionv2.GetMonitorV2Request +import org.opensearch.alerting.actionv2.GetMonitorV2Response +import org.opensearch.alerting.core.settings.AlertingV2Settings.Companion.ALERTING_V2_ENABLED +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.alerting.transport.SecureTransportAction +import org.opensearch.alerting.transport.TransportGetMonitorAction +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.inject.Inject +import org.opensearch.common.settings.Settings +import org.opensearch.common.xcontent.LoggingDeprecationHandler +import org.opensearch.common.xcontent.XContentHelper +import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.alerting.model.ScheduledJob +import org.opensearch.commons.alerting.util.AlertingException +import org.opensearch.core.action.ActionListener +import org.opensearch.core.rest.RestStatus +import org.opensearch.core.xcontent.NamedXContentRegistry +import org.opensearch.tasks.Task +import org.opensearch.transport.TransportService +import org.opensearch.transport.client.Client + +private val log = LogManager.getLogger(TransportGetMonitorAction::class.java) +private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) + +/** + * Transport action that contains the core logic for getting a monitor v2 by its ID. + * + * @opensearch.experimental + */ +class TransportGetMonitorV2Action @Inject constructor( + transportService: TransportService, + val client: Client, + actionFilters: ActionFilters, + val xContentRegistry: NamedXContentRegistry, + val clusterService: ClusterService, + settings: Settings, +) : HandledTransportAction( + GetMonitorV2Action.NAME, + transportService, + actionFilters, + ::GetMonitorV2Request +), + SecureTransportAction { + + @Volatile private var alertingV2Enabled = ALERTING_V2_ENABLED.get(settings) + + @Volatile override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) + + init { + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_ENABLED) { alertingV2Enabled = it } + listenFilterBySettingChange(clusterService) + } + + override fun doExecute(task: Task, request: GetMonitorV2Request, actionListener: ActionListener) { + if (!alertingV2Enabled) { + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "Alerting V2 is currently disabled, please enable it with the " + + "cluster setting: ${ALERTING_V2_ENABLED.key}", + RestStatus.FORBIDDEN + ), + ) + ) + return + } + + val getRequest = GetRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, request.monitorV2Id) + .version(request.version) + .fetchSourceContext(request.srcContext) + + val user = readUserFromThreadContext(client) + + if (!validateUserBackendRoles(user, actionListener)) { + return + } + + client.threadPool().threadContext.stashContext().use { + client.get( + getRequest, + object : ActionListener { + override fun onResponse(response: GetResponse) { + if (!response.isExists) { + actionListener.onFailure( + AlertingException.wrap(OpenSearchStatusException("MonitorV2 not found.", RestStatus.NOT_FOUND)) + ) + return + } + + if (response.isSourceEmpty) { + actionListener.onFailure( + AlertingException.wrap(OpenSearchStatusException("MonitorV2 found but was empty.", RestStatus.NO_CONTENT)) + ) + return + } + + val xcp = XContentHelper.createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, + response.sourceAsBytesRef, + XContentType.JSON + ) + + val scheduledJob = ScheduledJob.parse(xcp, response.id, response.version) + + validateMonitorV2(scheduledJob)?.let { + actionListener.onFailure(AlertingException.wrap(it)) + return + } + + val monitorV2 = scheduledJob as MonitorV2 + + // security is enabled and filterby is enabled + if (!checkUserPermissionsWithResource( + user, + monitorV2.user, + actionListener, + "monitor", + request.monitorV2Id + ) + ) { + return + } + + actionListener.onResponse( + GetMonitorV2Response( + response.id, + response.version, + response.seqNo, + response.primaryTerm, + monitorV2 + ) + ) + } + + override fun onFailure(e: Exception) { + if (isIndexNotFoundException(e)) { + log.error("Index not found while getting monitor V2", e) + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException("Monitor V2 not found. Backing index is missing.", RestStatus.NOT_FOUND, e) + ) + ) + } else { + log.error("Unexpected error while getting monitor", e) + actionListener.onFailure(AlertingException.wrap(e)) + } + } + } + ) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportIndexMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportIndexMonitorV2Action.kt new file mode 100644 index 000000000..e12637fc6 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportIndexMonitorV2Action.kt @@ -0,0 +1,868 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.transportv2 + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.newSingleThreadContext +import kotlinx.coroutines.withContext +import org.apache.logging.log4j.LogManager +import org.opensearch.ExceptionsHelper +import org.opensearch.OpenSearchException +import org.opensearch.OpenSearchStatusException +import org.opensearch.ResourceAlreadyExistsException +import org.opensearch.action.admin.cluster.health.ClusterHealthAction +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse +import org.opensearch.action.admin.indices.create.CreateIndexResponse +import org.opensearch.action.admin.indices.mapping.get.GetMappingsRequest +import org.opensearch.action.get.GetRequest +import org.opensearch.action.get.GetResponse +import org.opensearch.action.index.IndexRequest +import org.opensearch.action.index.IndexResponse +import org.opensearch.action.search.SearchRequest +import org.opensearch.action.search.SearchResponse +import org.opensearch.action.support.ActionFilters +import org.opensearch.action.support.HandledTransportAction +import org.opensearch.action.support.clustermanager.AcknowledgedResponse +import org.opensearch.alerting.AlertingV2Utils.validateMonitorV2 +import org.opensearch.alerting.PPLUtils.appendCustomCondition +import org.opensearch.alerting.PPLUtils.executePplQuery +import org.opensearch.alerting.PPLUtils.findEvalResultVar +import org.opensearch.alerting.PPLUtils.findEvalResultVarIdxInSchema +import org.opensearch.alerting.PPLUtils.getIndicesFromPplQuery +import org.opensearch.alerting.actionv2.IndexMonitorV2Action +import org.opensearch.alerting.actionv2.IndexMonitorV2Request +import org.opensearch.alerting.actionv2.IndexMonitorV2Response +import org.opensearch.alerting.core.ScheduledJobIndices +import org.opensearch.alerting.core.settings.AlertingV2Settings.Companion.ALERTING_V2_ENABLED +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.modelv2.MonitorV2.Companion.MONITOR_V2_TYPE +import org.opensearch.alerting.modelv2.PPLSQLMonitor +import org.opensearch.alerting.modelv2.PPLSQLTrigger.ConditionType +import org.opensearch.alerting.opensearchapi.suspendUntil +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_EXPIRE_DURATION +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_LOOK_BACK_WINDOW +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_MONITORS +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_QUERY_LENGTH +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_THROTTLE_DURATION +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS +import org.opensearch.alerting.settings.AlertingSettings.Companion.INDEX_TIMEOUT +import org.opensearch.alerting.settings.AlertingSettings.Companion.NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH +import org.opensearch.alerting.settings.AlertingSettings.Companion.NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH +import org.opensearch.alerting.settings.AlertingSettings.Companion.REQUEST_TIMEOUT +import org.opensearch.alerting.transport.SecureTransportAction +import org.opensearch.alerting.util.IndexUtils +import org.opensearch.alerting.util.use +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.inject.Inject +import org.opensearch.common.settings.Settings +import org.opensearch.common.xcontent.LoggingDeprecationHandler +import org.opensearch.common.xcontent.XContentFactory.jsonBuilder +import org.opensearch.common.xcontent.XContentHelper +import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.alerting.model.ScheduledJob +import org.opensearch.commons.alerting.model.ScheduledJob.Companion.SCHEDULED_JOBS_INDEX +import org.opensearch.commons.alerting.model.userErrorMessage +import org.opensearch.commons.alerting.util.AlertingException +import org.opensearch.commons.authuser.User +import org.opensearch.core.action.ActionListener +import org.opensearch.core.common.io.stream.NamedWriteableRegistry +import org.opensearch.core.rest.RestStatus +import org.opensearch.core.xcontent.NamedXContentRegistry +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.index.query.QueryBuilders +import org.opensearch.rest.RestRequest +import org.opensearch.search.builder.SearchSourceBuilder +import org.opensearch.tasks.Task +import org.opensearch.transport.TransportService +import org.opensearch.transport.client.Client + +private val log = LogManager.getLogger(TransportIndexMonitorV2Action::class.java) +private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) + +/** + * Transport action that contains the core logic for creating and updating v2 monitors. + * + * @opensearch.experimental + */ +class TransportIndexMonitorV2Action @Inject constructor( + val transportService: TransportService, + val client: Client, + actionFilters: ActionFilters, + val scheduledJobIndices: ScheduledJobIndices, + val clusterService: ClusterService, + val settings: Settings, + val xContentRegistry: NamedXContentRegistry, + val namedWriteableRegistry: NamedWriteableRegistry, +) : HandledTransportAction( + IndexMonitorV2Action.NAME, transportService, actionFilters, ::IndexMonitorV2Request +), + SecureTransportAction { + + // adjustable limits (via settings) + @Volatile private var alertingV2Enabled = ALERTING_V2_ENABLED.get(settings) + @Volatile private var maxMonitors = ALERTING_V2_MAX_MONITORS.get(settings) + @Volatile private var maxThrottleDuration = ALERTING_V2_MAX_THROTTLE_DURATION.get(settings) + @Volatile private var maxExpireDuration = ALERTING_V2_MAX_EXPIRE_DURATION.get(settings) + @Volatile private var maxLookBackWindow = ALERTING_V2_MAX_LOOK_BACK_WINDOW.get(settings) + @Volatile private var maxQueryLength = ALERTING_V2_MAX_QUERY_LENGTH.get(settings) + @Volatile private var maxQueryResults = ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS.get(settings) + @Volatile private var notificationSubjectMaxLength = NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH.get(settings) + @Volatile private var notificationMessageMaxLength = NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH.get(settings) + @Volatile private var requestTimeout = REQUEST_TIMEOUT.get(settings) + @Volatile private var indexTimeout = INDEX_TIMEOUT.get(settings) + @Volatile override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) + + init { + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_ENABLED) { alertingV2Enabled = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_MAX_MONITORS) { maxMonitors = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_MAX_THROTTLE_DURATION) { maxThrottleDuration = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_MAX_EXPIRE_DURATION) { maxExpireDuration = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_MAX_LOOK_BACK_WINDOW) { maxLookBackWindow = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_MAX_QUERY_LENGTH) { maxQueryLength = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS) { maxQueryResults = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH) { + notificationSubjectMaxLength = it + } + clusterService.clusterSettings.addSettingsUpdateConsumer(NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH) { + notificationMessageMaxLength = it + } + clusterService.clusterSettings.addSettingsUpdateConsumer(REQUEST_TIMEOUT) { requestTimeout = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(INDEX_TIMEOUT) { indexTimeout = it } + listenFilterBySettingChange(clusterService) + } + + override fun doExecute( + task: Task, + indexMonitorV2Request: IndexMonitorV2Request, + actionListener: ActionListener + ) { + if (!alertingV2Enabled) { + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "Alerting V2 is currently disabled, please enable it with the " + + "cluster setting: ${ALERTING_V2_ENABLED.key}", + RestStatus.FORBIDDEN + ), + ) + ) + return + } + + // read the user from thread context immediately, before + // downstream flows spin up new threads with fresh context + val user = readUserFromThreadContext(client) + + // validate the MonitorV2 based on its type + when (indexMonitorV2Request.monitorV2) { + is PPLSQLMonitor -> validatePplSqlMonitorUserPermissionsAndQuery( + indexMonitorV2Request, + user, + object : ActionListener { // validationListener + override fun onResponse(response: Unit) { + // user permissions to indices have already been checked + // proceed without the context of the user, otherwise, + // we would get permissions errors trying to search the alerting-config + // index as the user. pass the user object itself so backend + // roles can be matched and checked downstream + client.threadPool().threadContext.stashContext().use { + val pplSqlMonitor = indexMonitorV2Request.monitorV2 as PPLSQLMonitor + if (user == null) { + indexMonitorV2Request.monitorV2 = pplSqlMonitor + .copy(user = User("", listOf(), listOf(), mapOf())) + } else { + indexMonitorV2Request.monitorV2 = pplSqlMonitor + .copy(user = User(user.name, user.backendRoles, user.roles, user.customAttributes)) + } + checkScheduledJobIndex(indexMonitorV2Request, actionListener, user) + } + } + + override fun onFailure(e: Exception) { + actionListener.onFailure(e) + } + } + ) + else -> actionListener.onFailure( + AlertingException.wrap( + IllegalStateException( + "unexpected MonitorV2 type: ${indexMonitorV2Request.monitorV2.javaClass.name}" + ) + ) + ) + } + } + + // validates the PPL Monitor, its query, and user's permissions to the indices it queries by submitting it to SQL/PPL plugin + private fun validatePplSqlMonitorUserPermissionsAndQuery( + indexMonitorV2Request: IndexMonitorV2Request, + user: User?, + validationListener: ActionListener + ) { + client.threadPool().threadContext.stashContext().use { + scope.launch { + val singleThreadContext = newSingleThreadContext("IndexMonitorV2ActionThread") + withContext(singleThreadContext) { + it.restore() + + val pplSqlMonitor = indexMonitorV2Request.monitorV2 as PPLSQLMonitor + + val pplQueryValid = validatePplSqlQuery(pplSqlMonitor, validationListener) + if (!pplQueryValid) { + return@withContext + } + + // run basic validations against the PPL/SQL Monitor + val pplSqlMonitorValid = validatePplSqlMonitor(pplSqlMonitor, validationListener) + if (!pplSqlMonitorValid) { + return@withContext + } + + // check the user for basic permissions + val userHasPermissions = checkUser(user, indexMonitorV2Request, validationListener) + if (!userHasPermissions) { + return@withContext + } + + // check that given timestamp field is valid + val timestampFieldValid = checkPplQueryIndicesForTimestampField(pplSqlMonitor, validationListener) + if (!timestampFieldValid) { + return@withContext + } + + validationListener.onResponse(Unit) + } + } + } + } + + private suspend fun validatePplSqlQuery( + pplSqlMonitor: PPLSQLMonitor, + validationListener: ActionListener + ): Boolean { + // first attempt to run the monitor query and all possible + // extensions of it (from custom conditions) + try { + // first run the base query as is. + // if there are any PPL syntax or index not found or other errors, + // this will throw an exception + executePplQuery(pplSqlMonitor.query, clusterService.state().nodes.localNode, transportService) + + // now scan all the triggers with custom conditions, and ensure each query constructed + // from the base query + custom condition is valid + for (pplTrigger in pplSqlMonitor.triggers) { + if (pplTrigger.conditionType != ConditionType.CUSTOM) { + continue + } + + val evalResultVar = findEvalResultVar(pplTrigger.customCondition!!) + + val queryWithCustomCondition = appendCustomCondition(pplSqlMonitor.query, pplTrigger.customCondition!!) + + val executePplQueryResponse = executePplQuery( + queryWithCustomCondition, + clusterService.state().nodes.localNode, + transportService + ) + + val evalResultVarIdx = findEvalResultVarIdxInSchema(executePplQueryResponse, evalResultVar) + + val resultVarType = executePplQueryResponse + .getJSONArray("schema") + .getJSONObject(evalResultVarIdx) + .getString("type") + + // custom conditions must evaluate to a boolean result, otherwise it's invalid + if (resultVarType != "boolean") { + validationListener.onFailure( + AlertingException.wrap( + IllegalArgumentException( + "Custom condition in trigger ${pplTrigger.name} is invalid because it does not " + + "evaluate to a boolean, but instead to type: $resultVarType" + ) + ) + ) + return false + } + } + } catch (e: Exception) { + validationListener.onFailure( + AlertingException.wrap( + IllegalArgumentException("Validation error for PPL Query in PPL Monitor: ${e.userErrorMessage()}") + ) + ) + return false + } + + return true + } + + private fun validatePplSqlMonitor(pplSqlMonitor: PPLSQLMonitor, validationListener: ActionListener): Boolean { + // ensure the trigger throttle and expire durations are valid + pplSqlMonitor.triggers.forEach { trigger -> + trigger.throttleDuration?.let { throttleDuration -> + if (throttleDuration > maxThrottleDuration) { + validationListener.onFailure( + AlertingException.wrap( + IllegalArgumentException( + "Throttle duration must be at most $maxThrottleDuration but was $throttleDuration" + ) + ) + ) + return false + } + } + + if (trigger.expireDuration > maxExpireDuration) { + validationListener.onFailure( + AlertingException.wrap( + IllegalArgumentException( + "Expire duration must be at most $maxExpireDuration but was ${trigger.expireDuration}" + ) + ) + ) + return false + } + + if (trigger.conditionType == ConditionType.NUMBER_OF_RESULTS && + trigger.numResultsValue!! > maxQueryResults + ) { + validationListener.onFailure( + AlertingException.wrap( + IllegalArgumentException( + "Trigger ${trigger.id} checks for number of results threshold of ${trigger.numResultsValue}, " + + "but Alerting V2 is configured only to retrieve $maxQueryResults query results maximum. " + + "Please lower the number of results value to one below this maximum value, or adjust the cluster " + + "setting: $ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS.key}" + ) + ) + ) + return false + } + + trigger.actions.forEach { action -> + if (action.subjectTemplate?.idOrCode?.length!! > notificationSubjectMaxLength) { + validationListener.onFailure( + AlertingException.wrap( + IllegalArgumentException( + "Notification subject source cannot exceed length: $notificationSubjectMaxLength" + ) + ) + ) + return false + } + + if (action.messageTemplate.idOrCode.length > notificationMessageMaxLength) { + validationListener.onFailure( + AlertingException.wrap( + IllegalArgumentException( + "Notification message source cannot exceed length: $notificationMessageMaxLength" + ) + ) + ) + return false + } + } + } + + // ensure the query length doesn't exceed the limit + if (pplSqlMonitor.query.length > maxQueryLength) { + validationListener.onFailure( + AlertingException.wrap( + IllegalArgumentException( + "PPL Query length must be at most $maxQueryLength but was ${pplSqlMonitor.query.length}" + ) + ) + ) + return false + } + + // ensure the look back window doesn't exceed the limit + pplSqlMonitor.lookBackWindow?.let { + if (pplSqlMonitor.lookBackWindow > maxLookBackWindow) { + validationListener.onFailure( + AlertingException.wrap( + IllegalArgumentException( + "Look back window must be at most $maxLookBackWindow minutes but was ${pplSqlMonitor.lookBackWindow}" + ) + ) + ) + return false + } + } + + return true + } + + private fun checkUser( + user: User?, + indexMonitorV2Request: IndexMonitorV2Request, + validationListener: ActionListener + ): Boolean { + /* check initial user permissions */ + if (!validateUserBackendRoles(user, validationListener)) { + return false + } + + if ( + user != null && + !isAdmin(user) && + indexMonitorV2Request.rbacRoles != null + ) { + if (indexMonitorV2Request.rbacRoles.stream().anyMatch { !user.backendRoles.contains(it) }) { + log.debug( + "User specified backend roles, ${indexMonitorV2Request.rbacRoles}, " + + "that they don't have access to. User backend roles: ${user.backendRoles}" + ) + validationListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "User specified backend roles that they don't have access to. Contact administrator", RestStatus.FORBIDDEN + ) + ) + ) + return false + } else if (indexMonitorV2Request.rbacRoles.isEmpty()) { + log.debug( + "Non-admin user are not allowed to specify an empty set of backend roles. " + + "Please don't pass in the parameter or pass in at least one backend role." + ) + validationListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "Non-admin user are not allowed to specify an empty set of backend roles.", RestStatus.FORBIDDEN + ) + ) + ) + return false + } + } + + return true + } + + // if look back window is specified, all the indices that the PPL query searches + // must contain the timestamp field specified in the PPL Monitor, and they must + // all be of OpenSearch data type "date" + private suspend fun checkPplQueryIndicesForTimestampField( + pplSqlMonitor: PPLSQLMonitor, + validationListener: ActionListener + ): Boolean { + if (pplSqlMonitor.lookBackWindow == null) { + // if no look back window was specified, no need + // to check for timestamp field in PPL query indices + return true + } + + val pplQuery = pplSqlMonitor.query + val timestampField = pplSqlMonitor.timestampField + + try { + val indices = getIndicesFromPplQuery(pplQuery) + val getMappingsRequest = GetMappingsRequest().indices(*indices.toTypedArray()) + val getMappingsResponse = client.suspendUntil { admin().indices().getMappings(getMappingsRequest, it) } + + val metadataMap = getMappingsResponse.mappings + + for (index in metadataMap.keys) { + val metadata = metadataMap[index]!!.sourceAsMap["properties"] as Map + if (!metadata.keys.contains(timestampField)) { + validationListener.onFailure( + AlertingException.wrap( + IllegalArgumentException("Query index $index don't contain given timestamp field: $timestampField") + ) + ) + return false + } + val typeInfo = metadata[timestampField] as Map + val type = typeInfo["type"] + val dateType = "date" + val dateNanosType = "date_nanos" + if (type != dateType && type != dateNanosType) { + validationListener.onFailure( + AlertingException.wrap( + IllegalArgumentException( + "Timestamp field: $timestampField is present in index $index " + + "but is type $type instead of $dateType or $dateNanosType" + ) + ) + ) + return false + } + } + } catch (e: Exception) { + log.error("failed to read query indices' fields when checking for timestamp field: $timestampField") + validationListener.onFailure( + AlertingException.wrap( + IllegalArgumentException("failed to read query indices' fields when checking for timestamp field: $timestampField", e) + ) + ) + return false + } + + return true + } + + private fun checkScheduledJobIndex( + indexMonitorRequest: IndexMonitorV2Request, + actionListener: ActionListener, + user: User? + ) { + // user permissions to indices have already been checked + // proceed without the context of the user, otherwise, + // we would get permissions errors trying to search the alerting-config + // index as the user + client.threadPool().threadContext.stashContext().use { + /* check to see if alerting-config index (scheduled job index) is created and updated before indexing MonitorV2 into it */ + if (!scheduledJobIndices.scheduledJobIndexExists()) { // if alerting-config index doesn't exist, send request to create it + scheduledJobIndices.initScheduledJobIndex(object : ActionListener { + override fun onResponse(response: CreateIndexResponse) { + onCreateMappingsResponse(response.isAcknowledged, indexMonitorRequest, actionListener, user) + } + + override fun onFailure(e: Exception) { + if (ExceptionsHelper.unwrapCause(e) is ResourceAlreadyExistsException) { + scope.launch { + // Wait for the yellow status + val clusterHealthRequest = ClusterHealthRequest() + .indices(SCHEDULED_JOBS_INDEX) + .waitForYellowStatus() + val response: ClusterHealthResponse = client.suspendUntil { + execute(ClusterHealthAction.INSTANCE, clusterHealthRequest, it) + } + if (response.isTimedOut) { + actionListener.onFailure( + OpenSearchException("Cannot determine that the $SCHEDULED_JOBS_INDEX index is healthy") + ) + } + // Retry mapping of monitor + onCreateMappingsResponse(true, indexMonitorRequest, actionListener, user) + } + } else { + actionListener.onFailure(AlertingException.wrap(e)) + } + } + }) + } else if (!IndexUtils.scheduledJobIndexUpdated) { + IndexUtils.updateIndexMapping( + SCHEDULED_JOBS_INDEX, + ScheduledJobIndices.scheduledJobMappings(), clusterService.state(), client.admin().indices(), + object : ActionListener { + override fun onResponse(response: AcknowledgedResponse) { + onUpdateMappingsResponse(response, indexMonitorRequest, actionListener, user) + } + override fun onFailure(t: Exception) { + actionListener.onFailure(AlertingException.wrap(t)) + } + } + ) + } else { + prepareMonitorIndexing(indexMonitorRequest, actionListener, user) + } + } + } + + private fun onCreateMappingsResponse( + isAcknowledged: Boolean, + request: IndexMonitorV2Request, + actionListener: ActionListener, + user: User? + ) { + if (isAcknowledged) { + log.info("Created $SCHEDULED_JOBS_INDEX with mappings.") + prepareMonitorIndexing(request, actionListener, user) + IndexUtils.scheduledJobIndexUpdated() + } else { + log.info("Create $SCHEDULED_JOBS_INDEX mappings call not acknowledged.") + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "Create $SCHEDULED_JOBS_INDEX mappings call not acknowledged", RestStatus.INTERNAL_SERVER_ERROR + ) + ) + ) + } + } + + private fun onUpdateMappingsResponse( + response: AcknowledgedResponse, + indexMonitorRequest: IndexMonitorV2Request, + actionListener: ActionListener, + user: User? + ) { + if (response.isAcknowledged) { + log.info("Updated $SCHEDULED_JOBS_INDEX with mappings.") + IndexUtils.scheduledJobIndexUpdated() + prepareMonitorIndexing(indexMonitorRequest, actionListener, user) + } else { + log.info("Update $SCHEDULED_JOBS_INDEX mappings call not acknowledged.") + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "Updated $SCHEDULED_JOBS_INDEX mappings call not acknowledged.", + RestStatus.INTERNAL_SERVER_ERROR + ) + ) + ) + } + } + + private fun prepareMonitorIndexing( + indexMonitorRequest: IndexMonitorV2Request, + actionListener: ActionListener, + user: User? + ) { + if (indexMonitorRequest.method == RestRequest.Method.PUT) { // update monitor case + scope.launch { + updateMonitor(indexMonitorRequest, actionListener, user) + } + } else { // create monitor case + val query = QueryBuilders.boolQuery().filter(QueryBuilders.existsQuery(MONITOR_V2_TYPE)) + val searchSource = SearchSourceBuilder().query(query).timeout(requestTimeout) + val searchRequest = SearchRequest(SCHEDULED_JOBS_INDEX).source(searchSource) + + client.search( + searchRequest, + object : ActionListener { + override fun onResponse(searchResponse: SearchResponse) { + onMonitorCountSearchResponse(searchResponse, indexMonitorRequest, actionListener, user) + } + + override fun onFailure(t: Exception) { + actionListener.onFailure(AlertingException.wrap(t)) + } + } + ) + } + } + + /* Functions for Update Monitor flow */ + + private suspend fun updateMonitor( + indexMonitorRequest: IndexMonitorV2Request, + actionListener: ActionListener, + user: User? + ) { + val getRequest = GetRequest(SCHEDULED_JOBS_INDEX, indexMonitorRequest.monitorId) + try { + val getResponse: GetResponse = client.suspendUntil { client.get(getRequest, it) } + if (!getResponse.isExists) { + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException("MonitorV2 with ${indexMonitorRequest.monitorId} is not found", RestStatus.NOT_FOUND) + ) + ) + return + } + val xcp = XContentHelper.createParser( + xContentRegistry, LoggingDeprecationHandler.INSTANCE, + getResponse.sourceAsBytesRef, XContentType.JSON + ) + val scheduledJob = ScheduledJob.parse(xcp, getResponse.id, getResponse.version) + + validateMonitorV2(scheduledJob)?.let { + actionListener.onFailure(AlertingException.wrap(it)) + return + } + + val monitorV2 = scheduledJob as MonitorV2 + + onGetMonitorResponseForUpdate(monitorV2, indexMonitorRequest, actionListener, user) + } catch (e: Exception) { + actionListener.onFailure(AlertingException.wrap(e)) + } + } + + private suspend fun onGetMonitorResponseForUpdate( + existingMonitorV2: MonitorV2, + indexMonitorRequest: IndexMonitorV2Request, + actionListener: ActionListener, + user: User? + ) { + log.info("user: $user") + log.info("monitor user: ${existingMonitorV2.user}") + if ( + !checkUserPermissionsWithResource( + user, + existingMonitorV2.user, + actionListener, + "monitor_v2", + indexMonitorRequest.monitorId + ) + ) { + return + } + + var newMonitorV2 = indexMonitorRequest.monitorV2 + + // If both are enabled, use the current existing monitor enabled time, + // otherwise the next execution will be incorrect. + if (newMonitorV2.enabled && existingMonitorV2.enabled) { + newMonitorV2 = newMonitorV2.makeCopy(enabledTime = existingMonitorV2.enabledTime) + } + + /** + * On update monitor check which backend roles to associate to the monitor. + * Below are 2 examples of how the logic works + * + * Example 1, say we have a Monitor with backend roles [a, b, c, d] associated with it. + * If I'm User A (non-admin user) and I have backend roles [a, b, c] associated with me and I make a request to update + * the Monitor's backend roles to [a, b]. This would mean that the roles to remove are [c] and the roles to add are [a, b]. + * The Monitor's backend roles would then be [a, b, d]. + * + * Example 2, say we have a Monitor with backend roles [a, b, c, d] associated with it. + * If I'm User A (admin user) and I have backend roles [a, b, c] associated with me and I make a request to update + * the Monitor's backend roles to [a, b]. This would mean that the roles to remove are [c, d] and the roles to add are [a, b]. + * The Monitor's backend roles would then be [a, b]. + */ + if (user != null) { + if (indexMonitorRequest.rbacRoles != null) { + if (isAdmin(user)) { + newMonitorV2 = newMonitorV2.makeCopy( + user = User(user.name, indexMonitorRequest.rbacRoles, user.roles, user.customAttributes) + ) + } else { + // rolesToRemove: these are the backend roles to remove from the monitor + val rolesToRemove = user.backendRoles - indexMonitorRequest.rbacRoles + // remove the monitor's roles with rolesToRemove and add any roles passed into the request.rbacRoles + val updatedRbac = existingMonitorV2.user?.backendRoles.orEmpty() - rolesToRemove + indexMonitorRequest.rbacRoles + newMonitorV2 = newMonitorV2.makeCopy( + user = User(user.name, updatedRbac, user.roles, user.customAttributes) + ) + } + } else { + newMonitorV2 = newMonitorV2 + .makeCopy(user = User(user.name, existingMonitorV2.user!!.backendRoles, user.roles, user.customAttributes)) + } + log.info("Update monitor backend roles to: ${newMonitorV2.user?.backendRoles}") + } + + newMonitorV2 = newMonitorV2.makeCopy(schemaVersion = IndexUtils.scheduledJobIndexSchemaVersion) + val indexRequest = IndexRequest(SCHEDULED_JOBS_INDEX) + .setRefreshPolicy(indexMonitorRequest.refreshPolicy) + .source(newMonitorV2.toXContentWithUser(jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + .id(indexMonitorRequest.monitorId) + .routing(indexMonitorRequest.monitorId) + .timeout(indexTimeout) + + log.info( + "Updating monitor, ${existingMonitorV2.id}, from: ${existingMonitorV2.toXContentWithUser( + jsonBuilder(), + ToXContent.MapParams(mapOf("with_type" to "true")) + )} \n to: ${newMonitorV2.toXContentWithUser(jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))}" + ) + + try { + val indexResponse: IndexResponse = client.suspendUntil { client.index(indexRequest, it) } + val failureReasons = IndexUtils.checkShardsFailure(indexResponse) + if (failureReasons != null) { + actionListener.onFailure( + AlertingException.wrap(OpenSearchStatusException(failureReasons.toString(), indexResponse.status())) + ) + return + } + + actionListener.onResponse( + IndexMonitorV2Response( + indexResponse.id, indexResponse.version, indexResponse.seqNo, + indexResponse.primaryTerm, newMonitorV2 + ) + ) + } catch (e: Exception) { + actionListener.onFailure(AlertingException.wrap(e)) + } + } + + /* Functions for Create Monitor flow */ + + /** + * After searching for all existing monitors we validate the system can support another monitor to be created. + */ + private fun onMonitorCountSearchResponse( + monitorCountSearchResponse: SearchResponse, + indexMonitorRequest: IndexMonitorV2Request, + actionListener: ActionListener, + user: User? + ) { + val totalHits = monitorCountSearchResponse.hits.totalHits?.value + if (totalHits != null && totalHits >= maxMonitors) { + log.info("This request would create more than the allowed monitors [$maxMonitors].") + actionListener.onFailure( + AlertingException.wrap( + IllegalArgumentException( + "This request would create more than the allowed monitors [$maxMonitors]." + ) + ) + ) + } else { + scope.launch { + indexMonitor(indexMonitorRequest, actionListener, user) + } + } + } + + private suspend fun indexMonitor( + indexMonitorRequest: IndexMonitorV2Request, + actionListener: ActionListener, + user: User? + ) { + var monitorV2 = indexMonitorRequest.monitorV2 + + if (user != null) { + // Use the backend roles which is an intersection of the requested backend roles and the user's backend roles. + // Admins can pass in any backend role. Also if no backend role is passed in, all the user's backend roles are used. + val rbacRoles = if (indexMonitorRequest.rbacRoles == null) user.backendRoles.toSet() + else if (!isAdmin(user)) indexMonitorRequest.rbacRoles.intersect(user.backendRoles).toSet() + else indexMonitorRequest.rbacRoles + + monitorV2 = monitorV2.makeCopy( + user = User(user.name, rbacRoles.toList(), user.roles, user.customAttributes) + ) + + log.debug("Created monitor's backend roles: $rbacRoles") + } + + val indexRequest = IndexRequest(SCHEDULED_JOBS_INDEX) + .setRefreshPolicy(indexMonitorRequest.refreshPolicy) + .source(monitorV2.toXContentWithUser(jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + .routing(indexMonitorRequest.monitorId) + .setIfSeqNo(indexMonitorRequest.seqNo) + .setIfPrimaryTerm(indexMonitorRequest.primaryTerm) + .timeout(indexTimeout) + + log.info( + "Creating new monitorV2: ${monitorV2.toXContentWithUser( + jsonBuilder(), + ToXContent.MapParams(mapOf("with_type" to "true")) + )}" + ) + + try { + val indexResponse: IndexResponse = client.suspendUntil { client.index(indexRequest, it) } + val failureReasons = IndexUtils.checkShardsFailure(indexResponse) + if (failureReasons != null) { + log.info(failureReasons.toString()) + actionListener.onFailure( + AlertingException.wrap(OpenSearchStatusException(failureReasons.toString(), indexResponse.status())) + ) + return + } + + actionListener.onResponse( + IndexMonitorV2Response( + indexResponse.id, indexResponse.version, indexResponse.seqNo, + indexResponse.primaryTerm, monitorV2 + ) + ) + } catch (t: Exception) { + actionListener.onFailure(AlertingException.wrap(t)) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportSearchMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportSearchMonitorV2Action.kt new file mode 100644 index 000000000..6a36e2874 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportSearchMonitorV2Action.kt @@ -0,0 +1,122 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.transportv2 + +import org.apache.logging.log4j.LogManager +import org.opensearch.OpenSearchStatusException +import org.opensearch.action.search.SearchResponse +import org.opensearch.action.support.ActionFilters +import org.opensearch.action.support.HandledTransportAction +import org.opensearch.alerting.AlertingV2Utils.getEmptySearchResponse +import org.opensearch.alerting.AlertingV2Utils.isIndexNotFoundException +import org.opensearch.alerting.actionv2.SearchMonitorV2Action +import org.opensearch.alerting.actionv2.SearchMonitorV2Request +import org.opensearch.alerting.core.settings.AlertingV2Settings.Companion.ALERTING_V2_ENABLED +import org.opensearch.alerting.modelv2.MonitorV2.Companion.MONITOR_V2_TYPE +import org.opensearch.alerting.modelv2.PPLSQLMonitor.Companion.PPL_SQL_MONITOR_TYPE +import org.opensearch.alerting.opensearchapi.addFilter +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.alerting.transport.SecureTransportAction +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.inject.Inject +import org.opensearch.common.settings.Settings +import org.opensearch.commons.alerting.util.AlertingException +import org.opensearch.core.action.ActionListener +import org.opensearch.core.common.io.stream.NamedWriteableRegistry +import org.opensearch.core.rest.RestStatus +import org.opensearch.index.query.BoolQueryBuilder +import org.opensearch.index.query.QueryBuilders +import org.opensearch.tasks.Task +import org.opensearch.transport.TransportService +import org.opensearch.transport.client.Client + +private val log = LogManager.getLogger(TransportSearchMonitorV2Action::class.java) + +/** + * Transport action that contains the core logic for searching monitor V2s via an OpenSearch search query. + * + * @opensearch.experimental + */ +class TransportSearchMonitorV2Action @Inject constructor( + transportService: TransportService, + val settings: Settings, + val client: Client, + clusterService: ClusterService, + actionFilters: ActionFilters, + val namedWriteableRegistry: NamedWriteableRegistry +) : HandledTransportAction( + SearchMonitorV2Action.NAME, transportService, actionFilters, ::SearchMonitorV2Request +), + SecureTransportAction { + + @Volatile private var alertingV2Enabled = ALERTING_V2_ENABLED.get(settings) + + @Volatile + override var filterByEnabled: Boolean = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) + + init { + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_ENABLED) { alertingV2Enabled = it } + listenFilterBySettingChange(clusterService) + } + + override fun doExecute(task: Task, request: SearchMonitorV2Request, actionListener: ActionListener) { + if (!alertingV2Enabled) { + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "Alerting V2 is currently disabled, please enable it with the " + + "cluster setting: ${ALERTING_V2_ENABLED.key}", + RestStatus.FORBIDDEN + ), + ) + ) + return + } + + val searchSourceBuilder = request.searchRequest.source() + + val queryBuilder = if (searchSourceBuilder.query() == null) BoolQueryBuilder() + else QueryBuilders.boolQuery().must(searchSourceBuilder.query()) + + // filter out MonitorV1s in the alerting config index + // only return MonitorV2s that match the user-given search query + queryBuilder.filter(QueryBuilders.existsQuery(MONITOR_V2_TYPE)) + + searchSourceBuilder.query(queryBuilder) + .seqNoAndPrimaryTerm(true) + .version(true) + + val user = readUserFromThreadContext(client) + client.threadPool().threadContext.stashContext().use { + // if user is null, security plugin is disabled or user is super-admin + // if doFilterForUser() is false, security is enabled but filterby is disabled + if (user != null && doFilterForUser(user)) { + log.info("Filtering result by: ${user.backendRoles}") + addFilter(user, request.searchRequest.source(), "$MONITOR_V2_TYPE.$PPL_SQL_MONITOR_TYPE.user.backend_roles.keyword") + } + + client.search( + request.searchRequest, + object : ActionListener { + override fun onResponse(response: SearchResponse) { + actionListener.onResponse(response) + } + + override fun onFailure(e: Exception) { + if (isIndexNotFoundException(e)) { + log.error("Index not found while searching monitor", e) + val emptyResponse = getEmptySearchResponse() + actionListener.onResponse(emptyResponse) + } else { + log.error("Unexpected error while searching monitor", e) + actionListener.onFailure(AlertingException.wrap(e)) + } + } + } + ) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/IndexUtils.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/IndexUtils.kt index 093b0bd39..b388ae757 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/IndexUtils.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/IndexUtils.kt @@ -6,9 +6,11 @@ package org.opensearch.alerting.util import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest +import org.opensearch.action.index.IndexResponse import org.opensearch.action.support.IndicesOptions import org.opensearch.action.support.clustermanager.AcknowledgedResponse import org.opensearch.alerting.alerts.AlertIndices +import org.opensearch.alerting.alertsv2.AlertV2Indices import org.opensearch.alerting.comments.CommentsIndices import org.opensearch.alerting.core.ScheduledJobIndices import org.opensearch.cluster.ClusterState @@ -38,6 +40,8 @@ class IndexUtils { private set var alertingCommentIndexSchemaVersion: Int private set + var alertV2IndexSchemaVersion: Int + private set var scheduledJobIndexUpdated: Boolean = false private set @@ -47,15 +51,20 @@ class IndexUtils { private set var commentsIndexUpdated: Boolean = false private set + var alertV2IndexUpdated: Boolean = false + private set + var lastUpdatedAlertHistoryIndex: String? = null var lastUpdatedFindingHistoryIndex: String? = null var lastUpdatedCommentsHistoryIndex: String? = null + var lastUpdatedAlertV2HistoryIndex: String? = null init { scheduledJobIndexSchemaVersion = getSchemaVersion(ScheduledJobIndices.scheduledJobMappings()) alertIndexSchemaVersion = getSchemaVersion(AlertIndices.alertMapping()) findingIndexSchemaVersion = getSchemaVersion(AlertIndices.findingMapping()) alertingCommentIndexSchemaVersion = getSchemaVersion(CommentsIndices.commentsMapping()) + alertV2IndexSchemaVersion = getSchemaVersion(AlertV2Indices.alertV2Mapping()) } @JvmStatic @@ -78,6 +87,11 @@ class IndexUtils { commentsIndexUpdated = true } + @JvmStatic + fun alertV2IndexUpdated() { + commentsIndexUpdated = true + } + @JvmStatic fun getSchemaVersion(mapping: String): Int { val xcp = XContentType.JSON.xContent().createParser( @@ -205,5 +219,18 @@ class IndexUtils { fun getCreationDateForIndex(index: String, clusterState: ClusterState): Long { return clusterState.metadata.index(index).creationDate } + + @JvmStatic + fun checkShardsFailure(response: IndexResponse): String? { + val failureReasons = StringBuilder() + if (response.shardInfo.failed > 0) { + response.shardInfo.failures.forEach { + entry -> + failureReasons.append(entry.reason()) + } + return failureReasons.toString() + } + return null + } } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/workflow/WorkflowRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/workflow/WorkflowRunner.kt index ea24da3a6..56a708444 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/workflow/WorkflowRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/workflow/WorkflowRunner.kt @@ -5,36 +5,21 @@ package org.opensearch.alerting.workflow -import org.opensearch.OpenSearchSecurityException +import org.opensearch.alerting.AlertingV2Utils.getConfigAndSendNotification import org.opensearch.alerting.MonitorRunnerExecutionContext import org.opensearch.alerting.MonitorRunnerService -import org.opensearch.alerting.action.GetDestinationsAction -import org.opensearch.alerting.action.GetDestinationsRequest -import org.opensearch.alerting.action.GetDestinationsResponse -import org.opensearch.alerting.model.destination.Destination import org.opensearch.alerting.opensearchapi.InjectorContextElement -import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.opensearchapi.withClosableContext import org.opensearch.alerting.script.ChainedAlertTriggerExecutionContext -import org.opensearch.alerting.util.destinationmigration.NotificationActionConfigs -import org.opensearch.alerting.util.destinationmigration.NotificationApiUtils -import org.opensearch.alerting.util.destinationmigration.getTitle -import org.opensearch.alerting.util.destinationmigration.publishLegacyNotification -import org.opensearch.alerting.util.destinationmigration.sendNotification -import org.opensearch.alerting.util.isAllowed -import org.opensearch.alerting.util.isTestAction import org.opensearch.alerting.util.use import org.opensearch.commons.alerting.model.ActionRunResult -import org.opensearch.commons.alerting.model.Table import org.opensearch.commons.alerting.model.Workflow import org.opensearch.commons.alerting.model.WorkflowRunResult import org.opensearch.commons.alerting.model.action.Action -import org.opensearch.commons.notifications.model.NotificationConfigInfo import org.opensearch.core.common.Strings import org.opensearch.script.Script import org.opensearch.script.TemplateScript import org.opensearch.transport.TransportService -import org.opensearch.transport.client.node.NodeClient import java.time.Instant abstract class WorkflowRunner { @@ -93,107 +78,6 @@ abstract class WorkflowRunner { } } - protected suspend fun getConfigAndSendNotification( - action: Action, - monitorCtx: MonitorRunnerExecutionContext, - subject: String?, - message: String - ): String { - val config = getConfigForNotificationAction(action, monitorCtx) - if (config.destination == null && config.channel == null) { - throw IllegalStateException("Unable to find a Notification Channel or Destination config with id [${action.destinationId}]") - } - - // Adding a check on TEST_ACTION Destination type here to avoid supporting it as a LegacyBaseMessage type - // just for Alerting integration tests - if (config.destination?.isTestAction() == true) { - return "test action" - } - - if (config.destination?.isAllowed(monitorCtx.allowList) == false) { - throw IllegalStateException( - "Monitor contains a Destination type that is not allowed: ${config.destination.type}" - ) - } - - var actionResponseContent = "" - actionResponseContent = config.channel - ?.sendNotification( - monitorCtx.client!!, - config.channel.getTitle(subject), - message - ) ?: actionResponseContent - - actionResponseContent = config.destination - ?.buildLegacyBaseMessage(subject, message, monitorCtx.destinationContextFactory!!.getDestinationContext(config.destination)) - ?.publishLegacyNotification(monitorCtx.client!!) - ?: actionResponseContent - - return actionResponseContent - } - - /** - * The "destination" ID referenced in a Monitor Action could either be a Notification config or a Destination config - * depending on whether the background migration process has already migrated it from a Destination to a Notification config. - * - * To cover both of these cases, the Notification config will take precedence and if it is not found, the Destination will be retrieved. - */ - private suspend fun getConfigForNotificationAction( - action: Action, - monitorCtx: MonitorRunnerExecutionContext - ): NotificationActionConfigs { - var destination: Destination? = null - var notificationPermissionException: Exception? = null - - var channel: NotificationConfigInfo? = null - try { - channel = NotificationApiUtils.getNotificationConfigInfo(monitorCtx.client as NodeClient, action.destinationId) - } catch (e: OpenSearchSecurityException) { - notificationPermissionException = e - } - - // If the channel was not found, try to retrieve the Destination - if (channel == null) { - destination = try { - val table = Table( - "asc", - "destination.name.keyword", - null, - 1, - 0, - null - ) - val getDestinationsRequest = GetDestinationsRequest( - action.destinationId, - 0L, - null, - table, - "ALL" - ) - - val getDestinationsResponse: GetDestinationsResponse = monitorCtx.client!!.suspendUntil { - monitorCtx.client!!.execute(GetDestinationsAction.INSTANCE, getDestinationsRequest, it) - } - getDestinationsResponse.destinations.firstOrNull() - } catch (e: IllegalStateException) { - // Catching the exception thrown when the Destination was not found so the NotificationActionConfigs object can be returned - null - } catch (e: OpenSearchSecurityException) { - if (notificationPermissionException != null) { - throw notificationPermissionException - } else { - throw e - } - } - - if (destination == null && notificationPermissionException != null) { - throw notificationPermissionException - } - } - - return NotificationActionConfigs(destination, channel) - } - internal fun compileTemplate(template: Script, ctx: ChainedAlertTriggerExecutionContext): String { return MonitorRunnerService.monitorCtx.scriptService!!.compile(template, TemplateScript.CONTEXT) .newInstance(template.params + mapOf("ctx" to ctx.asTemplateArg())) diff --git a/alerting/src/main/resources/org/opensearch/alerting/alertsv2/alert_v2_mapping.json b/alerting/src/main/resources/org/opensearch/alerting/alertsv2/alert_v2_mapping.json new file mode 100644 index 000000000..5543a289c --- /dev/null +++ b/alerting/src/main/resources/org/opensearch/alerting/alertsv2/alert_v2_mapping.json @@ -0,0 +1,118 @@ +{ + "dynamic": "strict", + "_routing": { + "required": true + }, + "_meta" : { + "schema_version": 1 + }, + "properties": { + "schema_version": { + "type": "integer" + }, + "monitor_v2_id": { + "type": "keyword" + }, + "monitor_v2_version": { + "type": "long" + }, + "id": { + "type": "keyword" + }, + "version": { + "type": "long" + }, + "severity": { + "type": "keyword" + }, + "monitor_v2_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "monitor_v2_user": { + "properties": { + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "backend_roles": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "roles": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "custom_attribute_names": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + } + } + }, + "execution_id": { + "type": "keyword" + }, + "trigger_v2_id": { + "type": "keyword" + }, + "trigger_v2_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "triggered_time": { + "type": "date" + }, + "error_message": { + "type": "text" + }, + "query": { + "type": "text" + }, + "query_results": { + "type": "nested", + "properties": { + "schema": { + "type": "nested", + "dynamic": true + }, + "datarows": { + "type": "object", + "enabled": false + }, + "total": { + "type": "integer" + }, + "size": { + "type": "integer" + } + } + } + } +} \ No newline at end of file diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/AccessRoles.kt b/alerting/src/test/kotlin/org/opensearch/alerting/AccessRoles.kt index d14473884..133504168 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/AccessRoles.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/AccessRoles.kt @@ -9,6 +9,7 @@ import org.opensearch.alerting.action.ExecuteWorkflowAction import org.opensearch.commons.alerting.action.AlertingActions val ALL_ACCESS_ROLE = "all_access" +val PPL_FULL_ACCESS_ROLE = "ppl_full_access" val READALL_AND_MONITOR_ROLE = "readall_and_monitor" val ALERTING_FULL_ACCESS_ROLE = "alerting_full_access" val ALERTING_ACK_ALERTS_ROLE = "alerting_ack_alerts" @@ -31,6 +32,7 @@ val ALERTING_GET_ALERTS_ACCESS = "alerting_get_alerts_access" val ALERTING_INDEX_WORKFLOW_ACCESS = "alerting_index_workflow_access" val ROLE_TO_PERMISSION_MAPPING = mapOf( + ALL_ACCESS_ROLE to "*", ALERTING_NO_ACCESS_ROLE to "", ALERTING_GET_EMAIL_ACCOUNT_ACCESS to "cluster:admin/opendistro/alerting/destination/email_account/get", ALERTING_SEARCH_EMAIL_ACCOUNT_ACCESS to "cluster:admin/opendistro/alerting/destination/email_account/search", diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt b/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt index 77ff20006..3a5953b00 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt @@ -17,8 +17,10 @@ import org.opensearch.action.search.SearchResponse import org.opensearch.alerting.AlertingPlugin.Companion.COMMENTS_BASE_URI import org.opensearch.alerting.AlertingPlugin.Companion.EMAIL_ACCOUNT_BASE_URI import org.opensearch.alerting.AlertingPlugin.Companion.EMAIL_GROUP_BASE_URI +import org.opensearch.alerting.AlertingPlugin.Companion.MONITOR_V2_BASE_URI import org.opensearch.alerting.alerts.AlertIndices import org.opensearch.alerting.alerts.AlertIndices.Companion.FINDING_HISTORY_WRITE_INDEX +import org.opensearch.alerting.alertsv2.AlertV2Indices import org.opensearch.alerting.core.settings.ScheduledJobSettings import org.opensearch.alerting.model.destination.Chime import org.opensearch.alerting.model.destination.CustomWebhook @@ -26,6 +28,9 @@ import org.opensearch.alerting.model.destination.Destination import org.opensearch.alerting.model.destination.Slack import org.opensearch.alerting.model.destination.email.EmailAccount import org.opensearch.alerting.model.destination.email.EmailGroup +import org.opensearch.alerting.modelv2.AlertV2 +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.modelv2.PPLSQLMonitor import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.settings.DestinationSettings import org.opensearch.alerting.util.DestinationType @@ -66,23 +71,25 @@ import org.opensearch.core.xcontent.ToXContent import org.opensearch.core.xcontent.XContentBuilder import org.opensearch.core.xcontent.XContentParser import org.opensearch.core.xcontent.XContentParserUtils +import org.opensearch.index.query.QueryBuilders import org.opensearch.search.SearchModule import org.opensearch.search.builder.SearchSourceBuilder +import org.opensearch.test.OpenSearchTestCase import java.net.URLEncoder import java.nio.file.Files import java.time.Instant import java.time.ZonedDateTime import java.time.format.DateTimeFormatter import java.time.temporal.ChronoUnit +import java.time.temporal.ChronoUnit.MILLIS import java.util.Locale import java.util.UUID +import java.util.concurrent.TimeUnit import java.util.stream.Collectors import javax.management.MBeanServerInvocationHandler import javax.management.ObjectName import javax.management.remote.JMXConnectorFactory import javax.management.remote.JMXServiceURL -import kotlin.collections.ArrayList -import kotlin.collections.HashMap /** * Superclass for tests that interact with an external test cluster using OpenSearch's RestClient @@ -103,6 +110,7 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return NamedXContentRegistry( mutableListOf( Monitor.XCONTENT_REGISTRY, + MonitorV2.XCONTENT_REGISTRY, SearchInput.XCONTENT_REGISTRY, DocLevelMonitorInput.XCONTENT_REGISTRY, QueryLevelTrigger.XCONTENT_REGISTRY, @@ -129,6 +137,17 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return StringEntity(jsonString, APPLICATION_JSON) } + private fun createMonitorV2EntityWithBackendRoles(monitorV2: MonitorV2, rbacRoles: List?): HttpEntity { + if (rbacRoles == null) { + return monitorV2.toHttpEntity() + } + val temp = monitorV2.toJsonString() + val toReplace = temp.lastIndexOf("}") + val rbacString = rbacRoles.joinToString { "\"$it\"" } + val jsonString = temp.substring(0, toReplace) + ", \"rbac_roles\": [$rbacString] }" + return StringEntity(jsonString, APPLICATION_JSON) + } + protected fun createMonitorWithClient( client: RestClient, monitor: Monitor, @@ -150,10 +169,47 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return getMonitor(monitorId = monitorJson["_id"] as String) } + protected fun createMonitorV2WithClient( + client: RestClient, + monitorV2: MonitorV2, + rbacRoles: List? = null + ): MonitorV2 { + // every random ppl monitor's query searches index TEST_INDEX_NAME + // by default, so create that first before creating the monitor + if (!indexExists(TEST_INDEX_NAME)) { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + } + + // be sure to use the passed in client to send the create monitor request, + // as the user stored in this client is the user whose permissions we want + // to test, not client()'s admin level user + val response = client.makeRequest( + "POST", MONITOR_V2_BASE_URI, emptyMap(), + createMonitorV2EntityWithBackendRoles(monitorV2, rbacRoles) + ) + assertEquals("Unable to create a new monitor v2", RestStatus.OK, response.restStatus()) + + val monitorV2Json = jsonXContent.createParser( + NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, + response.entity.content + ).map() + assertUserNull(monitorV2Json as HashMap) + + return getMonitorV2(monitorV2Id = monitorV2Json["_id"] as String) + } + protected fun createMonitor(monitor: Monitor, refresh: Boolean = true): Monitor { return createMonitorWithClient(client(), monitor, emptyList(), refresh) } + protected fun createMonitorV2(monitorV2: MonitorV2): MonitorV2 { + val client = client() + val response = client.makeRequest("POST", MONITOR_V2_BASE_URI, emptyMap(), monitorV2.toHttpEntity()) + assertEquals("Unable to create a new monitor", RestStatus.OK, response.restStatus()) + + return getMonitorV2(monitorV2Id = response.asMap()["_id"] as String) + } + protected fun deleteMonitor(monitor: Monitor, refresh: Boolean = true): Response { val response = client().makeRequest( "DELETE", "$ALERTING_BASE_URI/${monitor.id}?refresh=$refresh", emptyMap(), @@ -164,6 +220,15 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return response } + protected fun deleteMonitorV2(monitorV2Id: String): Response { + val response = client().makeRequest( + "DELETE", "$MONITOR_V2_BASE_URI/$monitorV2Id?refresh=true", emptyMap() + ) + assertEquals("Unable to delete a monitor", RestStatus.OK, response.restStatus()) + + return response + } + protected fun deleteWorkflow(workflow: Workflow, deleteDelegates: Boolean = false, refresh: Boolean = true): Response { val response = client().makeRequest( "DELETE", @@ -535,6 +600,19 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return getMonitor(monitorId = monitorId) } + protected fun createRandomPPLMonitor(pplMonitorConfig: PPLSQLMonitor = randomPPLMonitor()): PPLSQLMonitor { + // every random ppl monitor's query searches index TEST_INDEX_NAME + // by default, so create that first before creating the monitor + val indexExistsResponse = adminClient().makeRequest("HEAD", TEST_INDEX_NAME) + if (indexExistsResponse.restStatus() == RestStatus.NOT_FOUND) { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + } + logger.info("ppl monitor: $pplMonitorConfig") + + val pplMonitorId = createMonitorV2(pplMonitorConfig).id + return getMonitorV2(monitorV2Id = pplMonitorId) as PPLSQLMonitor + } + protected fun createRandomDocumentMonitor(refresh: Boolean = false, withMetadata: Boolean = false): Monitor { val monitor = randomDocumentLevelMonitor(withMetadata = withMetadata) val monitorId = createMonitor(monitor, refresh).id @@ -568,6 +646,16 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return getWorkflow(workflowId = workflow.id) } + @Suppress("UNCHECKED_CAST") + protected fun updateMonitorV2(monitorV2: MonitorV2, refresh: Boolean = false): MonitorV2 { + val response = client().makeRequest( + "PUT", "$MONITOR_V2_BASE_URI/${monitorV2.id}?refresh=$refresh", + emptyMap(), monitorV2.toHttpEntity() + ) + assertEquals("Unable to update a monitorV2", RestStatus.OK, response.restStatus()) + return getMonitorV2(monitorV2Id = monitorV2.id) + } + protected fun updateMonitorWithClient( client: RestClient, monitor: Monitor, @@ -635,6 +723,33 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return monitor.copy(id = id, version = version) } + protected fun getMonitorV2( + monitorV2Id: String, + header: BasicHeader = BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ): MonitorV2 { + val response = client().makeRequest("GET", "$MONITOR_V2_BASE_URI/$monitorV2Id", null, header) + assertEquals("Unable to get monitorV2 $monitorV2Id", RestStatus.OK, response.restStatus()) + + val parser = createParser(XContentType.JSON.xContent(), response.entity.content) + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser) + + lateinit var id: String + var version: Long = 0 + lateinit var monitorV2: MonitorV2 + + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + parser.nextToken() + + when (parser.currentName()) { + "_id" -> id = parser.text() + "_version" -> version = parser.longValue() + "monitorV2" -> monitorV2 = MonitorV2.parse(parser) + } + } + + return monitorV2.makeCopy(id = id, version = version) + } + // TODO: understand why doc alerts wont work with the normal search Alerts function protected fun searchAlertsWithFilter( monitor: Monitor, @@ -644,9 +759,9 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { if (refresh) refreshIndex(indices) val request = """ - { "version" : true, - "query": { "match_all": {} } - } + { "version" : true, + "query": { "match_all": {} } + } """.trimIndent() val httpResponse = adminClient().makeRequest("GET", "/$indices/_search", StringEntity(request, APPLICATION_JSON)) assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus()) @@ -691,9 +806,9 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { if (refresh) refreshIndex(indices) val request = """ - { "version" : true, - "query": { "match_all": {} } - } + { "version" : true, + "query": { "match_all": {} } + } """.trimIndent() val httpResponse = adminClient().makeRequest("GET", "/$indices/_search", StringEntity(request, APPLICATION_JSON)) assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus()) @@ -716,9 +831,9 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { // If this is a test monitor (it doesn't have an ID) and no alerts will be saved for it. val searchParams = if (monitor.id != Monitor.NO_ID) mapOf("routing" to monitor.id) else mapOf() val request = """ - { "version" : true, - "query" : { "term" : { "${Alert.MONITOR_ID_FIELD}" : "${monitor.id}" } } - } + { "version" : true, + "query" : { "term" : { "${Alert.MONITOR_ID_FIELD}" : "${monitor.id}" } } + } """.trimIndent() val httpResponse = adminClient().makeRequest("GET", "/$indices/_search", searchParams, StringEntity(request, APPLICATION_JSON)) assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus()) @@ -730,6 +845,35 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { } } + protected fun searchAlertV2s( + monitorV2Id: String, + indices: String = AlertV2Indices.ALERT_V2_INDEX, + refresh: Boolean = true + ): List { + try { + if (refresh) refreshIndex(indices) + } catch (e: Exception) { + logger.warn("Could not refresh index $indices because: ${e.message}") + return emptyList() + } + + // If this is a test monitor (it doesn't have an ID) and no alerts will be saved for it. + val searchParams = if (monitorV2Id != MonitorV2.NO_ID) mapOf("routing" to monitorV2Id) else mapOf() + val request = """ + { "version" : true, + "query" : { "term" : { "${AlertV2.MONITOR_V2_ID_FIELD}" : "$monitorV2Id" } } + } + """.trimIndent() + val httpResponse = adminClient().makeRequest("GET", "/$indices/_search", searchParams, StringEntity(request, APPLICATION_JSON)) + assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus()) + + val searchResponse = SearchResponse.fromXContent(createParser(jsonXContent, httpResponse.entity.content)) + return searchResponse.hits.hits.map { + val xcp = createParser(jsonXContent, it.sourceRef) + AlertV2.parse(xcp, it.id, it.version) + } + } + protected fun acknowledgeAlerts(monitor: Monitor, vararg alerts: Alert): Response { val request = XContentFactory.jsonBuilder().startObject() .array("alerts", *alerts.map { it.id }.toTypedArray()) @@ -782,6 +926,17 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return getAlerts(client(), dataMap, header) } + protected fun getAlertV2s(): Response { + val response = client().makeRequest( + "GET", + "$MONITOR_V2_BASE_URI/alerts", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + assertEquals("Get call failed.", RestStatus.OK, response.restStatus()) + return response + } + protected fun refreshIndex(index: String): Response { val response = client().makeRequest("POST", "/$index/_refresh?expand_wildcards=all") assertEquals("Unable to refresh index", RestStatus.OK, response.restStatus()) @@ -843,6 +998,9 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { protected fun executeMonitor(client: RestClient, monitor: Monitor, params: Map = mapOf()): Response = client.makeRequest("POST", "$ALERTING_BASE_URI/_execute", params, monitor.toHttpEntityWithUser()) + protected fun executeMonitorV2(monitorId: String, params: Map = mutableMapOf()): Response = + client().makeRequest("POST", "$MONITOR_V2_BASE_URI/$monitorId/_execute", params) + protected fun searchFindings(params: Map = mutableMapOf()): GetFindingsResponse { var baseEndpoint = "${AlertingPlugin.FINDING_BASE_URI}/_search?" @@ -880,9 +1038,9 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { protected fun searchMonitors(): SearchResponse { var baseEndpoint = "${AlertingPlugin.MONITOR_BASE_URI}/_search?" val request = """ - { "version" : true, - "query": { "match_all": {} } - } + { "version" : true, + "query": { "match_all": {} } + } """.trimIndent() val httpResponse = adminClient().makeRequest("POST", baseEndpoint, StringEntity(request, APPLICATION_JSON)) assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus()) @@ -1215,9 +1373,9 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(twoMinsAgo) val testDoc = """ { - "test_strict_date_time": "$testTime", - "test_field": "$value", - "number": "$i" + "test_strict_date_time": "$testTime", + "test_field": "$value", + "number": "$i" } """.trimIndent() // Indexing documents with deterministic doc id to allow for easy selected deletion during testing @@ -1231,9 +1389,9 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(time) val testDoc = """ { - "test_strict_date_time": "$testTime", - "test_field": "$value", - "number": "$i" + "test_strict_date_time": "$testTime", + "test_field": "$value", + "number": "$i" } """.trimIndent() // Indexing documents with deterministic doc id to allow for easy selected deletion during testing @@ -1250,9 +1408,9 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(time) val testDoc = """ { - "test_strict_date_time": "$testTime", - "test_field": "$value", - "number": "$i" + "test_strict_date_time": "$testTime", + "test_field": "$value", + "number": "$i" } """.trimIndent() // Indexing documents with deterministic doc id to allow for easy selected deletion during testing @@ -1282,6 +1440,14 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { createIndex(encodedHistoryIndex, settings, mappingHack, "\"${AlertIndices.FINDING_HISTORY_WRITE_INDEX}\" : {}") } + fun putAlertV2Mappings(mapping: String? = null) { + val mappingHack = if (mapping != null) mapping else AlertV2Indices.alertV2Mapping().trimStart('{').trimEnd('}') + val encodedHistoryIndex = URLEncoder.encode(AlertV2Indices.ALERT_V2_HISTORY_INDEX_PATTERN, Charsets.UTF_8.toString()) + val settings = Settings.builder().put("index.hidden", true).build() + createIndex(AlertV2Indices.ALERT_V2_INDEX, settings, mappingHack) + createIndex(encodedHistoryIndex, settings, mappingHack, "\"${AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX}\" : {}") + } + fun scheduledJobMappings(): String { return javaClass.classLoader.getResource("mappings/scheduled-jobs.json").readText() } @@ -1318,6 +1484,23 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return shuffleXContent(toXContentWithUser(builder, ToXContent.EMPTY_PARAMS)).string() } + protected fun MonitorV2.toHttpEntity(): HttpEntity { + return StringEntity(toJsonString(), APPLICATION_JSON) + } + + private fun MonitorV2.toJsonString(): String { + return shuffleXContent(toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)).string() + } + + protected fun MonitorV2.toHttpEntityWithUser(): HttpEntity { + return StringEntity(toJsonStringWithUser(), APPLICATION_JSON) + } + + private fun MonitorV2.toJsonStringWithUser(): String { + val builder = jsonBuilder() + return shuffleXContent(toXContentWithUser(builder, ToXContent.EMPTY_PARAMS)).string() + } + protected fun Destination.toHttpEntity(): HttpEntity { return StringEntity(toJsonString(), APPLICATION_JSON) } @@ -1433,8 +1616,9 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return map[key] } - fun getAlertingStats(metrics: String = ""): Map { - val monitorStatsResponse = client().makeRequest("GET", "/_plugins/_alerting/stats$metrics") + fun getAlertingStats(metrics: String = "", alertingVersion: String? = null): Map { + val endpoint = "/_plugins/_alerting/stats$metrics${alertingVersion?.let { "?version=$it" }.orEmpty()}" + val monitorStatsResponse = client().makeRequest("GET", endpoint) val responseMap = createParser(XContentType.JSON.xContent(), monitorStatsResponse.entity.content).map() return responseMap } @@ -1512,11 +1696,13 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { val customAttributesString = customAttributes.entries.joinToString(prefix = "{", separator = ", ", postfix = "}") { "\"${it.key}\": \"${it.value}\"" } - var entity = " {\n" + - "\"password\": \"$password\",\n" + - "\"backend_roles\": [$broles],\n" + - "\"attributes\": $customAttributesString\n" + - "} " + var entity = """ + { + "password": "$password", + "backend_roles": [$broles], + "attributes": $customAttributesString + } + """.trimIndent() request.setJsonEntity(entity) client().performRequest(request) } @@ -1524,61 +1710,52 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { fun patchUserBackendRoles(name: String, backendRoles: Array) { val request = Request("PATCH", "/_plugins/_security/api/internalusers/$name") val broles = backendRoles.joinToString { "\"$it\"" } - var entity = " [{\n" + - "\"op\": \"replace\",\n" + - "\"path\": \"/backend_roles\",\n" + - "\"value\": [$broles]\n" + - "}]" + var entity = """ + [{ + "op": "replace", + "path": "/backend_roles", + "value": [$broles] + }] + """.trimIndent() request.setJsonEntity(entity) client().performRequest(request) } fun createIndexRole(name: String, index: String) { val request = Request("PUT", "/_plugins/_security/api/roles/$name") - var entity = "{\n" + - "\"cluster_permissions\": [\n" + - "],\n" + - "\"index_permissions\": [\n" + - "{\n" + - "\"index_patterns\": [\n" + - "\"$index\"\n" + - "],\n" + - "\"dls\": \"\",\n" + - "\"fls\": [],\n" + - "\"masked_fields\": [],\n" + - "\"allowed_actions\": [\n" + - "\"crud\"\n" + - "]\n" + - "}\n" + - "],\n" + - "\"tenant_permissions\": []\n" + - "}" + var entity = """ + { + "cluster_permissions": [], + "index_permissions": [{ + "index_patterns": ["$index"], + "dls": "", + "fls": [], + "masked_fields": [], + "allowed_actions": ["crud"] + }], + "tenant_permissions": [] + } + """.trimIndent() request.setJsonEntity(entity) client().performRequest(request) } fun createCustomIndexRole(name: String, index: String, clusterPermissions: String?) { val request = Request("PUT", "/_plugins/_security/api/roles/$name") - val clusterPermissionsArray = if (clusterPermissions.isNullOrBlank()) "" else "\"$clusterPermissions\"" - var entity = "{\n" + - "\"cluster_permissions\": [\n" + - "$clusterPermissionsArray\n" + - "],\n" + - "\"index_permissions\": [\n" + - "{\n" + - "\"index_patterns\": [\n" + - "\"$index\"\n" + - "],\n" + - "\"dls\": \"\",\n" + - "\"fls\": [],\n" + - "\"masked_fields\": [],\n" + - "\"allowed_actions\": [\n" + - "\"crud\"\n" + - "]\n" + - "}\n" + - "],\n" + - "\"tenant_permissions\": []\n" + - "}" + val clusterPerms = if (clusterPermissions.isNullOrEmpty()) "[]" else "[\"$clusterPermissions\"]" + var entity = """ + { + "cluster_permissions": $clusterPerms, + "index_permissions": [{ + "index_patterns": ["$index"], + "dls": "", + "fls": [], + "masked_fields": [], + "allowed_actions": ["crud"] + }], + "tenant_permissions": [] + } + """.trimIndent() request.setJsonEntity(entity) client().performRequest(request) } @@ -1587,55 +1764,43 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { val request = Request("PUT", "/_plugins/_security/api/roles/$name") val clusterPermissionsStr = - clusterPermissions.stream() - .filter { p -> !p.isNullOrBlank() } - .map { p: String? -> "\"" + p + "\"" } - .collect(Collectors.joining(",")) - - var entity = "{\n" + - "\"cluster_permissions\": [\n" + - "$clusterPermissionsStr\n" + - "],\n" + - "\"index_permissions\": [\n" + - "{\n" + - "\"index_patterns\": [\n" + - "\"$index\"\n" + - "],\n" + - "\"dls\": \"\",\n" + - "\"fls\": [],\n" + - "\"masked_fields\": [],\n" + - "\"allowed_actions\": [\n" + - "\"crud\"\n" + - "]\n" + - "}\n" + - "],\n" + - "\"tenant_permissions\": []\n" + - "}" + clusterPermissions.stream().map { p: String? -> "\"" + p + "\"" }.collect( + Collectors.joining(",") + ) + + var entity = """ + { + "cluster_permissions": [$clusterPermissionsStr], + "index_permissions": [{ + "index_patterns": ["$index"], + "dls": "", + "fls": [], + "masked_fields": [], + "allowed_actions": ["crud"] + }], + "tenant_permissions": [] + } + """.trimIndent() request.setJsonEntity(entity) client().performRequest(request) } fun createIndexRoleWithDocLevelSecurity(name: String, index: String, dlsQuery: String, clusterPermissions: String? = "") { val request = Request("PUT", "/_plugins/_security/api/roles/$name") - var entity = "{\n" + - "\"cluster_permissions\": [\n" + - "\"$clusterPermissions\"\n" + - "],\n" + - "\"index_permissions\": [\n" + - "{\n" + - "\"index_patterns\": [\n" + - "\"$index\"\n" + - "],\n" + - "\"dls\": \"$dlsQuery\",\n" + - "\"fls\": [],\n" + - "\"masked_fields\": [],\n" + - "\"allowed_actions\": [\n" + - "\"crud\"\n" + - "]\n" + - "}\n" + - "],\n" + - "\"tenant_permissions\": []\n" + - "}" + val clusterPerms = if (clusterPermissions.isNullOrEmpty()) "[]" else "[\"$clusterPermissions\"]" + var entity = """ + { + "cluster_permissions": $clusterPerms, + "index_permissions": [{ + "index_patterns": ["$index"], + "dls": "$dlsQuery", + "fls": [], + "masked_fields": [], + "allowed_actions": ["crud"] + }], + "tenant_permissions": [] + } + """.trimIndent() request.setJsonEntity(entity) client().performRequest(request) } @@ -1647,25 +1812,19 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { ) val request = Request("PUT", "/_plugins/_security/api/roles/$name") - var entity = "{\n" + - "\"cluster_permissions\": [\n" + - "$clusterPermissionsStr\n" + - "],\n" + - "\"index_permissions\": [\n" + - "{\n" + - "\"index_patterns\": [\n" + - "\"$index\"\n" + - "],\n" + - "\"dls\": \"$dlsQuery\",\n" + - "\"fls\": [],\n" + - "\"masked_fields\": [],\n" + - "\"allowed_actions\": [\n" + - "\"crud\"\n" + - "]\n" + - "}\n" + - "],\n" + - "\"tenant_permissions\": []\n" + - "}" + var entity = """ + { + "cluster_permissions": [$clusterPermissionsStr], + "index_permissions": [{ + "index_patterns": ["$index"], + "dls": "$dlsQuery", + "fls": [], + "masked_fields": [], + "allowed_actions": ["crud"] + }], + "tenant_permissions": [] + } + """.trimIndent() request.setJsonEntity(entity) client().performRequest(request) } @@ -1673,11 +1832,13 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { fun createUserRolesMapping(role: String, users: Array) { val request = Request("PUT", "/_plugins/_security/api/rolesmapping/$role") val usersStr = users.joinToString { it -> "\"$it\"" } - var entity = "{ \n" + - " \"backend_roles\" : [ ],\n" + - " \"hosts\" : [ ],\n" + - " \"users\" : [$usersStr]\n" + - "}" + var entity = """ + { + "backend_roles": [], + "hosts": [], + "users": [$usersStr] + } + """.trimIndent() request.setJsonEntity(entity) client().performRequest(request) } @@ -1688,11 +1849,13 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { val op = if (addUser) "add" else "remove" - val entity = "[{\n" + - " \"op\" : \"$op\",\n" + - " \"path\" : \"/users\",\n" + - " \"value\" : [$usersStr]\n" + - "}]" + val entity = """ + [{ + "op": "$op", + "path": "/users", + "value": [$usersStr] + }] + """.trimIndent() request.setJsonEntity(entity) client().performRequest(request) @@ -2007,4 +2170,94 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return deletedCommentId } + + protected fun isMonitorScheduled(monitorId: String, alertingStatsResponse: Map): Boolean { + val nodesInfo = alertingStatsResponse["nodes"] as Map + for (nodeId in nodesInfo.keys) { + val nodeInfo = nodesInfo[nodeId] as Map + val jobsInfo = nodeInfo["jobs_info"] as Map + if (jobsInfo.keys.contains(monitorId)) { + return true + } + } + + return false + } + + // this function is used for PPL Alerting testing. + // precondition: TEST_INDEX_NAME must be created before calling this + // indexes a doc from some time ago into index TEST_INDEX_NAME. + // this function only works on the TEST_INDEX_NAME index created + // specifically for this IT suite. It has fields + // "timestamp" (date), "abc" (string), "number" (integer) + protected fun indexDocFromSomeTimeAgo(timeValue: Long, timeUnit: ChronoUnit, abc: String, number: Int) { + val someTimeAgo = ZonedDateTime.now().minus(timeValue, timeUnit).truncatedTo(MILLIS) + val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(someTimeAgo) // the timestamp string is given a random timezone offset + val testDoc = """{ "timestamp" : "$testTime", "abc": "$abc", "number" : "$number" }""" + indexDoc(TEST_INDEX_NAME, UUID.randomUUID().toString(), testDoc) + } + + protected fun ensureNumMonitorV2s(expectedNum: Int) { + // if a validation error is thrown but a monitor is still accidentally created, + // what happens is that this check runs before the workflows to create + // alerting-config index and index the monitor complete, meaning this check gets + // no search results, then afterwards, the monitor is created, leading this function + // to falsely believe no monitor was create. wait some amount of time to let the + // workflows incorrectly create whatever monitors it will + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 10, TimeUnit.SECONDS) + + val search = SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).toString() + val searchResponse = client().makeRequest( + "POST", "$MONITOR_V2_BASE_URI/_search", + StringEntity(search, APPLICATION_JSON) + ) + + assertEquals("Search monitor failed", RestStatus.OK, searchResponse.restStatus()) + val xcp = createParser(XContentType.JSON.xContent(), searchResponse.entity.content) + val hits = xcp.map()["hits"]!! as Map> + val numberDocsFound = hits["total"]?.get("value") + assertEquals("Unexpected number of PPL Monitors found in Search Monitors", expectedNum, numberDocsFound) + } + + // takes in an execute monitor API response and returns true if the + // trigger condition was met. assumes the monitor executed only had 1 trigger + protected fun isTriggered(pplMonitor: PPLSQLMonitor, executeResponse: Response): Boolean { + val executeResponseMap = entityAsMap(executeResponse) + val triggerResultsObj = (executeResponseMap["trigger_results"] as Map)[pplMonitor.triggers[0].id] as Map + return triggerResultsObj["triggered"] as Boolean + } + + // takes in a get alerts API response and returns the current number of active alerts + protected fun numAlerts(getAlertsResponse: Response): Int { + logger.info("get alerts response: ${entityAsMap(getAlertsResponse)}") + return entityAsMap(getAlertsResponse)["total_alerts_v2"] as Int + } + + protected fun containsErrorAlert(getAlertsResponse: Response): Boolean { + val getAlertsMap = entityAsMap(getAlertsResponse) + val alertsList = getAlertsMap["alerts_v2"] as List> + alertsList.forEach { alert -> + val errorMessage = alert["error_message"] as String? + if (errorMessage != null) return true + } + return false + } + + protected fun getAlertV2HistoryDocCount(): Long { + val request = """ + { + "query": { + "match_all": {} + } + } + """.trimIndent() + val response = adminClient().makeRequest( + "POST", "${AlertV2Indices.ALERT_V2_HISTORY_ALL}/_search", emptyMap(), + StringEntity(request, APPLICATION_JSON) + ) + assertEquals("Request to get alert v2 history failed", RestStatus.OK, response.restStatus()) + return SearchResponse.fromXContent(createParser(jsonXContent, response.entity.content)).hits.totalHits!!.value + } } diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/PPLSQLMonitorRunnerIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/PPLSQLMonitorRunnerIT.kt new file mode 100644 index 000000000..070073994 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/PPLSQLMonitorRunnerIT.kt @@ -0,0 +1,496 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting + +import org.junit.Before +import org.opensearch.alerting.core.settings.AlertingV2Settings +import org.opensearch.alerting.modelv2.PPLSQLTrigger.ConditionType +import org.opensearch.alerting.modelv2.PPLSQLTrigger.NumResultsCondition +import org.opensearch.alerting.modelv2.PPLSQLTrigger.TriggerMode +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.common.settings.Settings +import org.opensearch.common.unit.TimeValue +import org.opensearch.commons.alerting.model.IntervalSchedule +import org.opensearch.test.OpenSearchTestCase +import java.time.temporal.ChronoUnit.MINUTES +import java.util.concurrent.TimeUnit + +/*** + * Create various kinds of monitors and ensures they all generate alerts + * under the expected circumstances + * + * Gradle command to run this suite: + * ./gradlew :alerting:integTest -Dhttps=true -Dsecurity=true -Duser=admin -Dpassword=admin \ + * --tests "org.opensearch.alerting.PPLMonitorRunnerIT" + */ +class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { + @Before + fun enableAlertingV2() { + client().updateSettings(AlertingV2Settings.ALERTING_V2_ENABLED.key, "true") + } + + fun `test monitor execution timeout generates error alert`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(2, MINUTES, "abc", 5) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + lookBackWindow = null, + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + expireDuration = 5, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + // set the monitor execution timebox to 1 nanosecond to guarantee a timeout + client().updateSettings(AlertingSettings.ALERT_V2_MONITOR_EXECUTION_MAX_DURATION.key, TimeValue.timeValueNanos(1L)) + + val executeMonitorResponse = executeMonitorV2(pplMonitor.id) + + val getAlertsResponse = getAlertV2s() + val alertsGenerated = numAlerts(getAlertsResponse) > 0 + val containsErrorAlert = containsErrorAlert(getAlertsResponse) + val executeResponseContainsError = (entityAsMap(executeMonitorResponse)["error"] as String?) != null + + assert(alertsGenerated) { "Alerts should have been generated but they weren't" } + assert(containsErrorAlert) { "Error alert should have been generated for timeout but wasn't" } + assert(executeResponseContainsError) { "Execute monitor response should've included an error message but didn't" } + } + + fun `test running number of results condition and result set mode ppl monitor`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(2, MINUTES, "abc", 5) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + lookBackWindow = null, + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + expireDuration = 5, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + val versionBefore = pplMonitor.version + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponse = getAlertV2s() + val alertsGenerated = numAlerts(getAlertsResponse) > 0 + + assert(triggered) { "Monitor should have triggered but it didn't" } + assert(alertsGenerated) { "Alerts should have been generated but they weren't" } + + val pplMonitorAfter = getMonitorV2(pplMonitor.id) + val versionAfter = pplMonitorAfter.version + + assert(versionBefore == versionAfter) { "Monitor version changed after monitor execution" } + } + + fun `test running number of results condition and per result mode ppl monitor`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) + indexDocFromSomeTimeAgo(2, MINUTES, "def", 10) + indexDocFromSomeTimeAgo(3, MINUTES, "ghi", 7) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + lookBackWindow = null, + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + expireDuration = 5, + mode = TriggerMode.PER_RESULT, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponse = getAlertV2s() + val alertsGenerated = numAlerts(getAlertsResponse) + + assert(triggered) { "Monitor should have triggered but it didn't" } + assertEquals( + "A number of alerts matching the number of docs ingested (3) should have been generated", + 3, alertsGenerated + ) + } + + fun `test running custom condition and result set mode ppl monitor`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 1) + indexDocFromSomeTimeAgo(2, MINUTES, "abc", 2) + indexDocFromSomeTimeAgo(3, MINUTES, "abc", 3) + indexDocFromSomeTimeAgo(4, MINUTES, "def", 4) + indexDocFromSomeTimeAgo(5, MINUTES, "def", 5) + indexDocFromSomeTimeAgo(6, MINUTES, "def", 6) + indexDocFromSomeTimeAgo(7, MINUTES, "ghi", 7) + indexDocFromSomeTimeAgo(8, MINUTES, "ghi", 8) + indexDocFromSomeTimeAgo(9, MINUTES, "ghi", 9) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + lookBackWindow = null, + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + expireDuration = 5, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.CUSTOM, + customCondition = "eval result = max_num > 5", + numResultsCondition = null, + numResultsValue = null + ) + ), + query = "source = $TEST_INDEX_NAME | stats max(number) as max_num by abc" + ) + ) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponse = getAlertV2s() + val alertsGenerated = numAlerts(getAlertsResponse) > 0 + + assert(triggered) { "Monitor should have triggered but it didn't" } + assert(alertsGenerated) { "Alerts should have been generated but they weren't" } + } + + fun `test running custom condition and per result mode ppl monitor`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 1) + indexDocFromSomeTimeAgo(2, MINUTES, "abc", 2) + indexDocFromSomeTimeAgo(3, MINUTES, "abc", 3) + indexDocFromSomeTimeAgo(4, MINUTES, "def", 4) + indexDocFromSomeTimeAgo(5, MINUTES, "def", 5) + indexDocFromSomeTimeAgo(6, MINUTES, "def", 6) + indexDocFromSomeTimeAgo(7, MINUTES, "ghi", 7) + indexDocFromSomeTimeAgo(8, MINUTES, "ghi", 8) + indexDocFromSomeTimeAgo(9, MINUTES, "ghi", 9) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + lookBackWindow = null, + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + expireDuration = 5, + mode = TriggerMode.PER_RESULT, + conditionType = ConditionType.CUSTOM, + customCondition = "eval evaluation = max_num > 5", + numResultsCondition = null, + numResultsValue = null + ) + ), + query = "source = $TEST_INDEX_NAME | stats max(number) as max_num by abc" + ) + ) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponse = getAlertV2s() + val alertsGenerated = numAlerts(getAlertsResponse) + + // when the indexed docs above are aggregated by field abc, we have: + // max("abc") = 3 + // max("def") = 6 + // max("ghi") = 9 + // only 2 of these buckets satisfy the custom condition max_num > 5, so + // only 2 alerts should be generated + + assert(triggered) { "Monitor should have triggered but it didn't" } + assertEquals( + "A number of alerts matching the number of docs ingested (2) should have been generated", + 2, alertsGenerated + ) + } + + fun `test running ppl monitor with lookback window and doc within lookback window`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(2, MINUTES, "abc", 5) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + lookBackWindow = 5, + timestampField = TIMESTAMP_FIELD, + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + expireDuration = 5, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponse = getAlertV2s() + val alertsGenerated = numAlerts(getAlertsResponse) > 0 + + assert(triggered) { "Monitor should have triggered but it didn't" } + assert(alertsGenerated) { "Alerts should have been generated but they weren't" } + } + + fun `test running ppl monitor with lookback window and doc beyond lookback window`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(10, MINUTES, "abc", 5) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + lookBackWindow = 5, + timestampField = TIMESTAMP_FIELD, + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + expireDuration = 5, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponse = getAlertV2s() + val alertsGenerated = numAlerts(getAlertsResponse) > 0 + + assert(!triggered) { "Monitor should not have triggered but it did" } + assert(!alertsGenerated) { "Alerts should not have been generated but they were" } + } + + fun `test execute api generated alert gets expired`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 20, unit = MINUTES), + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + expireDuration = 1L, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponsePreExpire = getAlertV2s() + val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 + + assert(triggered) { "Monitor should have triggered but it didn't" } + assert(alertsGeneratedPreExpire) { "Alerts should have been generated but they weren't" } + + // sleep briefly so alert mover can expire the alert + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 2, TimeUnit.MINUTES) + + val getAlertsResponsePostExpire = getAlertV2s() + val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 + assert(!alertsGeneratedPostExpire) + } + + fun `test scheduled job generated alert gets expired`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) + + // the monitor should generate 1 alert, then not generate + // any alerts for the rest of the test + createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + triggers = listOf( + randomPPLTrigger( + throttleDuration = 100L, + expireDuration = 1L, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + // sleep briefly so scheduled job can generate the alert + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 2, TimeUnit.MINUTES) + + val getAlertsResponsePreExpire = getAlertV2s() + val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 + + assert(alertsGeneratedPreExpire) { "Alerts should have been generated but they weren't" } + + // sleep briefly so alert mover can expire the alert + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 2, TimeUnit.MINUTES) + + val getAlertsResponsePostExpire = getAlertV2s() + logger.info("num alerts: ${numAlerts(getAlertsResponsePostExpire)}") + val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 + assert(!alertsGeneratedPostExpire) + } + + fun `test scheduled job monitor execution gets throttled`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + triggers = listOf( + randomPPLTrigger( + throttleDuration = 10, + expireDuration = 5, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponsePreThrottle = getAlertV2s() + val numAlertsPreThrottle = numAlerts(getAlertsResponsePreThrottle) + + assert(triggered) { "Monitor should have triggered but it didn't" } + assertEquals("Alerts should have been generated but they weren't", 1, numAlertsPreThrottle) + + // sleep briefly to give the monitor to execute again + // automatically and get throttled + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 2, TimeUnit.MINUTES) + + val getAlertsResponsePostThrottled = getAlertV2s() + val numAlertsPostThrottled = numAlerts(getAlertsResponsePostThrottled) + assertEquals("A new alert was generated when it should have been throttled", 1, numAlertsPostThrottled) + } + + fun `test manual monitor execution bypasses throttle`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 30, unit = MINUTES), + triggers = listOf( + randomPPLTrigger( + throttleDuration = 20, + expireDuration = 5, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponse = getAlertV2s() + val numAlerts = numAlerts(getAlertsResponse) + + assert(triggered) { "Monitor should have triggered but it didn't" } + assertEquals("Alerts should have been generated but they weren't", 1, numAlerts) + + // sleep briefly to get comfortable inside + // the throttle window + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 10, TimeUnit.SECONDS) + + val executeAgainResponse = executeMonitorV2(pplMonitor.id) + val triggeredAgain = isTriggered(pplMonitor, executeAgainResponse) + + val getAlertsAgainResponse = getAlertV2s() + val numAlertsAgain = numAlerts(getAlertsAgainResponse) + + assert(triggeredAgain) { "Monitor should have triggered again but it didn't" } + assertEquals("A new alert should have been generated but was instead throttled", 2, numAlertsAgain) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt b/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt index 93d4a62c5..751f69a03 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt @@ -12,6 +12,14 @@ import org.opensearch.alerting.model.AlertContext import org.opensearch.alerting.model.destination.email.EmailAccount import org.opensearch.alerting.model.destination.email.EmailEntry import org.opensearch.alerting.model.destination.email.EmailGroup +import org.opensearch.alerting.modelv2.AlertV2 +import org.opensearch.alerting.modelv2.PPLSQLMonitor +import org.opensearch.alerting.modelv2.PPLSQLMonitor.QueryLanguage +import org.opensearch.alerting.modelv2.PPLSQLTrigger +import org.opensearch.alerting.modelv2.PPLSQLTrigger.ConditionType +import org.opensearch.alerting.modelv2.PPLSQLTrigger.NumResultsCondition +import org.opensearch.alerting.modelv2.PPLSQLTrigger.TriggerMode +import org.opensearch.alerting.modelv2.TriggerV2.Severity import org.opensearch.alerting.util.getBucketKeysHash import org.opensearch.client.Request import org.opensearch.client.RequestOptions @@ -79,10 +87,18 @@ import org.opensearch.search.builder.SearchSourceBuilder import org.opensearch.test.OpenSearchTestCase.randomBoolean import org.opensearch.test.OpenSearchTestCase.randomInt import org.opensearch.test.OpenSearchTestCase.randomIntBetween +import org.opensearch.test.OpenSearchTestCase.randomLongBetween import org.opensearch.test.rest.OpenSearchRestTestCase +import org.opensearch.test.rest.OpenSearchRestTestCase.assertEquals import java.time.Instant import java.time.temporal.ChronoUnit +// constants for PPL Alerting tests +const val TIMESTAMP_FIELD = "timestamp" +const val TEST_INDEX_NAME = "index" +const val TEST_INDEX_MAPPINGS = + """"properties":{"timestamp":{"type":"date"},"abc":{"type":"keyword"},"number":{"type":"integer"}}""" + fun randomQueryLevelMonitor( name: String = OpenSearchRestTestCase.randomAlphaOfLength(10), user: User = randomUser(), @@ -292,6 +308,36 @@ fun randomWorkflowWithDelegates( ) } +fun randomPPLMonitor( + name: String = OpenSearchRestTestCase.randomAlphaOfLength(10), + enabled: Boolean = randomBoolean(), + schedule: Schedule = IntervalSchedule(interval = 5, unit = ChronoUnit.MINUTES), + lookBackWindow: Long? = randomLongBetween(10, 100), + timestampField: String? = lookBackWindow?.let { TIMESTAMP_FIELD }, + lastUpdateTime: Instant = Instant.now().truncatedTo(ChronoUnit.MILLIS), + enabledTime: Instant? = if (enabled) Instant.now().truncatedTo(ChronoUnit.MILLIS) else null, + description: String? = if (randomBoolean()) "some description" else null, + triggers: List = List(randomIntBetween(1, 5)) { randomPPLTrigger() }, + user: User? = randomUser(), + queryLanguage: QueryLanguage = QueryLanguage.PPL, + query: String = "source = $TEST_INDEX_NAME | head 10" +): PPLSQLMonitor { + return PPLSQLMonitor( + name = name, + enabled = enabled, + schedule = schedule, + lookBackWindow = lookBackWindow, + timestampField = timestampField, + lastUpdateTime = lastUpdateTime, + enabledTime = enabledTime, + description = description, + triggers = triggers, + user = user, + queryLanguage = queryLanguage, + query = query + ) +} + fun randomQueryLevelTrigger( id: String = UUIDs.base64UUID(), name: String = OpenSearchRestTestCase.randomAlphaOfLength(10), @@ -348,6 +394,42 @@ fun randomDocumentLevelTrigger( ) } +// random PPLTrigger defaults to a number_of_results trigger, because a custom condition +// would require knowledge of the PPL Monitor's query +// it is on the caller to be explicit and pass in valid arguments that would create either +// a valid PPL Monitor or one that intentionally throws an error. +// e.g. to create a valid PPL Monitor, if conditionType is CUSTOM, +// numResultsCondition and numResultsValue must be null, while +// customCondition must not be null. +fun randomPPLTrigger( + id: String = UUIDs.base64UUID(), + name: String = OpenSearchRestTestCase.randomAlphaOfLength(10), + severity: Severity = Severity.entries.random(), + throttleDuration: Long? = randomLongBetween(1, 100), + expireDuration: Long = randomLongBetween(1, 100), + actions: List = mutableListOf(), + mode: TriggerMode = TriggerMode.entries.random(), + conditionType: ConditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition: NumResultsCondition? = NumResultsCondition.entries.random(), + numResultsValue: Long? = randomLongBetween(1L, 50L), + customCondition: String? = null +): PPLSQLTrigger { + return PPLSQLTrigger( + id = id, + name = name, + severity = severity, + throttleDuration = throttleDuration, + expireDuration = expireDuration, + lastTriggeredTime = null, + actions = actions, + mode = mode, + conditionType = conditionType, + numResultsCondition = numResultsCondition, + numResultsValue = numResultsValue, + customCondition = customCondition + ) +} + fun randomBucketSelectorExtAggregationBuilder( name: String = OpenSearchRestTestCase.randomAlphaOfLength(10), bucketsPathsMap: MutableMap = mutableMapOf("avg" to "10"), @@ -424,10 +506,11 @@ fun randomTemplateScript( fun randomAction( name: String = OpenSearchRestTestCase.randomUnicodeOfLength(10), template: Script = randomTemplateScript("Hello World"), - destinationId: String = "", + subjectTemplate: Script = template, + destinationId: String = "abc", throttleEnabled: Boolean = false, throttle: Throttle = randomThrottle() -) = Action(name, destinationId, template, template, throttleEnabled, throttle, actionExecutionPolicy = null) +) = Action(name, destinationId, subjectTemplate, template, throttleEnabled, throttle, actionExecutionPolicy = null) fun randomActionWithPolicy( name: String = OpenSearchRestTestCase.randomUnicodeOfLength(10), @@ -472,6 +555,42 @@ fun randomAlert(monitor: Monitor = randomQueryLevelMonitor()): Alert { ) } +fun randomAlertV2( + id: String = UUIDs.base64UUID(), + version: Long = randomLongBetween(1, 10), + schemaVersion: Int = randomIntBetween(1, 10), + monitorId: String = UUIDs.base64UUID(), + monitorName: String = UUIDs.base64UUID(), + monitorVersion: Long = randomLongBetween(1, 10), + monitorUser: User? = randomUser(), + triggerId: String = UUIDs.base64UUID(), + triggerName: String = UUIDs.base64UUID(), + query: String = "source = $TEST_INDEX_NAME | head 10", + queryResults: Map = mapOf(), + triggeredTime: Instant = Instant.now().truncatedTo(ChronoUnit.MILLIS), + errorMessage: String? = "sample error message", + severity: Severity = Severity.entries.random(), + executionId: String? = UUIDs.base64UUID() +): AlertV2 { + return AlertV2( + id = id, + version = version, + schemaVersion = schemaVersion, + monitorId = monitorId, + monitorName = monitorName, + monitorVersion = monitorVersion, + monitorUser = monitorUser, + triggerId = triggerId, + triggerName = triggerName, + query = query, + queryResults = queryResults, + triggeredTime = triggeredTime, + errorMessage = errorMessage, + severity = severity, + executionId = executionId, + ) +} + fun randomDocLevelQuery( id: String = OpenSearchRestTestCase.randomAlphaOfLength(10), query: String = OpenSearchRestTestCase.randomAlphaOfLength(10), @@ -810,3 +929,161 @@ fun randomAlertContext( fun Map.objectMap(key: String): Map> { return this[key] as Map> } + +fun assertPplMonitorsEqual(pplMonitor1: PPLSQLMonitor, pplMonitor2: PPLSQLMonitor) { + // note: Get and Search Monitor responses do not include User information by + // design, so that check is skipped + + // note: Update Monitor API intentionally overrides the enabledTime of the new given monitor + // with the enabledTime of the existing monitor being updated to ensure execution correctness, + // so that check is skipped + + assertEquals("Monitor enabled fields not equal", pplMonitor1.enabled, pplMonitor2.enabled) + assertEquals("Monitor schedules not equal", pplMonitor1.schedule, pplMonitor2.schedule) + assertEquals("Monitor lookback windows not equal", pplMonitor1.lookBackWindow, pplMonitor2.lookBackWindow) + assertEquals("Monitor timestamp fields not equal", pplMonitor1.timestampField, pplMonitor2.timestampField) + assertEquals("Monitor last updated times are not equal", pplMonitor1.lastUpdateTime, pplMonitor2.lastUpdateTime) + assertEquals("Monitor query languages not equal", pplMonitor1.queryLanguage, pplMonitor2.queryLanguage) + assertEquals("Monitor queries not equal", pplMonitor1.query, pplMonitor2.query) + assertEquals("Number of triggers in monitor not equal", pplMonitor1.triggers.size, pplMonitor2.triggers.size) + + val sortedTriggers1 = pplMonitor1.triggers.sortedBy { it.id } + val sortedTriggers2 = pplMonitor2.triggers.sortedBy { it.id } + for (i in sortedTriggers1.indices) { + assertPplTriggersEqual(sortedTriggers1[i], sortedTriggers2[i]) + } +} + +fun assertPplTriggersEqual(pplTrigger1: PPLSQLTrigger, pplTrigger2: PPLSQLTrigger) { + assertEquals( + "Monitor trigger IDs not equal", + pplTrigger1.id, + pplTrigger2.id + ) + + val id = pplTrigger1.id + + assertEquals( + "Monitor trigger $id names not equal", + pplTrigger1.name, + pplTrigger2.name + ) + assertEquals( + "Monitor trigger $id severities not equal", + pplTrigger1.severity, + pplTrigger2.severity + ) + assertEquals( + "Monitor trigger $id throttle durations not equal", + pplTrigger1.throttleDuration, + pplTrigger2.throttleDuration + ) + assertEquals( + "Monitor trigger $id expire durations not equal", + pplTrigger1.expireDuration, + pplTrigger2.expireDuration + ) + assertEquals( + "Monitor trigger $id modes not equal", + pplTrigger1.mode, + pplTrigger2.mode + ) + assertEquals( + "Monitor trigger $id condition types not equal", + pplTrigger1.conditionType, + pplTrigger2.conditionType + ) + assertEquals( + "Monitor trigger $id number_of_results conditions not equal", + pplTrigger1.numResultsCondition, + pplTrigger2.numResultsCondition + ) + assertEquals( + "Monitor trigger $id number_of_results values not equal", + pplTrigger1.numResultsValue, + pplTrigger2.numResultsValue + ) + assertEquals( + "Monitor trigger $id custom conditions not equal", + pplTrigger1.customCondition, + pplTrigger2.customCondition + ) +} + +fun assertAlertV2sEqual(alert1: AlertV2, alert2: AlertV2) { + assertEquals( + "AlertV2 IDs are not equal", + alert1.id, + alert2.id + ) + assertEquals( + "AlertV2 versions are not equal", + alert1.version, + alert2.version + ) + assertEquals( + "AlertV2 schema versions are not equal", + alert1.schemaVersion, + alert2.schemaVersion + ) + assertEquals( + "AlertV2 monitor IDs are not equal", + alert1.monitorId, + alert2.monitorId + ) + assertEquals( + "AlertV2 monitor names are not equal", + alert1.monitorName, + alert2.monitorName + ) + assertEquals( + "AlertV2 monitor versions are not equal", + alert1.monitorVersion, + alert2.monitorVersion + ) + assertEquals( + "AlertV2 monitor users are not equal", + alert1.monitorUser.toString(), + alert2.monitorUser.toString() + ) + assertEquals( + "AlertV2 trigger IDs are not equal", + alert1.triggerId, + alert2.triggerId + ) + assertEquals( + "AlertV2 trigger names are not equal", + alert1.triggerName, + alert2.triggerName + ) + assertEquals( + "AlertV2 queries are not equal", + alert1.query, + alert2.query + ) + assertEquals( + "AlertV2 query results are not equal", + alert1.queryResults, + alert2.queryResults + ) + assertEquals( + "AlertV2 triggered times are not equal", + alert1.triggeredTime, + alert2.triggeredTime + ) + assertEquals( + "AlertV2 error messages are not equal", + alert1.errorMessage, + alert2.errorMessage + ) + assertEquals( + "AlertV2 severities are not equal", + alert1.severity, + alert2.severity + ) + assertEquals( + "AlertV2 execution IDs are not equal", + alert1.executionId, + alert2.executionId + ) +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2RequestTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2RequestTests.kt new file mode 100644 index 000000000..727225f93 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2RequestTests.kt @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.support.WriteRequest +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.test.OpenSearchTestCase + +class DeleteMonitorV2RequestTests : OpenSearchTestCase() { + fun `test get monitor v2 request as stream`() { + val req = DeleteMonitorV2Request( + monitorV2Id = "abc", + refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE + ) + assertNotNull(req) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = DeleteMonitorV2Request(sin) + + assertEquals(req.monitorV2Id, newReq.monitorV2Id) + assertEquals(req.refreshPolicy, newReq.refreshPolicy) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2ResponseTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2ResponseTests.kt new file mode 100644 index 000000000..f10082d8d --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2ResponseTests.kt @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.test.OpenSearchTestCase + +class DeleteMonitorV2ResponseTests : OpenSearchTestCase() { + fun `test get monitor v2 request as stream`() { + val req = DeleteMonitorV2Response( + id = "abc", + version = 3L + ) + assertNotNull(req) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = DeleteMonitorV2Response(sin) + + assertEquals(req.id, newReq.id) + assertEquals(req.version, newReq.version) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2RequestTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2RequestTests.kt new file mode 100644 index 000000000..b2eb38a93 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2RequestTests.kt @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.alerting.assertPplMonitorsEqual +import org.opensearch.alerting.modelv2.PPLSQLMonitor +import org.opensearch.alerting.randomPPLMonitor +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.common.unit.TimeValue +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.test.OpenSearchTestCase + +class ExecuteMonitorV2RequestTests : OpenSearchTestCase() { + fun `test execute monitor v2 request`() { + val req = ExecuteMonitorV2Request( + dryrun = true, + manual = false, + monitorV2Id = "abc", + monitorV2 = randomPPLMonitor(), + requestEnd = TimeValue.timeValueMinutes(30L) + ) + assertNotNull(req) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = ExecuteMonitorV2Request(sin) + + assertEquals(req.dryrun, newReq.dryrun) + assertEquals(req.manual, newReq.manual) + assertEquals(req.monitorV2Id, newReq.monitorV2Id) + assertPplMonitorsEqual(req.monitorV2 as PPLSQLMonitor, newReq.monitorV2 as PPLSQLMonitor) + assertEquals(req.requestEnd, newReq.requestEnd) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2ResponseTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2ResponseTests.kt new file mode 100644 index 000000000..e4f932170 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2ResponseTests.kt @@ -0,0 +1,57 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.alerting.modelv2.PPLSQLMonitorRunResult +import org.opensearch.alerting.modelv2.PPLSQLTriggerRunResult +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.test.OpenSearchTestCase + +class ExecuteMonitorV2ResponseTests : OpenSearchTestCase() { + fun `test execute monitor response`() { + val monitorRunResult = PPLSQLMonitorRunResult( + monitorName = "some-monitor", + error = IllegalArgumentException("some-error"), + triggerResults = mapOf( + "some-trigger-id" to PPLSQLTriggerRunResult( + triggerName = "some-trigger", + triggered = true, + error = IllegalArgumentException("some-error") + ) + ), + pplQueryResults = mapOf("some-result" to mapOf("some-field" to 3)) + ) + val response = ExecuteMonitorV2Response(monitorRunResult) + assertNotNull(response) + + val out = BytesStreamOutput() + response.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newResponse = ExecuteMonitorV2Response(sin) + + assertEquals(response.monitorV2RunResult.monitorName, newResponse.monitorV2RunResult.monitorName) + assertEquals(response.monitorV2RunResult.error?.message, newResponse.monitorV2RunResult.error?.message) + assert(response.monitorV2RunResult.triggerResults.containsKey("some-trigger-id")) + assert(newResponse.monitorV2RunResult.triggerResults.containsKey("some-trigger-id")) + assertEquals( + response.monitorV2RunResult.triggerResults["some-trigger-id"]!!.triggerName, + newResponse.monitorV2RunResult.triggerResults["some-trigger-id"]!!.triggerName + ) + assertEquals( + response.monitorV2RunResult.triggerResults["some-trigger-id"]!!.triggered, + newResponse.monitorV2RunResult.triggerResults["some-trigger-id"]!!.triggered + ) + assertEquals( + response.monitorV2RunResult.triggerResults["some-trigger-id"]!!.error?.message, + newResponse.monitorV2RunResult.triggerResults["some-trigger-id"]!!.error?.message + ) + assertEquals( + (response.monitorV2RunResult as PPLSQLMonitorRunResult).pplQueryResults, + (newResponse.monitorV2RunResult as PPLSQLMonitorRunResult).pplQueryResults + ) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2RequestTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2RequestTests.kt new file mode 100644 index 000000000..aa963c8a0 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2RequestTests.kt @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.commons.alerting.model.Table +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.test.OpenSearchTestCase + +class GetAlertsV2RequestTests : OpenSearchTestCase() { + fun `test get alerts request as stream`() { + val table = Table("asc", "sortString", null, 1, 0, "") + + val req = GetAlertsV2Request( + table = table, + severityLevel = "1", + monitorV2Ids = listOf("1", "2"), + ) + assertNotNull(req) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = GetAlertsV2Request(sin) + + assertEquals("1", newReq.severityLevel) + assertEquals(table, newReq.table) + assertTrue(newReq.monitorV2Ids!!.contains("1")) + assertTrue(newReq.monitorV2Ids!!.contains("2")) + } + + fun `test get alerts request with filter as stream`() { + val table = Table("asc", "sortString", null, 1, 0, "") + val req = GetAlertsV2Request(table, "1", null) + assertNotNull(req) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = GetAlertsV2Request(sin) + + assertEquals("1", newReq.severityLevel) + assertNull(newReq.monitorV2Ids) + assertEquals(table, newReq.table) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2ResponseTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2ResponseTests.kt new file mode 100644 index 000000000..8f803bc00 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2ResponseTests.kt @@ -0,0 +1,93 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.alerting.modelv2.AlertV2 +import org.opensearch.alerting.modelv2.TriggerV2 +import org.opensearch.alerting.randomUser +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.alerting.util.string +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.XContentBuilder +import org.opensearch.test.OpenSearchTestCase +import java.time.Instant +import java.util.Collections + +class GetAlertsV2ResponseTests : OpenSearchTestCase() { + fun `test get alerts response with no alerts`() { + val req = GetAlertsV2Response(Collections.emptyList(), 0) + assertNotNull(req) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = GetAlertsV2Response(sin) + assertTrue(newReq.alertV2s.isEmpty()) + assertEquals(0, newReq.totalAlertV2s) + } + + fun `test get alerts response with alerts`() { + val alert = AlertV2( + monitorId = "id", + monitorName = "name", + monitorVersion = AlertV2.NO_VERSION, + monitorUser = randomUser(), + triggerId = "triggerId", + triggerName = "triggerNamer", + query = "source = some_index", + queryResults = mapOf(), + triggeredTime = Instant.now(), + errorMessage = null, + severity = TriggerV2.Severity.LOW, + executionId = "executionId" + ) + val res = GetAlertsV2Response(listOf(alert), 1) + assertNotNull(res) + + val out = BytesStreamOutput() + res.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newRes = GetAlertsV2Response(sin) + assertEquals(1, newRes.alertV2s.size) + assertEquals(alert, newRes.alertV2s[0]) + assertEquals(1, newRes.totalAlertV2s) + } + + fun `test toXContent for get alerts response`() { + val now = Instant.now() + val alert = AlertV2( + monitorId = "id", + monitorName = "name", + monitorVersion = AlertV2.NO_VERSION, + monitorUser = randomUser(), + triggerId = "triggerId", + triggerName = "triggerName", + query = "source = some_index", + queryResults = mapOf(), + triggeredTime = now, + errorMessage = null, + severity = TriggerV2.Severity.LOW, + executionId = "executionId" + ) + + val req = GetAlertsV2Response(listOf(alert), 1) + var actualXContentString = req.toXContent( + XContentBuilder.builder(XContentType.JSON.xContent()), + ToXContent.EMPTY_PARAMS + ).string() + val expectedXContentString = "{\"alerts_v2\":[{\"id\":\"\",\"version\":-1,\"monitor_v2_id\":\"id\",\"schema_version\":0," + + "\"monitor_v2_version\":-1,\"monitor_v2_name\":\"name\",\"execution_id\":\"executionId\",\"trigger_v2_id\":\"triggerId\"," + + "\"trigger_v2_name\":\"triggerName\",\"query\":\"source = some_index\",\"query_results\":{},\"error_message\":null," + + "\"severity\":\"low\",\"triggered_time\":${now.toEpochMilli()}}],\"total_alerts_v2\":1}" + + logger.info("expected: $expectedXContentString") + logger.info("actual: $actualXContentString") + + assertEquals(expectedXContentString, actualXContentString) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2RequestTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2RequestTests.kt new file mode 100644 index 000000000..a4ede1ad2 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2RequestTests.kt @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.search.fetch.subphase.FetchSourceContext +import org.opensearch.test.OpenSearchTestCase + +class GetMonitorV2RequestTests : OpenSearchTestCase() { + fun `test get monitor v2 request as stream`() { + val req = GetMonitorV2Request( + monitorV2Id = "abc", + version = 2L, + srcContext = FetchSourceContext.FETCH_SOURCE + ) + assertNotNull(req) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = GetMonitorV2Request(sin) + + assertEquals(req.monitorV2Id, newReq.monitorV2Id) + assertEquals(req.version, newReq.version) + assertEquals(req.srcContext, newReq.srcContext) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2ResponseTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2ResponseTests.kt new file mode 100644 index 000000000..11a8c218b --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2ResponseTests.kt @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.randomPPLMonitor +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.test.OpenSearchTestCase + +class GetMonitorV2ResponseTests : OpenSearchTestCase() { + fun `test get monitor v2 response as stream`() { + val req = GetMonitorV2Response( + id = "abc", + version = 2L, + seqNo = 1L, + primaryTerm = 2L, + monitorV2 = randomPPLMonitor() as MonitorV2 + ) + assertNotNull(req) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = GetMonitorV2Response(sin) + + assertEquals(req.id, newReq.id) + assertEquals(req.version, newReq.version) + assertEquals(req.seqNo, newReq.seqNo) + assertEquals(req.primaryTerm, newReq.primaryTerm) + assertEquals(req.monitorV2, newReq.monitorV2) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2RequestTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2RequestTests.kt new file mode 100644 index 000000000..1e7c7cd08 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2RequestTests.kt @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.support.WriteRequest +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.randomPPLMonitor +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.rest.RestRequest +import org.opensearch.test.OpenSearchTestCase + +class IndexMonitorV2RequestTests : OpenSearchTestCase() { + fun `test index monitor v2 request as stream`() { + val req = IndexMonitorV2Request( + monitorId = "abc", + seqNo = 1L, + primaryTerm = 1L, + refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, + method = RestRequest.Method.POST, + monitorV2 = randomPPLMonitor() as MonitorV2, + rbacRoles = listOf("role-a", "role-b") + ) + assertNotNull(req) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = IndexMonitorV2Request(sin) + + assertEquals(req.monitorId, newReq.monitorId) + assertEquals(req.seqNo, newReq.seqNo) + assertEquals(req.primaryTerm, newReq.primaryTerm) + assertEquals(req.refreshPolicy, newReq.refreshPolicy) + assertEquals(req.method, newReq.method) + assertEquals(req.monitorV2, newReq.monitorV2) + assertEquals(req.rbacRoles, newReq.rbacRoles) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2ResponseTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2ResponseTests.kt new file mode 100644 index 000000000..2de07698e --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2ResponseTests.kt @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.randomPPLMonitor +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.test.OpenSearchTestCase + +class IndexMonitorV2ResponseTests : OpenSearchTestCase() { + fun `test index monitor v2 response as stream`() { + val req = IndexMonitorV2Response( + id = "abc", + version = 2L, + seqNo = 1L, + primaryTerm = 1L, + monitorV2 = randomPPLMonitor() as MonitorV2 + ) + assertNotNull(req) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = IndexMonitorV2Response(sin) + + assertEquals(req.id, newReq.id) + assertEquals(req.version, newReq.version) + assertEquals(req.seqNo, newReq.seqNo) + assertEquals(req.primaryTerm, newReq.primaryTerm) + assertEquals(req.monitorV2, newReq.monitorV2) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2RequestTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2RequestTests.kt new file mode 100644 index 000000000..987a738a2 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2RequestTests.kt @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.search.SearchRequest +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.common.unit.TimeValue +import org.opensearch.commons.alerting.action.SearchMonitorRequest +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.search.builder.SearchSourceBuilder +import org.opensearch.test.OpenSearchTestCase +import org.opensearch.test.rest.OpenSearchRestTestCase +import java.util.concurrent.TimeUnit + +class SearchMonitorV2RequestTests : OpenSearchTestCase() { + fun `test search monitors request`() { + val searchSourceBuilder = SearchSourceBuilder().from(0).size(100).timeout(TimeValue(60, TimeUnit.SECONDS)) + val searchRequest = SearchRequest().indices(OpenSearchRestTestCase.randomAlphaOfLength(10)).source(searchSourceBuilder) + val req = SearchMonitorV2Request(searchRequest) + assertNotNull(req) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = SearchMonitorRequest(sin) + + assertNotNull(newReq.searchRequest) + assertEquals(1, newReq.searchRequest.indices().size) + assertEquals(req.searchRequest, newReq.searchRequest) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/alerts/AlertIndicesIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/alerts/AlertIndicesIT.kt index 69a7e0363..9e1e2437f 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/alerts/AlertIndicesIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/alerts/AlertIndicesIT.kt @@ -64,7 +64,7 @@ class AlertIndicesIT : AlertingRestTestCase() { executeMonitor(createRandomMonitor()) assertIndexExists(AlertIndices.ALERT_INDEX) assertIndexExists(AlertIndices.ALERT_HISTORY_WRITE_INDEX) - verifyIndexSchemaVersion(ScheduledJob.SCHEDULED_JOBS_INDEX, 8) + verifyIndexSchemaVersion(ScheduledJob.SCHEDULED_JOBS_INDEX, 9) verifyIndexSchemaVersion(AlertIndices.ALERT_INDEX, 5) verifyIndexSchemaVersion(AlertIndices.ALERT_HISTORY_WRITE_INDEX, 5) } @@ -88,7 +88,7 @@ class AlertIndicesIT : AlertingRestTestCase() { val trueMonitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger))) executeMonitor(trueMonitor.id) assertIndexExists(AlertIndices.FINDING_HISTORY_WRITE_INDEX) - verifyIndexSchemaVersion(ScheduledJob.SCHEDULED_JOBS_INDEX, 8) + verifyIndexSchemaVersion(ScheduledJob.SCHEDULED_JOBS_INDEX, 9) verifyIndexSchemaVersion(AlertIndices.FINDING_HISTORY_WRITE_INDEX, 4) } diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/alertsv2/AlertV2IndicesIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/alertsv2/AlertV2IndicesIT.kt new file mode 100644 index 000000000..672bc3902 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/alertsv2/AlertV2IndicesIT.kt @@ -0,0 +1,450 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.alertsv2 + +import org.junit.Before +import org.opensearch.alerting.AlertingRestTestCase +import org.opensearch.alerting.TEST_INDEX_MAPPINGS +import org.opensearch.alerting.TEST_INDEX_NAME +import org.opensearch.alerting.core.settings.AlertingV2Settings +import org.opensearch.alerting.makeRequest +import org.opensearch.alerting.modelv2.PPLSQLMonitor +import org.opensearch.alerting.modelv2.PPLSQLTrigger +import org.opensearch.alerting.modelv2.PPLSQLTrigger.ConditionType +import org.opensearch.alerting.modelv2.PPLSQLTrigger.NumResultsCondition +import org.opensearch.alerting.modelv2.PPLSQLTrigger.TriggerMode +import org.opensearch.alerting.randomPPLMonitor +import org.opensearch.alerting.randomPPLTrigger +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.common.settings.Settings +import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.alerting.model.IntervalSchedule +import org.opensearch.commons.alerting.model.ScheduledJob +import org.opensearch.core.rest.RestStatus +import org.opensearch.test.OpenSearchTestCase +import java.time.temporal.ChronoUnit.MINUTES +import java.util.concurrent.TimeUnit + +/** + * Tests AlertV2 history migration, AlertV2 deletion, and AlertV2 expiration functionality + * + * Gradle command to run this suite: + * ./gradlew :alerting:integTest -Dhttps=true -Dsecurity=true -Duser=admin -Dpassword=admin \ + * --tests "org.opensearch.alerting.alertsv2.AlertV2IndicesIT" + */ +class AlertV2IndicesIT : AlertingRestTestCase() { + @Before + fun enableAlertingV2() { + client().updateSettings(AlertingV2Settings.ALERTING_V2_ENABLED.key, "true") + } + + fun `test create alert v2 index`() { + generateAlertV2s() + + assertIndexExists(AlertV2Indices.ALERT_V2_INDEX) + assertIndexExists(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) + } + + fun `test update alert v2 index mapping with new schema version`() { + wipeAllODFEIndices() + assertIndexDoesNotExist(AlertV2Indices.ALERT_V2_INDEX) + assertIndexDoesNotExist(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) + + putAlertV2Mappings( + AlertV2Indices.alertV2Mapping().trimStart('{').trimEnd('}') + .replace("\"schema_version\": 1", "\"schema_version\": 0") + ) + assertIndexExists(AlertV2Indices.ALERT_V2_INDEX) + assertIndexExists(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) + verifyIndexSchemaVersion(AlertV2Indices.ALERT_V2_INDEX, 0) + verifyIndexSchemaVersion(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX, 0) + + wipeAllODFEIndices() + + generateAlertV2s() + assertIndexExists(AlertV2Indices.ALERT_V2_INDEX) + assertIndexExists(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) + verifyIndexSchemaVersion(ScheduledJob.SCHEDULED_JOBS_INDEX, 9) + verifyIndexSchemaVersion(AlertV2Indices.ALERT_V2_INDEX, 1) + verifyIndexSchemaVersion(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX, 1) + } + + fun `test alert v2 index gets recreated automatically if deleted`() { + wipeAllODFEIndices() + assertIndexDoesNotExist(AlertV2Indices.ALERT_V2_INDEX) + + generateAlertV2s() + + assertIndexExists(AlertV2Indices.ALERT_V2_INDEX) + assertIndexExists(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) + wipeAllODFEIndices() + assertIndexDoesNotExist(AlertV2Indices.ALERT_V2_INDEX) + assertIndexDoesNotExist(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) + + // ensure execute monitor succeeds even after alert indices are deleted + generateAlertV2s() + } + + fun `test rollover alert v2 history index`() { + // Update the rollover check to be every 1 second and the index max age to be 1 second + client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_ROLLOVER_PERIOD.key, "1s") + client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_INDEX_MAX_AGE.key, "1s") + + generateAlertV2s() + + // Allow for a rollover index. + OpenSearchTestCase.waitUntil({ + return@waitUntil (getAlertV2Indices().size >= 3) + }, 2, TimeUnit.SECONDS) + + assertTrue("Did not find 3 alert v2 indices", getAlertV2Indices().size >= 3) + } + + fun `test alert v2 history disabled`() { + resetHistorySettings() + + // Disable alert history + client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_ENABLED.key, "false") + + val pplMonitorId = generateAlertV2s( + randomPPLMonitor( + schedule = IntervalSchedule(interval = 30, unit = MINUTES), + query = "source = $TEST_INDEX_NAME | head 3", + triggers = listOf( + randomPPLTrigger( + mode = PPLSQLTrigger.TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + expireDuration = 1L + ) + ) + ) + ) + + val alerts1 = searchAlertV2s(pplMonitorId) + assertEquals("1 alert should be present", 1, alerts1.size) + + // wait for alert to expire. + // since alert history is disabled, this should result + // in hard deletion + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 2, TimeUnit.MINUTES) + + // Since history is disabled, the alert should be hard deleted by now + val alerts2 = searchAlertV2s(pplMonitorId, AlertV2Indices.ALL_ALERT_V2_INDEX_PATTERN) + assertTrue("There should be no alerts, but alerts were found", alerts2.isEmpty()) + } + + fun `test short retention period`() { + resetHistorySettings() + + val pplMonitorId = generateAlertV2s( + randomPPLMonitor( + schedule = IntervalSchedule(interval = 30, unit = MINUTES), + query = "source = $TEST_INDEX_NAME | head 3", + triggers = listOf( + randomPPLTrigger( + mode = PPLSQLTrigger.TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + expireDuration = 1L + ) + ) + ) + ) + + val alerts1 = searchAlertV2s(pplMonitorId) + assertEquals("1 alert should be present", 1, alerts1.size) + + // history index should be created but empty + assertEquals(0, getAlertV2HistoryDocCount()) + + // wait for alert to expire. + // since alert history is enabled, this should result + // in the alert being archived in history index + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 2, TimeUnit.MINUTES) + + assertTrue(searchAlertV2s(pplMonitorId).isEmpty()) + assertEquals(1, getAlertV2HistoryDocCount()) + + // update rollover check and max docs as well as decreasing the retention period + client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_ROLLOVER_PERIOD.key, "3s") + client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_MAX_DOCS.key, 1) + client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_RETENTION_PERIOD.key, "1s") + + // give some time for newly updated settings to take effect + OpenSearchTestCase.waitUntil({ + return@waitUntil getAlertV2HistoryDocCount() == 0L + }, 40, TimeUnit.SECONDS) + + // Given the max_docs and retention settings above, the history index will rollover and the non-write index will be deleted. + // This leaves two indices: active alerts index and an empty history write index + assertEquals("Did not find 2 alert v2 indices", 2, getAlertV2Indices().size) + assertEquals(0, getAlertV2HistoryDocCount()) + } + + fun `test generated alert gets expired because monitor was deleted with alert history enabled`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 20, unit = MINUTES), + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + // for this test, configured expire can't be the reason for alert expiration + expireDuration = 1000L, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponsePreExpire = getAlertV2s() + val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 + + assert(triggered) { "Monitor should have triggered but it didn't" } + assert(alertsGeneratedPreExpire) { "Alerts should have been generated but they weren't" } + + // delete the monitor + deleteMonitorV2(pplMonitor.id) + + // sleep so postDelete can expire the generated alert + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 5, TimeUnit.SECONDS) + + val getAlertsResponsePostExpire = getAlertV2s() + val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 + assert(!alertsGeneratedPostExpire) + + assertEquals(1, getAlertV2HistoryDocCount()) + } + + fun `test generated alert gets expired because monitor was edited with alert history enabled`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) + + // first create a ppl monitor that's guaranteed to generate an alert + val initialPplTrigger = randomPPLTrigger( + id = "initialID", + throttleDuration = null, + // for this test, configured expire can't be the reason for alert expiration + expireDuration = 1000L, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + + val initialPplMonitorConfig = randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 20, unit = MINUTES), + triggers = listOf(initialPplTrigger), + query = "source = $TEST_INDEX_NAME | head 10" + ) + + val pplMonitor = createRandomPPLMonitor(initialPplMonitorConfig) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponsePreExpire = getAlertV2s() + val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 + + assert(triggered) { "Monitor should have triggered but it didn't" } + assert(alertsGeneratedPreExpire) { "Alerts should have been generated but they weren't" } + + // update the monitor to any new config, + // and more importantly, updated triggers + updateMonitorV2(randomPPLMonitor().makeCopy(pplMonitor.id, pplMonitor.version)) + + // sleep so postIndex can expire the generated alert + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 5, TimeUnit.SECONDS) + + val getAlertsResponsePostExpire = getAlertV2s() + val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 + assert(!alertsGeneratedPostExpire) + + assertEquals(1, getAlertV2HistoryDocCount()) + } + + fun `test generated alert gets expired because monitor was deleted with alert history disabled`() { + client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_ENABLED.key, "false") + + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 20, unit = MINUTES), + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + // for this test, configured expire can't be the reason for alert expiration + expireDuration = 1000L, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponsePreExpire = getAlertV2s() + val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 + + assert(triggered) { "Monitor should have triggered but it didn't" } + assert(alertsGeneratedPreExpire) { "Alerts should have been generated but they weren't" } + + // delete the monitor + deleteMonitorV2(pplMonitor.id) + + // sleep so postDelete can expire the generated alert + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 5, TimeUnit.SECONDS) + + val getAlertsResponsePostExpire = getAlertV2s() + val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 + assert(!alertsGeneratedPostExpire) + + assertEquals(0, getAlertV2HistoryDocCount()) + } + + fun `test generated alert gets expired because monitor was edited with alert history disabled`() { + client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_ENABLED.key, "false") + + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) + + // first create a ppl monitor that's guaranteed to generate an alert + val initialPplTrigger = randomPPLTrigger( + id = "initialID", + throttleDuration = null, + // for this test, configured expire can't be the reason for alert expiration + expireDuration = 1000L, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + + val initialPplMonitorConfig = randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 20, unit = MINUTES), + triggers = listOf(initialPplTrigger), + query = "source = $TEST_INDEX_NAME | head 10" + ) + + val pplMonitor = createRandomPPLMonitor(initialPplMonitorConfig) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponsePreExpire = getAlertV2s() + val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 + + assert(triggered) { "Monitor should have triggered but it didn't" } + assert(alertsGeneratedPreExpire) { "Alerts should have been generated but they weren't" } + + // update the monitor to any new config + updateMonitorV2(randomPPLMonitor().makeCopy(pplMonitor.id, pplMonitor.version)) + + // sleep so postIndex can expire the generated alert + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 5, TimeUnit.SECONDS) + + val getAlertsResponsePostExpire = getAlertV2s() + val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 + assert(!alertsGeneratedPostExpire) + + assertEquals(0, getAlertV2HistoryDocCount()) + } + + private fun assertIndexExists(index: String) { + val response = client().makeRequest("HEAD", index) + assertEquals("Index $index does not exist.", RestStatus.OK, response.restStatus()) + } + + private fun assertIndexDoesNotExist(index: String) { + val response = client().makeRequest("HEAD", index) + assertEquals("Index $index exists when it shouldn't.", RestStatus.NOT_FOUND, response.restStatus()) + } + + private fun resetHistorySettings() { + client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_ENABLED.key, "true") + client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_ROLLOVER_PERIOD.key, "60s") + client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_RETENTION_PERIOD.key, "60s") + } + + private fun getAlertV2Indices(): List { + val response = client().makeRequest("GET", "/_cat/indices/${AlertV2Indices.ALL_ALERT_V2_INDEX_PATTERN}?format=json") + val xcp = createParser(XContentType.JSON.xContent(), response.entity.content) + val responseList = xcp.list() + val indices = mutableListOf() + responseList.filterIsInstance>().forEach { indices.add(it["index"] as String) } + + return indices + } + + // generates alerts by creating then executing a monitor + private fun generateAlertV2s( + pplMonitorConfig: PPLSQLMonitor = randomPPLMonitor( + query = "source = $TEST_INDEX_NAME | head 3", + triggers = listOf( + randomPPLTrigger( + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L + ) + ) + ) + ): String { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) + indexDocFromSomeTimeAgo(2, MINUTES, "def", 10) + indexDocFromSomeTimeAgo(3, MINUTES, "ghi", 7) + + val pplMonitor = createRandomPPLMonitor(pplMonitorConfig) + + val executeResponse = executeMonitorV2(pplMonitor.id) + + // ensure execute call succeeded + val xcp = createParser(XContentType.JSON.xContent(), executeResponse.entity.content) + val output = xcp.map() + assertNull("Error running monitor v2", output["error"]) + + return pplMonitor.id + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/AlertV2Tests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/AlertV2Tests.kt new file mode 100644 index 000000000..fade152bc --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/AlertV2Tests.kt @@ -0,0 +1,59 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.modelv2 + +import org.opensearch.alerting.assertAlertV2sEqual +import org.opensearch.alerting.modelv2.AlertV2.Companion.ALERT_V2_ID_FIELD +import org.opensearch.alerting.modelv2.AlertV2.Companion.ALERT_V2_VERSION_FIELD +import org.opensearch.alerting.modelv2.AlertV2.Companion.ERROR_MESSAGE_FIELD +import org.opensearch.alerting.modelv2.AlertV2.Companion.EXECUTION_ID_FIELD +import org.opensearch.alerting.modelv2.AlertV2.Companion.SEVERITY_FIELD +import org.opensearch.alerting.randomAlertV2 +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.test.OpenSearchTestCase + +class AlertV2Tests : OpenSearchTestCase() { + fun `test alertv2 as stream`() { + val alertV2 = randomAlertV2() + val out = BytesStreamOutput() + alertV2.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newAlertV2 = AlertV2(sin) + assertAlertV2sEqual(alertV2, newAlertV2) + } + + fun `test alertv2 asTemplateArgs`() { + val alertV2 = randomAlertV2() + val templateArgs = alertV2.asTemplateArg() + + assertEquals( + "Template args field $ALERT_V2_ID_FIELD doesn't match", + alertV2.id, + templateArgs[ALERT_V2_ID_FIELD] + ) + assertEquals( + "Template args field $ALERT_V2_VERSION_FIELD doesn't match", + alertV2.version, + templateArgs[ALERT_V2_VERSION_FIELD] + ) + assertEquals( + "Template args field $ERROR_MESSAGE_FIELD doesn't match", + alertV2.errorMessage, + templateArgs[ERROR_MESSAGE_FIELD] + ) + assertEquals( + "Template args field $EXECUTION_ID_FIELD doesn't match", + alertV2.executionId, + templateArgs[EXECUTION_ID_FIELD] + ) + assertEquals( + "Template args field $SEVERITY_FIELD doesn't match", + alertV2.severity.value, + templateArgs[SEVERITY_FIELD] + ) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/MonitorV2Tests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/MonitorV2Tests.kt new file mode 100644 index 000000000..998131ad2 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/MonitorV2Tests.kt @@ -0,0 +1,172 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.modelv2 + +import org.opensearch.alerting.assertPplMonitorsEqual +import org.opensearch.alerting.modelv2.MonitorV2.Companion.ALERTING_V2_MAX_NAME_LENGTH +import org.opensearch.alerting.modelv2.MonitorV2.Companion.ENABLED_FIELD +import org.opensearch.alerting.modelv2.MonitorV2.Companion.ENABLED_TIME_FIELD +import org.opensearch.alerting.modelv2.MonitorV2.Companion.LAST_UPDATE_TIME_FIELD +import org.opensearch.alerting.modelv2.MonitorV2.Companion.LOOK_BACK_WINDOW_FIELD +import org.opensearch.alerting.modelv2.MonitorV2.Companion.MONITOR_V2_MIN_LOOK_BACK_WINDOW +import org.opensearch.alerting.modelv2.MonitorV2.Companion.NAME_FIELD +import org.opensearch.alerting.modelv2.MonitorV2.Companion.SCHEDULE_FIELD +import org.opensearch.alerting.modelv2.PPLSQLMonitor.Companion.QUERY_FIELD +import org.opensearch.alerting.randomPPLMonitor +import org.opensearch.alerting.randomPPLTrigger +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.commons.alerting.util.IndexUtils.Companion._ID +import org.opensearch.commons.alerting.util.IndexUtils.Companion._VERSION +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.test.OpenSearchTestCase +import java.lang.IllegalArgumentException +import java.time.Instant + +class MonitorV2Tests : OpenSearchTestCase() { + fun `test enabled time`() { + val pplMonitor = randomPPLMonitor(enabled = true, enabledTime = Instant.now()) + try { + pplMonitor.makeCopy(enabled = false) + fail("Disabling monitor with enabled time set should fail.") + } catch (_: IllegalArgumentException) {} + + val disabledMonitor = pplMonitor.copy(enabled = false, enabledTime = null) + + try { + disabledMonitor.makeCopy(enabled = true) + fail("Enabling monitor without enabled time should fail") + } catch (_: IllegalArgumentException) {} + } + + fun `test max triggers`() { + val tooManyTriggers = mutableListOf() + for (i in 0..10) { // 11 times + tooManyTriggers.add(randomPPLTrigger()) + } + + try { + randomPPLMonitor(triggers = tooManyTriggers) + fail("Monitor with too many triggers should be rejected.") + } catch (_: IllegalArgumentException) {} + } + + fun `test monitor name too long`() { + var monitorName = "" + for (i in 0 until ALERTING_V2_MAX_NAME_LENGTH + 1) { + monitorName += "a" + } + + try { + randomPPLMonitor(name = monitorName) + fail("Monitor with too long a name should be rejected.") + } catch (_: IllegalArgumentException) {} + } + + fun `test monitor min look back window`() { + try { + randomPPLMonitor( + lookBackWindow = MONITOR_V2_MIN_LOOK_BACK_WINDOW - 1 + ) + fail("Monitor with too long a name should be rejected.") + } catch (_: IllegalArgumentException) {} + } + + fun `test monitor no triggers`() { + try { + randomPPLMonitor( + triggers = listOf() + ) + fail("Monitor without triggers be rejected.") + } catch (_: IllegalArgumentException) {} + } + + fun `test monitor with look back window without timestamp field`() { + try { + randomPPLMonitor( + lookBackWindow = randomLongBetween(1, 10), + timestampField = null + ) + fail("Monitor with look back window but without timestamp field be rejected.") + } catch (_: IllegalArgumentException) {} + } + + fun `test monitor without look back window with timestamp field`() { + try { + randomPPLMonitor( + lookBackWindow = null, + timestampField = "some_timestamp_field" + ) + fail("Monitor without look back window but with timestamp field be rejected.") + } catch (_: IllegalArgumentException) {} + } + + fun `test monitor v2 as stream`() { + val pplMonitor = randomPPLMonitor() + val monitorV2 = pplMonitor as MonitorV2 + val out = BytesStreamOutput() + MonitorV2.writeTo(out, monitorV2) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newMonitorV2 = MonitorV2.readFrom(sin) + val newPplMonitor = newMonitorV2 as PPLSQLMonitor + assertPplMonitorsEqual(pplMonitor, newPplMonitor) + } + + fun `test ppl monitor as stream`() { + val pplMonitor = randomPPLMonitor() + val out = BytesStreamOutput() + pplMonitor.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newPplMonitor = PPLSQLMonitor(sin) + assertPplMonitorsEqual(pplMonitor, newPplMonitor) + } + + fun `test ppl monitor asTemplateArgs`() { + val pplMonitor = randomPPLMonitor() + val templateArgs = pplMonitor.asTemplateArg() + + assertEquals( + "Template args field $_ID doesn't match", + pplMonitor.id, + templateArgs[_ID] + ) + assertEquals( + "Template args field $_VERSION doesn't match", + pplMonitor.version, + templateArgs[_VERSION] + ) + assertEquals( + "Template args field $NAME_FIELD doesn't match", + pplMonitor.name, + templateArgs[NAME_FIELD] + ) + assertEquals( + "Template args field $ENABLED_FIELD doesn't match", + pplMonitor.enabled, + templateArgs[ENABLED_FIELD] + ) + assertNotNull(templateArgs[SCHEDULE_FIELD]) + assertEquals( + "Template args field $LOOK_BACK_WINDOW_FIELD doesn't match", + pplMonitor.lookBackWindow, + templateArgs[LOOK_BACK_WINDOW_FIELD] + ) + assertEquals( + "Template args field $LAST_UPDATE_TIME_FIELD doesn't match", + pplMonitor.lastUpdateTime.toEpochMilli(), + templateArgs[LAST_UPDATE_TIME_FIELD] + ) + assertEquals( + "Template args field $ENABLED_TIME_FIELD doesn't match", + pplMonitor.enabledTime?.toEpochMilli(), + templateArgs[ENABLED_TIME_FIELD] + ) + assertEquals( + "Template args field $QUERY_FIELD doesn't match", + pplMonitor.query, + templateArgs[QUERY_FIELD] + ) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/RunResultV2Tests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/RunResultV2Tests.kt new file mode 100644 index 000000000..4290af27c --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/RunResultV2Tests.kt @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.modelv2 + +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.test.OpenSearchTestCase + +class RunResultV2Tests : OpenSearchTestCase() { + fun `test ppl sql trigger run result as stream`() { + val runResult = PPLSQLTriggerRunResult( + triggerName = "some-trigger", + triggered = true, + error = IllegalArgumentException("some-error") + ) + val out = BytesStreamOutput() + runResult.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newRunResult = PPLSQLTriggerRunResult(sin) + assertEquals(runResult.triggerName, newRunResult.triggerName) + } + + fun `test ppl sql monitor run result as monitor v2 run result as stream`() { + val monitorRunResult = PPLSQLMonitorRunResult( + monitorName = "some-monitor", + error = IllegalArgumentException("some-error"), + triggerResults = mapOf( + "some-trigger-id" to PPLSQLTriggerRunResult( + triggerName = "some-trigger", + triggered = true, + error = IllegalArgumentException("some-error") + ) + ), + pplQueryResults = mapOf("some-result" to mapOf("some-field" to 2)) + ) as MonitorV2RunResult + + val out = BytesStreamOutput() + MonitorV2RunResult.writeTo(out, monitorRunResult) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newMonitorRunResult = MonitorV2RunResult.readFrom(sin) + assertEquals(monitorRunResult.monitorName, newMonitorRunResult.monitorName) + assertEquals(monitorRunResult.error?.message, newMonitorRunResult.error?.message) + assert(monitorRunResult.triggerResults.containsKey("some-trigger-id")) + assert(newMonitorRunResult.triggerResults.containsKey("some-trigger-id")) + assertEquals( + monitorRunResult.triggerResults["some-trigger-id"]!!.triggerName, + newMonitorRunResult.triggerResults["some-trigger-id"]!!.triggerName + ) + assertEquals( + monitorRunResult.triggerResults["some-trigger-id"]!!.triggered, + newMonitorRunResult.triggerResults["some-trigger-id"]!!.triggered + ) + assertEquals( + monitorRunResult.triggerResults["some-trigger-id"]!!.error?.message, + newMonitorRunResult.triggerResults["some-trigger-id"]!!.error?.message + ) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/TriggerV2Tests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/TriggerV2Tests.kt new file mode 100644 index 000000000..430572254 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/TriggerV2Tests.kt @@ -0,0 +1,254 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.modelv2 + +import org.opensearch.alerting.assertPplTriggersEqual +import org.opensearch.alerting.modelv2.MonitorV2.Companion.ALERTING_V2_MAX_NAME_LENGTH +import org.opensearch.alerting.modelv2.PPLSQLTrigger.Companion.CONDITION_TYPE_FIELD +import org.opensearch.alerting.modelv2.PPLSQLTrigger.Companion.CUSTOM_CONDITION_FIELD +import org.opensearch.alerting.modelv2.PPLSQLTrigger.Companion.MODE_FIELD +import org.opensearch.alerting.modelv2.PPLSQLTrigger.Companion.NUM_RESULTS_CONDITION_FIELD +import org.opensearch.alerting.modelv2.PPLSQLTrigger.Companion.NUM_RESULTS_VALUE_FIELD +import org.opensearch.alerting.modelv2.PPLSQLTrigger.ConditionType +import org.opensearch.alerting.modelv2.PPLSQLTrigger.NumResultsCondition +import org.opensearch.alerting.modelv2.TriggerV2.Companion.ACTIONS_FIELD +import org.opensearch.alerting.modelv2.TriggerV2.Companion.EXPIRE_FIELD +import org.opensearch.alerting.modelv2.TriggerV2.Companion.ID_FIELD +import org.opensearch.alerting.modelv2.TriggerV2.Companion.MONITOR_V2_MIN_EXPIRE_DURATION_MINUTES +import org.opensearch.alerting.modelv2.TriggerV2.Companion.MONITOR_V2_MIN_THROTTLE_DURATION_MINUTES +import org.opensearch.alerting.modelv2.TriggerV2.Companion.NAME_FIELD +import org.opensearch.alerting.modelv2.TriggerV2.Companion.NOTIFICATIONS_ID_MAX_LENGTH +import org.opensearch.alerting.modelv2.TriggerV2.Companion.SEVERITY_FIELD +import org.opensearch.alerting.modelv2.TriggerV2.Companion.THROTTLE_FIELD +import org.opensearch.alerting.randomAction +import org.opensearch.alerting.randomPPLTrigger +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.test.OpenSearchTestCase +import java.lang.IllegalArgumentException + +class TriggerV2Tests : OpenSearchTestCase() { + fun `test min throttle duration`() { + try { + randomPPLTrigger( + throttleDuration = MONITOR_V2_MIN_THROTTLE_DURATION_MINUTES - 1 + ) + fail("Trigger with throttle duration less than 1 should be rejected") + } catch (_: IllegalArgumentException) {} + } + + fun `test min expire duration`() { + try { + randomPPLTrigger( + expireDuration = MONITOR_V2_MIN_EXPIRE_DURATION_MINUTES - 1 + ) + fail("Trigger with expire duration less than 1 should be rejected") + } catch (_: IllegalArgumentException) {} + } + + fun `test trigger name too long`() { + var triggerName = "" + for (i in 0 until ALERTING_V2_MAX_NAME_LENGTH + 1) { + triggerName += "a" + } + + try { + randomPPLTrigger(name = triggerName) + fail("Trigger with too long a name should be rejected.") + } catch (_: IllegalArgumentException) {} + } + + fun `test number of results trigger with negative number of results value`() { + try { + randomPPLTrigger( + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsValue = -1L, + numResultsCondition = NumResultsCondition.GREATER_THAN + ) + fail("Number of results trigger with negative number of results value should be rejected.") + } catch (_: IllegalArgumentException) {} + } + + fun `test trigger action name too long`() { + var actionName = "" + for (i in 0 until ALERTING_V2_MAX_NAME_LENGTH + 1) { + actionName += "a" + } + + try { + randomPPLTrigger( + actions = listOf( + randomAction( + name = actionName + ) + ) + ) + fail("Trigger action with too long a name should be rejected.") + } catch (_: IllegalArgumentException) {} + } + + fun `test trigger action channel ID too long`() { + var channelId = "" + for (i in 0 until NOTIFICATIONS_ID_MAX_LENGTH + 1) { + channelId += "a" + } + + try { + randomPPLTrigger( + actions = listOf( + randomAction( + destinationId = channelId + ) + ) + ) + fail("Trigger action with too long a channel ID should be rejected.") + } catch (_: IllegalArgumentException) {} + } + + fun `test number_of_results trigger has no number_of_results value field`() { + try { + randomPPLTrigger( + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.entries.random(), + numResultsValue = null, + customCondition = null + ) + fail("Number of results trigger that has no number of results value should be rejected.") + } catch (_: IllegalArgumentException) {} + } + + fun `test number_of_results trigger has no number_of_results condition field`() { + try { + randomPPLTrigger( + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = null, + numResultsValue = randomLongBetween(1, 10), + customCondition = null + ) + fail("Number of results trigger that has no number of results condition should be rejected.") + } catch (_: IllegalArgumentException) {} + } + + fun `test number_of_results trigger has custom_condition value field`() { + try { + randomPPLTrigger( + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = null, + numResultsValue = null, + customCondition = "eval result = something > 5" + ) + fail("Number of results trigger that has custom condition should be rejected.") + } catch (_: IllegalArgumentException) {} + } + + fun `test custom trigger has number_of_results value field`() { + try { + randomPPLTrigger( + conditionType = ConditionType.CUSTOM, + numResultsCondition = NumResultsCondition.entries.random(), + numResultsValue = null, + customCondition = null + ) + fail("Number of results trigger that has no number of results value should be rejected.") + } catch (_: IllegalArgumentException) {} + } + + fun `test custom trigger has number_of_results condition field`() { + try { + randomPPLTrigger( + conditionType = ConditionType.CUSTOM, + numResultsCondition = null, + numResultsValue = randomLongBetween(1, 10), + customCondition = null + ) + fail("Number of results trigger that has no number of results condition should be rejected.") + } catch (_: IllegalArgumentException) {} + } + + fun `test custom trigger has no custom_condition value field`() { + try { + randomPPLTrigger( + conditionType = ConditionType.CUSTOM, + numResultsCondition = null, + numResultsValue = null, + customCondition = null + ) + fail("Number of results trigger that has custom condition should be rejected.") + } catch (_: IllegalArgumentException) {} + } + + fun `test ppl trigger as stream`() { + val pplTrigger = randomPPLTrigger() + val out = BytesStreamOutput() + pplTrigger.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newPplTrigger = PPLSQLTrigger(sin) + assertPplTriggersEqual(pplTrigger, newPplTrigger) + } + + fun `test ppl trigger asTemplateArgs`() { + val pplTrigger = randomPPLTrigger() + val templateArgs = pplTrigger.asTemplateArg() + + assertEquals( + "Template args field $ID_FIELD doesn't match", + pplTrigger.id, + templateArgs[ID_FIELD] + ) + assertEquals( + "Template args field $NAME_FIELD doesn't match", + pplTrigger.name, + templateArgs[NAME_FIELD] + ) + assertEquals( + "Template args field $SEVERITY_FIELD doesn't match", + pplTrigger.severity.value, + templateArgs[SEVERITY_FIELD] + ) + assertEquals( + "Template args field $THROTTLE_FIELD doesn't match", + pplTrigger.throttleDuration, + templateArgs[THROTTLE_FIELD] + ) + assertEquals( + "Template args field $EXPIRE_FIELD doesn't match", + pplTrigger.expireDuration, + templateArgs[EXPIRE_FIELD] + ) + assertEquals( + "Template args field $EXPIRE_FIELD doesn't match", + pplTrigger.expireDuration, + templateArgs[EXPIRE_FIELD] + ) + val actions = templateArgs[ACTIONS_FIELD] as List<*> + assertEquals("number of trigger actions doesn't match", pplTrigger.actions.size, actions.size) + assertEquals( + "Template args field $MODE_FIELD doesn't match", + pplTrigger.mode.value, + templateArgs[MODE_FIELD] + ) + assertEquals( + "Template args field $CONDITION_TYPE_FIELD doesn't match", + pplTrigger.conditionType.value, + templateArgs[CONDITION_TYPE_FIELD] + ) + assertEquals( + "Template args field $NUM_RESULTS_CONDITION_FIELD doesn't match", + pplTrigger.numResultsCondition?.value, + templateArgs[NUM_RESULTS_CONDITION_FIELD] + ) + assertEquals( + "Template args field $NUM_RESULTS_VALUE_FIELD doesn't match", + pplTrigger.numResultsValue, + templateArgs[NUM_RESULTS_VALUE_FIELD] + ) + assertEquals( + "Template args field $CUSTOM_CONDITION_FIELD doesn't match", + pplTrigger.customCondition, + templateArgs[CUSTOM_CONDITION_FIELD] + ) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt index 2e3743572..deb1b7cd0 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt @@ -72,7 +72,9 @@ import java.util.concurrent.TimeUnit @Suppress("UNCHECKED_CAST") class MonitorRestApiIT : AlertingRestTestCase() { - val USE_TYPED_KEYS = ToXContent.MapParams(mapOf("with_type" to "true")) + companion object { + val USE_TYPED_KEYS = ToXContent.MapParams(mapOf("with_type" to "true")) + } @Throws(Exception::class) fun `test plugin is loaded`() { @@ -1534,19 +1536,6 @@ class MonitorRestApiIT : AlertingRestTestCase() { assertEquals("More than $numberOfNodes successful node", numberOfNodes, nodesResponse["successful"]) } - private fun isMonitorScheduled(monitorId: String, alertingStatsResponse: Map): Boolean { - val nodesInfo = alertingStatsResponse["nodes"] as Map - for (nodeId in nodesInfo.keys) { - val nodeInfo = nodesInfo[nodeId] as Map - val jobsInfo = nodeInfo["jobs_info"] as Map - if (jobsInfo.keys.contains(monitorId)) { - return true - } - } - - return false - } - private fun assertAlertingStatsSweeperEnabled(alertingStatsResponse: Map, expected: Boolean) { assertEquals( "Legacy scheduled job enabled field is not set to $expected", diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorV2RestApiIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorV2RestApiIT.kt new file mode 100644 index 000000000..dbf379e91 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorV2RestApiIT.kt @@ -0,0 +1,524 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.resthandler + +import org.apache.hc.core5.http.ContentType +import org.apache.hc.core5.http.io.entity.StringEntity +import org.junit.Before +import org.opensearch.alerting.AlertingPlugin.Companion.MONITOR_V2_BASE_URI +import org.opensearch.alerting.AlertingRestTestCase +import org.opensearch.alerting.TEST_INDEX_MAPPINGS +import org.opensearch.alerting.TEST_INDEX_NAME +import org.opensearch.alerting.assertPplMonitorsEqual +import org.opensearch.alerting.core.settings.AlertingV2Settings +import org.opensearch.alerting.makeRequest +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.modelv2.PPLSQLMonitor +import org.opensearch.alerting.modelv2.PPLSQLTrigger.ConditionType +import org.opensearch.alerting.randomAction +import org.opensearch.alerting.randomPPLMonitor +import org.opensearch.alerting.randomPPLTrigger +import org.opensearch.alerting.randomQueryLevelMonitor +import org.opensearch.alerting.randomTemplateScript +import org.opensearch.alerting.resthandler.MonitorRestApiIT.Companion.USE_TYPED_KEYS +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_EXPIRE_DURATION +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_LOOK_BACK_WINDOW +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_MONITORS +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_QUERY_LENGTH +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_THROTTLE_DURATION +import org.opensearch.alerting.settings.AlertingSettings.Companion.NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH +import org.opensearch.alerting.settings.AlertingSettings.Companion.NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH +import org.opensearch.client.ResponseException +import org.opensearch.common.UUIDs +import org.opensearch.common.settings.Settings +import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.alerting.model.ScheduledJob +import org.opensearch.core.common.bytes.BytesReference +import org.opensearch.core.rest.RestStatus +import org.opensearch.core.xcontent.XContentBuilder +import org.opensearch.core.xcontent.XContentParser +import org.opensearch.core.xcontent.XContentParserUtils +import org.opensearch.index.query.QueryBuilders +import org.opensearch.search.builder.SearchSourceBuilder +import org.opensearch.test.junit.annotations.TestLogging +import java.time.temporal.ChronoUnit.MINUTES + +/*** + * Tests Alerting V2 CRUD and validations + * + * Gradle command to run this suite: + * ./gradlew :alerting:integTest -Dhttps=true -Dsecurity=true -Duser=admin -Dpassword=admin \ + * --tests "org.opensearch.alerting.resthandler.MonitorV2RestApiIT" + */ +@TestLogging("level:DEBUG", reason = "Debug for tests.") +@Suppress("UNCHECKED_CAST") +class MonitorV2RestApiIT : AlertingRestTestCase() { + @Before + fun enableAlertingV2() { + client().updateSettings(AlertingV2Settings.ALERTING_V2_ENABLED.key, "true") + } + + /* Simple Case Tests */ + fun `test create ppl monitor`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + val pplMonitor = randomPPLMonitor() + + val response = client().makeRequest("POST", MONITOR_V2_BASE_URI, emptyMap(), pplMonitor.toHttpEntity()) + assertEquals("Unable to create a new monitor v2", RestStatus.OK, response.restStatus()) + + val responseBody = response.asMap() + val createdId = responseBody["_id"] as String + val createdVersion = responseBody["_version"] as Int + assertNotEquals("response is missing Id", MonitorV2.NO_ID, createdId) + assertEquals("incorrect version", 1, createdVersion) + } + + fun `test update ppl monitor`() { + val originalMonitor = createRandomPPLMonitor() + + val newMonitorConfig = randomPPLMonitor() + + val updateResponse = client().makeRequest( + "PUT", + "$MONITOR_V2_BASE_URI/${originalMonitor.id}", + emptyMap(), newMonitorConfig.toHttpEntity() + ) + + assertEquals("Update monitor failed", RestStatus.OK, updateResponse.restStatus()) + val responseBody = updateResponse.asMap() + assertEquals("Updated monitor id doesn't match", originalMonitor.id, responseBody["_id"] as String) + assertEquals("Version not incremented", (originalMonitor.version + 1).toInt(), responseBody["_version"] as Int) + + val updatedMonitor = getMonitorV2(originalMonitor.id) as PPLSQLMonitor + assertPplMonitorsEqual(newMonitorConfig, updatedMonitor) + } + + fun `test get ppl monitor`() { + // first create the monitor + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + val pplMonitor = randomPPLMonitor() + + val createResponse = client().makeRequest("POST", MONITOR_V2_BASE_URI, emptyMap(), pplMonitor.toHttpEntity()) + assertEquals("Unable to create a new monitor v2", RestStatus.OK, createResponse.restStatus()) + + val responseBody = createResponse.asMap() + val pplMonitorId = responseBody["_id"] as String + val pplMonitorVersion = (responseBody["_version"] as Int).toLong() + + // then attempt to get it + val response = client().makeRequest("GET", "$MONITOR_V2_BASE_URI/$pplMonitorId") + assertEquals("Unable to get monitorV2 $pplMonitorId", RestStatus.OK, response.restStatus()) + + val parser = createParser(XContentType.JSON.xContent(), response.entity.content) + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser) + + lateinit var id: String + var version: Long = 0 + lateinit var storedPplMonitor: PPLSQLMonitor + + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + parser.nextToken() + + when (parser.currentName()) { + "_id" -> id = parser.text() + "_version" -> version = parser.longValue() + "monitorV2" -> storedPplMonitor = MonitorV2.parse(parser) as PPLSQLMonitor + } + } + + assertEquals( + "Monitor V2 ID from Get Monitor doesn't match one from Create Monitor response", + pplMonitorId, id + ) + assertEquals( + "Monitor V2 version from Get Monitor doesn't match one from Create Monitor response", + pplMonitorVersion, version + ) + assertPplMonitorsEqual(pplMonitor, storedPplMonitor) + } + + fun `test head ppl monitor`() { + val submittedPplMonitor = createRandomPPLMonitor() + val response = client().makeRequest("HEAD", "$MONITOR_V2_BASE_URI/${submittedPplMonitor.id}") + assertEquals("Unable to get monitorV2 ${submittedPplMonitor.id}", RestStatus.NO_CONTENT, response.restStatus()) + } + + fun `test search ppl monitor with GET and match_all`() { + createRandomPPLMonitor() + + val search = SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).toString() + val searchResponse = client().makeRequest( + "GET", "$MONITOR_V2_BASE_URI/_search", + emptyMap(), StringEntity(search, ContentType.APPLICATION_JSON) + ) + + assertEquals("Search monitor failed", RestStatus.OK, searchResponse.restStatus()) + val xcp = createParser(XContentType.JSON.xContent(), searchResponse.entity.content) + val hits = xcp.map()["hits"]!! as Map> + val numberDocsFound = hits["total"]?.get("value") + assertEquals("PPL Monitor not found during search", 1, numberDocsFound) + } + + fun `test search ppl monitor with POST and term query on ID`() { + val pplMonitor = createRandomPPLMonitor() + + val search = SearchSourceBuilder().query(QueryBuilders.termQuery("_id", pplMonitor.id)).toString() + val searchResponse = client().makeRequest( + "POST", "$MONITOR_V2_BASE_URI/_search", + emptyMap(), StringEntity(search, ContentType.APPLICATION_JSON) + ) + + assertEquals("Search monitor failed", RestStatus.OK, searchResponse.restStatus()) + val xcp = createParser(XContentType.JSON.xContent(), searchResponse.entity.content) + val hits = xcp.map()["hits"]!! as Map> + val numberDocsFound = hits["total"]?.get("value") + assertEquals("PPL Monitor not found during search", 1, numberDocsFound) + } + + fun `test delete ppl monitor`() { + val pplMonitor = createRandomPPLMonitor() + + val deleteResponse = client().makeRequest("DELETE", "$MONITOR_V2_BASE_URI/${pplMonitor.id}") + assertEquals("Delete failed", RestStatus.OK, deleteResponse.restStatus()) + + val getResponse = client().makeRequest("HEAD", "$MONITOR_V2_BASE_URI/${pplMonitor.id}") + assertEquals("Deleted monitor still exists", RestStatus.NOT_FOUND, getResponse.restStatus()) + } + + fun `test parsing ppl monitor as a scheduled job`() { + val monitorV2 = createRandomPPLMonitor() + + val builder = monitorV2.toXContentWithUser(XContentBuilder.builder(XContentType.JSON.xContent()), USE_TYPED_KEYS) + val string = BytesReference.bytes(builder).utf8ToString() + val xcp = createParser(XContentType.JSON.xContent(), string) + val scheduledJob = ScheduledJob.parse(xcp, monitorV2.id, monitorV2.version) + assertEquals(monitorV2, scheduledJob) + } + + fun `test monitor stats v1 and v2 only return stats for their respective monitors`() { + enableScheduledJob() + + val monitorV1Id = createMonitor(randomQueryLevelMonitor(enabled = true)).id + val monitorV2Id = createRandomPPLMonitor(randomPPLMonitor(enabled = true)).id + + val statsAllResponse = getAlertingStats(alertingVersion = null) + val statsV1Response = getAlertingStats(alertingVersion = "v1") + val statsV2Response = getAlertingStats(alertingVersion = "v2") + + logger.info("all stats: $statsAllResponse") + logger.info("v1 stats: $statsV1Response") + logger.info("v2 stats: $statsV2Response") + + assertTrue("All stats does not contain V1 Monitor", isMonitorScheduled(monitorV1Id, statsAllResponse)) + assertTrue("All stats does not contain V2 Monitor", isMonitorScheduled(monitorV2Id, statsAllResponse)) + + assertTrue("V1 stats does not contain V1 Monitor", isMonitorScheduled(monitorV1Id, statsV1Response)) + assertFalse("V1 stats contains V2 Monitor", isMonitorScheduled(monitorV2Id, statsV1Response)) + + assertTrue("V2 stats does not contain V2 Monitor", isMonitorScheduled(monitorV2Id, statsV2Response)) + assertFalse("V2 stats contains V1 Monitor", isMonitorScheduled(monitorV1Id, statsV2Response)) + } + + /* Validation Tests */ + fun `test create ppl monitor that queries nonexistent index fails`() { + val pplMonitorConfig = randomPPLMonitor( + query = "source = nonexistent_index | head 10" + ) + + // ensure the request fails + try { + createRandomPPLMonitor(pplMonitorConfig) + fail("Expected request to fail with BAD_REQUEST but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + } + + // ensure no monitor was created + ensureNumMonitorV2s(0) + } + + fun `test create ppl monitor with more than max allowed monitors fails`() { + adminClient().updateSettings(ALERTING_V2_MAX_MONITORS.key, 1) + + createRandomPPLMonitor() + + // ensure the request fails + try { + createRandomPPLMonitor() + fail("Expected request to fail with BAD_REQUEST but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + } + + // ensure no monitor was created + ensureNumMonitorV2s(1) + } + + fun `test create ppl monitor with throttle greater than max fails`() { + val maxThrottleDuration = 60L + client().updateSettings(ALERTING_V2_MAX_THROTTLE_DURATION.key, maxThrottleDuration) + + // ensure the request fails + try { + createRandomPPLMonitor( + randomPPLMonitor( + triggers = listOf( + randomPPLTrigger(throttleDuration = maxThrottleDuration + 10) + ) + ) + ) + fail("Expected request to fail with BAD_REQUEST but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + } + + // ensure no monitor was created + ensureNumMonitorV2s(0) + } + + fun `test create ppl monitor with expire greater than max fails`() { + val maxExpireDuration = 60L + client().updateSettings(ALERTING_V2_MAX_EXPIRE_DURATION.key, maxExpireDuration) + + // ensure the request fails + try { + createRandomPPLMonitor( + randomPPLMonitor( + triggers = listOf( + randomPPLTrigger(expireDuration = maxExpireDuration + 10) + ) + ) + ) + fail("Expected request to fail with BAD_REQUEST but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + } + + // ensure no monitor was created + ensureNumMonitorV2s(0) + } + + fun `test create ppl monitor with look back window greater than max fails`() { + val maxLookBackWindow = 60L + client().updateSettings(ALERTING_V2_MAX_LOOK_BACK_WINDOW.key, maxLookBackWindow) + + // ensure the request fails + try { + createRandomPPLMonitor( + randomPPLMonitor( + lookBackWindow = maxLookBackWindow + 10 + ) + ) + fail("Expected request to fail with BAD_REQUEST but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + } + + // ensure no monitor was created + ensureNumMonitorV2s(0) + } + + fun `test create ppl monitor with invalid query fails`() { + // ensure the request fails + try { + createRandomPPLMonitor( + randomPPLMonitor( + query = "source = $TEST_INDEX_NAME | not valid ppl" + ) + ) + fail("Expected request to fail with BAD_REQUEST but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + } + + // ensure no monitor was created + ensureNumMonitorV2s(0) + } + + fun `test create ppl monitor with query that's too long fails`() { + adminClient().updateSettings(ALERTING_V2_MAX_QUERY_LENGTH.key, 1) + + // ensure the request fails + try { + createRandomPPLMonitor( + randomPPLMonitor( + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + fail("Expected request to fail with BAD_REQUEST but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + } + + // ensure no monitor was created + ensureNumMonitorV2s(0) + } + + fun `test create ppl monitor with invalid custom condition fails`() { + // ensure the request fails + try { + createRandomPPLMonitor( + randomPPLMonitor( + triggers = listOf( + randomPPLTrigger( + conditionType = ConditionType.CUSTOM, + customCondition = "not a valid PPL custom condition", + numResultsCondition = null, + numResultsValue = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + fail("Expected request to fail with BAD_REQUEST but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + } + + // ensure no monitor was created + ensureNumMonitorV2s(0) + } + + fun `test create ppl monitor with custom condition that evals to num not bool fails`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 1) + indexDocFromSomeTimeAgo(2, MINUTES, "abc", 2) + + // ensure the request fails + try { + createRandomPPLMonitor( + randomPPLMonitor( + triggers = listOf( + randomPPLTrigger( + conditionType = ConditionType.CUSTOM, + customCondition = "eval something = sum * 2", + numResultsCondition = null, + numResultsValue = null + ) + ), + query = "source = $TEST_INDEX_NAME | stats sum(number) as sum by abc" + ) + ) + fail("Expected request to fail with BAD_REQUEST but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + } + + // ensure no monitor was created + ensureNumMonitorV2s(0) + } + + fun `test create ppl monitor with notification subject source too long fails`() { + adminClient().updateSettings(NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH.key, 100) + + var subjectTooLong = "" + for (i in 0 until 101) { + subjectTooLong += "a" + } + + // ensure the request fails + try { + createRandomPPLMonitor( + randomPPLMonitor( + triggers = listOf( + randomPPLTrigger( + actions = listOf( + randomAction( + template = randomTemplateScript( + source = "some message" + ), + subjectTemplate = randomTemplateScript( + source = subjectTooLong + ) + ) + ) + ) + ) + ) + ) + fail("Expected request to fail with BAD_REQUEST but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + } + + // ensure no monitor was created + ensureNumMonitorV2s(0) + } + + fun `test create ppl monitor with notification message source too long fails`() { + adminClient().updateSettings(NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH.key, 1000) + + var messageTooLong = "" + for (i in 0 until 1001) { + messageTooLong += "a" + } + + // ensure the request fails + try { + createRandomPPLMonitor( + randomPPLMonitor( + triggers = listOf( + randomPPLTrigger( + actions = listOf( + randomAction( + template = randomTemplateScript( + source = messageTooLong + ), + subjectTemplate = randomTemplateScript( + source = "some subject" + ) + ) + ) + ) + ) + ) + ) + fail("Expected request to fail with BAD_REQUEST but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + } + + // ensure no monitor was created + ensureNumMonitorV2s(0) + } + + fun `test get ppl monitor with invalid monitor ID length`() { + val badId = UUIDs.base64UUID() + "extra" + try { + client().makeRequest("GET", "$MONITOR_V2_BASE_URI/$badId") + fail("Expected request to fail with BAD_REQUEST but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + } + } + + fun `test update nonexistent ppl monitor fails`() { + // the random monitor query searches index TEST_INDEX_NAME, + // so we need to create that first to ensure at least the request body is valid + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + + val monitorV2 = randomPPLMonitor() + val randomId = UUIDs.base64UUID() + + try { + client().makeRequest("PUT", "$MONITOR_V2_BASE_URI/$randomId", emptyMap(), monitorV2.toHttpEntity()) + fail("Expected request to fail with NOT_FOUND but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.NOT_FOUND, e.response.restStatus()) + } + } + + fun `test delete nonexistent ppl monitor fails`() { + val randomId = UUIDs.base64UUID() + + try { + client().makeRequest("DELETE", "$MONITOR_V2_BASE_URI/$randomId") + fail("Expected request to fail with NOT_FOUND but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.NOT_FOUND, e.response.restStatus()) + } + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureMonitorV2RestApiIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureMonitorV2RestApiIT.kt new file mode 100644 index 000000000..ef82094bb --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureMonitorV2RestApiIT.kt @@ -0,0 +1,875 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.resthandler + +import org.apache.hc.core5.http.ContentType +import org.apache.hc.core5.http.HttpHeaders +import org.apache.hc.core5.http.io.entity.StringEntity +import org.apache.hc.core5.http.message.BasicHeader +import org.junit.After +import org.junit.Before +import org.junit.BeforeClass +import org.opensearch.alerting.ALERTING_FULL_ACCESS_ROLE +import org.opensearch.alerting.ALL_ACCESS_ROLE +import org.opensearch.alerting.AlertingPlugin.Companion.MONITOR_V2_BASE_URI +import org.opensearch.alerting.AlertingRestTestCase +import org.opensearch.alerting.PPL_FULL_ACCESS_ROLE +import org.opensearch.alerting.ROLE_TO_PERMISSION_MAPPING +import org.opensearch.alerting.TEST_INDEX_MAPPINGS +import org.opensearch.alerting.TEST_INDEX_NAME +import org.opensearch.alerting.core.settings.AlertingV2Settings +import org.opensearch.alerting.makeRequest +import org.opensearch.alerting.modelv2.PPLSQLMonitor +import org.opensearch.alerting.modelv2.PPLSQLTrigger.ConditionType +import org.opensearch.alerting.modelv2.PPLSQLTrigger.NumResultsCondition +import org.opensearch.alerting.modelv2.PPLSQLTrigger.TriggerMode +import org.opensearch.alerting.randomPPLMonitor +import org.opensearch.alerting.randomPPLTrigger +import org.opensearch.client.ResponseException +import org.opensearch.client.RestClient +import org.opensearch.common.settings.Settings +import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.alerting.model.IntervalSchedule +import org.opensearch.commons.rest.SecureRestClientBuilder +import org.opensearch.core.rest.RestStatus +import org.opensearch.index.query.QueryBuilders +import org.opensearch.search.builder.SearchSourceBuilder +import java.time.temporal.ChronoUnit.MINUTES + +/*** + * Tests Alerting V2 CRUD with role-based access control + * + * Gradle command to run this suite: + * ./gradlew :alerting:integTest -Dhttps=true -Dsecurity=true -Duser=admin -Dpassword=admin \ + * --tests "org.opensearch.alerting.resthandler.SecureMonitorV2RestApiIT" + */ +class SecureMonitorV2RestApiIT : AlertingRestTestCase() { + + companion object { + @BeforeClass + @JvmStatic fun setup() { + // things to execute once and keep around for the class + org.junit.Assume.assumeTrue(System.getProperty("security", "false")!!.toBoolean()) + } + } + + val user = "userD" + var userClient: RestClient? = null + + @Before + fun create() { + client().updateSettings(AlertingV2Settings.ALERTING_V2_ENABLED.key, "true") + if (userClient == null) { + createUser(user, arrayOf()) + userClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), user, password) + .setSocketTimeout(60000) + .setConnectionRequestTimeout(180000) + .build() + } + } + + @After + fun cleanup() { + userClient?.close() + deleteUser(user) + } + + fun `test create monitor as user without alerting access fails`() { + if (!isHttps()) { + return + } + + val pplMonitorConfig = randomPPLMonitor() + + createUserWithTestDataAndCustomRole( + user, + TEST_INDEX_NAME, + "custom_role", + listOf(), + null + ) + + try { + createMonitorV2WithClient( + userClient!!, + monitorV2 = pplMonitorConfig + ) + fail("Expected create monitor to fail as user does not have permissions to call alerting APIs") + } catch (e: ResponseException) { + assertEquals("Unexpected error status", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) + } + + ensureNumMonitorV2s(0) + } + + fun `test create monitor that queries index user doesn't have access to fails`() { + if (!isHttps()) { + return + } + + createIndex("some_index", Settings.EMPTY) + + val pplMonitorConfig = randomPPLMonitor( + query = "source = some_index | head 10" + ) + + createUserWithTestDataAndCustomRole( + user, + "other_index", + "custom_role", + listOf(), + getClusterPermissionsFromCustomRole(ALL_ACCESS_ROLE) + ) + + try { + createMonitorV2WithClient( + userClient!!, + monitorV2 = pplMonitorConfig + ) + fail("Expected create monitor to fail as user does not have permissions to index that monitor queries") + } catch (e: ResponseException) { + assertEquals("Unexpected error status", RestStatus.BAD_REQUEST.status, e.response.statusLine.statusCode) + } + + ensureNumMonitorV2s(0) + } + + fun `test update monitor that queries index user doesn't have access to fails`() { + if (!isHttps()) { + return + } + + // RBAC is out of scope for this test, so give all users and requests the same one + val backendRole = "backend_role_a" + + val pplMonitorConfig = randomPPLMonitor() + + // first create the monitor with a user that has access to all indices + // (the FULL_ACCESS_ROLEs include full index permissions) + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf(backendRole), + false + ) + + // this function automatically creates index TEST_INDEX_NAME, then a monitor that queries it + val pplMonitor = createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf(backendRole)) + + /* + user: String, + index: String, + role: String, + backendRoles: List, + clusterPermissions: String?, + */ + + // prepare a user that has full access to all cluster actions, + // but only access to a specific unrelated index + val noIndicesUser = "noIndicesUser" + createUserWithTestDataAndCustomRole( + noIndicesUser, + "unrelated_index", + "unrelated_role", + listOf(backendRole), + listOf(ROLE_TO_PERMISSION_MAPPING[ALL_ACCESS_ROLE]) + ) + + val noIndicesUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), noIndicesUser, password) + .setSocketTimeout(60000) + .setConnectionRequestTimeout(180000) + .build() + + // update some field that isn't the PPL query and the index it's querying + val newMonitor = pplMonitorConfig.makeCopy(name = "some_random_name") + + try { + // noIndicesUser, who only has access to index unrelated_index, should be blocked + // from updating a monitor that queries index TEST_INDEX_NAME because noIndicesUser + // has no access to TEST_INDEX_NAME + noIndicesUserClient!!.makeRequest( + "PUT", + "$MONITOR_V2_BASE_URI/${pplMonitor.id}", + newMonitor.toHttpEntity(), + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + fail("Expected update monitor to fail as user does not have permissions to index that monitor queries") + } catch (e: ResponseException) { + assertEquals("Unexpected error status", RestStatus.BAD_REQUEST.status, e.response.statusLine.statusCode) + } + + // cleanup + noIndicesUserClient.close() + } + + fun `test RBAC create monitor with backend roles user has access to succeeds`() { + enableFilterBy() + if (!isHttps()) { + return + } + + val pplMonitorConfig = randomPPLMonitor(enabled = true) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a", "backend_role_b"), + false + ) + + createMonitorV2WithClient(userClient!!, monitorV2 = pplMonitorConfig, listOf("backend_role_a")) + + ensureNumMonitorV2s(1) + } + + fun `test RBAC create monitor with backend roles user has no access to fails`() { + enableFilterBy() + if (!isHttps()) { + return + } + + val pplMonitorConfig = randomPPLMonitor(enabled = true) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a", "backend_role_b"), + false + ) + + try { + createMonitorV2WithClient( + userClient!!, + monitorV2 = pplMonitorConfig, + listOf("backend_role_a", "backend_role_b", "backend_role_c") + ) + fail("Expected create monitor to fail as user does not have backend_role_c backend role") + } catch (e: ResponseException) { + assertEquals("Unexpected error status", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) + } + + ensureNumMonitorV2s(0) + } + + fun `test RBAC update monitorV2 as user with correct backend roles succeeds`() { + enableFilterBy() + if (!isHttps()) { + return + } + val pplMonitorConfig = randomPPLMonitor(enabled = true) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a", "backend_role_b"), + false + ) + + val pplMonitor = createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) + + // getUser should have access to the monitor above created by user + val updateUser = "updateUser" + + createUserWithRoles( + updateUser, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a"), + true + ) + + val getUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), updateUser, password) + .setSocketTimeout(60000) + .setConnectionRequestTimeout(180000) + .build() + + val newMonitor = randomPPLMonitor() + val updateMonitorResponse = getUserClient!!.makeRequest( + "PUT", + "$MONITOR_V2_BASE_URI/${pplMonitor.id}", + newMonitor.toHttpEntity(), + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + assertEquals("Update monitorV2 failed", RestStatus.OK, updateMonitorResponse.restStatus()) + + // cleanup + getUserClient.close() + } + + fun `test RBAC update monitorV2 as user without correct backend roles fails`() { + enableFilterBy() + if (!isHttps()) { + return + } + val pplMonitorConfig = randomPPLMonitor(enabled = true) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a", "backend_role_b"), + false + ) + + val pplMonitor = createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) + + // updateUser should have access to the monitor above created by user + val updateUser = "updateUser" + + createUserWithRoles( + updateUser, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_c"), + true + ) + + val getUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), updateUser, password) + .setSocketTimeout(60000) + .setConnectionRequestTimeout(180000) + .build() + + val newMonitor = randomPPLMonitor() + + try { + getUserClient!!.makeRequest( + "PUT", + "$MONITOR_V2_BASE_URI/${pplMonitor.id}", + newMonitor.toHttpEntity(), + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + fail("Expected update monitor to fail as user does not have the correct backend roles") + } catch (e: ResponseException) { + assertEquals("Unexpected error status", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) + } + + // cleanup + getUserClient.close() + } + + fun `test RBAC get monitorV2 as user with correct backend roles succeeds`() { + enableFilterBy() + if (!isHttps()) { + return + } + val pplMonitorConfig = randomPPLMonitor(enabled = true) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a", "backend_role_b"), + false + ) + + val pplMonitor = createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) + + // getUser should have access to the monitor above created by user + val getUser = "getUser" + + createUserWithRoles( + getUser, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a"), + true + ) + + val getUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), getUser, password) + .setSocketTimeout(60000) + .setConnectionRequestTimeout(180000) + .build() + + val getMonitorResponse = getUserClient!!.makeRequest( + "GET", + "$MONITOR_V2_BASE_URI/${pplMonitor.id}", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + assertEquals("Get monitorV2 failed", RestStatus.OK, getMonitorResponse.restStatus()) + + // cleanup + getUserClient.close() + } + + fun `test RBAC get monitorV2 as user without correct backend roles fails`() { + enableFilterBy() + if (!isHttps()) { + return + } + val pplMonitorConfig = randomPPLMonitor(enabled = true) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a", "backend_role_b"), + false + ) + + val pplMonitor = createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) + + // getUser should not have access to the monitor above created by user + val getUser = "getUser" + + createUserWithRoles( + getUser, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_c"), + true + ) + + val getUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), getUser, password) + .setSocketTimeout(60000) + .setConnectionRequestTimeout(180000) + .build() + + try { + getUserClient!!.makeRequest( + "GET", + "$MONITOR_V2_BASE_URI/${pplMonitor.id}", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + fail("Expected Forbidden exception") + } catch (e: ResponseException) { + assertEquals("Unexpected get monitor status", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) + } finally { + getUserClient?.close() + } + } + + fun `test RBAC search monitorV2 as user with correct backend roles returns results`() { + enableFilterBy() + if (!isHttps()) { + return + } + val pplMonitorConfig = randomPPLMonitor(enabled = true) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a", "backend_role_b"), + false + ) + + createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) + + // getUser should have access to the monitor above created by user + val searchUser = "searchUser" + + createUserWithRoles( + searchUser, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a"), + true + ) + + val searchUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), searchUser, password) + .setSocketTimeout(60000) + .setConnectionRequestTimeout(180000) + .build() + + val search = SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).toString() + val searchMonitorResponse = searchUserClient!!.makeRequest( + "POST", + "$MONITOR_V2_BASE_URI/_search", + StringEntity(search, ContentType.APPLICATION_JSON), + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + assertEquals("Search monitorV2 failed", RestStatus.OK, searchMonitorResponse.restStatus()) + + createParser(XContentType.JSON.xContent(), searchMonitorResponse.entity.content).use { xcp -> + val hits = xcp.map()["hits"]!! as Map> + logger.info("hits: $hits") + val numberDocsFound = hits["total"]?.get("value") + assertEquals("Created PPL Monitor should be visible but was not", 1, numberDocsFound) + } + + // cleanup + searchUserClient.close() + } + + fun `test RBAC search monitorV2 as user without correct backend roles returns no results`() { + enableFilterBy() + if (!isHttps()) { + return + } + val pplMonitorConfig = randomPPLMonitor(enabled = true) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a", "backend_role_b"), + false + ) + + createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) + + // getUser should have access to the monitor above created by user + val searchUser = "searchUser" + + createUserWithRoles( + searchUser, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_c"), + true + ) + + val searchUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), searchUser, password) + .setSocketTimeout(60000) + .setConnectionRequestTimeout(180000) + .build() + + val search = SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).toString() + val searchMonitorResponse = searchUserClient!!.makeRequest( + "POST", + "$MONITOR_V2_BASE_URI/_search", + StringEntity(search, ContentType.APPLICATION_JSON), + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + assertEquals("Search monitorV2 failed", RestStatus.OK, searchMonitorResponse.restStatus()) + + createParser(XContentType.JSON.xContent(), searchMonitorResponse.entity.content).use { xcp -> + val hits = xcp.map()["hits"]!! as Map> + val numberDocsFound = hits["total"]?.get("value") + assertEquals("Created PPL Monitor should be visible but was not", 0, numberDocsFound) + } + + // cleanup + searchUserClient.close() + } + + fun `test RBAC execute monitorV2 as user with correct backend roles succeeds`() { + enableFilterBy() + if (!isHttps()) { + return + } + val pplMonitorConfig = randomPPLMonitor(enabled = true) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a", "backend_role_b"), + false + ) + + val pplMonitor = createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) + + // getUser should have access to the monitor above created by user + val executeUser = "executeUser" + + createUserWithRoles( + executeUser, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a"), + true + ) + + val getUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), executeUser, password) + .setSocketTimeout(60000) + .setConnectionRequestTimeout(180000) + .build() + + val getMonitorResponse = getUserClient!!.makeRequest( + "POST", + "$MONITOR_V2_BASE_URI/${pplMonitor.id}/_execute", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + assertEquals("Get monitorV2 failed", RestStatus.OK, getMonitorResponse.restStatus()) + + // cleanup + getUserClient.close() + } + + fun `test RBAC execute monitorV2 as user without correct backend roles fails`() { + enableFilterBy() + if (!isHttps()) { + return + } + val pplMonitorConfig = randomPPLMonitor(enabled = true) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a", "backend_role_b"), + false + ) + + val pplMonitor = createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) + + // getUser should not have access to the monitor above created by user + val executeUser = "executeUser" + + createUserWithRoles( + executeUser, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_c"), + true + ) + + val getUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), executeUser, password) + .setSocketTimeout(60000) + .setConnectionRequestTimeout(180000) + .build() + + try { + getUserClient!!.makeRequest( + "POST", + "$MONITOR_V2_BASE_URI/${pplMonitor.id}/_execute", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + fail("Expected Forbidden exception") + } catch (e: ResponseException) { + assertEquals("Unexpected delete monitor status", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) + } finally { + getUserClient?.close() + } + } + + fun `test RBAC get alerts v2 as user with correct backend roles succeeds`() { + enableFilterBy() + if (!isHttps()) { + return + } + + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(2, MINUTES, "abc", 5) + + val pplMonitorConfig = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + lookBackWindow = null, + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + expireDuration = 5, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a", "backend_role_b"), + false + ) + + val pplMonitor = createMonitorV2WithClient( + userClient!!, + pplMonitorConfig, + null + ) as PPLSQLMonitor + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + assertTrue(triggered) + + // TODO: creating this user overrides the ALERTING_FULL_ACCESS mapping and displaces "user" + // TODO: above, even though passing in isExistingRole = true should trigger an update + // TODO: role mappings call. doesn't block the test because "user" isn't used for the + // TODO: rest of the test, but this could lead to unexpected behavior for future test writers + // the get alerts user should be able to see the alerts + val getAlertsUser = "getAlertsUser" + createUserWithRoles( + getAlertsUser, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a"), + true + ) + + val getAlertsUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), getAlertsUser, password) + .setSocketTimeout(60000) + .setConnectionRequestTimeout(180000) + .build() + + val getAlertsResponse = getAlertsUserClient!!.makeRequest( + "GET", + "$MONITOR_V2_BASE_URI/alerts", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + assertEquals("Get alerts v2 failed", RestStatus.OK, getAlertsResponse.restStatus()) + + val alertsGenerated = numAlerts(getAlertsResponse) > 0 + assert(alertsGenerated) + + // cleanup + getAlertsUserClient.close() + } + + fun `test RBAC get alerts v2 as user without correct backend roles fails`() { + enableFilterBy() + if (!isHttps()) { + return + } + + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(2, MINUTES, "abc", 5) + + val pplMonitorConfig = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + lookBackWindow = null, + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + expireDuration = 5, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a", "backend_role_b"), + false + ) + + val pplMonitor = createMonitorV2WithClient( + userClient!!, + pplMonitorConfig, + null + ) as PPLSQLMonitor + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + assertTrue(triggered) + + // the get alerts user should be able to see the alerts + val getAlertsUser = "getAlertsUser" + createUserWithRoles( + getAlertsUser, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_c"), + true + ) + + val getAlertsUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), getAlertsUser, password) + .setSocketTimeout(60000) + .setConnectionRequestTimeout(180000) + .build() + + val getAlertsResponse = getAlertsUserClient!!.makeRequest( + "GET", + "$MONITOR_V2_BASE_URI/alerts", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + assertEquals("Get alerts v2 failed", RestStatus.OK, getAlertsResponse.restStatus()) + + val alertsGenerated = numAlerts(getAlertsResponse) > 0 + assert(!alertsGenerated) + + // cleanup + getAlertsUserClient.close() + } + + fun `test RBAC delete monitorV2 as user with correct backend roles succeeds`() { + enableFilterBy() + if (!isHttps()) { + return + } + val pplMonitorConfig = randomPPLMonitor(enabled = true) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a", "backend_role_b"), + false + ) + + val pplMonitor = createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) + + // getUser should have access to the monitor above created by user + val deleteUser = "deleteUser" + + createUserWithRoles( + deleteUser, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a"), + true + ) + + val deleteUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), deleteUser, password) + .setSocketTimeout(60000) + .setConnectionRequestTimeout(180000) + .build() + + val getMonitorResponse = deleteUserClient!!.makeRequest( + "DELETE", + "$MONITOR_V2_BASE_URI/${pplMonitor.id}", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + assertEquals("Get monitorV2 failed", RestStatus.OK, getMonitorResponse.restStatus()) + + ensureNumMonitorV2s(0) + + // cleanup + deleteUserClient.close() + } + + fun `test RBAC delete monitorV2 as user without correct backend roles fails`() { + enableFilterBy() + if (!isHttps()) { + return + } + val pplMonitorConfig = randomPPLMonitor(enabled = true) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a", "backend_role_b"), + false + ) + + val pplMonitor = createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) + + // getUser should not have access to the monitor above created by user + val deleteUser = "deleteUser" + + createUserWithRoles( + deleteUser, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_c"), + true + ) + + val deleteUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), deleteUser, password) + .setSocketTimeout(60000) + .setConnectionRequestTimeout(180000) + .build() + + try { + deleteUserClient!!.makeRequest( + "DELETE", + "$MONITOR_V2_BASE_URI/${pplMonitor.id}", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + fail("Expected Forbidden exception") + } catch (e: ResponseException) { + assertEquals("Unexpected delete monitor status", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) + } finally { + deleteUserClient?.close() + } + + ensureNumMonitorV2s(1) + } +} diff --git a/core/build.gradle b/core/build.gradle index cfce74c42..9aad7da88 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -8,7 +8,8 @@ apply plugin: 'opensearch.java-rest-test' apply plugin: 'org.jetbrains.kotlin.jvm' apply plugin: 'jacoco' -configurations{ +configurations { + zipArchive all { resolutionStrategy { // force commons-beanutils to a non-vulnerable version @@ -17,6 +18,18 @@ configurations{ } } +def sqlJarDirectory = "$buildDir/dependencies/opensearch-sql-plugin" + +task addJarsToClasspath(type: Copy) { + from(fileTree(dir: sqlJarDirectory)) { + include "opensearch-sql-${opensearch_build}.jar" + include "ppl-${opensearch_build}.jar" + include "protocol-${opensearch_build}.jar" + include "core-${opensearch_build}.jar" + } + into("$buildDir/classes") +} + dependencies { compileOnly "org.opensearch:opensearch:${opensearch_version}" implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}" @@ -26,8 +39,44 @@ dependencies { api "org.opensearch.client:opensearch-rest-client:${opensearch_version}" api "org.opensearch:common-utils:${common_utils_version}@jar" implementation 'commons-validator:commons-validator:1.7' + implementation 'org.json:json:20240303' + + api fileTree(dir: sqlJarDirectory, include: ["opensearch-sql-thin-${opensearch_build}.jar", "ppl-${opensearch_build}.jar", "protocol-${opensearch_build}.jar", "core-${opensearch_build}.jar"]) + + zipArchive group: 'org.opensearch.plugin', name:'opensearch-sql-plugin', version: "${opensearch_build}" testImplementation "org.opensearch.test:framework:${opensearch_version}" testImplementation "org.jetbrains.kotlin:kotlin-test:${kotlin_version}" testImplementation "org.jetbrains.kotlin:kotlin-test-junit:${kotlin_version}" } + +task extractSqlJar(type: Copy) { + mustRunAfter() + from(zipTree(configurations.zipArchive.find { it.name.startsWith("opensearch-sql-plugin") })) + into sqlJarDirectory +} + +task extractSqlClass(type: Copy, dependsOn: [extractSqlJar]) { + from zipTree("${sqlJarDirectory}/opensearch-sql-${opensearch_build}.jar") + into("$buildDir/opensearch-sql") + include 'org/opensearch/sql/**' +} + +task replaceSqlJar(type: Jar, dependsOn: [extractSqlClass]) { + from("$buildDir/opensearch-sql") + archiveFileName = "opensearch-sql-thin-${opensearch_build}.jar" + destinationDirectory = file(sqlJarDirectory) + doLast { + file("${sqlJarDirectory}/opensearch-sql-${opensearch_build}.jar").delete() + } +} + +tasks.addJarsToClasspath.dependsOn(replaceSqlJar) + +compileJava { + dependsOn addJarsToClasspath +} + +compileKotlin { + dependsOn addJarsToClasspath +} diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobStats.kt b/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobStats.kt index 07792d553..0e02817fc 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobStats.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobStats.kt @@ -7,7 +7,8 @@ package org.opensearch.alerting.core.action.node import org.opensearch.action.support.nodes.BaseNodeResponse import org.opensearch.alerting.core.JobSweeperMetrics -import org.opensearch.alerting.core.resthandler.RestScheduledJobStatsHandler +import org.opensearch.alerting.core.resthandler.StatsRequestUtils.JOBS_INFO +import org.opensearch.alerting.core.resthandler.StatsRequestUtils.JOB_SCHEDULING_METRICS import org.opensearch.alerting.core.schedule.JobSchedulerMetrics import org.opensearch.cluster.node.DiscoveryNode import org.opensearch.core.common.io.stream.StreamInput @@ -69,13 +70,13 @@ class ScheduledJobStats : BaseNodeResponse, ToXContentFragment { builder.field("schedule_status", status) builder.field("roles", node.roles.map { it.roleName().uppercase(Locale.getDefault()) }) if (jobSweeperMetrics != null) { - builder.startObject(RestScheduledJobStatsHandler.JOB_SCHEDULING_METRICS) + builder.startObject(JOB_SCHEDULING_METRICS) jobSweeperMetrics!!.toXContent(builder, params) builder.endObject() } if (jobInfos != null) { - builder.startObject(RestScheduledJobStatsHandler.JOBS_INFO) + builder.startObject(JOBS_INFO) for (job in jobInfos!!) { builder.startObject(job.scheduledJobId) job.toXContent(builder, params) diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsRequest.kt b/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsRequest.kt index 6a82e8204..1d9bd0578 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsRequest.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsRequest.kt @@ -17,18 +17,25 @@ import java.io.IOException class ScheduledJobsStatsRequest : BaseNodesRequest { var jobSchedulingMetrics: Boolean = true var jobsInfo: Boolean = true + // show Alerting V2 scheduled jobs if true, Alerting V1 scheduled jobs if false, all scheduled jobs if null + var showAlertingV2ScheduledJobs: Boolean? = null constructor(si: StreamInput) : super(si) { jobSchedulingMetrics = si.readBoolean() jobsInfo = si.readBoolean() + showAlertingV2ScheduledJobs = si.readOptionalBoolean() + } + + constructor(nodeIds: Array, showAlertingV2ScheduledJobs: Boolean?) : super(*nodeIds) { + this.showAlertingV2ScheduledJobs = showAlertingV2ScheduledJobs } - constructor(nodeIds: Array) : super(*nodeIds) @Throws(IOException::class) override fun writeTo(out: StreamOutput) { super.writeTo(out) out.writeBoolean(jobSchedulingMetrics) out.writeBoolean(jobsInfo) + out.writeOptionalBoolean(showAlertingV2ScheduledJobs) } fun all(): ScheduledJobsStatsRequest { diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsTransportAction.kt b/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsTransportAction.kt index f2ed94623..398f5634a 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsTransportAction.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsTransportAction.kt @@ -93,7 +93,8 @@ class ScheduledJobsStatsTransportAction : TransportNodesAction, + ) { + + val responseReader = Writeable.Reader { + TransportPPLQueryResponse(it) + } + + val wrappedListener = object : ActionListener { + override fun onResponse(response: ActionResponse) { + val recreated = recreateObject(response) { TransportPPLQueryResponse(it) } + listener.onResponse(recreated) + } + + override fun onFailure(exception: Exception) { + listener.onFailure(exception) + } + } + + transportService.sendRequest( + localNode, + PPLQueryAction.NAME, + request, + TransportRequestOptions + .builder() + .withTimeout(TimeValue.timeValueMinutes(1)) + .build(), + object : ActionListenerResponseHandler( + wrappedListener, + responseReader + ) { + override fun handleResponse(response: ActionResponse) { + wrappedListener.onResponse(response) + } + + override fun handleException(e: TransportException) { + wrappedListener.onFailure(e) + } + } + ) + } +} diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/RestScheduledJobStatsHandler.kt b/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/RestScheduledJobStatsHandler.kt index fbe57ab19..1ee6e3bde 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/RestScheduledJobStatsHandler.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/RestScheduledJobStatsHandler.kt @@ -6,8 +6,6 @@ package org.opensearch.alerting.core.resthandler import org.opensearch.alerting.core.action.node.ScheduledJobsStatsAction -import org.opensearch.alerting.core.action.node.ScheduledJobsStatsRequest -import org.opensearch.core.common.Strings import org.opensearch.rest.BaseRestHandler import org.opensearch.rest.BaseRestHandler.RestChannelConsumer import org.opensearch.rest.RestHandler @@ -16,23 +14,12 @@ import org.opensearch.rest.RestRequest import org.opensearch.rest.RestRequest.Method.GET import org.opensearch.rest.action.RestActions import org.opensearch.transport.client.node.NodeClient -import java.util.Locale -import java.util.TreeSet /** * RestScheduledJobStatsHandler is handler for getting ScheduledJob Stats. */ class RestScheduledJobStatsHandler(private val path: String) : BaseRestHandler() { - companion object { - const val JOB_SCHEDULING_METRICS: String = "job_scheduling_metrics" - const val JOBS_INFO: String = "jobs_info" - private val METRICS = mapOf Unit>( - JOB_SCHEDULING_METRICS to { it -> it.jobSchedulingMetrics = true }, - JOBS_INFO to { it -> it.jobsInfo = true } - ) - } - override fun getName(): String { return "${path}_jobs_stats" } @@ -71,7 +58,14 @@ class RestScheduledJobStatsHandler(private val path: String) : BaseRestHandler() } override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { - val scheduledJobNodesStatsRequest = getRequest(request) + val alertingVersion = request.param("version") + if (alertingVersion != null && alertingVersion !in listOf("v1", "v2")) { + throw IllegalArgumentException("Version parameter must be one of v1 or v2") + } + + val showV2ScheduledJobs: Boolean? = alertingVersion?.let { it == "v2" } + + val scheduledJobNodesStatsRequest = StatsRequestUtils.getStatsRequest(request, showV2ScheduledJobs, this::unrecognized) return RestChannelConsumer { channel -> client.execute( ScheduledJobsStatsAction.INSTANCE, @@ -80,43 +74,4 @@ class RestScheduledJobStatsHandler(private val path: String) : BaseRestHandler() ) } } - - private fun getRequest(request: RestRequest): ScheduledJobsStatsRequest { - val nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId")) - val metrics = Strings.tokenizeByCommaToSet(request.param("metric")) - val scheduledJobsStatsRequest = ScheduledJobsStatsRequest(nodesIds) - scheduledJobsStatsRequest.timeout(request.param("timeout")) - - if (metrics.isEmpty()) { - return scheduledJobsStatsRequest - } else if (metrics.size == 1 && metrics.contains("_all")) { - scheduledJobsStatsRequest.all() - } else if (metrics.contains("_all")) { - throw IllegalArgumentException( - String.format( - Locale.ROOT, - "request [%s] contains _all and individual metrics [%s]", - request.path(), - request.param("metric") - ) - ) - } else { - // use a sorted set so the unrecognized parameters appear in a reliable sorted order - scheduledJobsStatsRequest.clear() - val invalidMetrics = TreeSet() - for (metric in metrics) { - val handler = METRICS[metric] - if (handler != null) { - handler.invoke(scheduledJobsStatsRequest) - } else { - invalidMetrics.add(metric) - } - } - - if (!invalidMetrics.isEmpty()) { - throw IllegalArgumentException(unrecognized(request, invalidMetrics, METRICS.keys, "metric")) - } - } - return scheduledJobsStatsRequest - } } diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/StatsRequestUtils.kt b/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/StatsRequestUtils.kt new file mode 100644 index 000000000..58b33d709 --- /dev/null +++ b/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/StatsRequestUtils.kt @@ -0,0 +1,63 @@ +package org.opensearch.alerting.core.resthandler + +import org.opensearch.alerting.core.action.node.ScheduledJobsStatsRequest +import org.opensearch.core.common.Strings +import org.opensearch.rest.RestRequest +import java.util.Locale +import java.util.TreeSet + +internal object StatsRequestUtils { + + const val JOB_SCHEDULING_METRICS: String = "job_scheduling_metrics" + const val JOBS_INFO: String = "jobs_info" + val METRICS = mapOf Unit>( + JOB_SCHEDULING_METRICS to { it.jobSchedulingMetrics = true }, + JOBS_INFO to { it.jobsInfo = true } + ) + + fun getStatsRequest( + request: RestRequest, + showAlertingV2ScheduledJobs: Boolean?, + unrecognizedFn: (RestRequest, Set, Set, String) -> String + ): ScheduledJobsStatsRequest { + val nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId")) + val metrics = Strings.tokenizeByCommaToSet(request.param("metric")) + val scheduledJobsStatsRequest = ScheduledJobsStatsRequest( + nodeIds = nodesIds, + showAlertingV2ScheduledJobs = showAlertingV2ScheduledJobs + ) + scheduledJobsStatsRequest.timeout(request.param("timeout")) + + if (metrics.isEmpty()) { + return scheduledJobsStatsRequest + } else if (metrics.size == 1 && metrics.contains("_all")) { + scheduledJobsStatsRequest.all() + } else if (metrics.contains("_all")) { + throw IllegalArgumentException( + String.format( + Locale.ROOT, + "request [%s] contains _all and individual metrics [%s]", + request.path(), + request.param("metric") + ) + ) + } else { + // use a sorted set so the unrecognized parameters appear in a reliable sorted order + scheduledJobsStatsRequest.clear() + val invalidMetrics = TreeSet() + for (metric in metrics) { + val handler = METRICS[metric] + if (handler != null) { + handler.invoke(scheduledJobsStatsRequest) + } else { + invalidMetrics.add(metric) + } + } + + if (!invalidMetrics.isEmpty()) { + throw IllegalArgumentException(unrecognizedFn(request, invalidMetrics, METRICS.keys, "metric")) + } + } + return scheduledJobsStatsRequest + } +} diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/schedule/JobScheduler.kt b/core/src/main/kotlin/org/opensearch/alerting/core/schedule/JobScheduler.kt index a4a729121..8245e1a78 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/core/schedule/JobScheduler.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/core/schedule/JobScheduler.kt @@ -30,6 +30,12 @@ class JobScheduler(private val threadPool: ThreadPool, private val jobRunner: Jo */ private val scheduledJobIdToInfo = ConcurrentHashMap() + /** + * The scheduled job type of Monitor V2s, for filtering + * out V1 vs V2 Monitors when collecting Monitor Stats + */ + private val monitorV2Type = "monitor_v2" + /** * Schedules the jobs in [jobsToSchedule] for execution. * @@ -191,8 +197,19 @@ class JobScheduler(private val threadPool: ThreadPool, private val jobRunner: Jo return true } - fun getJobSchedulerMetric(): List { - return scheduledJobIdToInfo.entries.stream() + fun getJobSchedulerMetric(showAlertingV2ScheduledJobs: Boolean?): List { + val scheduledJobEntries = scheduledJobIdToInfo.entries + + val filteredScheduledJobEntries = if (showAlertingV2ScheduledJobs == null) { + // if no alerting version was specified, do not filter + scheduledJobEntries + } else if (showAlertingV2ScheduledJobs) { + scheduledJobEntries.filter { it.value.scheduledJob.type == monitorV2Type } + } else { + scheduledJobEntries.filter { it.value.scheduledJob.type != monitorV2Type } + } + + return filteredScheduledJobEntries.stream() .map { entry -> JobSchedulerMetrics( entry.value.scheduledJobId, diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/settings/AlertingV2Settings.kt b/core/src/main/kotlin/org/opensearch/alerting/core/settings/AlertingV2Settings.kt new file mode 100644 index 000000000..fd92dd5e9 --- /dev/null +++ b/core/src/main/kotlin/org/opensearch/alerting/core/settings/AlertingV2Settings.kt @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.core.settings + +import org.opensearch.common.settings.Setting + +/** + * This class exclusively houses the Alerting V2 enabled setting, so that both Monitor V2 Stats + * and the rest of the CRUD APIs can read it + */ +class AlertingV2Settings { + companion object { + val ALERTING_V2_ENABLED = Setting.boolSetting( + "plugins.alerting.v2.enabled", + true, + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + } +} diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/util/XContentExtensions.kt b/core/src/main/kotlin/org/opensearch/alerting/core/util/XContentExtensions.kt new file mode 100644 index 000000000..9ca03ed6b --- /dev/null +++ b/core/src/main/kotlin/org/opensearch/alerting/core/util/XContentExtensions.kt @@ -0,0 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.core.util + +import org.opensearch.core.xcontent.XContentBuilder +import java.time.Instant + +fun XContentBuilder.nonOptionalTimeField(name: String, instant: Instant): XContentBuilder { + return this.timeField(name, "${name}_in_millis", instant.toEpochMilli()) +} diff --git a/core/src/main/kotlin/org/opensearch/alerting/opensearchapi/OpenSearchExtensions.kt b/core/src/main/kotlin/org/opensearch/alerting/opensearchapi/OpenSearchExtensions.kt index 582d13fbe..fd500ef1d 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/opensearchapi/OpenSearchExtensions.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/opensearchapi/OpenSearchExtensions.kt @@ -14,6 +14,7 @@ import org.opensearch.OpenSearchException import org.opensearch.action.bulk.BackoffPolicy import org.opensearch.action.search.SearchResponse import org.opensearch.action.search.ShardSearchFailure +import org.opensearch.alerting.core.ppl.PPLPluginInterface import org.opensearch.common.settings.Settings import org.opensearch.common.util.concurrent.ThreadContext import org.opensearch.common.xcontent.XContentHelper @@ -170,6 +171,20 @@ suspend fun NotificationsPluginInterface.suspendUntil(block: NotificationsPl }) } +/** + * Converts [PPLPluginInterface] methods that take a callback into a kotlin suspending function. + * + * @param block - a block of code that is passed an [ActionListener] that should be passed to the PPLPluginInterface API. + */ +suspend fun PPLPluginInterface.suspendUntil(block: PPLPluginInterface.(ActionListener) -> Unit): T = + suspendCoroutine { cont -> + block(object : ActionListener { + override fun onResponse(response: T) = cont.resume(response) + + override fun onFailure(e: Exception) = cont.resumeWithException(e) + }) + } + class InjectorContextElement( id: String, settings: Settings, diff --git a/core/src/main/resources/mappings/scheduled-jobs.json b/core/src/main/resources/mappings/scheduled-jobs.json index 6e3d31c51..ba5c0010d 100644 --- a/core/src/main/resources/mappings/scheduled-jobs.json +++ b/core/src/main/resources/mappings/scheduled-jobs.json @@ -1,6 +1,6 @@ { "_meta" : { - "schema_version": 8 + "schema_version": 9 }, "properties": { "monitor": { @@ -450,6 +450,201 @@ } } }, + "monitor_v2": { + "dynamic": "false", + "properties": { + "ppl_monitor": { + "dynamic": "false", + "properties": { + "schema_version": { + "type": "integer" + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "enabled": { + "type": "boolean" + }, + "schedule": { + "properties": { + "period": { + "properties": { + "interval": { + "type": "integer" + }, + "unit": { + "type": "keyword" + } + } + }, + "cron": { + "properties": { + "expression": { + "type": "text" + }, + "timezone": { + "type": "keyword" + } + } + } + } + }, + "look_back_window_minutes": { + "type": "long" + }, + "timestamp_field": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "last_update_time": { + "type": "date", + "format": "strict_date_time||epoch_millis" + }, + "enabled_time": { + "type": "date", + "format": "strict_date_time||epoch_millis" + }, + "description": { + "type": "text" + }, + "user": { + "properties": { + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "backend_roles": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "roles": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "custom_attribute_names": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + } + } + }, + "query_language": { + "type": "keyword" + }, + "query": { + "type": "text" + }, + "triggers": { + "type": "nested", + "properties": { + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "severity": { + "type": "keyword" + }, + "throttle_minutes": { + "type": "long" + }, + "expires_minutes": { + "type": "long" + }, + "last_triggered_time": { + "type": "date", + "format": "strict_date_time||epoch_millis" + }, + "mode": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "num_results_condition": { + "type": "keyword" + }, + "num_results_value": { + "type": "long" + }, + "custom_condition": { + "type": "text" + }, + "actions": { + "type": "nested", + "properties": { + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "destination_id": { + "type": "keyword" + }, + "subject_template": { + "type": "object", + "enabled": false + }, + "message_template": { + "type": "object", + "enabled": false + }, + "throttle_enabled": { + "type": "boolean" + }, + "throttle": { + "properties": { + "value": { + "type": "integer" + }, + "unit": { + "type": "keyword" + } + } + } + } + } + } + } + } + } + } + }, "destination": { "dynamic": "false", "properties": { diff --git a/release-notes/opensearch-alerting.release-notes-3.5.0.0.md b/release-notes/opensearch-alerting.release-notes-3.5.0.0.md deleted file mode 100644 index e9c00745e..000000000 --- a/release-notes/opensearch-alerting.release-notes-3.5.0.0.md +++ /dev/null @@ -1,7 +0,0 @@ -## Version 3.5.0 Release Notes - -Compatible with OpenSearch and OpenSearch Dashboards version 3.5.0 - -### Features - -* Access control for results in trigger execution context ([#1991](https://github.com/opensearch-project/alerting/pull/1991)) \ No newline at end of file From 9e8ba3e1fc0f14d72d4cc91d68c68e13a5992f88 Mon Sep 17 00:00:00 2001 From: Dennis Toepker Date: Mon, 9 Feb 2026 15:02:04 -0800 Subject: [PATCH 2/9] PPL Alerting: refactoring PPL Alerting to be behind V1 APIs, and retiring V2 APIs --- alerting/build.gradle | 198 ++-- .../org/opensearch/alerting/AlertingPlugin.kt | 47 +- .../opensearch/alerting/AlertingV2Utils.kt | 219 ----- .../alerting/BucketLevelMonitorRunner.kt | 6 + .../alerting/DocumentLevelMonitorRunner.kt | 6 + .../org/opensearch/alerting/MonitorRunner.kt | 3 +- .../alerting/MonitorRunnerService.kt | 131 +-- .../opensearch/alerting/MonitorV2Runner.kt | 29 - .../alerting/PPLSQLMonitorRunner.kt | 195 ++-- .../alerting/QueryLevelMonitorRunner.kt | 6 + .../opensearch/alerting/WorkflowService.kt | 8 +- .../alerting/action/ExecuteMonitorRequest.kt | 5 + .../alerting/action/ExecuteWorkflowRequest.kt | 5 + .../actionv2/DeleteMonitorV2Action.kt | 15 - .../actionv2/DeleteMonitorV2Request.kt | 39 - .../actionv2/DeleteMonitorV2Response.kt | 43 - .../actionv2/ExecuteMonitorV2Action.kt | 15 - .../actionv2/ExecuteMonitorV2Request.kt | 71 -- .../actionv2/ExecuteMonitorV2Response.kt | 38 - .../alerting/actionv2/GetAlertsV2Action.kt | 15 - .../alerting/actionv2/GetAlertsV2Request.kt | 47 - .../alerting/actionv2/GetAlertsV2Response.kt | 52 -- .../alerting/actionv2/GetMonitorV2Action.kt | 15 - .../alerting/actionv2/GetMonitorV2Request.kt | 52 -- .../alerting/actionv2/GetMonitorV2Response.kt | 80 -- .../alerting/actionv2/IndexMonitorV2Action.kt | 15 - .../actionv2/IndexMonitorV2Request.kt | 69 -- .../actionv2/IndexMonitorV2Response.kt | 73 -- .../actionv2/SearchMonitorV2Action.kt | 16 - .../actionv2/SearchMonitorV2Request.kt | 37 - .../alerting/alertsv2/AlertV2Indices.kt | 15 +- .../alerting/alertsv2/AlertV2Mover.kt | 111 +-- .../opensearch/alerting/modelv2/AlertV2.kt | 257 ------ .../opensearch/alerting/modelv2/MonitorV2.kt | 157 ---- .../alerting/modelv2/MonitorV2RunResult.kt | 49 - .../alerting/modelv2/PPLSQLMonitor.kt | 410 --------- .../modelv2/PPLSQLMonitorRunResult.kt | 74 -- .../alerting/modelv2/PPLSQLTrigger.kt | 413 --------- .../modelv2/PPLSQLTriggerRunResult.kt | 60 -- .../opensearch/alerting/modelv2/TriggerV2.kt | 70 -- .../alerting/modelv2/TriggerV2RunResult.kt | 25 - .../RemoteDocumentLevelMonitorRunner.kt | 1 + .../resthandler/RestExecuteMonitorAction.kt | 4 +- .../resthandler/RestExecuteWorkflowAction.kt | 4 +- .../resthandler/RestGetAlertsAction.kt | 8 +- .../resthandler/RestIndexMonitorAction.kt | 13 +- .../RestDeleteMonitorV2Action.kt | 60 -- .../RestExecuteMonitorV2Action.kt | 82 -- .../resthandlerv2/RestGetAlertsV2Action.kt | 72 -- .../resthandlerv2/RestGetMonitorV2Action.kt | 73 -- .../resthandlerv2/RestIndexMonitorV2Action.kt | 86 -- .../RestSearchMonitorV2Action.kt | 128 --- .../script/PPLTriggerExecutionContext.kt | 25 +- .../script/TriggerV2ExecutionContext.kt | 21 - .../alerting/service/DeleteMonitorService.kt | 15 - .../alerting/settings/AlertingSettings.kt | 14 - .../TransportDeleteWorkflowAction.kt | 16 +- .../TransportExecuteMonitorAction.kt | 18 +- .../TransportExecuteWorkflowAction.kt | 9 +- .../transport/TransportGetAlertsAction.kt | 13 +- .../transport/TransportIndexMonitorAction.kt | 267 +++++- .../transport/TransportIndexWorkflowAction.kt | 14 +- .../transport/TransportSearchMonitorAction.kt | 2 + .../TransportDeleteMonitorV2Action.kt | 148 --- .../TransportExecuteMonitorV2Action.kt | 229 ----- .../transportv2/TransportGetAlertsV2Action.kt | 211 ----- .../TransportGetMonitorV2Action.kt | 172 ---- .../TransportIndexMonitorV2Action.kt | 868 ------------------ .../TransportSearchMonitorV2Action.kt | 122 --- .../opensearch/alerting/util/AlertingUtils.kt | 132 ++- .../opensearch/alerting/util/IndexUtils.kt | 4 +- .../workflow/CompositeWorkflowRunner.kt | 11 + .../alerting/workflow/WorkflowRunner.kt | 3 +- .../alerting/alerts/alert_mapping.json | 24 +- .../alerting/alertsv2/alert_v2_mapping.json | 118 --- .../alerting/AlertingRestTestCase.kt | 193 +--- .../alerting/DocumentMonitorRunnerIT.kt | 2 +- .../alerting/MonitorDataSourcesIT.kt | 6 +- .../alerting/PPLSQLMonitorRunnerIT.kt | 137 +-- .../org/opensearch/alerting/TestHelpers.kt | 212 +---- .../action/ExecuteMonitorRequestTests.kt | 4 +- .../actionv2/DeleteMonitorV2RequestTests.kt | 29 - .../actionv2/DeleteMonitorV2ResponseTests.kt | 28 - .../actionv2/ExecuteMonitorV2RequestTests.kt | 38 - .../actionv2/ExecuteMonitorV2ResponseTests.kt | 57 -- .../actionv2/GetAlertsV2RequestTests.kt | 49 - .../actionv2/GetAlertsV2ResponseTests.kt | 93 -- .../actionv2/GetMonitorV2RequestTests.kt | 31 - .../actionv2/GetMonitorV2ResponseTests.kt | 36 - .../actionv2/IndexMonitorV2RequestTests.kt | 42 - .../actionv2/IndexMonitorV2ResponseTests.kt | 36 - .../actionv2/SearchMonitorV2RequestTests.kt | 34 - .../alerting/alerts/AlertIndicesIT.kt | 6 +- .../alerting/alertsv2/AlertV2IndicesIT.kt | 80 +- .../alerting/modelv2/AlertV2Tests.kt | 59 -- .../alerting/modelv2/MonitorV2Tests.kt | 172 ---- .../alerting/modelv2/RunResultV2Tests.kt | 61 -- .../alerting/modelv2/TriggerV2Tests.kt | 254 ----- .../resthandler/MonitorV2RestApiIT.kt | 140 +-- .../resthandler/SecureMonitorV2RestApiIT.kt | 101 +- .../transport/AlertingSingleNodeTestCase.kt | 4 +- build.gradle | 2 +- .../core/action/node/ScheduledJobStats.kt | 4 +- .../action/node/ScheduledJobsStatsRequest.kt | 9 +- .../node/ScheduledJobsStatsTransportAction.kt | 3 +- .../RestScheduledJobStatsHandler.kt | 62 +- .../core/resthandler/StatsRequestUtils.kt | 63 -- .../alerting/core/schedule/JobScheduler.kt | 21 +- .../core/settings/AlertingV2Settings.kt | 22 - .../alerting/core/util/XContentExtensions.kt | 13 - 110 files changed, 1110 insertions(+), 7171 deletions(-) delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/AlertingV2Utils.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/MonitorV2Runner.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Action.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Request.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Response.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Action.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Request.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Response.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Action.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Request.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Response.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Action.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Request.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Response.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Action.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Request.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Response.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2Action.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2Request.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/modelv2/AlertV2.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2RunResult.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitor.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitorRunResult.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTrigger.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTriggerRunResult.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2RunResult.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestDeleteMonitorV2Action.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestExecuteMonitorV2Action.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetAlertsV2Action.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetMonitorV2Action.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestIndexMonitorV2Action.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestSearchMonitorV2Action.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/script/TriggerV2ExecutionContext.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportDeleteMonitorV2Action.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportExecuteMonitorV2Action.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetAlertsV2Action.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetMonitorV2Action.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportIndexMonitorV2Action.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportSearchMonitorV2Action.kt delete mode 100644 alerting/src/main/resources/org/opensearch/alerting/alertsv2/alert_v2_mapping.json delete mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2RequestTests.kt delete mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2ResponseTests.kt delete mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2RequestTests.kt delete mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2ResponseTests.kt delete mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2RequestTests.kt delete mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2ResponseTests.kt delete mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2RequestTests.kt delete mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2ResponseTests.kt delete mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2RequestTests.kt delete mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2ResponseTests.kt delete mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2RequestTests.kt delete mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/modelv2/AlertV2Tests.kt delete mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/modelv2/MonitorV2Tests.kt delete mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/modelv2/RunResultV2Tests.kt delete mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/modelv2/TriggerV2Tests.kt delete mode 100644 core/src/main/kotlin/org/opensearch/alerting/core/resthandler/StatsRequestUtils.kt delete mode 100644 core/src/main/kotlin/org/opensearch/alerting/core/settings/AlertingV2Settings.kt delete mode 100644 core/src/main/kotlin/org/opensearch/alerting/core/util/XContentExtensions.kt diff --git a/alerting/build.gradle b/alerting/build.gradle index 74c2c8d4a..4674c1e98 100644 --- a/alerting/build.gradle +++ b/alerting/build.gradle @@ -445,105 +445,105 @@ task prepareBwcTests { } } -// Create two test clusters with 3 nodes of the old version -2.times {i -> - task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) { - dependsOn 'prepareBwcTests' - useCluster testClusters."${baseName}$i" - filter { - includeTestsMatching "org.opensearch.alerting.bwc.*IT" - } - systemProperty 'tests.rest.bwcsuite', 'old_cluster' - systemProperty 'tests.rest.bwcsuite_round', 'old' - systemProperty 'tests.plugin_bwc_version', bwcVersion - nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}$i".allHttpSocketURI.join(",")}") - nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}$i".getName()}") - } -} - -// Upgrade one node of the old cluster to new OpenSearch version with upgraded plugin version. -// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node. -// This is also used as a one third upgraded cluster for a rolling upgrade. -task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) { - useCluster testClusters."${baseName}0" - dependsOn "${baseName}#oldVersionClusterTask0" - doFirst { - testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) - } - filter { - includeTestsMatching "org.opensearch.alerting.bwc.*IT" - } - systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' - systemProperty 'tests.rest.bwcsuite_round', 'first' - systemProperty 'tests.plugin_bwc_version', bwcVersion - nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}") - nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}") -} - -// Upgrade the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded. -// This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes. -// This is used for rolling upgrade. -task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) { - dependsOn "${baseName}#mixedClusterTask" - useCluster testClusters."${baseName}0" - doFirst { - testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) - } - filter { - includeTestsMatching "org.opensearch.alerting.bwc.*IT" - } - systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' - systemProperty 'tests.rest.bwcsuite_round', 'second' - systemProperty 'tests.plugin_bwc_version', bwcVersion - nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}") - nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}") -} - -// Upgrade the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded. -// This results in a fully upgraded cluster. -// This is used for rolling upgrade. -task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) { - dependsOn "${baseName}#twoThirdsUpgradedClusterTask" - useCluster testClusters."${baseName}0" - doFirst { - testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) - } - filter { - includeTestsMatching "org.opensearch.alerting.bwc.*IT" - } - mustRunAfter "${baseName}#mixedClusterTask" - systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' - systemProperty 'tests.rest.bwcsuite_round', 'third' - systemProperty 'tests.plugin_bwc_version', bwcVersion - nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}") - nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}") -} - -// Upgrade all the nodes of the old cluster to new OpenSearch version with upgraded plugin version -// at the same time resulting in a fully upgraded cluster. -task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) { - dependsOn "${baseName}#oldVersionClusterTask1" - useCluster testClusters."${baseName}1" - doFirst { - testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins) - } - filter { - includeTestsMatching "org.opensearch.alerting.bwc.*IT" - } - systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster' - systemProperty 'tests.plugin_bwc_version', bwcVersion - nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}") - nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}") -} - -// A bwc test suite which runs all the bwc tasks combined -task bwcTestSuite(type: StandaloneRestIntegTestTask) { - exclude '**/*Test*' - exclude '**/*IT*' - dependsOn tasks.named("${baseName}#mixedClusterTask") - dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask") - dependsOn tasks.named("${baseName}#fullRestartClusterTask") -} +//// Create two test clusters with 3 nodes of the old version +//2.times {i -> +// task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) { +// dependsOn 'prepareBwcTests' +// useCluster testClusters."${baseName}$i" +// filter { +// includeTestsMatching "org.opensearch.alerting.bwc.*IT" +// } +// systemProperty 'tests.rest.bwcsuite', 'old_cluster' +// systemProperty 'tests.rest.bwcsuite_round', 'old' +// systemProperty 'tests.plugin_bwc_version', bwcVersion +// nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}$i".allHttpSocketURI.join(",")}") +// nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}$i".getName()}") +// } +//} +// +//// Upgrade one node of the old cluster to new OpenSearch version with upgraded plugin version. +//// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node. +//// This is also used as a one third upgraded cluster for a rolling upgrade. +//task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) { +// useCluster testClusters."${baseName}0" +// dependsOn "${baseName}#oldVersionClusterTask0" +// doFirst { +// testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) +// } +// filter { +// includeTestsMatching "org.opensearch.alerting.bwc.*IT" +// } +// systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' +// systemProperty 'tests.rest.bwcsuite_round', 'first' +// systemProperty 'tests.plugin_bwc_version', bwcVersion +// nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}") +// nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}") +//} +// +//// Upgrade the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded. +//// This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes. +//// This is used for rolling upgrade. +//task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) { +// dependsOn "${baseName}#mixedClusterTask" +// useCluster testClusters."${baseName}0" +// doFirst { +// testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) +// } +// filter { +// includeTestsMatching "org.opensearch.alerting.bwc.*IT" +// } +// systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' +// systemProperty 'tests.rest.bwcsuite_round', 'second' +// systemProperty 'tests.plugin_bwc_version', bwcVersion +// nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}") +// nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}") +//} +// +//// Upgrade the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded. +//// This results in a fully upgraded cluster. +//// This is used for rolling upgrade. +//task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) { +// dependsOn "${baseName}#twoThirdsUpgradedClusterTask" +// useCluster testClusters."${baseName}0" +// doFirst { +// testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) +// } +// filter { +// includeTestsMatching "org.opensearch.alerting.bwc.*IT" +// } +// mustRunAfter "${baseName}#mixedClusterTask" +// systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' +// systemProperty 'tests.rest.bwcsuite_round', 'third' +// systemProperty 'tests.plugin_bwc_version', bwcVersion +// nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}") +// nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}") +//} +// +//// Upgrade all the nodes of the old cluster to new OpenSearch version with upgraded plugin version +//// at the same time resulting in a fully upgraded cluster. +//task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) { +// dependsOn "${baseName}#oldVersionClusterTask1" +// useCluster testClusters."${baseName}1" +// doFirst { +// testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins) +// } +// filter { +// includeTestsMatching "org.opensearch.alerting.bwc.*IT" +// } +// systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster' +// systemProperty 'tests.plugin_bwc_version', bwcVersion +// nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}") +// nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}") +//} +// +//// A bwc test suite which runs all the bwc tasks combined +//task bwcTestSuite(type: StandaloneRestIntegTestTask) { +// exclude '**/*Test*' +// exclude '**/*IT*' +// dependsOn tasks.named("${baseName}#mixedClusterTask") +// dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask") +// dependsOn tasks.named("${baseName}#fullRestartClusterTask") +//} run { doFirst { diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index d80d7b114..24e8943da 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -14,12 +14,6 @@ import org.opensearch.alerting.action.GetEmailGroupAction import org.opensearch.alerting.action.GetRemoteIndexesAction import org.opensearch.alerting.action.SearchEmailAccountAction import org.opensearch.alerting.action.SearchEmailGroupAction -import org.opensearch.alerting.actionv2.DeleteMonitorV2Action -import org.opensearch.alerting.actionv2.ExecuteMonitorV2Action -import org.opensearch.alerting.actionv2.GetAlertsV2Action -import org.opensearch.alerting.actionv2.GetMonitorV2Action -import org.opensearch.alerting.actionv2.IndexMonitorV2Action -import org.opensearch.alerting.actionv2.SearchMonitorV2Action import org.opensearch.alerting.alerts.AlertIndices import org.opensearch.alerting.alerts.AlertIndices.Companion.ALL_ALERT_INDEX_PATTERN import org.opensearch.alerting.alertsv2.AlertV2Indices @@ -34,10 +28,8 @@ import org.opensearch.alerting.core.action.node.ScheduledJobsStatsTransportActio import org.opensearch.alerting.core.lock.LockService import org.opensearch.alerting.core.resthandler.RestScheduledJobStatsHandler import org.opensearch.alerting.core.schedule.JobScheduler -import org.opensearch.alerting.core.settings.AlertingV2Settings import org.opensearch.alerting.core.settings.LegacyOpenDistroScheduledJobSettings import org.opensearch.alerting.core.settings.ScheduledJobSettings -import org.opensearch.alerting.modelv2.MonitorV2 import org.opensearch.alerting.remote.monitors.RemoteMonitorRegistry import org.opensearch.alerting.resthandler.RestAcknowledgeAlertAction import org.opensearch.alerting.resthandler.RestAcknowledgeChainedAlertAction @@ -62,12 +54,6 @@ import org.opensearch.alerting.resthandler.RestSearchAlertingCommentAction import org.opensearch.alerting.resthandler.RestSearchEmailAccountAction import org.opensearch.alerting.resthandler.RestSearchEmailGroupAction import org.opensearch.alerting.resthandler.RestSearchMonitorAction -import org.opensearch.alerting.resthandlerv2.RestDeleteMonitorV2Action -import org.opensearch.alerting.resthandlerv2.RestExecuteMonitorV2Action -import org.opensearch.alerting.resthandlerv2.RestGetAlertsV2Action -import org.opensearch.alerting.resthandlerv2.RestGetMonitorV2Action -import org.opensearch.alerting.resthandlerv2.RestIndexMonitorV2Action -import org.opensearch.alerting.resthandlerv2.RestSearchMonitorV2Action import org.opensearch.alerting.script.TriggerScript import org.opensearch.alerting.service.DeleteMonitorService import org.opensearch.alerting.settings.AlertingSettings @@ -105,12 +91,6 @@ import org.opensearch.alerting.transport.TransportSearchAlertingCommentAction import org.opensearch.alerting.transport.TransportSearchEmailAccountAction import org.opensearch.alerting.transport.TransportSearchEmailGroupAction import org.opensearch.alerting.transport.TransportSearchMonitorAction -import org.opensearch.alerting.transportv2.TransportDeleteMonitorV2Action -import org.opensearch.alerting.transportv2.TransportExecuteMonitorV2Action -import org.opensearch.alerting.transportv2.TransportGetAlertsV2Action -import org.opensearch.alerting.transportv2.TransportGetMonitorV2Action -import org.opensearch.alerting.transportv2.TransportIndexMonitorV2Action -import org.opensearch.alerting.transportv2.TransportSearchMonitorV2Action import org.opensearch.alerting.util.DocLevelMonitorQueries import org.opensearch.alerting.util.destinationmigration.DestinationMigrationCoordinator import org.opensearch.cluster.metadata.IndexNameExpressionResolver @@ -131,6 +111,8 @@ import org.opensearch.commons.alerting.model.ClusterMetricsInput import org.opensearch.commons.alerting.model.DocLevelMonitorInput import org.opensearch.commons.alerting.model.DocumentLevelTrigger import org.opensearch.commons.alerting.model.Monitor +import org.opensearch.commons.alerting.model.PPLSQLInput +import org.opensearch.commons.alerting.model.PPLSQLTrigger import org.opensearch.commons.alerting.model.QueryLevelTrigger import org.opensearch.commons.alerting.model.ScheduledJob import org.opensearch.commons.alerting.model.ScheduledJob.Companion.SCHEDULED_JOBS_INDEX @@ -193,7 +175,6 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R @JvmField val UI_METADATA_EXCLUDE = arrayOf("monitor.${Monitor.UI_METADATA_FIELD}") @JvmField val TENANT_ID_HEADER = "x-tenant-id" @JvmField val MONITOR_BASE_URI = "/_plugins/_alerting/monitors" - @JvmField val MONITOR_V2_BASE_URI = "/_plugins/_alerting/v2/monitors" @JvmField val WORKFLOW_BASE_URI = "/_plugins/_alerting/workflows" @JvmField val REMOTE_BASE_URI = "/_plugins/_alerting/remote" @JvmField val DESTINATION_BASE_URI = "/_plugins/_alerting/destinations" @@ -206,7 +187,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R @JvmField val FINDING_BASE_URI = "/_plugins/_alerting/findings" @JvmField val COMMENTS_BASE_URI = "/_plugins/_alerting/comments" - @JvmField val ALERTING_JOB_TYPES = listOf("monitor", "workflow", "monitor_v2") + @JvmField val ALERTING_JOB_TYPES = listOf("monitor", "workflow") } lateinit var runner: MonitorRunnerService @@ -258,14 +239,6 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R RestIndexAlertingCommentAction(), RestSearchAlertingCommentAction(), RestDeleteAlertingCommentAction(), - - // Alerting V2 - RestIndexMonitorV2Action(), - RestExecuteMonitorV2Action(), - RestDeleteMonitorV2Action(), - RestGetMonitorV2Action(), - RestSearchMonitorV2Action(settings, clusterService), - RestGetAlertsV2Action() ) } @@ -299,29 +272,22 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R ActionPlugin.ActionHandler(ExecuteWorkflowAction.INSTANCE, TransportExecuteWorkflowAction::class.java), ActionPlugin.ActionHandler(GetRemoteIndexesAction.INSTANCE, TransportGetRemoteIndexesAction::class.java), ActionPlugin.ActionHandler(DocLevelMonitorFanOutAction.INSTANCE, TransportDocLevelMonitorFanOutAction::class.java), - - // Alerting V2 - ActionPlugin.ActionHandler(IndexMonitorV2Action.INSTANCE, TransportIndexMonitorV2Action::class.java), - ActionPlugin.ActionHandler(GetMonitorV2Action.INSTANCE, TransportGetMonitorV2Action::class.java), - ActionPlugin.ActionHandler(SearchMonitorV2Action.INSTANCE, TransportSearchMonitorV2Action::class.java), - ActionPlugin.ActionHandler(DeleteMonitorV2Action.INSTANCE, TransportDeleteMonitorV2Action::class.java), - ActionPlugin.ActionHandler(ExecuteMonitorV2Action.INSTANCE, TransportExecuteMonitorV2Action::class.java), - ActionPlugin.ActionHandler(GetAlertsV2Action.INSTANCE, TransportGetAlertsV2Action::class.java) ) } override fun getNamedXContent(): List { return listOf( Monitor.XCONTENT_REGISTRY, - MonitorV2.XCONTENT_REGISTRY, SearchInput.XCONTENT_REGISTRY, DocLevelMonitorInput.XCONTENT_REGISTRY, + PPLSQLInput.XCONTENT_REGISTRY, QueryLevelTrigger.XCONTENT_REGISTRY, BucketLevelTrigger.XCONTENT_REGISTRY, ClusterMetricsInput.XCONTENT_REGISTRY, DocumentLevelTrigger.XCONTENT_REGISTRY, ChainedAlertTrigger.XCONTENT_REGISTRY, RemoteMonitorTrigger.XCONTENT_REGISTRY, + PPLSQLTrigger.XCONTENT_REGISTRY, Workflow.XCONTENT_REGISTRY ) } @@ -524,17 +490,14 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R AlertingSettings.ALERT_V2_HISTORY_MAX_DOCS, AlertingSettings.ALERT_V2_HISTORY_RETENTION_PERIOD, AlertingSettings.ALERT_V2_MONITOR_EXECUTION_MAX_DURATION, - AlertingSettings.ALERTING_V2_MAX_MONITORS, AlertingSettings.ALERTING_V2_MAX_THROTTLE_DURATION, AlertingSettings.ALERTING_V2_MAX_EXPIRE_DURATION, - AlertingSettings.ALERTING_V2_MAX_LOOK_BACK_WINDOW, AlertingSettings.ALERTING_V2_MAX_QUERY_LENGTH, AlertingSettings.ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS, AlertingSettings.ALERT_V2_QUERY_RESULTS_MAX_SIZE, AlertingSettings.ALERT_V2_PER_RESULT_TRIGGER_MAX_ALERTS, AlertingSettings.NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH, AlertingSettings.NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH, - AlertingV2Settings.ALERTING_V2_ENABLED, ) } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingV2Utils.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingV2Utils.kt deleted file mode 100644 index 8a3936eb6..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingV2Utils.kt +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting - -import org.apache.lucene.search.TotalHits -import org.apache.lucene.search.TotalHits.Relation -import org.opensearch.OpenSearchSecurityException -import org.opensearch.action.search.SearchResponse -import org.opensearch.action.search.ShardSearchFailure -import org.opensearch.alerting.AlertingPlugin.Companion.MONITOR_BASE_URI -import org.opensearch.alerting.AlertingPlugin.Companion.MONITOR_V2_BASE_URI -import org.opensearch.alerting.action.GetDestinationsAction -import org.opensearch.alerting.action.GetDestinationsRequest -import org.opensearch.alerting.action.GetDestinationsResponse -import org.opensearch.alerting.model.destination.Destination -import org.opensearch.alerting.modelv2.MonitorV2 -import org.opensearch.alerting.opensearchapi.suspendUntil -import org.opensearch.alerting.util.destinationmigration.NotificationActionConfigs -import org.opensearch.alerting.util.destinationmigration.NotificationApiUtils.Companion.getNotificationConfigInfo -import org.opensearch.alerting.util.destinationmigration.getTitle -import org.opensearch.alerting.util.destinationmigration.publishLegacyNotification -import org.opensearch.alerting.util.destinationmigration.sendNotification -import org.opensearch.alerting.util.isAllowed -import org.opensearch.alerting.util.isTestAction -import org.opensearch.commons.alerting.model.Monitor -import org.opensearch.commons.alerting.model.ScheduledJob -import org.opensearch.commons.alerting.model.Table -import org.opensearch.commons.alerting.model.Workflow -import org.opensearch.commons.alerting.model.action.Action -import org.opensearch.commons.notifications.model.NotificationConfigInfo -import org.opensearch.index.IndexNotFoundException -import org.opensearch.search.SearchHits -import org.opensearch.search.aggregations.InternalAggregations -import org.opensearch.search.internal.InternalSearchResponse -import org.opensearch.search.profile.SearchProfileShardResults -import org.opensearch.search.suggest.Suggest -import org.opensearch.transport.RemoteTransportException -import org.opensearch.transport.client.node.NodeClient -import java.util.Collections - -object AlertingV2Utils { - // Validates that the given scheduled job is a Monitor - // returns the exception to pass into actionListener.onFailure if not. - fun validateMonitorV1(scheduledJob: ScheduledJob): Exception? { - if (scheduledJob is MonitorV2) { - return IllegalStateException( - "The ID given corresponds to an Alerting V2 Monitor, but a V1 Monitor was expected. " + - "If you wish to operate on a V2 Monitor (e.g. PPL Monitor), please use " + - "the Alerting V2 APIs with endpoint prefix: $MONITOR_V2_BASE_URI." - ) - } else if (scheduledJob !is Monitor && scheduledJob !is Workflow) { - return IllegalStateException( - "The ID given corresponds to a scheduled job of unknown type: ${scheduledJob.javaClass.name}. " + - "Please validate the ID and ensure it corresponds to a valid Monitor." - ) - } - return null - } - - // Validates that the given scheduled job is a MonitorV2 - // returns the exception to pass into actionListener.onFailure if not. - fun validateMonitorV2(scheduledJob: ScheduledJob): Exception? { - if (scheduledJob is Monitor || scheduledJob is Workflow) { - return IllegalStateException( - "The ID given corresponds to an Alerting V1 Monitor, but a V2 Monitor was expected. " + - "If you wish to operate on a V1 Monitor (e.g. Per Query, Per Document, etc), please use " + - "the Alerting V1 APIs with endpoint prefix: $MONITOR_BASE_URI." - ) - } else if (scheduledJob !is MonitorV2) { - return IllegalStateException( - "The ID given corresponds to a scheduled job of unknown type: ${scheduledJob.javaClass.name}. " + - "Please validate the ID and ensure it corresponds to a valid Monitor." - ) - } - return null - } - - // Checks if the exception is caused by an IndexNotFoundException (directly or nested). - // Used in Get and Search monitor functionalities to determine whether a "no results" - // response should be returned - fun isIndexNotFoundException(e: Exception): Boolean { - if (e is IndexNotFoundException) { - return true - } - if (e is RemoteTransportException) { - val cause = e.cause - if (cause is IndexNotFoundException) { - return true - } - } - return false - } - - // Used in Get and Search monitor functionalities to return a "no results" response - fun getEmptySearchResponse(): SearchResponse { - val internalSearchResponse = InternalSearchResponse( - SearchHits(emptyArray(), TotalHits(0L, Relation.EQUAL_TO), 0.0f), - InternalAggregations.from(Collections.emptyList()), - Suggest(Collections.emptyList()), - SearchProfileShardResults(Collections.emptyMap()), - false, - false, - 0 - ) - - return SearchResponse( - internalSearchResponse, - "", - 0, - 0, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ) - } - - suspend fun getConfigAndSendNotification( - action: Action, - monitorCtx: MonitorRunnerExecutionContext, - subject: String?, - message: String - ): String { - val config = getConfigForNotificationAction(action, monitorCtx) - if (config.destination == null && config.channel == null) { - throw IllegalStateException("Unable to find a Notification Channel or Destination config with id [${action.destinationId}]") - } - - // Adding a check on TEST_ACTION Destination type here to avoid supporting it as a LegacyBaseMessage type - // just for Alerting integration tests - if (config.destination?.isTestAction() == true) { - return "test action" - } - - if (config.destination?.isAllowed(monitorCtx.allowList) == false) { - throw IllegalStateException( - "Monitor contains a Destination type that is not allowed: ${config.destination.type}" - ) - } - - var actionResponseContent = "" - actionResponseContent = config.channel - ?.sendNotification( - monitorCtx.client!!, - config.channel.getTitle(subject), - message - ) ?: actionResponseContent - - actionResponseContent = config.destination - ?.buildLegacyBaseMessage(subject, message, monitorCtx.destinationContextFactory!!.getDestinationContext(config.destination)) - ?.publishLegacyNotification(monitorCtx.client!!) - ?: actionResponseContent - - return actionResponseContent - } - - /** - * The "destination" ID referenced in a Monitor Action could either be a Notification config or a Destination config - * depending on whether the background migration process has already migrated it from a Destination to a Notification config. - * - * To cover both of these cases, the Notification config will take precedence and if it is not found, the Destination will be retrieved. - */ - private suspend fun getConfigForNotificationAction( - action: Action, - monitorCtx: MonitorRunnerExecutionContext - ): NotificationActionConfigs { - var destination: Destination? = null - var notificationPermissionException: Exception? = null - - var channel: NotificationConfigInfo? = null - try { - channel = getNotificationConfigInfo(monitorCtx.client as NodeClient, action.destinationId) - } catch (e: OpenSearchSecurityException) { - notificationPermissionException = e - } - - // If the channel was not found, try to retrieve the Destination - if (channel == null) { - destination = try { - val table = Table( - "asc", - "destination.name.keyword", - null, - 1, - 0, - null - ) - val getDestinationsRequest = GetDestinationsRequest( - action.destinationId, - 0L, - null, - table, - "ALL" - ) - - val getDestinationsResponse: GetDestinationsResponse = monitorCtx.client!!.suspendUntil { - monitorCtx.client!!.execute(GetDestinationsAction.INSTANCE, getDestinationsRequest, it) - } - getDestinationsResponse.destinations.firstOrNull() - } catch (e: IllegalStateException) { - // Catching the exception thrown when the Destination was not found so the NotificationActionConfigs object can be returned - null - } catch (e: OpenSearchSecurityException) { - if (notificationPermissionException != null) - throw notificationPermissionException - else - throw e - } - - if (destination == null && notificationPermissionException != null) - throw notificationPermissionException - } - - return NotificationActionConfigs(destination, channel) - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/BucketLevelMonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/BucketLevelMonitorRunner.kt index 416401520..11e963e63 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/BucketLevelMonitorRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/BucketLevelMonitorRunner.kt @@ -72,6 +72,7 @@ object BucketLevelMonitorRunner : MonitorRunner() { periodStart: Instant, periodEnd: Instant, dryrun: Boolean, + manual: Boolean, workflowRunContext: WorkflowRunContext?, executionId: String, transportService: TransportService @@ -85,6 +86,11 @@ object BucketLevelMonitorRunner : MonitorRunner() { var monitorResult = MonitorRunResult(monitor.name, periodStart, periodEnd) val currentAlerts = try { + // create stateless alert indices as well to prevent get alerts from returning error because + // stateless alerts indices couldn't be found + monitorCtx.alertV2Indices!!.createOrUpdateAlertV2Index() + monitorCtx.alertV2Indices!!.createOrUpdateInitialAlertV2HistoryIndex() + monitorCtx.alertIndices!!.createOrUpdateAlertIndex(monitor.dataSources) monitorCtx.alertIndices!!.createOrUpdateInitialAlertHistoryIndex(monitor.dataSources) if (monitor.dataSources.findingsEnabled == true) { diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt index 8a961b3b9..873184d6e 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt @@ -58,6 +58,7 @@ class DocumentLevelMonitorRunner : MonitorRunner() { periodStart: Instant, periodEnd: Instant, dryrun: Boolean, + manual: Boolean, workflowRunContext: WorkflowRunContext?, executionId: String, transportService: TransportService @@ -69,6 +70,11 @@ class DocumentLevelMonitorRunner : MonitorRunner() { monitorCtx.findingsToTriggeredQueries = mutableMapOf() try { + // create stateless alert indices as well to prevent get alerts from returning error because + // stateless alerts indices couldn't be found + monitorCtx.alertV2Indices!!.createOrUpdateAlertV2Index() + monitorCtx.alertV2Indices!!.createOrUpdateInitialAlertV2HistoryIndex() + monitorCtx.alertIndices!!.createOrUpdateAlertIndex(monitor.dataSources) monitorCtx.alertIndices!!.createOrUpdateInitialAlertHistoryIndex(monitor.dataSources) monitorCtx.alertIndices!!.createOrUpdateInitialFindingHistoryIndex(monitor.dataSources) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunner.kt index 04aaac098..e0d97345a 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunner.kt @@ -5,11 +5,11 @@ package org.opensearch.alerting -import org.opensearch.alerting.AlertingV2Utils.getConfigAndSendNotification import org.opensearch.alerting.opensearchapi.InjectorContextElement import org.opensearch.alerting.opensearchapi.withClosableContext import org.opensearch.alerting.script.QueryLevelTriggerExecutionContext import org.opensearch.alerting.script.TriggerExecutionContext +import org.opensearch.alerting.util.getConfigAndSendNotification import org.opensearch.alerting.util.use import org.opensearch.commons.ConfigConstants import org.opensearch.commons.alerting.model.ActionRunResult @@ -29,6 +29,7 @@ abstract class MonitorRunner { periodStart: Instant, periodEnd: Instant, dryRun: Boolean, + manual: Boolean, workflowRunContext: WorkflowRunContext? = null, executionId: String, transportService: TransportService diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt index 10e277957..561a36c92 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt @@ -21,9 +21,6 @@ import org.opensearch.alerting.action.ExecuteMonitorResponse import org.opensearch.alerting.action.ExecuteWorkflowAction import org.opensearch.alerting.action.ExecuteWorkflowRequest import org.opensearch.alerting.action.ExecuteWorkflowResponse -import org.opensearch.alerting.actionv2.ExecuteMonitorV2Action -import org.opensearch.alerting.actionv2.ExecuteMonitorV2Request -import org.opensearch.alerting.actionv2.ExecuteMonitorV2Response import org.opensearch.alerting.alerts.AlertIndices import org.opensearch.alerting.alerts.AlertMover.Companion.moveAlerts import org.opensearch.alerting.alertsv2.AlertV2Indices @@ -33,16 +30,11 @@ import org.opensearch.alerting.core.ScheduledJobIndices import org.opensearch.alerting.core.lock.LockModel import org.opensearch.alerting.core.lock.LockService import org.opensearch.alerting.model.destination.DestinationContextFactory -import org.opensearch.alerting.modelv2.MonitorV2 -import org.opensearch.alerting.modelv2.MonitorV2RunResult -import org.opensearch.alerting.modelv2.PPLSQLMonitor -import org.opensearch.alerting.modelv2.PPLSQLMonitor.Companion.PPL_SQL_MONITOR_TYPE import org.opensearch.alerting.opensearchapi.retry import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.remote.monitors.RemoteDocumentLevelMonitorRunner import org.opensearch.alerting.remote.monitors.RemoteMonitorRegistry import org.opensearch.alerting.script.TriggerExecutionContext -import org.opensearch.alerting.script.TriggerV2ExecutionContext import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_BACKOFF_COUNT import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_BACKOFF_MILLIS @@ -82,6 +74,7 @@ import org.opensearch.commons.alerting.util.AlertingException import org.opensearch.commons.alerting.util.IndexPatternUtils import org.opensearch.commons.alerting.util.isBucketLevelMonitor import org.opensearch.commons.alerting.util.isMonitorOfStandardType +import org.opensearch.commons.alerting.util.isPplSqlMonitor import org.opensearch.core.action.ActionListener import org.opensearch.core.rest.RestStatus import org.opensearch.core.xcontent.NamedXContentRegistry @@ -321,6 +314,9 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon if (monitorCtx.alertIndices!!.isAlertInitialized(job.dataSources)) { moveAlerts(monitorCtx.client!!, job.id, job) } + if (monitorCtx.alertV2Indices!!.isAlertV2Initialized()) { + moveAlertV2s(job.id, job, monitorCtx) + } } } catch (e: Exception) { logger.error("Failed to move active alerts for monitor [${job.id}].", e) @@ -336,18 +332,6 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon logger.error("Failed to move active alerts for monitor [${job.id}].", e) } } - } else if (job is MonitorV2) { - launch { - try { - monitorCtx.moveAlertsRetryPolicy!!.retry(logger) { - if (monitorCtx.alertV2Indices!!.isAlertV2Initialized()) { - moveAlertV2s(job.id, job, monitorCtx) - } - } - } catch (e: Exception) { - logger.error("Failed to move active alertV2s for monitorV2 [${job.id}].", e) - } - } } else { throw IllegalArgumentException("Invalid job type") } @@ -398,6 +382,7 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon monitorCtx.client!!.execute( ExecuteWorkflowAction.INSTANCE, ExecuteWorkflowRequest( + false, false, TimeValue(periodEnd.toEpochMilli()), job.id, @@ -428,6 +413,7 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon monitorCtx.clusterService!!.state().nodes().localNode.id ) val executeMonitorRequest = ExecuteMonitorRequest( + false, false, TimeValue(periodEnd.toEpochMilli()), job.id, @@ -449,44 +435,6 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon } } } - is MonitorV2 -> { - if (job !is PPLSQLMonitor) { - throw IllegalStateException("Invalid MonitorV2 type: ${job.javaClass.name}") - } - - launch { - var monitorLock: LockModel? = null - try { - monitorLock = monitorCtx.client!!.suspendUntil { - monitorCtx.lockService!!.acquireLock(job, it) - } ?: return@launch - logger.debug("lock ${monitorLock!!.lockId} acquired") - logger.debug( - "PERF_DEBUG: executing $PPL_SQL_MONITOR_TYPE ${job.id} on node " + - monitorCtx.clusterService!!.state().nodes().localNode.id - ) - val executeMonitorV2Request = ExecuteMonitorV2Request( - false, - false, - job.id, // only need to pass in MonitorV2 ID - null, // no need to pass in MonitorV2 object itself - TimeValue(periodEnd.toEpochMilli()) - ) - monitorCtx.client!!.suspendUntil { - monitorCtx.client!!.execute( - ExecuteMonitorV2Action.INSTANCE, - executeMonitorV2Request, - it - ) - } - } catch (e: Exception) { - logger.error("MonitorV2 run failed for monitor with id ${job.id}", e) - } finally { - monitorCtx.client!!.suspendUntil { monitorCtx.lockService!!.release(monitorLock, it) } - logger.debug("lock ${monitorLock?.lockId} released") - } - } - } else -> { throw IllegalArgumentException("Invalid job type") } @@ -498,9 +446,10 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon periodStart: Instant, periodEnd: Instant, dryrun: Boolean, + manual: Boolean, transportService: TransportService ): WorkflowRunResult { - return CompositeWorkflowRunner.runWorkflow(workflow, monitorCtx, periodStart, periodEnd, dryrun, transportService) + return CompositeWorkflowRunner.runWorkflow(workflow, monitorCtx, periodStart, periodEnd, dryrun, manual, transportService) } suspend fun runJob( @@ -508,6 +457,7 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon periodStart: Instant, periodEnd: Instant, dryrun: Boolean, + manual: Boolean, transportService: TransportService ): MonitorRunResult<*> { // Updating the scheduled job index at the start of monitor execution runs for when there is an upgrade the the schema mapping @@ -516,7 +466,7 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon if (job is Workflow) { logger.info("Executing scheduled workflow - id: ${job.id}, periodStart: $periodStart, periodEnd: $periodEnd, dryrun: $dryrun") - CompositeWorkflowRunner.runWorkflow(workflow = job, monitorCtx, periodStart, periodEnd, dryrun, transportService) + CompositeWorkflowRunner.runWorkflow(workflow = job, monitorCtx, periodStart, periodEnd, dryrun, manual, transportService) } val monitor = job as Monitor val executionId = "${monitor.id}_${LocalDateTime.now(ZoneOffset.UTC)}_${UUID.randomUUID()}" @@ -538,13 +488,25 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon "Executing scheduled monitor - id: ${monitor.id}, type: ${monitor.monitorType}, periodStart: $periodStart, " + "periodEnd: $periodEnd, dryrun: $dryrun, executionId: $executionId" ) - val runResult = if (monitor.isBucketLevelMonitor()) { + val runResult = if (monitor.isPplSqlMonitor()) { + PPLSQLMonitorRunner.runMonitor( + monitor, + monitorCtx, + periodStart, + periodEnd, + dryrun, + manual, + executionId = executionId, + transportService = transportService, + ) + } else if (monitor.isBucketLevelMonitor()) { BucketLevelMonitorRunner.runMonitor( monitor, monitorCtx, periodStart, periodEnd, dryrun, + manual, executionId = executionId, transportService = transportService ) @@ -555,6 +517,7 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon periodStart, periodEnd, dryrun, + manual, executionId = executionId, transportService = transportService ) @@ -565,6 +528,7 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon periodStart, periodEnd, dryrun, + manual, executionId = executionId, transportService = transportService ) @@ -580,6 +544,7 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon periodStart, periodEnd, dryrun, + manual, executionId = executionId, transportService = transportService ) @@ -605,44 +570,6 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon } } - // after the above JobRunner interface override runJob calls ExecuteMonitorV2 API, - // the ExecuteMonitorV2 transport action calls this function to call the PPLSQLMonitorRunner, - // where the core PPL/SQL Monitor execution logic resides - suspend fun runJobV2( - monitorV2: MonitorV2, - periodEnd: Instant, - dryrun: Boolean, - manual: Boolean, - transportService: TransportService, - ): MonitorV2RunResult<*> { - updateAlertingConfigIndexSchema() - - val executionId = "${monitorV2.id}_${LocalDateTime.now(ZoneOffset.UTC)}_${UUID.randomUUID()}" - val monitorV2Type = when (monitorV2) { - is PPLSQLMonitor -> PPL_SQL_MONITOR_TYPE - else -> throw IllegalStateException("Unexpected MonitorV2 type: ${monitorV2.javaClass.name}") - } - - logger.info( - "Executing scheduled monitor v2 - id: ${monitorV2.id}, type: $monitorV2Type, " + - "periodEnd: $periodEnd, dryrun: $dryrun, manual: $manual, executionId: $executionId" - ) - - // for now, always call PPLSQLMonitorRunner since only PPL Monitors are initially supported - // to introduce new MonitorV2 type, create its MonitorRunner, and if/else branch - // to the corresponding MonitorRunners based on type. For now, default to PPLSQLMonitorRunner - val runResult = PPLSQLMonitorRunner.runMonitorV2( - monitorV2, - monitorCtx, - periodEnd, - dryrun, - manual, - executionId = executionId, - transportService = transportService, - ) - return runResult - } - // TODO: See if we can move below methods (or few of these) to a common utils internal fun getRolesForMonitor(monitor: Monitor): List { /* @@ -687,12 +614,6 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon .execute() } - internal fun compileTemplateV2(template: Script, ctx: TriggerV2ExecutionContext): String { - return monitorCtx.scriptService!!.compile(template, TemplateScript.CONTEXT) - .newInstance(template.params + mapOf("ctx" to ctx.asTemplateArg())) - .execute() - } - private fun updateAlertingConfigIndexSchema() { if (!IndexUtils.scheduledJobIndexUpdated && monitorCtx.clusterService != null && monitorCtx.client != null) { IndexUtils.updateIndexMapping( diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorV2Runner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorV2Runner.kt deleted file mode 100644 index ccf933148..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorV2Runner.kt +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting - -import org.opensearch.alerting.modelv2.MonitorV2 -import org.opensearch.alerting.modelv2.MonitorV2RunResult -import org.opensearch.transport.TransportService -import java.time.Instant - -/** - * Interface for monitor V2 runners. All monitor v2 runner classes that house - * a specific v2 monitor type's execution logic must implement this interface. - * - * @opensearch.experimental - */ -interface MonitorV2Runner { - suspend fun runMonitorV2( - monitorV2: MonitorV2, - monitorCtx: MonitorRunnerExecutionContext, // MonitorV2 reads from same context as Monitor does - periodEnd: Instant, - dryRun: Boolean, - manual: Boolean, - executionId: String, - transportService: TransportService - ): MonitorV2RunResult<*> -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/PPLSQLMonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/PPLSQLMonitorRunner.kt index d183389ff..18cbc2ef1 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/PPLSQLMonitorRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/PPLSQLMonitorRunner.kt @@ -17,7 +17,6 @@ import org.opensearch.action.bulk.BulkRequest import org.opensearch.action.bulk.BulkResponse import org.opensearch.action.index.IndexRequest import org.opensearch.action.support.WriteRequest -import org.opensearch.alerting.AlertingV2Utils.getConfigAndSendNotification import org.opensearch.alerting.PPLUtils.appendCustomCondition import org.opensearch.alerting.PPLUtils.appendDataRowsLimit import org.opensearch.alerting.PPLUtils.capPPLQueryResultsSize @@ -25,27 +24,29 @@ import org.opensearch.alerting.PPLUtils.executePplQuery import org.opensearch.alerting.PPLUtils.findEvalResultVar import org.opensearch.alerting.PPLUtils.findEvalResultVarIdxInSchema import org.opensearch.alerting.alertsv2.AlertV2Indices -import org.opensearch.alerting.modelv2.AlertV2 -import org.opensearch.alerting.modelv2.MonitorV2 -import org.opensearch.alerting.modelv2.MonitorV2RunResult -import org.opensearch.alerting.modelv2.PPLSQLMonitor -import org.opensearch.alerting.modelv2.PPLSQLMonitorRunResult -import org.opensearch.alerting.modelv2.PPLSQLTrigger -import org.opensearch.alerting.modelv2.PPLSQLTrigger.ConditionType -import org.opensearch.alerting.modelv2.PPLSQLTrigger.NumResultsCondition -import org.opensearch.alerting.modelv2.PPLSQLTrigger.TriggerMode -import org.opensearch.alerting.modelv2.PPLSQLTriggerRunResult -import org.opensearch.alerting.modelv2.TriggerV2.Severity import org.opensearch.alerting.opensearchapi.InjectorContextElement import org.opensearch.alerting.opensearchapi.retry import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.opensearchapi.withClosableContext import org.opensearch.alerting.script.PPLTriggerExecutionContext import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.alerting.util.getConfigAndSendNotification import org.opensearch.common.xcontent.XContentFactory import org.opensearch.commons.alerting.alerts.AlertError import org.opensearch.commons.alerting.model.Alert +import org.opensearch.commons.alerting.model.InputRunResults +import org.opensearch.commons.alerting.model.Monitor +import org.opensearch.commons.alerting.model.Monitor.MonitorType +import org.opensearch.commons.alerting.model.MonitorRunResult +import org.opensearch.commons.alerting.model.PPLSQLInput +import org.opensearch.commons.alerting.model.PPLSQLTrigger +import org.opensearch.commons.alerting.model.PPLSQLTrigger.ConditionType +import org.opensearch.commons.alerting.model.PPLSQLTrigger.NumResultsCondition +import org.opensearch.commons.alerting.model.PPLSQLTrigger.Severity +import org.opensearch.commons.alerting.model.PPLSQLTrigger.TriggerMode +import org.opensearch.commons.alerting.model.PPLSQLTriggerRunResult import org.opensearch.commons.alerting.model.ScheduledJob.Companion.SCHEDULED_JOBS_INDEX +import org.opensearch.commons.alerting.model.WorkflowRunContext import org.opensearch.commons.alerting.model.action.Action import org.opensearch.commons.alerting.model.userErrorMessage import org.opensearch.core.common.Strings @@ -56,10 +57,7 @@ import org.opensearch.transport.TransportService import org.opensearch.transport.client.node.NodeClient import java.time.Duration import java.time.Instant -import java.time.ZoneOffset.UTC -import java.time.format.DateTimeFormatter import java.time.temporal.ChronoUnit -import java.util.Locale import kotlin.math.min import kotlin.time.measureTimedValue @@ -73,32 +71,36 @@ import kotlin.time.measureTimedValue * @opensearch.experimental */ -object PPLSQLMonitorRunner : MonitorV2Runner { +object PPLSQLMonitorRunner : MonitorRunner() { private val logger = LogManager.getLogger(javaClass) - override suspend fun runMonitorV2( - monitorV2: MonitorV2, - monitorCtx: MonitorRunnerExecutionContext, // MonitorV2 reads from same context as Monitor + override suspend fun runMonitor( + monitor: Monitor, + monitorCtx: MonitorRunnerExecutionContext, + periodStart: Instant, periodEnd: Instant, dryRun: Boolean, manual: Boolean, + workflowRunContext: WorkflowRunContext?, executionId: String, transportService: TransportService, - ): MonitorV2RunResult<*> { - if (monitorV2 !is PPLSQLMonitor) { - throw IllegalStateException("Unexpected monitor type: ${monitorV2.javaClass.name}") + ): MonitorRunResult { + if (monitor.monitorType != MonitorType.PPL_MONITOR.value) { + throw IllegalStateException("Unexpected monitor type: ${monitor.javaClass.name}") } - if (monitorV2.id == MonitorV2.NO_ID) { + if (monitor.id == Monitor.NO_ID) { throw IllegalStateException("Received PPL Monitor to execute that unexpectedly has no ID") } - logger.debug("Running PPL Monitor: ${monitorV2.id}. Thread: ${Thread.currentThread().name}") + logger.debug("Running PPL Monitor: ${monitor.id}. Thread: ${Thread.currentThread().name}") + + var monitorResult = MonitorRunResult(monitor.name, periodStart, periodEnd) + val pplSqlInput = monitor.inputs[0] as PPLSQLInput // time the monitor execution run for informational logging val monitorRunStart = Instant.now() - val pplSqlMonitor = monitorV2 val nodeClient = monitorCtx.client as NodeClient // create some objects that will be used later @@ -114,27 +116,15 @@ object PPLSQLMonitorRunner : MonitorV2Runner { try { monitorCtx.alertV2Indices!!.createOrUpdateAlertV2Index() monitorCtx.alertV2Indices!!.createOrUpdateInitialAlertV2HistoryIndex() + + // create stateful alert indices as well to prevent get alerts from returning error because + // stateful alerts indices couldn't be found + monitorCtx.alertIndices!!.createOrUpdateAlertIndex(monitor.dataSources) + monitorCtx.alertIndices!!.createOrUpdateInitialAlertHistoryIndex(monitor.dataSources) } catch (e: Exception) { - val id = if (pplSqlMonitor.id.trim().isEmpty()) "_na_" else pplSqlMonitor.id + val id = if (monitor.id.trim().isEmpty()) "_na_" else monitor.id logger.error("Error loading alerts for monitorV2: $id", e) - return PPLSQLMonitorRunResult(pplSqlMonitor.name, e, mapOf(), mapOf()) - } - - val timeFilteredQuery = if (pplSqlMonitor.lookBackWindow != null) { - logger.debug("look back window specified for PPL Monitor: ${monitorV2.id}, injecting look back window time filter") - // if lookback window is specified, inject a top level lookback window time filter - // into the PPL query - val lookBackWindow = pplSqlMonitor.lookBackWindow!! - val lookbackPeriodStart = periodEnd.minus(lookBackWindow, ChronoUnit.MINUTES) - val timeFilteredQuery = addTimeFilter(pplSqlMonitor.query, lookbackPeriodStart, periodEnd, pplSqlMonitor.timestampField!!) - logger.debug("time filtered query: $timeFilteredQuery") - timeFilteredQuery - } else { - logger.debug("look back window not specified for PPL Monitor: ${monitorV2.id}, proceeding with original base query") - // otherwise, don't inject any time filter whatsoever - // unless the query itself has user-specified time filters, this query - // will return all applicable data in the cluster - pplSqlMonitor.query + return monitorResult.copy(error = e) } val monitorExecutionDuration = monitorCtx @@ -150,11 +140,13 @@ object PPLSQLMonitorRunner : MonitorV2Runner { try { withTimeout(monitorExecutionDuration.millis) { runTriggers( - pplSqlMonitor, - timeFilteredQuery, + monitor, + pplSqlInput.query, timeOfCurrentExecution, manual, dryRun, + periodStart, + periodEnd, triggerResults, pplSqlQueryResults, executionId, @@ -170,8 +162,8 @@ object PPLSQLMonitorRunner : MonitorV2Runner { // the above time frame and generate their own alerts monitorCtx.retryPolicy?.let { saveAlertsV2( - generateErrorAlert(null, pplSqlMonitor, e, executionId, timeOfCurrentExecution), - pplSqlMonitor, + generateErrorAlert(null, monitor, e, executionId, timeOfCurrentExecution), + monitor, it, nodeClient ) @@ -183,29 +175,33 @@ object PPLSQLMonitorRunner : MonitorV2Runner { // for throttle checking purposes, reindex the PPL Monitor into the alerting-config index // with updated last triggered times for each of its triggers if (triggerResults.any { it.value.triggered }) { - updateMonitorWithLastTriggeredTimes(pplSqlMonitor, nodeClient) + updateMonitorWithLastTriggeredTimes(monitor, nodeClient) } val monitorRunEnd = Instant.now() val monitorRunTime = Duration.between(monitorRunStart, monitorRunEnd) - logger.info("monitor ${pplSqlMonitor.id} execution $executionId run time: $monitorRunTime") + logger.info("monitor ${monitor.id} execution $executionId run time: $monitorRunTime") - return PPLSQLMonitorRunResult( - pplSqlMonitor.name, - exception, - triggerResults, - pplSqlQueryResults + return monitorResult.copy( + inputResults = InputRunResults( + results = listOf(pplSqlQueryResults), + error = exception, + aggTriggersAfterKey = null // aggTriggersAfterKey not relevant for PPL/SQL Monitors + ), + triggerResults = triggerResults ) } suspend fun runTriggers( - pplSqlMonitor: PPLSQLMonitor, + pplSqlMonitor: Monitor, timeFilteredQuery: String, timeOfCurrentExecution: Instant, manual: Boolean, dryRun: Boolean, + periodStart: Instant, + periodEnd: Instant, triggerResults: MutableMap, pplSqlQueryResults: MutableMap>, executionId: String, @@ -213,7 +209,9 @@ object PPLSQLMonitorRunner : MonitorV2Runner { nodeClient: NodeClient, transportService: TransportService ) { - for (pplSqlTrigger in pplSqlMonitor.triggers) { + for (trigger in pplSqlMonitor.triggers) { + val pplSqlTrigger = trigger as PPLSQLTrigger + try { // check for throttle and skip execution // before even running the trigger itself @@ -222,7 +220,7 @@ object PPLSQLMonitorRunner : MonitorV2Runner { logger.info("throttling trigger ${pplSqlTrigger.id} from monitor ${pplSqlMonitor.id}") // automatically return that this trigger is untriggered - triggerResults[pplSqlTrigger.id] = PPLSQLTriggerRunResult(pplSqlTrigger.name, false, null) + triggerResults[pplSqlTrigger.id] = PPLSQLTriggerRunResult(pplSqlTrigger.name, null, false) continue } @@ -288,7 +286,7 @@ object PPLSQLMonitorRunner : MonitorV2Runner { logger.debug("PPLTrigger ${pplSqlTrigger.name} with ID ${pplSqlTrigger.id} triggered: $triggered") // store the trigger execution results for Execute Monitor API response - triggerResults[pplSqlTrigger.id] = PPLSQLTriggerRunResult(pplSqlTrigger.name, triggered, null) + triggerResults[pplSqlTrigger.id] = PPLSQLTriggerRunResult(pplSqlTrigger.name, null, triggered) if (triggered) { logger.debug("generating alerts for PPLTrigger ${pplSqlTrigger.name} with ID ${pplSqlTrigger.id}") @@ -325,8 +323,11 @@ object PPLSQLMonitorRunner : MonitorV2Runner { val pplTriggerExecutionContext = PPLTriggerExecutionContext( pplSqlMonitor, null, - pplSqlTrigger, - queryResult + listOf(queryResult.toMap()), + periodStart, + periodEnd, + monitorCtx.clusterService!!.clusterSettings, + pplSqlTrigger ) runAction( @@ -376,7 +377,7 @@ object PPLSQLMonitorRunner : MonitorV2Runner { // the interval between throttledTimeBound and now is the throttle window // i.e. any PPLTrigger whose last trigger time is in this window must be throttled val throttleTimeBound = pplTrigger.throttleDuration?.let { - timeOfCurrentExecution.minus(pplTrigger.throttleDuration, ChronoUnit.MINUTES) + timeOfCurrentExecution.minus(pplTrigger.throttleDuration!!, ChronoUnit.MINUTES) } // the trigger must be throttled if... @@ -385,36 +386,6 @@ object PPLSQLMonitorRunner : MonitorV2Runner { pplTrigger.lastTriggeredTime!!.isAfter(throttleTimeBound!!) // and it's not yet out of its throttle window } - // adds monitor schedule-based time filter - // query: the raw PPL Monitor query - // lookbackPeriodStart: the lower bound of the query interval based on monitor schedule and look back window - // periodEnd: the upper bound of the initially computed query interval based on monitor schedule - // timestampField: the timestamp field that will be used to time bound the query results - private fun addTimeFilter(query: String, lookbackPeriodStart: Instant, periodEnd: Instant, timestampField: String): String { - // PPL plugin only accepts timestamp strings in this format - val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss", Locale.ROOT).withZone(UTC) - - val periodStartPplTimestamp = formatter.format(lookbackPeriodStart) - val periodEndPplTimeStamp = formatter.format(periodEnd) - - val timeFilterAppend = "| where $timestampField > TIMESTAMP('$periodStartPplTimestamp') and " + - "$timestampField < TIMESTAMP('$periodEndPplTimeStamp')" - val timeFilterReplace = "$timeFilterAppend |" - - val timeFilteredQuery: String = if (query.contains("|")) { - // if Monitor query contains piped statements, inject the time filter - // as the first piped statement (i.e. before more complex statements - // like aggregations can take effect later in the query) - query.replaceFirst("|", timeFilterReplace) - } else { - // otherwise the query contains no piped statements and is simply a - // `search source=` statement, simply append time filter at the end - query + timeFilterAppend - } - - return timeFilteredQuery - } - private fun evaluateNumResultsTrigger( pplQueryResponse: JSONObject, numResultsCondition: NumResultsCondition, @@ -522,22 +493,23 @@ object PPLSQLMonitorRunner : MonitorV2Runner { private fun generateAlerts( pplSqlTrigger: PPLSQLTrigger, - pplSqlMonitor: PPLSQLMonitor, + pplSqlMonitor: Monitor, preparedQueryResults: List, executionId: String, timeOfCurrentExecution: Instant - ): List { - val alertV2s = mutableListOf() + ): List { + val alertV2s = mutableListOf() for (queryResult in preparedQueryResults) { - val alertV2 = AlertV2( + val alertV2 = Alert( monitorId = pplSqlMonitor.id, monitorName = pplSqlMonitor.name, monitorVersion = pplSqlMonitor.version, monitorUser = pplSqlMonitor.user, triggerId = pplSqlTrigger.id, triggerName = pplSqlTrigger.name, - query = pplSqlMonitor.query, - queryResults = queryResult.toMap(), + pplQuery = (pplSqlMonitor.inputs[0] as PPLSQLInput).query, + pplQueryResults = queryResult.toMap(), + state = Alert.State.ACTIVE, triggeredTime = timeOfCurrentExecution, severity = pplSqlTrigger.severity, executionId = executionId @@ -550,27 +522,28 @@ object PPLSQLMonitorRunner : MonitorV2Runner { private fun generateErrorAlert( pplSqlTrigger: PPLSQLTrigger?, - pplSqlMonitor: PPLSQLMonitor, + pplSqlMonitor: Monitor, exception: Exception, executionId: String, timeOfCurrentExecution: Instant - ): List { + ): List { val errorMessage = "Failed to run PPL Monitor ${pplSqlMonitor.id}, PPL Trigger ${pplSqlTrigger?.id}: " + exception.userErrorMessage() val obfuscatedErrorMessage = AlertError.obfuscateIPAddresses(errorMessage) - val alertV2 = AlertV2( + val alertV2 = Alert( monitorId = pplSqlMonitor.id, monitorName = pplSqlMonitor.name, monitorVersion = pplSqlMonitor.version, monitorUser = pplSqlMonitor.user, triggerId = pplSqlTrigger?.id ?: "", triggerName = pplSqlTrigger?.name ?: "", - query = pplSqlMonitor.query, - queryResults = mapOf(), + pplQuery = (pplSqlMonitor.inputs[0] as PPLSQLInput).query, + pplQueryResults = mapOf(), + state = Alert.State.ERROR, triggeredTime = timeOfCurrentExecution, errorMessage = obfuscatedErrorMessage, - severity = Severity.ERROR, + severity = Severity.ERROR.value, executionId = executionId ) @@ -578,8 +551,8 @@ object PPLSQLMonitorRunner : MonitorV2Runner { } private suspend fun saveAlertsV2( - alerts: List, - pplSqlMonitor: PPLSQLMonitor, + alerts: List, + pplSqlMonitor: Monitor, retryPolicy: BackoffPolicy, client: NodeClient ) { @@ -601,7 +574,7 @@ object PPLSQLMonitorRunner : MonitorV2Runner { val bulkResponse: BulkResponse = client.suspendUntil { client.bulk(bulkRequest, it) } val failedResponses = (bulkResponse.items ?: arrayOf()).filter { it.isFailed } failedResponses.forEach { - logger.debug("write alerts failed responses: ${it.failureMessage}") + logger.info("write alerts failed responses: ${it.failureMessage}") } requestsToRetry = failedResponses.filter { it.status() == RestStatus.TOO_MANY_REQUESTS } .map { bulkRequest.requests()[it.itemId] as IndexRequest } @@ -616,7 +589,7 @@ object PPLSQLMonitorRunner : MonitorV2Runner { // during monitor execution, the ppl sql monitor object stored in memory had its triggers updated // with their last trigger times. this function simply indexes those updated triggers into the // alerting-config index - private suspend fun updateMonitorWithLastTriggeredTimes(pplSqlMonitor: PPLSQLMonitor, client: NodeClient) { + private suspend fun updateMonitorWithLastTriggeredTimes(pplSqlMonitor: Monitor, client: NodeClient) { val indexRequest = IndexRequest(SCHEDULED_JOBS_INDEX) .id(pplSqlMonitor.id) .source( @@ -640,17 +613,17 @@ object PPLSQLMonitorRunner : MonitorV2Runner { action: Action, triggerCtx: PPLTriggerExecutionContext, monitorCtx: MonitorRunnerExecutionContext, - pplSqlMonitor: PPLSQLMonitor, + pplSqlMonitor: Monitor, dryrun: Boolean ) { // this function can throw an exception, which is caught by the try // catch in runMonitor() to generate an error alert val notifSubject = if (action.subjectTemplate != null) - MonitorRunnerService.compileTemplateV2(action.subjectTemplate!!, triggerCtx) + MonitorRunnerService.compileTemplate(action.subjectTemplate!!, triggerCtx) else "" - var notifMessage = MonitorRunnerService.compileTemplateV2(action.messageTemplate, triggerCtx) + var notifMessage = MonitorRunnerService.compileTemplate(action.messageTemplate, triggerCtx) if (Strings.isNullOrEmpty(notifMessage)) { throw IllegalStateException("Message content missing in the Destination with id: ${action.destinationId}") } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/QueryLevelMonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/QueryLevelMonitorRunner.kt index 197b0103b..44370228e 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/QueryLevelMonitorRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/QueryLevelMonitorRunner.kt @@ -34,6 +34,7 @@ object QueryLevelMonitorRunner : MonitorRunner() { periodStart: Instant, periodEnd: Instant, dryrun: Boolean, + manual: Boolean, workflowRunContext: WorkflowRunContext?, executionId: String, transportService: TransportService @@ -47,6 +48,11 @@ object QueryLevelMonitorRunner : MonitorRunner() { var monitorResult = MonitorRunResult(monitor.name, periodStart, periodEnd) val currentAlerts = try { + // create stateless alert indices as well to prevent get alerts from returning error because + // stateless alerts indices couldn't be found + monitorCtx.alertV2Indices!!.createOrUpdateAlertV2Index() + monitorCtx.alertV2Indices!!.createOrUpdateInitialAlertV2HistoryIndex() + monitorCtx.alertIndices!!.createOrUpdateAlertIndex(monitor.dataSources) monitorCtx.alertIndices!!.createOrUpdateInitialAlertHistoryIndex(monitor.dataSources) monitorCtx.alertService!!.loadCurrentAlertsForQueryLevelMonitor(monitor, workflowRunContext) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/WorkflowService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/WorkflowService.kt index 83f8b56e0..15f1e192e 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/WorkflowService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/WorkflowService.kt @@ -11,7 +11,6 @@ import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse import org.opensearch.action.search.SearchRequest import org.opensearch.action.search.SearchResponse -import org.opensearch.alerting.AlertingV2Utils.validateMonitorV1 import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.common.xcontent.LoggingDeprecationHandler import org.opensearch.common.xcontent.XContentType @@ -133,12 +132,7 @@ class WorkflowService( xContentRegistry, LoggingDeprecationHandler.INSTANCE, hit.sourceAsString ).use { hitsParser -> - val scheduledJob = ScheduledJob.parse(hitsParser, hit.id, hit.version) - validateMonitorV1(scheduledJob)?.let { - throw OpenSearchException(it) - } - - val monitor = scheduledJob as Monitor + val monitor = ScheduledJob.parse(hitsParser, hit.id, hit.version) as Monitor monitors.add(monitor) } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/action/ExecuteMonitorRequest.kt b/alerting/src/main/kotlin/org/opensearch/alerting/action/ExecuteMonitorRequest.kt index c7b699dfc..1d2c69218 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/action/ExecuteMonitorRequest.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/action/ExecuteMonitorRequest.kt @@ -15,6 +15,7 @@ import java.io.IOException class ExecuteMonitorRequest : ActionRequest { val dryrun: Boolean + val manual: Boolean val requestEnd: TimeValue val monitorId: String? val monitor: Monitor? @@ -22,12 +23,14 @@ class ExecuteMonitorRequest : ActionRequest { constructor( dryrun: Boolean, + manual: Boolean, requestEnd: TimeValue, monitorId: String?, monitor: Monitor?, requestStart: TimeValue? = null ) : super() { this.dryrun = dryrun + this.manual = manual this.requestEnd = requestEnd this.monitorId = monitorId this.monitor = monitor @@ -37,6 +40,7 @@ class ExecuteMonitorRequest : ActionRequest { @Throws(IOException::class) constructor(sin: StreamInput) : this( sin.readBoolean(), // dryrun + sin.readBoolean(), // manual sin.readTimeValue(), // requestEnd sin.readOptionalString(), // monitorId if (sin.readBoolean()) { @@ -52,6 +56,7 @@ class ExecuteMonitorRequest : ActionRequest { @Throws(IOException::class) override fun writeTo(out: StreamOutput) { out.writeBoolean(dryrun) + out.writeBoolean(manual) out.writeTimeValue(requestEnd) out.writeOptionalString(monitorId) if (monitor != null) { diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/action/ExecuteWorkflowRequest.kt b/alerting/src/main/kotlin/org/opensearch/alerting/action/ExecuteWorkflowRequest.kt index 104448cce..fdf962048 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/action/ExecuteWorkflowRequest.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/action/ExecuteWorkflowRequest.kt @@ -19,6 +19,7 @@ import java.io.IOException */ class ExecuteWorkflowRequest : ActionRequest { val dryrun: Boolean + val manual: Boolean val requestEnd: TimeValue val workflowId: String? val workflow: Workflow? @@ -26,12 +27,14 @@ class ExecuteWorkflowRequest : ActionRequest { constructor( dryrun: Boolean, + manual: Boolean, requestEnd: TimeValue, workflowId: String?, workflow: Workflow?, requestStart: TimeValue? = null, ) : super() { this.dryrun = dryrun + this.manual = manual this.requestEnd = requestEnd this.requestStart = requestStart this.workflowId = workflowId @@ -40,6 +43,7 @@ class ExecuteWorkflowRequest : ActionRequest { @Throws(IOException::class) constructor(sin: StreamInput) : this( + sin.readBoolean(), sin.readBoolean(), sin.readTimeValue(), sin.readOptionalString(), @@ -62,6 +66,7 @@ class ExecuteWorkflowRequest : ActionRequest { @Throws(IOException::class) override fun writeTo(out: StreamOutput) { out.writeBoolean(dryrun) + out.writeBoolean(manual) out.writeTimeValue(requestEnd) out.writeOptionalTimeValue(requestStart) out.writeOptionalString(workflowId) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Action.kt deleted file mode 100644 index b182d87d4..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Action.kt +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.action.ActionType - -class DeleteMonitorV2Action private constructor() : ActionType(NAME, ::DeleteMonitorV2Response) { - companion object { - val INSTANCE = DeleteMonitorV2Action() - const val NAME = "cluster:admin/opensearch/alerting/v2/monitor/delete" - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Request.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Request.kt deleted file mode 100644 index 7024842ac..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Request.kt +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.action.ActionRequest -import org.opensearch.action.ActionRequestValidationException -import org.opensearch.action.support.WriteRequest -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.core.common.io.stream.StreamOutput -import java.io.IOException - -class DeleteMonitorV2Request : ActionRequest { - val monitorV2Id: String - val refreshPolicy: WriteRequest.RefreshPolicy - - constructor(monitorV2Id: String, refreshPolicy: WriteRequest.RefreshPolicy) : super() { - this.monitorV2Id = monitorV2Id - this.refreshPolicy = refreshPolicy - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - monitorV2Id = sin.readString(), - refreshPolicy = WriteRequest.RefreshPolicy.readFrom(sin) - ) - - override fun validate(): ActionRequestValidationException? { - return null - } - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - out.writeString(monitorV2Id) - refreshPolicy.writeTo(out) - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Response.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Response.kt deleted file mode 100644 index b4850b662..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2Response.kt +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.commons.alerting.util.IndexUtils -import org.opensearch.commons.notifications.action.BaseResponse -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.core.common.io.stream.StreamOutput -import org.opensearch.core.xcontent.ToXContent -import org.opensearch.core.xcontent.XContentBuilder - -class DeleteMonitorV2Response : BaseResponse { - var id: String - var version: Long - - constructor( - id: String, - version: Long - ) : super() { - this.id = id - this.version = version - } - - constructor(sin: StreamInput) : this( - sin.readString(), // id - sin.readLong() // version - ) - - override fun writeTo(out: StreamOutput) { - out.writeString(id) - out.writeLong(version) - } - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - return builder.startObject() - .field(IndexUtils._ID, id) - .field(IndexUtils._VERSION, version) - .endObject() - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Action.kt deleted file mode 100644 index f0de80e8d..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Action.kt +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.action.ActionType - -class ExecuteMonitorV2Action private constructor() : ActionType(NAME, ::ExecuteMonitorV2Response) { - companion object { - val INSTANCE = ExecuteMonitorV2Action() - const val NAME = "cluster:admin/opensearch/alerting/v2/monitor/execute" - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Request.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Request.kt deleted file mode 100644 index 99a01667c..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Request.kt +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.action.ActionRequest -import org.opensearch.action.ActionRequestValidationException -import org.opensearch.action.ValidateActions -import org.opensearch.alerting.modelv2.MonitorV2 -import org.opensearch.common.unit.TimeValue -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.core.common.io.stream.StreamOutput -import java.io.IOException - -class ExecuteMonitorV2Request : ActionRequest { - val dryrun: Boolean - val manual: Boolean - val monitorV2Id: String? // exactly one of monitorId or monitor must be non-null - val monitorV2: MonitorV2? - val requestEnd: TimeValue - - constructor( - dryrun: Boolean, - manual: Boolean, // if execute was called by user or by scheduled job - monitorV2Id: String?, - monitorV2: MonitorV2?, - requestEnd: TimeValue - ) : super() { - this.dryrun = dryrun - this.manual = manual - this.monitorV2Id = monitorV2Id - this.monitorV2 = monitorV2 - this.requestEnd = requestEnd - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - sin.readBoolean(), // dryrun - sin.readBoolean(), // manual - sin.readOptionalString(), // monitorV2Id - if (sin.readBoolean()) { - MonitorV2.readFrom(sin) // monitorV2 - } else { - null - }, - sin.readTimeValue() // requestEnd - ) - - override fun validate(): ActionRequestValidationException? = - if (monitorV2 == null && monitorV2Id == null) { - ValidateActions.addValidationError("Neither a monitor ID nor monitor object was supplied", null) - } else { - null - } - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - out.writeBoolean(dryrun) - out.writeBoolean(manual) - out.writeOptionalString(monitorV2Id) - if (monitorV2 != null) { - out.writeBoolean(true) - MonitorV2.writeTo(out, monitorV2) - } else { - out.writeBoolean(false) - } - out.writeTimeValue(requestEnd) - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Response.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Response.kt deleted file mode 100644 index 7d6eb8b1f..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Response.kt +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.alerting.modelv2.MonitorV2RunResult -import org.opensearch.core.action.ActionResponse -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.core.common.io.stream.StreamOutput -import org.opensearch.core.xcontent.ToXContent -import org.opensearch.core.xcontent.ToXContentObject -import org.opensearch.core.xcontent.XContentBuilder -import java.io.IOException - -class ExecuteMonitorV2Response : ActionResponse, ToXContentObject { - val monitorV2RunResult: MonitorV2RunResult<*> - - constructor(monitorV2RunResult: MonitorV2RunResult<*>) : super() { - this.monitorV2RunResult = monitorV2RunResult - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - MonitorV2RunResult.readFrom(sin) // monitorRunResult - ) - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - MonitorV2RunResult.writeTo(out, monitorV2RunResult) - } - - @Throws(IOException::class) - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - return monitorV2RunResult.toXContent(builder, ToXContent.EMPTY_PARAMS) - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Action.kt deleted file mode 100644 index e656d6a71..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Action.kt +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.action.ActionType - -class GetAlertsV2Action private constructor() : ActionType(NAME, ::GetAlertsV2Response) { - companion object { - val INSTANCE = GetAlertsV2Action() - const val NAME = "cluster:admin/opensearch/alerting/v2/alerts/get" - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Request.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Request.kt deleted file mode 100644 index 008057aa4..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Request.kt +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.action.ActionRequest -import org.opensearch.action.ActionRequestValidationException -import org.opensearch.commons.alerting.model.Table -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.core.common.io.stream.StreamOutput -import java.io.IOException - -class GetAlertsV2Request : ActionRequest { - val table: Table - val severityLevel: String - val monitorV2Ids: List? - - constructor( - table: Table, - severityLevel: String, - monitorV2Ids: List? = null, - ) : super() { - this.table = table - this.severityLevel = severityLevel - this.monitorV2Ids = monitorV2Ids - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - table = Table.readFrom(sin), - severityLevel = sin.readString(), - monitorV2Ids = sin.readOptionalStringList(), - ) - - override fun validate(): ActionRequestValidationException? { - return null - } - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - table.writeTo(out) - out.writeString(severityLevel) - out.writeOptionalStringCollection(monitorV2Ids) - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Response.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Response.kt deleted file mode 100644 index 0de492496..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2Response.kt +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.alerting.modelv2.AlertV2 -import org.opensearch.commons.notifications.action.BaseResponse -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.core.common.io.stream.StreamOutput -import org.opensearch.core.xcontent.ToXContent -import org.opensearch.core.xcontent.XContentBuilder -import java.io.IOException -import java.util.Collections - -class GetAlertsV2Response : BaseResponse { - val alertV2s: List - - // totalAlertV2s is not the same as the size of alertV2s because there can be 30 alerts from the request, but - // the request only asked for 5 alerts, so totalAlertV2s will be 30, but alertV2s will only contain 5 alerts - val totalAlertV2s: Int? - - constructor( - alertV2s: List, - totalAlertV2s: Int? - ) : super() { - this.alertV2s = alertV2s - this.totalAlertV2s = totalAlertV2s - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - alertV2s = Collections.unmodifiableList(sin.readList(::AlertV2)), - totalAlertV2s = sin.readOptionalInt() - ) - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - out.writeCollection(alertV2s) - out.writeOptionalInt(totalAlertV2s) - } - - @Throws(IOException::class) - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - builder.startObject() - .field("alerts_v2", alertV2s) - .field("total_alerts_v2", totalAlertV2s) - - return builder.endObject() - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Action.kt deleted file mode 100644 index 58b6a1cb9..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Action.kt +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.action.ActionType - -class GetMonitorV2Action private constructor() : ActionType(NAME, ::GetMonitorV2Response) { - companion object { - val INSTANCE = GetMonitorV2Action() - const val NAME = "cluster:admin/opensearch/alerting/v2/monitor/get" - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Request.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Request.kt deleted file mode 100644 index 7f051ff1e..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Request.kt +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.action.ActionRequest -import org.opensearch.action.ActionRequestValidationException -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.core.common.io.stream.StreamOutput -import org.opensearch.search.fetch.subphase.FetchSourceContext -import java.io.IOException - -class GetMonitorV2Request : ActionRequest { - val monitorV2Id: String - val version: Long - val srcContext: FetchSourceContext? - - constructor( - monitorV2Id: String, - version: Long, - srcContext: FetchSourceContext? - ) : super() { - this.monitorV2Id = monitorV2Id - this.version = version - this.srcContext = srcContext - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - sin.readString(), // monitorV2Id - sin.readLong(), // version - if (sin.readBoolean()) { - FetchSourceContext(sin) // srcContext - } else { - null - } - ) - - override fun validate(): ActionRequestValidationException? { - return null - } - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - out.writeString(monitorV2Id) - out.writeLong(version) - out.writeBoolean(srcContext != null) - srcContext?.writeTo(out) - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Response.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Response.kt deleted file mode 100644 index 5b6df334f..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Response.kt +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.alerting.modelv2.MonitorV2 -import org.opensearch.commons.alerting.util.IndexUtils.Companion._ID -import org.opensearch.commons.alerting.util.IndexUtils.Companion._PRIMARY_TERM -import org.opensearch.commons.alerting.util.IndexUtils.Companion._SEQ_NO -import org.opensearch.commons.alerting.util.IndexUtils.Companion._VERSION -import org.opensearch.commons.notifications.action.BaseResponse -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.core.common.io.stream.StreamOutput -import org.opensearch.core.xcontent.ToXContent -import org.opensearch.core.xcontent.XContentBuilder -import java.io.IOException - -class GetMonitorV2Response : BaseResponse { - var id: String - var version: Long - var seqNo: Long - var primaryTerm: Long - var monitorV2: MonitorV2? - - constructor( - id: String, - version: Long, - seqNo: Long, - primaryTerm: Long, - monitorV2: MonitorV2? - ) : super() { - this.id = id - this.version = version - this.seqNo = seqNo - this.primaryTerm = primaryTerm - this.monitorV2 = monitorV2 - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - id = sin.readString(), // id - version = sin.readLong(), // version - seqNo = sin.readLong(), // seqNo - primaryTerm = sin.readLong(), // primaryTerm - monitorV2 = if (sin.readBoolean()) { - MonitorV2.readFrom(sin) // monitorV2 - } else { - null - } - ) - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - out.writeString(id) - out.writeLong(version) - out.writeLong(seqNo) - out.writeLong(primaryTerm) - if (monitorV2 != null) { - out.writeBoolean(true) - MonitorV2.writeTo(out, monitorV2!!) - } else { - out.writeBoolean(false) - } - } - - @Throws(IOException::class) - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - builder.startObject() - .field(_ID, id) - .field(_VERSION, version) - .field(_SEQ_NO, seqNo) - .field(_PRIMARY_TERM, primaryTerm) - if (monitorV2 != null) { - builder.field("monitorV2", monitorV2) - } - return builder.endObject() - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Action.kt deleted file mode 100644 index aab23b631..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Action.kt +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.action.ActionType - -class IndexMonitorV2Action private constructor() : ActionType(NAME, ::IndexMonitorV2Response) { - companion object { - val INSTANCE = IndexMonitorV2Action() - const val NAME = "cluster:admin/opensearch/alerting/v2/monitor/write" - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Request.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Request.kt deleted file mode 100644 index 105408d07..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Request.kt +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.action.ActionRequest -import org.opensearch.action.ActionRequestValidationException -import org.opensearch.action.support.WriteRequest -import org.opensearch.alerting.modelv2.MonitorV2 -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.core.common.io.stream.StreamOutput -import org.opensearch.rest.RestRequest -import java.io.IOException - -class IndexMonitorV2Request : ActionRequest { - val monitorId: String - val seqNo: Long - val primaryTerm: Long - val refreshPolicy: WriteRequest.RefreshPolicy - val method: RestRequest.Method - var monitorV2: MonitorV2 - val rbacRoles: List? - - constructor( - monitorId: String, - seqNo: Long, - primaryTerm: Long, - refreshPolicy: WriteRequest.RefreshPolicy, - method: RestRequest.Method, - monitorV2: MonitorV2, - rbacRoles: List? = null - ) : super() { - this.monitorId = monitorId - this.seqNo = seqNo - this.primaryTerm = primaryTerm - this.refreshPolicy = refreshPolicy - this.method = method - this.monitorV2 = monitorV2 - this.rbacRoles = rbacRoles - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - monitorId = sin.readString(), - seqNo = sin.readLong(), - primaryTerm = sin.readLong(), - refreshPolicy = WriteRequest.RefreshPolicy.readFrom(sin), - method = sin.readEnum(RestRequest.Method::class.java), - monitorV2 = MonitorV2.readFrom(sin), - rbacRoles = sin.readOptionalStringList() - ) - - override fun validate(): ActionRequestValidationException? { - return null - } - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - out.writeString(monitorId) - out.writeLong(seqNo) - out.writeLong(primaryTerm) - refreshPolicy.writeTo(out) - out.writeEnum(method) - MonitorV2.writeTo(out, monitorV2) - out.writeOptionalStringCollection(rbacRoles) - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Response.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Response.kt deleted file mode 100644 index 99d076334..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2Response.kt +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.alerting.modelv2.MonitorV2 -import org.opensearch.commons.alerting.util.IndexUtils.Companion._ID -import org.opensearch.commons.alerting.util.IndexUtils.Companion._PRIMARY_TERM -import org.opensearch.commons.alerting.util.IndexUtils.Companion._SEQ_NO -import org.opensearch.commons.alerting.util.IndexUtils.Companion._VERSION -import org.opensearch.commons.notifications.action.BaseResponse -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.core.common.io.stream.StreamOutput -import org.opensearch.core.xcontent.ToXContent -import org.opensearch.core.xcontent.XContentBuilder -import java.io.IOException - -class IndexMonitorV2Response : BaseResponse { - var id: String - var version: Long - var seqNo: Long - var primaryTerm: Long - var monitorV2: MonitorV2 - - constructor( - id: String, - version: Long, - seqNo: Long, - primaryTerm: Long, - monitorV2: MonitorV2 - ) : super() { - this.id = id - this.version = version - this.seqNo = seqNo - this.primaryTerm = primaryTerm - this.monitorV2 = monitorV2 - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - sin.readString(), // id - sin.readLong(), // version - sin.readLong(), // seqNo - sin.readLong(), // primaryTerm - MonitorV2.readFrom(sin) // monitorV2 - ) - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - out.writeString(id) - out.writeLong(version) - out.writeLong(seqNo) - out.writeLong(primaryTerm) - MonitorV2.writeTo(out, monitorV2) - } - - @Throws(IOException::class) - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - return builder.startObject() - .field(_ID, id) - .field(_VERSION, version) - .field(_SEQ_NO, seqNo) - .field(_PRIMARY_TERM, primaryTerm) - .field(MONITOR_V2_FIELD, monitorV2) - .endObject() - } - - companion object { - const val MONITOR_V2_FIELD = "monitor_v2" - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2Action.kt deleted file mode 100644 index bc4d9e572..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2Action.kt +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.action.ActionType -import org.opensearch.action.search.SearchResponse - -class SearchMonitorV2Action private constructor() : ActionType(NAME, ::SearchResponse) { - companion object { - val INSTANCE = SearchMonitorV2Action() - const val NAME = "cluster:admin/opensearch/alerting/v2/monitor/search" - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2Request.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2Request.kt deleted file mode 100644 index c16b62fd8..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2Request.kt +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.action.ActionRequest -import org.opensearch.action.ActionRequestValidationException -import org.opensearch.action.search.SearchRequest -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.core.common.io.stream.StreamOutput -import java.io.IOException - -class SearchMonitorV2Request : ActionRequest { - val searchRequest: SearchRequest - - constructor( - searchRequest: SearchRequest - ) : super() { - this.searchRequest = searchRequest - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - searchRequest = SearchRequest(sin) - ) - - override fun validate(): ActionRequestValidationException? { - return null - } - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - searchRequest.writeTo(out) - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Indices.kt b/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Indices.kt index 75257fc25..5e3d93fc3 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Indices.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Indices.kt @@ -24,6 +24,7 @@ import org.opensearch.action.admin.indices.rollover.RolloverRequest import org.opensearch.action.admin.indices.rollover.RolloverResponse import org.opensearch.action.support.IndicesOptions import org.opensearch.action.support.clustermanager.AcknowledgedResponse +import org.opensearch.alerting.alerts.AlertIndices import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_V2_HISTORY_ENABLED import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_V2_HISTORY_INDEX_MAX_AGE @@ -94,8 +95,8 @@ class AlertV2Indices( const val ALL_ALERT_V2_INDEX_PATTERN = ".opensearch-alerting-v2-alert*" @JvmStatic - fun alertV2Mapping() = - AlertV2Indices::class.java.getResource("alert_v2_mapping.json").readText() + fun alertMapping() = + AlertIndices::class.java.getResource("alert_mapping.json").readText() } @Volatile private var alertV2HistoryEnabled = ALERT_V2_HISTORY_ENABLED.get(settings) @@ -170,24 +171,24 @@ class AlertV2Indices( suspend fun createOrUpdateAlertV2Index() { if (!alertV2IndexInitialized) { - alertV2IndexInitialized = createIndex(ALERT_V2_INDEX, alertV2Mapping()) + alertV2IndexInitialized = createIndex(ALERT_V2_INDEX, alertMapping()) if (alertV2IndexInitialized) IndexUtils.alertIndexUpdated() } else { - if (!IndexUtils.alertIndexUpdated) updateIndexMapping(ALERT_V2_INDEX, alertV2Mapping()) + if (!IndexUtils.alertIndexUpdated) updateIndexMapping(ALERT_V2_INDEX, alertMapping()) } alertV2IndexInitialized } suspend fun createOrUpdateInitialAlertV2HistoryIndex() { if (!alertV2HistoryIndexInitialized) { - alertV2HistoryIndexInitialized = createIndex(ALERT_V2_HISTORY_INDEX_PATTERN, alertV2Mapping(), ALERT_V2_HISTORY_WRITE_INDEX) + alertV2HistoryIndexInitialized = createIndex(ALERT_V2_HISTORY_INDEX_PATTERN, alertMapping(), ALERT_V2_HISTORY_WRITE_INDEX) if (alertV2HistoryIndexInitialized) IndexUtils.lastUpdatedAlertV2HistoryIndex = IndexUtils.getIndexNameWithAlias( clusterService.state(), ALERT_V2_HISTORY_WRITE_INDEX ) } else { - updateIndexMapping(ALERT_V2_HISTORY_WRITE_INDEX, alertV2Mapping(), true) + updateIndexMapping(ALERT_V2_HISTORY_WRITE_INDEX, alertMapping(), true) } alertV2HistoryIndexInitialized } @@ -299,7 +300,7 @@ class AlertV2Indices( alertV2HistoryIndexInitialized, ALERT_V2_HISTORY_WRITE_INDEX, ALERT_V2_HISTORY_INDEX_PATTERN, - alertV2Mapping(), + alertMapping(), alertV2HistoryMaxDocs, alertV2HistoryMaxAge, ALERT_V2_HISTORY_WRITE_INDEX diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Mover.kt b/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Mover.kt index bf5aa9fad..7f768d310 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Mover.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Mover.kt @@ -18,15 +18,6 @@ import org.opensearch.action.search.SearchResponse import org.opensearch.alerting.MonitorRunnerExecutionContext import org.opensearch.alerting.alertsv2.AlertV2Indices.Companion.ALERT_V2_HISTORY_WRITE_INDEX import org.opensearch.alerting.alertsv2.AlertV2Indices.Companion.ALERT_V2_INDEX -import org.opensearch.alerting.modelv2.AlertV2 -import org.opensearch.alerting.modelv2.AlertV2.Companion.TRIGGERED_TIME_FIELD -import org.opensearch.alerting.modelv2.AlertV2.Companion.TRIGGER_V2_ID_FIELD -import org.opensearch.alerting.modelv2.MonitorV2 -import org.opensearch.alerting.modelv2.MonitorV2.Companion.MONITOR_V2_TYPE -import org.opensearch.alerting.modelv2.MonitorV2.Companion.TRIGGERS_FIELD -import org.opensearch.alerting.modelv2.PPLSQLMonitor.Companion.PPL_SQL_MONITOR_TYPE -import org.opensearch.alerting.modelv2.TriggerV2.Companion.EXPIRE_FIELD -import org.opensearch.alerting.modelv2.TriggerV2.Companion.ID_FIELD import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_V2_HISTORY_ENABLED import org.opensearch.alerting.util.MAX_SEARCH_SIZE @@ -39,12 +30,25 @@ import org.opensearch.common.xcontent.LoggingDeprecationHandler import org.opensearch.common.xcontent.XContentFactory import org.opensearch.common.xcontent.XContentHelper import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.alerting.model.Alert +import org.opensearch.commons.alerting.model.Alert.Companion.MONITOR_ID_FIELD +import org.opensearch.commons.alerting.model.Alert.Companion.START_TIME_FIELD +import org.opensearch.commons.alerting.model.Alert.Companion.TRIGGER_ID_FIELD +import org.opensearch.commons.alerting.model.Monitor +import org.opensearch.commons.alerting.model.Monitor.Companion.MONITOR_TYPE +import org.opensearch.commons.alerting.model.Monitor.Companion.MONITOR_TYPE_FIELD +import org.opensearch.commons.alerting.model.Monitor.Companion.TRIGGERS_FIELD +import org.opensearch.commons.alerting.model.Monitor.MonitorType +import org.opensearch.commons.alerting.model.PPLSQLTrigger.Companion.EXPIRE_FIELD +import org.opensearch.commons.alerting.model.PPLSQLTrigger.Companion.PPL_SQL_TRIGGER_FIELD import org.opensearch.commons.alerting.model.ScheduledJob.Companion.SCHEDULED_JOBS_INDEX +import org.opensearch.commons.alerting.model.Trigger.Companion.ID_FIELD import org.opensearch.core.common.bytes.BytesReference import org.opensearch.core.rest.RestStatus import org.opensearch.core.xcontent.NamedXContentRegistry import org.opensearch.core.xcontent.ToXContent import org.opensearch.core.xcontent.XContentParser +import org.opensearch.core.xcontent.XContentParserUtils import org.opensearch.index.VersionType import org.opensearch.index.query.QueryBuilders import org.opensearch.search.builder.SearchSourceBuilder @@ -63,6 +67,12 @@ private val logger = LogManager.getLogger(AlertV2Mover::class.java) * permanently deleting them (if alert v2 history disabled). It also contains the * logic for moving alerts in response to a monitor update or deletion. * + * Lifecycle: + * * 1. AlertV2 is generated when a TriggerV2's condition is met. The TriggerV2 fires and forgets the AlertV2. + * * 2. AlertV2 is stored in the alerts index. AlertV2s are stateless. (e.g. they are never ACTIVE or COMPLETED) + * * 3. AlertV2 is soft deleted after its expire duration (determined by its trigger), and archived in an alert history index + * * 4. Based on the alert v2 history retention period, the AlertV2 is permanently deleted + * * @opensearch.experimental */ class AlertV2Mover( @@ -148,16 +158,16 @@ class AlertV2Mover( } } - private suspend fun searchForExpiredAlerts(): List { + private suspend fun searchForExpiredAlerts(): List { logger.debug("beginning search for expired alerts") /* first collect all triggers and their expire durations */ // when searching the alerting-config index, only trigger IDs and their expire durations are needed val monitorV2sSearchQuery = SearchSourceBuilder.searchSource() - .query(QueryBuilders.existsQuery(MONITOR_V2_TYPE)) + .query(QueryBuilders.termQuery("$MONITOR_TYPE.$MONITOR_TYPE_FIELD", MonitorType.PPL_MONITOR.value)) .fetchSource( arrayOf( - "$MONITOR_V2_TYPE.$PPL_SQL_MONITOR_TYPE.$TRIGGERS_FIELD.$ID_FIELD", - "$MONITOR_V2_TYPE.$PPL_SQL_MONITOR_TYPE.$TRIGGERS_FIELD.$EXPIRE_FIELD" + "$MONITOR_TYPE.$TRIGGERS_FIELD.$PPL_SQL_TRIGGER_FIELD.$ID_FIELD", + "$MONITOR_TYPE.$TRIGGERS_FIELD.$PPL_SQL_TRIGGER_FIELD.$EXPIRE_FIELD" ), null ) @@ -166,21 +176,22 @@ class AlertV2Mover( val monitorV2sRequest = SearchRequest(SCHEDULED_JOBS_INDEX) .source(monitorV2sSearchQuery) val searchMonitorV2sResponse: SearchResponse = client.suspendUntil { search(monitorV2sRequest, it) } + logger.debug("search monitor response num hits: ${searchMonitorV2sResponse.hits.totalHits.value}") logger.debug("searching triggers for their expire durations") // construct a map that stores each trigger's expiration time // TODO: create XContent parser specifically for responses to the above search to avoid casting val triggerToExpireDuration = mutableMapOf() searchMonitorV2sResponse.hits.forEach { hit -> - val monitorV2Obj = hit.sourceAsMap[MONITOR_V2_TYPE] as Map - val pplMonitorObj = monitorV2Obj[PPL_SQL_MONITOR_TYPE] as Map - val triggers = pplMonitorObj[TRIGGERS_FIELD] as List> - for (trigger in triggers) { - val triggerId = trigger[ID_FIELD] as String - val expireDuration = (trigger[EXPIRE_FIELD] as Int).toLong() - logger.debug("triggerId: $triggerId") + val monitorObj = hit.sourceAsMap[MONITOR_TYPE] as Map + val triggers = monitorObj[TRIGGERS_FIELD] as List> + triggers.forEach { trigger -> + val pplSqlTrigger = trigger[PPL_SQL_TRIGGER_FIELD] as Map + val pplSqlTriggerId = pplSqlTrigger[ID_FIELD] as String + val expireDuration = (pplSqlTrigger[EXPIRE_FIELD] as Int).toLong() + logger.debug("triggerId: $pplSqlTriggerId") logger.debug("triggerExpires: $expireDuration") - triggerToExpireDuration[triggerId] = expireDuration + triggerToExpireDuration[pplSqlTriggerId] = expireDuration } } @@ -202,8 +213,8 @@ class AlertV2Mover( expiredAlertsBoolQuery.should( QueryBuilders.boolQuery() - .must(QueryBuilders.termQuery(TRIGGER_V2_ID_FIELD, triggerId)) - .must(QueryBuilders.rangeQuery(TRIGGERED_TIME_FIELD).lte(maxValidTime)) + .must(QueryBuilders.termQuery(TRIGGER_ID_FIELD, triggerId)) + .must(QueryBuilders.rangeQuery(START_TIME_FIELD).lte(maxValidTime)) ) } @@ -216,7 +227,7 @@ class AlertV2Mover( // now nonexistent trigger was somehow found expiredAlertsBoolQuery.should( QueryBuilders.boolQuery() - .mustNot(QueryBuilders.termsQuery(TRIGGER_V2_ID_FIELD, triggerToExpireDuration.keys.toList())) + .mustNot(QueryBuilders.termsQuery(TRIGGER_ID_FIELD, triggerToExpireDuration.keys.toList())) ) // Explicitly specify that at least one should clause must match @@ -233,10 +244,10 @@ class AlertV2Mover( // parse the search results into full alert docs, as they will need to be // indexed into alert history indices - val expiredAlertV2s = mutableListOf() + val expiredAlertV2s = mutableListOf() expiredAlertsResponse.hits.forEach { hit -> expiredAlertV2s.add( - AlertV2.parse(alertV2ContentParser(hit.sourceRef), hit.id, hit.version) + Alert.parse(alertContentParser(hit.sourceRef), hit.id, hit.version) ) } @@ -245,7 +256,7 @@ class AlertV2Mover( return expiredAlertV2s } - private suspend fun deleteExpiredAlerts(expiredAlerts: List): BulkResponse? { + private suspend fun deleteExpiredAlerts(expiredAlerts: List): BulkResponse? { logger.debug("beginning to hard delete expired alerts permanently") // If no expired alerts are found, simply return if (expiredAlerts.isEmpty()) { @@ -265,7 +276,7 @@ class AlertV2Mover( return deleteResponse } - private suspend fun copyExpiredAlerts(expiredAlerts: List): BulkResponse? { + private suspend fun copyExpiredAlerts(expiredAlerts: List): BulkResponse? { logger.debug("beginning to copy expired alerts to history write index") // If no expired alerts are found, simply return if (expiredAlerts.isEmpty()) { @@ -287,7 +298,7 @@ class AlertV2Mover( return copyResponse } - private suspend fun deleteExpiredAlertsThatWereCopied(copyResponse: BulkResponse?, expiredAlerts: List): BulkResponse? { + private suspend fun deleteExpiredAlertsThatWereCopied(copyResponse: BulkResponse?, expiredAlerts: List): BulkResponse? { logger.debug("beginning to delete expired alerts that were copied to history write index") // if there were no expired alerts to copy, skip deleting anything if (copyResponse == null) { @@ -296,7 +307,7 @@ class AlertV2Mover( // pre-index the alerts so retrieving their // monitor IDs for routing is easier - val alertsById: Map = expiredAlerts.associateBy { it.id } + val alertsById: Map = expiredAlerts.associateBy { it.id } val deleteRequests = copyResponse.items.filterNot { it.isFailed }.map { DeleteRequest(ALERT_V2_INDEX, it.id) @@ -324,30 +335,32 @@ class AlertV2Mover( } } - private fun alertV2ContentParser(bytesReference: BytesReference): XContentParser { - return XContentHelper.createParser( - NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, - bytesReference, XContentType.JSON - ) - } - private fun areAlertV2IndicesPresent(): Boolean { return alertV2IndexInitialized && alertV2HistoryIndexInitialized } companion object { + private fun alertContentParser(bytesReference: BytesReference): XContentParser { + val xcp = XContentHelper.createParser( + NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, + bytesReference, XContentType.JSON + ) + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp) + return xcp + } + // this method is used by MonitorRunnerService's postIndex and postDelete // functions to move (in the case of alert v2 history enabled) or delete // (in the case of alert v2 history disabled) the alerts generated by // a monitor in response to the event that the monitor gets updated // or deleted - suspend fun moveAlertV2s(monitorV2Id: String, monitorV2: MonitorV2?, monitorCtx: MonitorRunnerExecutionContext) { + suspend fun moveAlertV2s(monitorV2Id: String, monitorV2: Monitor?, monitorCtx: MonitorRunnerExecutionContext) { logger.debug("beginning to move alerts for postIndex or postDelete of monitor: $monitorV2Id") val client = monitorCtx.client!! // first collect all alerts that came from this updated or deleted monitor val boolQuery = QueryBuilders.boolQuery() - .filter(QueryBuilders.termQuery(AlertV2.MONITOR_V2_ID_FIELD, monitorV2Id)) + .filter(QueryBuilders.termQuery(MONITOR_ID_FIELD, monitorV2Id)) /* this monitorV2 != null case happens when this function is called by postIndex. if the monitor is updated, @@ -365,7 +378,7 @@ class AlertV2Mover( meaning this logic will pick up those updated triggers and correctly move/delete the alerts */ if (monitorV2 != null) { - boolQuery.mustNot(QueryBuilders.termsQuery(TRIGGER_V2_ID_FIELD, monitorV2.triggers.map { it.id })) + boolQuery.mustNot(QueryBuilders.termsQuery(TRIGGER_ID_FIELD, monitorV2.triggers.map { it.id })) } val alertsSearchQuery = SearchSourceBuilder.searchSource() @@ -379,14 +392,11 @@ class AlertV2Mover( // If no alerts are found, simply return if (searchAlertsResponse.hits.totalHits?.value == 0L) return - val activeAlerts = mutableListOf() + val activeAlerts = mutableListOf() searchAlertsResponse.hits.forEach { hit -> activeAlerts.add( - AlertV2.parse( - XContentHelper.createParser( - NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, - hit.sourceRef, XContentType.JSON - ), + Alert.parse( + alertContentParser(hit.sourceRef), hit.id, hit.version ) @@ -395,7 +405,7 @@ class AlertV2Mover( // pre-index the alerts so retrieving their // monitor IDs for routing is easier - val alertsById: Map = activeAlerts.associateBy { it.id } + val alertsById: Map = activeAlerts.associateBy { it.id } val alertV2HistoryEnabled = monitorCtx.clusterService!!.clusterSettings.get(ALERT_V2_HISTORY_ENABLED) @@ -405,15 +415,10 @@ class AlertV2Mover( if (alertV2HistoryEnabled) { logger.debug("alert v2 history enabled, copying alerts to history write index") val indexRequests = searchAlertsResponse.hits.map { hit -> - val xcp = XContentHelper.createParser( - NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, - hit.sourceRef, XContentType.JSON - ) - IndexRequest(ALERT_V2_HISTORY_WRITE_INDEX) .routing(monitorV2Id) .source( - AlertV2.parse(xcp, hit.id, hit.version) + Alert.parse(alertContentParser(hit.sourceRef), hit.id, hit.version) .toXContentWithUser(XContentFactory.jsonBuilder()) ) .version(hit.version) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/AlertV2.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/AlertV2.kt deleted file mode 100644 index af188f180..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/AlertV2.kt +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.modelv2 - -import org.opensearch.alerting.core.util.nonOptionalTimeField -import org.opensearch.alerting.modelv2.TriggerV2.Severity -import org.opensearch.common.lucene.uid.Versions -import org.opensearch.commons.alerting.util.instant -import org.opensearch.commons.alerting.util.optionalUserField -import org.opensearch.commons.authuser.User -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.core.common.io.stream.StreamOutput -import org.opensearch.core.common.io.stream.Writeable -import org.opensearch.core.xcontent.ToXContent -import org.opensearch.core.xcontent.XContentBuilder -import org.opensearch.core.xcontent.XContentParser -import org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken -import java.io.IOException -import java.time.Instant - -/** - * Alert generated by Alerting V2 - * An alert is created when a Trigger's trigger conditions are met. - * - * @property id Alert ID. Defaults to [NO_ID]. - * @property version Version number of the Alert. Defaults to [NO_VERSION]. - * @property schemaVersion Version of the alerting-alerts index schema when this Alert was indexed. Defaults to [NO_SCHEMA_VERSION]. - * @property monitorId ID of the Monitor that generated this Alert. - * @property monitorName Name of the Monitor that generated this Alert. - * @property monitorVersion Version of the Monitor at the time it generated this Alert. - * @property triggerId ID of the Trigger in the Monitor that generated this alert. - * @property triggerName Name of the trigger in the Monitor that generated this alert. - * @property queryResults Results from the Monitor's query that caused the Trigger to fire. - * @property triggeredTime Timestamp for when the Alert was generated. - * @property errorMessage Optional error message if there were issues during Trigger execution. - * Null indicates no errors occurred. - * @property severity Severity level of the alert (e.g., "HIGH", "MEDIUM", "LOW"). - * @property executionId Optional ID for the Monitor execution that generated this Alert. - * - * @see MonitorV2 For the monitor that generates alerts - * @see TriggerV2 For the trigger conditions that create alerts - * - * Lifecycle: - * 1. AlertV2 is generated when a TriggerV2's condition is met. The TriggerV2 fires and forgets the AlertV2. - * 2. AlertV2 is stored in the alerts index. AlertV2s are stateless. (e.g. they are never ACTIVE or COMPLETED) - * 3. AlertV2 is soft deleted after its expire duration (determined by its trigger), and archived in an alert history index - * 4. Based on the alert v2 history retention period, the AlertV2 is permanently deleted - * - * @opensearch.experimental - */ -data class AlertV2( - val id: String = NO_ID, - val version: Long = NO_VERSION, - val schemaVersion: Int = NO_SCHEMA_VERSION, - val monitorId: String, - val monitorName: String, - val monitorVersion: Long, - val monitorUser: User?, - val triggerId: String, - val triggerName: String, - val query: String, - val queryResults: Map, - val triggeredTime: Instant, - val errorMessage: String? = null, - val severity: Severity, - val executionId: String? = null -) : Writeable, ToXContent { - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - id = sin.readString(), - version = sin.readLong(), - schemaVersion = sin.readInt(), - monitorId = sin.readString(), - monitorName = sin.readString(), - monitorVersion = sin.readLong(), - monitorUser = if (sin.readBoolean()) { - User(sin) - } else { - null - }, - triggerId = sin.readString(), - triggerName = sin.readString(), - query = sin.readString(), - queryResults = sin.readMap(), - triggeredTime = sin.readInstant(), - errorMessage = sin.readOptionalString(), - severity = sin.readEnum(Severity::class.java), - executionId = sin.readOptionalString() - ) - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - out.writeString(id) - out.writeLong(version) - out.writeInt(schemaVersion) - out.writeString(monitorId) - out.writeString(monitorName) - out.writeLong(monitorVersion) - out.writeBoolean(monitorUser != null) - monitorUser?.writeTo(out) - out.writeString(triggerId) - out.writeString(triggerName) - out.writeString(query) - out.writeMap(queryResults) - out.writeInstant(triggeredTime) - out.writeOptionalString(errorMessage) - out.writeEnum(severity) - out.writeOptionalString(executionId) - } - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - return createXContentBuilder(builder, false) - } - - fun toXContentWithUser(builder: XContentBuilder): XContentBuilder { - return createXContentBuilder(builder, true) - } - - private fun createXContentBuilder(builder: XContentBuilder, withUser: Boolean): XContentBuilder { - builder.startObject() - .field(ALERT_V2_ID_FIELD, id) - .field(ALERT_V2_VERSION_FIELD, version) - .field(MONITOR_V2_ID_FIELD, monitorId) - .field(SCHEMA_VERSION_FIELD, schemaVersion) - .field(MONITOR_V2_VERSION_FIELD, monitorVersion) - .field(MONITOR_V2_NAME_FIELD, monitorName) - .field(EXECUTION_ID_FIELD, executionId) - .field(TRIGGER_V2_ID_FIELD, triggerId) - .field(TRIGGER_V2_NAME_FIELD, triggerName) - .field(QUERY_FIELD, query) - .field(QUERY_RESULTS_FIELD, queryResults) - .field(ERROR_MESSAGE_FIELD, errorMessage) - .field(SEVERITY_FIELD, severity.value) - .nonOptionalTimeField(TRIGGERED_TIME_FIELD, triggeredTime) - - if (withUser) { - builder.optionalUserField(MONITOR_V2_USER_FIELD, monitorUser) - } - - builder.endObject() - - return builder - } - - fun asTemplateArg(): Map { - return mapOf( - ALERT_V2_ID_FIELD to id, - ALERT_V2_VERSION_FIELD to version, - ERROR_MESSAGE_FIELD to errorMessage, - EXECUTION_ID_FIELD to executionId, - SEVERITY_FIELD to severity.value - ) - } - - companion object { - const val ALERT_V2_ID_FIELD = "id" - const val ALERT_V2_VERSION_FIELD = "version" - const val MONITOR_V2_ID_FIELD = "monitor_v2_id" - const val MONITOR_V2_VERSION_FIELD = "monitor_v2_version" - const val MONITOR_V2_NAME_FIELD = "monitor_v2_name" - const val MONITOR_V2_USER_FIELD = "monitor_v2_user" - const val TRIGGER_V2_ID_FIELD = "trigger_v2_id" - const val TRIGGER_V2_NAME_FIELD = "trigger_v2_name" - const val TRIGGERED_TIME_FIELD = "triggered_time" - const val QUERY_FIELD = "query" - const val QUERY_RESULTS_FIELD = "query_results" - const val ERROR_MESSAGE_FIELD = "error_message" - const val EXECUTION_ID_FIELD = "execution_id" - const val SEVERITY_FIELD = "severity" - const val SCHEMA_VERSION_FIELD = "schema_version" - - const val NO_ID = "" - const val NO_VERSION = Versions.NOT_FOUND - const val NO_SCHEMA_VERSION = 0 - - @JvmStatic - @JvmOverloads - @Throws(IOException::class) - fun parse(xcp: XContentParser, id: String = NO_ID, version: Long = NO_VERSION): AlertV2 { - var schemaVersion = NO_SCHEMA_VERSION - lateinit var monitorId: String - lateinit var monitorName: String - var monitorVersion: Long = Versions.NOT_FOUND - var monitorUser: User? = null - lateinit var triggerId: String - lateinit var triggerName: String - lateinit var query: String - var queryResults: Map = mapOf() - lateinit var severity: Severity - var triggeredTime: Instant? = null - var errorMessage: String? = null - var executionId: String? = null - - ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp) - while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { - val fieldName = xcp.currentName() - xcp.nextToken() - - when (fieldName) { - MONITOR_V2_ID_FIELD -> monitorId = xcp.text() - SCHEMA_VERSION_FIELD -> schemaVersion = xcp.intValue() - MONITOR_V2_NAME_FIELD -> monitorName = xcp.text() - MONITOR_V2_VERSION_FIELD -> monitorVersion = xcp.longValue() - MONITOR_V2_USER_FIELD -> - monitorUser = if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { - null - } else { - User.parse(xcp) - } - TRIGGER_V2_ID_FIELD -> triggerId = xcp.text() - TRIGGER_V2_NAME_FIELD -> triggerName = xcp.text() - QUERY_FIELD -> query = xcp.text() - QUERY_RESULTS_FIELD -> queryResults = xcp.map() - TRIGGERED_TIME_FIELD -> triggeredTime = xcp.instant() - ERROR_MESSAGE_FIELD -> errorMessage = xcp.textOrNull() - EXECUTION_ID_FIELD -> executionId = xcp.textOrNull() - TriggerV2.SEVERITY_FIELD -> { - val input = xcp.text() - val enumMatchResult = Severity.enumFromString(input) - ?: throw IllegalArgumentException( - "Invalid value for ${TriggerV2.SEVERITY_FIELD}: $input. " + - "Supported values are ${Severity.entries.map { it.value }}" - ) - severity = enumMatchResult - } - } - } - - return AlertV2( - id = id, - version = version, - schemaVersion = schemaVersion, - monitorId = requireNotNull(monitorId), - monitorName = requireNotNull(monitorName), - monitorVersion = monitorVersion, - monitorUser = monitorUser, - triggerId = requireNotNull(triggerId), - triggerName = requireNotNull(triggerName), - query = requireNotNull(query), - queryResults = requireNotNull(queryResults), - triggeredTime = requireNotNull(triggeredTime), - errorMessage = errorMessage, - severity = severity, - executionId = executionId - ) - } - - @JvmStatic - @Throws(IOException::class) - fun readFrom(sin: StreamInput): AlertV2 { - return AlertV2(sin) - } - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2.kt deleted file mode 100644 index d7b1ef16c..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2.kt +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.modelv2 - -import org.opensearch.alerting.modelv2.PPLSQLMonitor.Companion.PPL_SQL_MONITOR_TYPE -import org.opensearch.common.CheckedFunction -import org.opensearch.commons.alerting.model.Schedule -import org.opensearch.commons.alerting.model.ScheduledJob -import org.opensearch.commons.authuser.User -import org.opensearch.core.ParseField -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.core.common.io.stream.StreamOutput -import org.opensearch.core.xcontent.NamedXContentRegistry -import org.opensearch.core.xcontent.ToXContent -import org.opensearch.core.xcontent.XContentBuilder -import org.opensearch.core.xcontent.XContentParser -import org.opensearch.core.xcontent.XContentParserUtils -import java.io.IOException -import java.time.Instant - -/** - * Monitor V2 interface. All v2 monitors of different types must implement this interface. - * - * @opensearch.experimental - */ -interface MonitorV2 : ScheduledJob { - override val id: String - override val version: Long - override val name: String - override val enabled: Boolean - override val schedule: Schedule - override val lastUpdateTime: Instant // required for scheduled job maintenance - override val enabledTime: Instant? // required for scheduled job maintenance - val description: String? - val user: User? - val triggers: List - val schemaVersion: Int // for updating monitors - val lookBackWindow: Long? // how far back to look when querying data during monitor execution - val timestampField: String? // field that will be used to inject lookback window time filter - - fun asTemplateArg(): Map - - fun toXContentWithUser(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder - - fun makeCopy( - id: String = this.id, - version: Long = this.version, - name: String = this.name, - enabled: Boolean = this.enabled, - schedule: Schedule = this.schedule, - lastUpdateTime: Instant = this.lastUpdateTime, - enabledTime: Instant? = this.enabledTime, - description: String? = this.description, - user: User? = this.user, - // no support for overriding triggers in interface-level makeCopy(), - // triggers can be copied at instance-level data class copy() - schemaVersion: Int = this.schemaVersion, - lookBackWindow: Long? = this.lookBackWindow, - timestampField: String? = this.timestampField - ): MonitorV2 - - enum class MonitorV2Type(val value: String) { - PPL_SQL_MONITOR(PPL_SQL_MONITOR_TYPE); - - override fun toString(): String { - return value - } - - companion object { - fun enumFromString(value: String): MonitorV2Type? { - return MonitorV2Type.entries.find { it.value == value } - } - } - } - - companion object { - // scheduled job field names - const val MONITOR_V2_TYPE = "monitor_v2" // scheduled job type is MonitorV2 - - // field names - const val NAME_FIELD = "name" - const val ENABLED_FIELD = "enabled" - const val SCHEDULE_FIELD = "schedule" - const val LAST_UPDATE_TIME_FIELD = "last_update_time" - const val ENABLED_TIME_FIELD = "enabled_time" - const val DESCRIPTION_FIELD = "description" - const val USER_FIELD = "user" - const val TRIGGERS_FIELD = "triggers" - const val SCHEMA_VERSION_FIELD = "schema_version" - const val LOOK_BACK_WINDOW_FIELD = "look_back_window_minutes" - const val TIMESTAMP_FIELD = "timestamp_field" - - // default values - const val NO_ID = "" - const val NO_VERSION = 1L - - // hard, nonadjustable limits - const val MONITOR_V2_MAX_TRIGGERS = 10 - const val MONITOR_V2_MIN_LOOK_BACK_WINDOW = 1L // 1 minute - const val ALERTING_V2_MAX_NAME_LENGTH = 30 // max length of any name for monitors, triggers, notif actions, etc - const val UUID_LENGTH = 20 // the length of a UUID generated by UUIDs.base64UUID() - const val DESCRIPTION_MAX_LENGTH = 2000 - - val XCONTENT_REGISTRY = NamedXContentRegistry.Entry( - ScheduledJob::class.java, - ParseField(MONITOR_V2_TYPE), - CheckedFunction { parse(it) } - ) - - @JvmStatic - @Throws(IOException::class) - fun parse(xcp: XContentParser, id: String = NO_ID, version: Long = NO_VERSION): MonitorV2 { - /* parse outer object for monitorV2 type, then delegate to correct monitorV2 parser */ - - XContentParserUtils.ensureExpectedToken( // outer monitor object start - XContentParser.Token.START_OBJECT, - xcp.currentToken(), - xcp - ) - - // monitor type field name - XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, xcp.nextToken(), xcp) - val monitorTypeText = xcp.currentName() - val monitorType = MonitorV2Type.enumFromString(monitorTypeText) - ?: throw IllegalStateException( - "when parsing MonitorV2, received invalid monitor type: $monitorTypeText. " + - "Please ensure monitor object is wrapped in an outer $PPL_SQL_MONITOR_TYPE object" - ) - - // inner monitor object start - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp) - - return when (monitorType) { - MonitorV2Type.PPL_SQL_MONITOR -> PPLSQLMonitor.parse(xcp, id, version) - } - } - - fun readFrom(sin: StreamInput): MonitorV2 { - return when (val monitorType = sin.readEnum(MonitorV2Type::class.java)) { - MonitorV2Type.PPL_SQL_MONITOR -> PPLSQLMonitor(sin) - else -> throw IllegalStateException("Unexpected input \"$monitorType\" when reading MonitorV2") - } - } - - fun writeTo(out: StreamOutput, monitorV2: MonitorV2) { - when (monitorV2) { - is PPLSQLMonitor -> { - out.writeEnum(MonitorV2Type.PPL_SQL_MONITOR) - monitorV2.writeTo(out) - } - } - } - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2RunResult.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2RunResult.kt deleted file mode 100644 index 28d91b297..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2RunResult.kt +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.modelv2 - -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.core.common.io.stream.StreamOutput -import org.opensearch.core.common.io.stream.Writeable -import org.opensearch.core.xcontent.ToXContent - -/** - * Monitor V2 run result interface. All classes that store the results - * of a monitor v2 run must implement this interface - * - * @opensearch.experimental - */ -interface MonitorV2RunResult : Writeable, ToXContent { - val monitorName: String - val error: Exception? - val triggerResults: Map - - enum class MonitorV2RunResultType { - PPL_SQL_MONITOR_RUN_RESULT; - } - - companion object { - const val ERROR_FIELD = "error" - const val TRIGGER_RESULTS_FIELD = "trigger_results" - - fun readFrom(sin: StreamInput): MonitorV2RunResult<*> { - val monitorRunResultType = sin.readEnum(MonitorV2RunResultType::class.java) - return when (monitorRunResultType) { - MonitorV2RunResultType.PPL_SQL_MONITOR_RUN_RESULT -> PPLSQLMonitorRunResult(sin) - else -> throw IllegalStateException("Unexpected input [$monitorRunResultType] when reading MonitorV2RunResult") - } - } - - fun writeTo(out: StreamOutput, monitorV2RunResult: MonitorV2RunResult<*>) { - when (monitorV2RunResult) { - is PPLSQLMonitorRunResult -> { - out.writeEnum(MonitorV2RunResultType.PPL_SQL_MONITOR_RUN_RESULT) - monitorV2RunResult.writeTo(out) - } - } - } - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitor.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitor.kt deleted file mode 100644 index 8b39069ae..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitor.kt +++ /dev/null @@ -1,410 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.modelv2 - -import org.opensearch.alerting.core.util.nonOptionalTimeField -import org.opensearch.alerting.modelv2.MonitorV2.Companion.ALERTING_V2_MAX_NAME_LENGTH -import org.opensearch.alerting.modelv2.MonitorV2.Companion.DESCRIPTION_FIELD -import org.opensearch.alerting.modelv2.MonitorV2.Companion.DESCRIPTION_MAX_LENGTH -import org.opensearch.alerting.modelv2.MonitorV2.Companion.ENABLED_FIELD -import org.opensearch.alerting.modelv2.MonitorV2.Companion.ENABLED_TIME_FIELD -import org.opensearch.alerting.modelv2.MonitorV2.Companion.LAST_UPDATE_TIME_FIELD -import org.opensearch.alerting.modelv2.MonitorV2.Companion.LOOK_BACK_WINDOW_FIELD -import org.opensearch.alerting.modelv2.MonitorV2.Companion.MONITOR_V2_MAX_TRIGGERS -import org.opensearch.alerting.modelv2.MonitorV2.Companion.MONITOR_V2_MIN_LOOK_BACK_WINDOW -import org.opensearch.alerting.modelv2.MonitorV2.Companion.NAME_FIELD -import org.opensearch.alerting.modelv2.MonitorV2.Companion.NO_ID -import org.opensearch.alerting.modelv2.MonitorV2.Companion.NO_VERSION -import org.opensearch.alerting.modelv2.MonitorV2.Companion.SCHEDULE_FIELD -import org.opensearch.alerting.modelv2.MonitorV2.Companion.SCHEMA_VERSION_FIELD -import org.opensearch.alerting.modelv2.MonitorV2.Companion.TIMESTAMP_FIELD -import org.opensearch.alerting.modelv2.MonitorV2.Companion.TRIGGERS_FIELD -import org.opensearch.alerting.modelv2.MonitorV2.Companion.USER_FIELD -import org.opensearch.commons.alerting.model.CronSchedule -import org.opensearch.commons.alerting.model.Schedule -import org.opensearch.commons.alerting.util.AlertingException -import org.opensearch.commons.alerting.util.IndexUtils -import org.opensearch.commons.alerting.util.instant -import org.opensearch.commons.alerting.util.optionalTimeField -import org.opensearch.commons.alerting.util.optionalUserField -import org.opensearch.commons.authuser.User -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.core.common.io.stream.StreamOutput -import org.opensearch.core.xcontent.ToXContent -import org.opensearch.core.xcontent.XContentBuilder -import org.opensearch.core.xcontent.XContentParser -import org.opensearch.core.xcontent.XContentParserUtils -import java.io.IOException -import java.time.Instant - -/** - * PPL/SQL Monitor for OpenSearch Alerting V2 - * - * @property id Monitor ID. Defaults to [NO_ID]. - * @property version Version number of the monitor. Defaults to [NO_VERSION]. - * @property name Display name of the monitor. - * @property enabled Boolean flag indicating whether the monitor is currently on or off. - * @property schedule Defines when and how often the monitor should run. Can be a CRON or interval schedule. - * @property lookBackWindow How far back each Monitor execution's query should look back when searching data. - * @property lastUpdateTime Timestamp of the last update to this monitor. - * @property enabledTime Timestamp when the monitor was last enabled. Null if never enabled. - * @property description Optional Monitor description. - * @property triggers List of [PPLTrigger]s associated with this monitor. - * @property schemaVersion Version of the alerting-config index schema used when this Monitor was indexed. Defaults to [NO_SCHEMA_VERSION]. - * @property queryLanguage The query language used. Defaults to [QueryLanguage.PPL]. - * @property query The query string to be executed by this monitor. - * - * @opensearch.experimental - */ -data class PPLSQLMonitor( - override val id: String = NO_ID, - override val version: Long = NO_VERSION, - override val name: String, - override val enabled: Boolean, - override val schedule: Schedule, - override val lookBackWindow: Long?, - override val timestampField: String?, - override val lastUpdateTime: Instant, - override val enabledTime: Instant?, - override val description: String?, - override val user: User?, - override val triggers: List, - override val schemaVersion: Int = IndexUtils.NO_SCHEMA_VERSION, - val queryLanguage: QueryLanguage = QueryLanguage.PPL, // default to PPL, SQL not currently supported - val query: String -) : MonitorV2 { - - // specify scheduled job type - override val type = MonitorV2.MONITOR_V2_TYPE - - override fun fromDocument(id: String, version: Long): PPLSQLMonitor = copy(id = id, version = version) - - init { - // SQL monitors are not yet supported - if (this.queryLanguage == QueryLanguage.SQL) { - throw IllegalArgumentException("SQL queries are not supported. Please use a PPL query.") - } - - require(this.name.length <= ALERTING_V2_MAX_NAME_LENGTH) { - "Monitor name too long, length must be less than $ALERTING_V2_MAX_NAME_LENGTH." - } - - if (this.lookBackWindow != null) { - requireNotNull(this.timestampField) { "If look back window is specified, timestamp field must not be null." } - } else { - require(this.timestampField == null) { "If look back window is not specified, timestamp field must not be specified." } - } - - require(this.triggers.isNotEmpty()) { "Monitor must include at least 1 trigger." } - require(this.triggers.size <= MONITOR_V2_MAX_TRIGGERS) { "Monitors can only have $MONITOR_V2_MAX_TRIGGERS triggers." } - - lookBackWindow?.let { - require(this.lookBackWindow >= MONITOR_V2_MIN_LOOK_BACK_WINDOW) { - "Monitors look back windows must be at least $MONITOR_V2_MIN_LOOK_BACK_WINDOW minute." - } - } - - this.description?.let { - require(this.description.length <= DESCRIPTION_MAX_LENGTH) { "Description must be under $DESCRIPTION_MAX_LENGTH characters." } - } - - // for checking trigger ID uniqueness - val triggerIds = mutableSetOf() - this.triggers.forEach { trigger -> - require(triggerIds.add(trigger.id)) { "Duplicate trigger id: ${trigger.id}. Trigger ids must be unique." } - } - - if (this.enabled) { - requireNotNull(this.enabledTime) - } else { - require(this.enabledTime == null) - } - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - id = sin.readString(), - version = sin.readLong(), - name = sin.readString(), - enabled = sin.readBoolean(), - schedule = Schedule.readFrom(sin), - lookBackWindow = sin.readOptionalLong(), - timestampField = sin.readOptionalString(), - lastUpdateTime = sin.readInstant(), - enabledTime = sin.readOptionalInstant(), - description = sin.readOptionalString(), - user = if (sin.readBoolean()) { - User(sin) - } else { - null - }, - triggers = sin.readList(PPLSQLTrigger.Companion::readFrom), - schemaVersion = sin.readInt(), - queryLanguage = sin.readEnum(QueryLanguage::class.java), - query = sin.readString() - ) - - override fun toXContentWithUser(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - return createXContentBuilder(builder, params, true) - } - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - return createXContentBuilder(builder, params, false) - } - - private fun createXContentBuilder(builder: XContentBuilder, params: ToXContent.Params, withUser: Boolean): XContentBuilder { - builder.startObject() // overall start object - - // if this is being written as ScheduledJob, add extra object layer and add ScheduledJob - // related metadata, default to false - if (params.paramAsBoolean("with_type", false)) { - builder.startObject(MonitorV2.MONITOR_V2_TYPE) - } - - // wrap PPLSQLMonitor in outer object named after its monitor type - // required for MonitorV2 XContentParser to first encounter this, - // read in monitor type, then delegate to correct parse() function - builder.startObject(PPL_SQL_MONITOR_TYPE) // monitor type start object - - builder.field(NAME_FIELD, name) - builder.field(SCHEDULE_FIELD, schedule) - builder.field(LOOK_BACK_WINDOW_FIELD, lookBackWindow) - builder.field(TIMESTAMP_FIELD, timestampField) - builder.field(ENABLED_FIELD, enabled) - builder.nonOptionalTimeField(LAST_UPDATE_TIME_FIELD, lastUpdateTime) - builder.optionalTimeField(ENABLED_TIME_FIELD, enabledTime) - builder.field(DESCRIPTION_FIELD, description) - builder.field(TRIGGERS_FIELD, triggers.toTypedArray()) - builder.field(SCHEMA_VERSION_FIELD, schemaVersion) - builder.field(QUERY_LANGUAGE_FIELD, queryLanguage.value) - builder.field(QUERY_FIELD, query) - - if (withUser) { - builder.optionalUserField(USER_FIELD, user) - } - - builder.endObject() // monitor type end object - - // if ScheduledJob metadata was added, end the extra object layer that was created - if (params.paramAsBoolean("with_type", false)) { - builder.endObject() - } - - builder.endObject() // overall end object - - return builder - } - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - out.writeString(id) - out.writeLong(version) - out.writeString(name) - out.writeBoolean(enabled) - - if (schedule is CronSchedule) { - out.writeEnum(Schedule.TYPE.CRON) - } else { - out.writeEnum(Schedule.TYPE.INTERVAL) - } - schedule.writeTo(out) - - out.writeOptionalLong(lookBackWindow) - out.writeOptionalString(timestampField) - out.writeInstant(lastUpdateTime) - out.writeOptionalInstant(enabledTime) - out.writeOptionalString(description) - - out.writeBoolean(user != null) - user?.writeTo(out) - - out.writeVInt(triggers.size) - triggers.forEach { it.writeTo(out) } - out.writeInt(schemaVersion) - out.writeEnum(queryLanguage) - out.writeString(query) - } - - override fun asTemplateArg(): Map { - return mapOf( - IndexUtils._ID to id, - IndexUtils._VERSION to version, - NAME_FIELD to name, - ENABLED_FIELD to enabled, - SCHEDULE_FIELD to schedule, - LOOK_BACK_WINDOW_FIELD to lookBackWindow, - LAST_UPDATE_TIME_FIELD to lastUpdateTime.toEpochMilli(), - ENABLED_TIME_FIELD to enabledTime?.toEpochMilli(), - QUERY_FIELD to query - ) - } - - override fun makeCopy( - id: String, - version: Long, - name: String, - enabled: Boolean, - schedule: Schedule, - lastUpdateTime: Instant, - enabledTime: Instant?, - description: String?, - user: User?, - schemaVersion: Int, - lookBackWindow: Long?, - timestampField: String? - ): PPLSQLMonitor { - return copy( - id = id, - version = version, - name = name, - enabled = enabled, - schedule = schedule, - lastUpdateTime = lastUpdateTime, - enabledTime = enabledTime, - description = description, - user = user, - schemaVersion = schemaVersion, - lookBackWindow = lookBackWindow, - timestampField = timestampField - ) - } - - enum class QueryLanguage(val value: String) { - PPL(PPL_QUERY_LANGUAGE), - SQL(SQL_QUERY_LANGUAGE); - - companion object { - fun enumFromString(value: String): QueryLanguage? = QueryLanguage.entries.firstOrNull { it.value == value } - } - } - - companion object { - // monitor type name - const val PPL_SQL_MONITOR_TYPE = "ppl_monitor" - - // query languages - const val PPL_QUERY_LANGUAGE = "ppl" - const val SQL_QUERY_LANGUAGE = "sql" - - // field names - const val QUERY_LANGUAGE_FIELD = "query_language" - const val QUERY_FIELD = "query" - - @JvmStatic - @JvmOverloads - @Throws(IOException::class) - fun parse(xcp: XContentParser, id: String = NO_ID, version: Long = NO_VERSION): PPLSQLMonitor { - var name: String? = null - var enabled = true - var schedule: Schedule? = null - var lookBackWindow: Long? = null - var timestampField: String? = null - var lastUpdateTime: Instant? = null - var enabledTime: Instant? = null - var description: String? = null - var user: User? = null - val triggers: MutableList = mutableListOf() - var schemaVersion = IndexUtils.NO_SCHEMA_VERSION - var queryLanguage: QueryLanguage = QueryLanguage.PPL // default to PPL - var query: String? = null - - /* parse */ - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) - while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { - val fieldName = xcp.currentName() - xcp.nextToken() - - when (fieldName) { - NAME_FIELD -> name = xcp.text() - ENABLED_FIELD -> enabled = xcp.booleanValue() - SCHEDULE_FIELD -> schedule = Schedule.parse(xcp) - LOOK_BACK_WINDOW_FIELD -> { - if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { - lookBackWindow = xcp.longValue() - } - } - TIMESTAMP_FIELD -> { - if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { - timestampField = xcp.text() - } - } - LAST_UPDATE_TIME_FIELD -> lastUpdateTime = xcp.instant() - ENABLED_TIME_FIELD -> enabledTime = xcp.instant() - DESCRIPTION_FIELD -> { - if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { - description = xcp.text() - } - } - USER_FIELD -> { - if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { - user = User.parse(xcp) - } - } - TRIGGERS_FIELD -> { - XContentParserUtils.ensureExpectedToken( - XContentParser.Token.START_ARRAY, - xcp.currentToken(), - xcp - ) - while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { - triggers.add(PPLSQLTrigger.parseInner(xcp)) - } - } - SCHEMA_VERSION_FIELD -> schemaVersion = xcp.intValue() - QUERY_LANGUAGE_FIELD -> { - val input = xcp.text() - val enumMatchResult = QueryLanguage.enumFromString(input) - ?: throw AlertingException.wrap( - IllegalArgumentException( - "Invalid value for $QUERY_LANGUAGE_FIELD: $input. " + - "Supported values are ${QueryLanguage.entries.map { it.value }}" - ) - ) - queryLanguage = enumMatchResult - } - QUERY_FIELD -> query = xcp.text() - else -> throw IllegalArgumentException("Unexpected field when parsing PPL/SQL Monitor: $fieldName") - } - } - - /* validations */ - - // if enabled, set time of MonitorV2 creation/update is set as enable time - if (enabled && enabledTime == null) { - enabledTime = Instant.now() - } else if (!enabled) { - enabledTime = null - } - - lastUpdateTime = lastUpdateTime ?: Instant.now() - - // check for required fields - requireNotNull(name) { "Monitor name is null" } - requireNotNull(schedule) { "Schedule is null" } - requireNotNull(query) { "Query is null" } - requireNotNull(lastUpdateTime) { "Last update time is null" } - - /* return PPLSQLMonitor */ - return PPLSQLMonitor( - id, - version, - name, - enabled, - schedule, - lookBackWindow, - timestampField, - lastUpdateTime, - enabledTime, - description, - user, - triggers, - schemaVersion, - queryLanguage, - query - ) - } - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitorRunResult.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitorRunResult.kt deleted file mode 100644 index 12a34c560..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitorRunResult.kt +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.modelv2 - -import org.opensearch.alerting.modelv2.AlertV2.Companion.MONITOR_V2_NAME_FIELD -import org.opensearch.alerting.modelv2.MonitorV2RunResult.Companion.ERROR_FIELD -import org.opensearch.alerting.modelv2.MonitorV2RunResult.Companion.TRIGGER_RESULTS_FIELD -import org.opensearch.commons.utils.STRING_READER -import org.opensearch.commons.utils.STRING_WRITER -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.core.common.io.stream.StreamOutput -import org.opensearch.core.common.io.stream.Writeable -import org.opensearch.core.xcontent.ToXContent -import org.opensearch.core.xcontent.XContentBuilder -import java.io.IOException - -/** - * A class that stores the run results of a PPL/SQL Monitor - * - * @opensearch.experimental - */ -data class PPLSQLMonitorRunResult( - override val monitorName: String, - override val error: Exception?, - override val triggerResults: Map, - val pplQueryResults: Map> // key: trigger id, value: query results -) : MonitorV2RunResult { - - @Throws(IOException::class) - @Suppress("UNCHECKED_CAST") - constructor(sin: StreamInput) : this( - sin.readString(), // monitorName - sin.readException(), // error - sin.readMap(STRING_READER, runResultReader()) as Map, // triggerResults - sin.readMap() as Map> // pplQueryResults - ) - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - builder.startObject() - builder.field(MONITOR_V2_NAME_FIELD, monitorName) - builder.field(ERROR_FIELD, error?.message) - builder.field(TRIGGER_RESULTS_FIELD, triggerResults) - builder.field(PPL_QUERY_RESULTS_FIELD, pplQueryResults) - builder.endObject() - return builder - } - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - out.writeString(monitorName) - out.writeException(error) - out.writeMap(triggerResults, STRING_WRITER, runResultWriter()) - out.writeMap(pplQueryResults) - } - - companion object { - const val PPL_QUERY_RESULTS_FIELD = "ppl_query_results" - - private fun runResultReader(): Writeable.Reader { - return Writeable.Reader { - PPLSQLTriggerRunResult.readFrom(it) - } - } - - private fun runResultWriter(): Writeable.Writer { - return Writeable.Writer { streamOutput: StreamOutput, runResult: PPLSQLTriggerRunResult -> - runResult.writeTo(streamOutput) - } - } - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTrigger.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTrigger.kt deleted file mode 100644 index b902d39a2..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTrigger.kt +++ /dev/null @@ -1,413 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.modelv2 - -import org.opensearch.alerting.modelv2.MonitorV2.Companion.ALERTING_V2_MAX_NAME_LENGTH -import org.opensearch.alerting.modelv2.MonitorV2.Companion.UUID_LENGTH -import org.opensearch.alerting.modelv2.TriggerV2.Companion.ACTIONS_FIELD -import org.opensearch.alerting.modelv2.TriggerV2.Companion.DEFAULT_EXPIRE_DURATION -import org.opensearch.alerting.modelv2.TriggerV2.Companion.EXPIRE_FIELD -import org.opensearch.alerting.modelv2.TriggerV2.Companion.ID_FIELD -import org.opensearch.alerting.modelv2.TriggerV2.Companion.LAST_TRIGGERED_FIELD -import org.opensearch.alerting.modelv2.TriggerV2.Companion.MONITOR_V2_MIN_EXPIRE_DURATION_MINUTES -import org.opensearch.alerting.modelv2.TriggerV2.Companion.MONITOR_V2_MIN_THROTTLE_DURATION_MINUTES -import org.opensearch.alerting.modelv2.TriggerV2.Companion.NAME_FIELD -import org.opensearch.alerting.modelv2.TriggerV2.Companion.NOTIFICATIONS_ID_MAX_LENGTH -import org.opensearch.alerting.modelv2.TriggerV2.Companion.SEVERITY_FIELD -import org.opensearch.alerting.modelv2.TriggerV2.Companion.THROTTLE_FIELD -import org.opensearch.alerting.modelv2.TriggerV2.Severity -import org.opensearch.common.CheckedFunction -import org.opensearch.common.UUIDs -import org.opensearch.commons.alerting.model.action.Action -import org.opensearch.commons.alerting.util.instant -import org.opensearch.commons.alerting.util.optionalTimeField -import org.opensearch.core.ParseField -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.core.common.io.stream.StreamOutput -import org.opensearch.core.xcontent.NamedXContentRegistry -import org.opensearch.core.xcontent.ToXContent -import org.opensearch.core.xcontent.XContentBuilder -import org.opensearch.core.xcontent.XContentParser -import org.opensearch.core.xcontent.XContentParserUtils -import java.io.IOException -import java.time.Instant - -/** - * The PPL/SQL Trigger for PPL/SQL Monitors - * - * There are two types of PPLTrigger conditions: NUMBER_OF_RESULTS and CUSTOM - * NUMBER_OF_RESULTS: triggers based on whether the number of query results returned by the PPLSQLMonitor - * query meets some threshold - * CUSTOM: triggers based on a custom condition that user specifies (a single ppl eval statement) - * - * PPLTriggers can run on two modes: RESULT_SET and PER_RESULT - * RESULT_SET: exactly one Alert is generated when the Trigger condition is met - * PER_RESULT: one Alert is generated per trigger condition-meeting query result row - * - * @property id Trigger ID, defaults to a base64 UUID. - * @property name Display name of the Trigger. - * @property severity The severity level of the Trigger. - * @property throttleDuration Optional duration (in minutes) for which alerts from this Trigger should be throttled/suppressed. - * Null indicates no throttling. - * @property expireDuration Duration (in minutes) after which alerts from this Trigger should be deleted permanently. - * @property lastTriggeredTime The last time this Trigger generated an Alert. Null if Trigger hasn't generated an Alert yet. - * @property actions List of notification-sending actions to run when the Trigger condition is met. - * @property mode Specifies whether the trigger evaluates the entire result set or each result individually. - * Can be either [TriggerMode.RESULT_SET] or [TriggerMode.PER_RESULT]. - * @property conditionType The type of condition to evaluate. - * Can be either [ConditionType.NUMBER_OF_RESULTS] or [ConditionType.CUSTOM]. - * @property numResultsCondition The comparison operator for NUMBER_OF_RESULTS conditions. Required if using NUMBER_OF_RESULTS conditions, - * required to be null otherwise. - * @property numResultsValue The threshold value for NUMBER_OF_RESULTS conditions. Required if using NUMBER_OF_RESULTS conditions, - * required to be null otherwise. - * @property customCondition A custom condition expression. Required if using CUSTOM conditions, - * required to be null otherwise. - * - * @opensearch.experimental - */ -data class PPLSQLTrigger( - override val id: String = UUIDs.base64UUID(), - override val name: String, - override val severity: Severity, - override val throttleDuration: Long?, - override val expireDuration: Long = DEFAULT_EXPIRE_DURATION, - override var lastTriggeredTime: Instant?, - override val actions: List, - val mode: TriggerMode, // RESULT_SET or PER_RESULT - val conditionType: ConditionType, // NUMBER_OF_RESULTS or CUSTOM - val numResultsCondition: NumResultsCondition?, - val numResultsValue: Long?, - val customCondition: String? -) : TriggerV2 { - - init { - requireNotNull(this.name) { "Trigger name must be included." } - requireNotNull(this.severity) { "Trigger severity must be included." } - requireNotNull(this.mode) { "Trigger mode must be included." } - requireNotNull(this.conditionType) { "Trigger condition type must be included." } - - require(this.id.length <= UUID_LENGTH) { - "Trigger ID too long, length must be less than $UUID_LENGTH." - } - - require(this.name.length <= ALERTING_V2_MAX_NAME_LENGTH) { - "Trigger name too long, length must be less than $ALERTING_V2_MAX_NAME_LENGTH." - } - - require(this.expireDuration >= MONITOR_V2_MIN_EXPIRE_DURATION_MINUTES) { - "expire duration cannot be less than $MONITOR_V2_MIN_EXPIRE_DURATION_MINUTES, was $expireDuration." - } - - this.throttleDuration?.let { - require(it >= MONITOR_V2_MIN_THROTTLE_DURATION_MINUTES) { - "Throttle duration cannot be less than $MONITOR_V2_MIN_THROTTLE_DURATION_MINUTES, was $throttleDuration." - } - } - - this.actions.forEach { - require(it.name.length <= ALERTING_V2_MAX_NAME_LENGTH) { - "Name of action with ID ${it.id} too long, length must be less than $ALERTING_V2_MAX_NAME_LENGTH." - } - require(it.destinationId.length <= NOTIFICATIONS_ID_MAX_LENGTH) { - "Channel ID of action with ID ${it.id} too long, length must be less than $NOTIFICATIONS_ID_MAX_LENGTH." - } - require(it.destinationId.isNotEmpty()) { - "Channel ID should not be empty." - } - require(it.destinationId.matches(validCharsRegex)) { - "Channel ID should only have alphanumeric characters, dashes, and underscores." - } - } - - when (this.conditionType) { - ConditionType.NUMBER_OF_RESULTS -> { - requireNotNull(this.numResultsCondition) { - "if trigger condition is of type ${ConditionType.NUMBER_OF_RESULTS.value}, " + - "$NUM_RESULTS_CONDITION_FIELD must be included." - } - requireNotNull(this.numResultsValue) { - "if trigger condition is of type ${ConditionType.NUMBER_OF_RESULTS.value}, " + - "$NUM_RESULTS_VALUE_FIELD must be included." - } - require(this.customCondition == null) { - "if trigger condition is of type ${ConditionType.NUMBER_OF_RESULTS.value}, " + - "$CUSTOM_CONDITION_FIELD must not be included." - } - } - ConditionType.CUSTOM -> { - requireNotNull(this.customCondition) { - "if trigger condition is of type ${ConditionType.CUSTOM.value}, " + - "$CUSTOM_CONDITION_FIELD must be included." - } - require(this.numResultsCondition == null) { - "if trigger condition is of type ${ConditionType.CUSTOM.value}, " + - "$NUM_RESULTS_CONDITION_FIELD must not be included." - } - require(this.numResultsValue == null) { - "if trigger condition is of type ${ConditionType.CUSTOM.value}, " + - "$NUM_RESULTS_VALUE_FIELD must not be included." - } - } - } - - if (conditionType == ConditionType.NUMBER_OF_RESULTS) { - require(this.numResultsValue!! >= 0L) { "Number of results to check for cannot be negative." } - } - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - sin.readString(), // id - sin.readString(), // name - sin.readEnum(Severity::class.java), // severity - sin.readOptionalLong(), // throttleDuration - sin.readLong(), // expireDuration - sin.readOptionalInstant(), // lastTriggeredTime - sin.readList(::Action), // actions - sin.readEnum(TriggerMode::class.java), // trigger mode - sin.readEnum(ConditionType::class.java), // condition type - if (sin.readBoolean()) sin.readEnum(NumResultsCondition::class.java) else null, // num results condition - sin.readOptionalLong(), // num results value - sin.readOptionalString() // custom condition - ) - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - out.writeString(id) - out.writeString(name) - out.writeEnum(severity) - out.writeOptionalLong(throttleDuration) - out.writeLong(expireDuration) - out.writeOptionalInstant(lastTriggeredTime) - out.writeCollection(actions) - out.writeEnum(mode) - out.writeEnum(conditionType) - - out.writeBoolean(numResultsCondition != null) - numResultsCondition?.let { out.writeEnum(numResultsCondition) } - - out.writeOptionalLong(numResultsValue) - out.writeOptionalString(customCondition) - } - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params?): XContentBuilder { - builder.startObject() - builder.field(ID_FIELD, id) - builder.field(NAME_FIELD, name) - builder.field(SEVERITY_FIELD, severity.value) - throttleDuration?.let { builder.field(THROTTLE_FIELD, throttleDuration) } - builder.field(EXPIRE_FIELD, expireDuration) - builder.optionalTimeField(LAST_TRIGGERED_FIELD, lastTriggeredTime) - builder.field(ACTIONS_FIELD, actions.toTypedArray()) - builder.field(MODE_FIELD, mode.value) - builder.field(CONDITION_TYPE_FIELD, conditionType.value) - numResultsCondition?.let { builder.field(NUM_RESULTS_CONDITION_FIELD, numResultsCondition.value) } - numResultsValue?.let { builder.field(NUM_RESULTS_VALUE_FIELD, numResultsValue) } - customCondition?.let { builder.field(CUSTOM_CONDITION_FIELD, customCondition) } - builder.endObject() - return builder - } - - fun asTemplateArg(): Map { - return mapOf( - ID_FIELD to id, - NAME_FIELD to name, - SEVERITY_FIELD to severity.value, - THROTTLE_FIELD to throttleDuration, - EXPIRE_FIELD to expireDuration, - ACTIONS_FIELD to actions.map { it.asTemplateArg() }, - MODE_FIELD to mode.value, - CONDITION_TYPE_FIELD to conditionType.value, - NUM_RESULTS_CONDITION_FIELD to numResultsCondition?.value, - NUM_RESULTS_VALUE_FIELD to numResultsValue, - CUSTOM_CONDITION_FIELD to customCondition - ) - } - - enum class TriggerMode(val value: String) { - RESULT_SET("result_set"), - PER_RESULT("per_result"); - - companion object { - fun enumFromString(value: String): TriggerMode? = entries.firstOrNull { it.value == value } - } - } - - enum class ConditionType(val value: String) { - NUMBER_OF_RESULTS("number_of_results"), - CUSTOM("custom"); - - companion object { - fun enumFromString(value: String): ConditionType? = entries.firstOrNull { it.value == value } - } - } - - enum class NumResultsCondition(val value: String) { - GREATER_THAN(">"), - GREATER_THAN_EQUAL(">="), - LESS_THAN("<"), - LESS_THAN_EQUAL("<="), - EQUAL("=="), - NOT_EQUAL("!="); - - companion object { - fun enumFromString(value: String): NumResultsCondition? = entries.firstOrNull { it.value == value } - } - } - - companion object { - // trigger wrapper object field name - const val PPL_SQL_TRIGGER_FIELD = "ppl_trigger" - - // field names - const val MODE_FIELD = "mode" - const val CONDITION_TYPE_FIELD = "type" - const val NUM_RESULTS_CONDITION_FIELD = "num_results_condition" - const val NUM_RESULTS_VALUE_FIELD = "num_results_value" - const val CUSTOM_CONDITION_FIELD = "custom_condition" - - // regular expression for validating that a string contains - // only valid chars (letters, numbers, -, _) - private val validCharsRegex = """^[a-zA-Z0-9_-]+$""".toRegex() - - val XCONTENT_REGISTRY = NamedXContentRegistry.Entry( - TriggerV2::class.java, - ParseField(PPL_SQL_TRIGGER_FIELD), - CheckedFunction { parseInner(it) } - ) - - @JvmStatic - @Throws(IOException::class) - fun parseInner(xcp: XContentParser): PPLSQLTrigger { - var id = UUIDs.base64UUID() // assign a default triggerId if one is not specified - var name: String? = null - var severity: Severity? = null - var throttleDuration: Long? = null - var expireDuration: Long = DEFAULT_EXPIRE_DURATION - var lastTriggeredTime: Instant? = null - val actions: MutableList = mutableListOf() - var mode: TriggerMode? = null - var conditionType: ConditionType? = null - var numResultsCondition: NumResultsCondition? = null - var numResultsValue: Long? = null - var customCondition: String? = null - - /* parse */ - XContentParserUtils.ensureExpectedToken( // outer trigger object start - XContentParser.Token.START_OBJECT, - xcp.currentToken(), xcp - ) - - while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { - val fieldName = xcp.currentName() - xcp.nextToken() - - when (fieldName) { - ID_FIELD -> id = xcp.text() - NAME_FIELD -> name = xcp.text() - SEVERITY_FIELD -> { - val input = xcp.text() - val enumMatchResult = Severity.enumFromString(input) - ?: throw IllegalArgumentException( - "Invalid value for $SEVERITY_FIELD: $input. " + - "Supported values are ${Severity.entries.map { it.value }}" - ) - severity = enumMatchResult - } - MODE_FIELD -> { - val input = xcp.text() - val enumMatchResult = TriggerMode.enumFromString(input) - ?: throw IllegalArgumentException( - "Invalid value for $MODE_FIELD: $input. " + - "Supported values are ${TriggerMode.entries.map { it.value }}" - ) - mode = enumMatchResult - } - CONDITION_TYPE_FIELD -> { - val input = xcp.text() - val enumMatchResult = ConditionType.enumFromString(input) - ?: throw IllegalArgumentException( - "Invalid value for $CONDITION_TYPE_FIELD: $input. " + - "Supported values are ${ConditionType.entries.map { it.value }}" - ) - conditionType = enumMatchResult - } - NUM_RESULTS_CONDITION_FIELD -> { - if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { - val input = xcp.text() - val enumMatchResult = NumResultsCondition.enumFromString(input) - ?: throw IllegalArgumentException( - "Invalid value for $NUM_RESULTS_CONDITION_FIELD: $input. " + - "Supported values are ${NumResultsCondition.entries.map { it.value }}" - ) - numResultsCondition = enumMatchResult - } - } - NUM_RESULTS_VALUE_FIELD -> { - if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { - numResultsValue = xcp.longValue() - } - } - CUSTOM_CONDITION_FIELD -> { - if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { - customCondition = xcp.text() - } - } - THROTTLE_FIELD -> { - if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { - throttleDuration = xcp.longValue() - } - } - EXPIRE_FIELD -> { - if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { - expireDuration = xcp.longValue() - } - } - LAST_TRIGGERED_FIELD -> lastTriggeredTime = xcp.instant() - ACTIONS_FIELD -> { - XContentParserUtils.ensureExpectedToken( - XContentParser.Token.START_ARRAY, - xcp.currentToken(), - xcp - ) - while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { - actions.add(Action.parse(xcp)) - } - } - else -> throw IllegalArgumentException("Unexpected field when parsing PPL Trigger: $fieldName") - } - } - - /* validations */ - requireNotNull(name) { "Trigger name must be included" } - requireNotNull(severity) { "Trigger severity must be included" } - requireNotNull(mode) { "Trigger mode must be included" } - requireNotNull(conditionType) { "Trigger condition type must be included" } - - // 3. prepare and return PPLTrigger object - return PPLSQLTrigger( - id, - name, - severity, - throttleDuration, - expireDuration, - lastTriggeredTime, - actions, - mode, - conditionType, - numResultsCondition, - numResultsValue, - customCondition - ) - } - - @JvmStatic - @Throws(IOException::class) - fun readFrom(sin: StreamInput): PPLSQLTrigger { - return PPLSQLTrigger(sin) - } - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTriggerRunResult.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTriggerRunResult.kt deleted file mode 100644 index 0c505d9e6..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTriggerRunResult.kt +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.modelv2 - -import org.opensearch.alerting.modelv2.TriggerV2RunResult.Companion.ERROR_FIELD -import org.opensearch.alerting.modelv2.TriggerV2RunResult.Companion.NAME_FIELD -import org.opensearch.alerting.modelv2.TriggerV2RunResult.Companion.TRIGGERED_FIELD -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.core.common.io.stream.StreamOutput -import org.opensearch.core.xcontent.ToXContent -import org.opensearch.core.xcontent.XContentBuilder -import java.io.IOException - -/** - * A class that stores the run results of an individual - * PPL/SQL trigger within a PPL/SQL monitor - * - * @opensearch.experimental - */ -data class PPLSQLTriggerRunResult( - override var triggerName: String, - override var triggered: Boolean, - override var error: Exception?, -) : TriggerV2RunResult { - - @Throws(IOException::class) - @Suppress("UNCHECKED_CAST") - constructor(sin: StreamInput) : this( - triggerName = sin.readString(), - triggered = sin.readBoolean(), - error = sin.readException() - ) - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - builder.startObject() - builder.field(NAME_FIELD, triggerName) - builder.field(TRIGGERED_FIELD, triggered) - builder.field(ERROR_FIELD, error?.message) - builder.endObject() - return builder - } - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - out.writeString(triggerName) - out.writeBoolean(triggered) - out.writeException(error) - } - - companion object { - @JvmStatic - @Throws(IOException::class) - fun readFrom(sin: StreamInput): PPLSQLTriggerRunResult { - return PPLSQLTriggerRunResult(sin) - } - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2.kt deleted file mode 100644 index a4fe90e14..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2.kt +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.modelv2 - -import org.opensearch.alerting.modelv2.PPLSQLTrigger.Companion.PPL_SQL_TRIGGER_FIELD -import org.opensearch.commons.alerting.model.action.Action -import org.opensearch.commons.notifications.model.BaseModel -import java.time.Instant - -/** - * Trigger V2 interface. All triggers of different v2 monitor - * types must implement this interface - * - * @opensearch.experimental - */ -interface TriggerV2 : BaseModel { - - val id: String - val name: String - val severity: Severity - val throttleDuration: Long? - val expireDuration: Long - var lastTriggeredTime: Instant? - val actions: List - - enum class TriggerV2Type(val value: String) { - PPL_TRIGGER(PPL_SQL_TRIGGER_FIELD); - - override fun toString(): String { - return value - } - } - - enum class Severity(val value: String) { - INFO("info"), - ERROR("error"), - LOW("low"), - MEDIUM("medium"), - HIGH("high"), - CRITICAL("critical"); - - companion object { - fun enumFromString(value: String): Severity? { - return entries.find { it.value == value } - } - } - } - - companion object { - // field names - const val ID_FIELD = "id" - const val NAME_FIELD = "name" - const val SEVERITY_FIELD = "severity" - const val THROTTLE_FIELD = "throttle_minutes" - const val LAST_TRIGGERED_FIELD = "last_triggered_time" - const val EXPIRE_FIELD = "expires_minutes" - const val ACTIONS_FIELD = "actions" - - // hard, nonadjustable limits - const val MONITOR_V2_MIN_THROTTLE_DURATION_MINUTES = 1L // one minute min duration to match scheduled job interval granularity - const val MONITOR_V2_MIN_EXPIRE_DURATION_MINUTES = 1L // one minute min duration to match scheduled job interval granularity - const val NOTIFICATIONS_ID_MAX_LENGTH = 512 // length limit for notifications channel custom ID at channel creation time - - // default fallback values of fields if none are passed in - const val DEFAULT_EXPIRE_DURATION = (7 * 24 * 60).toLong() // 7 days in minutes - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2RunResult.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2RunResult.kt deleted file mode 100644 index 8e3069972..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2RunResult.kt +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.modelv2 - -import org.opensearch.core.common.io.stream.Writeable -import org.opensearch.core.xcontent.ToXContent - -/** - * Trigger V2 Run Result interface. All classes that store the run results - * of an individual v2 trigger must implement this interface - */ -interface TriggerV2RunResult : Writeable, ToXContent { - val triggerName: String - val triggered: Boolean - val error: Exception? - - companion object { - const val NAME_FIELD = "name" - const val TRIGGERED_FIELD = "triggered" - const val ERROR_FIELD = "error" - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/remote/monitors/RemoteDocumentLevelMonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/remote/monitors/RemoteDocumentLevelMonitorRunner.kt index c12356cab..495b914da 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/remote/monitors/RemoteDocumentLevelMonitorRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/remote/monitors/RemoteDocumentLevelMonitorRunner.kt @@ -41,6 +41,7 @@ class RemoteDocumentLevelMonitorRunner : MonitorRunner() { periodStart: Instant, periodEnd: Instant, dryRun: Boolean, + manual: Boolean, workflowRunContext: WorkflowRunContext?, executionId: String, transportService: TransportService diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestExecuteMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestExecuteMonitorAction.kt index 4dd4f588b..792f86824 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestExecuteMonitorAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestExecuteMonitorAction.kt @@ -60,7 +60,7 @@ class RestExecuteMonitorAction : BaseRestHandler() { if (request.hasParam("monitorID")) { val monitorId = request.param("monitorID") - val execMonitorRequest = ExecuteMonitorRequest(dryrun, requestEnd, monitorId, null) + val execMonitorRequest = ExecuteMonitorRequest(dryrun, true, requestEnd, monitorId, null) client.execute(ExecuteMonitorAction.INSTANCE, execMonitorRequest, RestToXContentListener(channel)) } else { val xcp = request.contentParser() @@ -73,7 +73,7 @@ class RestExecuteMonitorAction : BaseRestHandler() { throw AlertingException.wrap(e) } - val execMonitorRequest = ExecuteMonitorRequest(dryrun, requestEnd, null, monitor) + val execMonitorRequest = ExecuteMonitorRequest(dryrun, true, requestEnd, null, monitor) client.execute(ExecuteMonitorAction.INSTANCE, execMonitorRequest, RestToXContentListener(channel)) } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestExecuteWorkflowAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestExecuteWorkflowAction.kt index 096f03010..60a3ccc86 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestExecuteWorkflowAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestExecuteWorkflowAction.kt @@ -41,13 +41,13 @@ class RestExecuteWorkflowAction : BaseRestHandler() { if (request.hasParam("workflowID")) { val workflowId = request.param("workflowID") - val execWorkflowRequest = ExecuteWorkflowRequest(dryrun, requestEnd, workflowId, null) + val execWorkflowRequest = ExecuteWorkflowRequest(dryrun, true, requestEnd, workflowId, null) client.execute(ExecuteWorkflowAction.INSTANCE, execWorkflowRequest, RestToXContentListener(channel)) } else { val xcp = request.contentParser() XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp) val workflow = Workflow.parse(xcp, Workflow.NO_ID, Workflow.NO_VERSION) - val execWorkflowRequest = ExecuteWorkflowRequest(dryrun, requestEnd, null, workflow) + val execWorkflowRequest = ExecuteWorkflowRequest(dryrun, true, requestEnd, null, workflow) client.execute(ExecuteWorkflowAction.INSTANCE, execWorkflowRequest, RestToXContentListener(channel)) } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetAlertsAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetAlertsAction.kt index d147aa299..96f15831f 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetAlertsAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetAlertsAction.kt @@ -11,7 +11,6 @@ import org.opensearch.commons.alerting.action.AlertingActions import org.opensearch.commons.alerting.action.GetAlertsRequest import org.opensearch.commons.alerting.model.Table import org.opensearch.rest.BaseRestHandler -import org.opensearch.rest.BaseRestHandler.RestChannelConsumer import org.opensearch.rest.RestHandler.ReplacedRoute import org.opensearch.rest.RestHandler.Route import org.opensearch.rest.RestRequest @@ -54,7 +53,7 @@ class RestGetAlertsAction : BaseRestHandler() { val size = request.paramAsInt("size", 20) val startIndex = request.paramAsInt("startIndex", 0) val searchString = request.param("searchString", "") - val severityLevel = request.param("severityLevel", "ALL") + val severityLevel = request.param("severityLevel", "ALL") // TODO: stateless and stateful operate on diff sevs val alertState = request.param("alertState", "ALL") val monitorId: String? = request.param("monitorId") val workflowId: String? = request.param("workflowIds") @@ -79,4 +78,9 @@ class RestGetAlertsAction : BaseRestHandler() { client.execute(AlertingActions.GET_ALERTS_ACTION_TYPE, getAlertsRequest, RestToXContentListener(channel)) } } + + companion object { + const val STATEFUL = "stateful" + const val STATELESS = "stateless" + } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestIndexMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestIndexMonitorAction.kt index 5f753edd1..815fbd85f 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestIndexMonitorAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestIndexMonitorAction.kt @@ -18,6 +18,7 @@ import org.opensearch.commons.alerting.model.BucketLevelTrigger import org.opensearch.commons.alerting.model.DocLevelMonitorInput import org.opensearch.commons.alerting.model.DocumentLevelTrigger import org.opensearch.commons.alerting.model.Monitor +import org.opensearch.commons.alerting.model.PPLSQLTrigger import org.opensearch.commons.alerting.model.QueryLevelTrigger import org.opensearch.commons.alerting.model.ScheduledJob import org.opensearch.commons.alerting.util.AlertingException @@ -30,7 +31,6 @@ import org.opensearch.core.xcontent.XContentParser.Token import org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken import org.opensearch.index.seqno.SequenceNumbers import org.opensearch.rest.BaseRestHandler -import org.opensearch.rest.BaseRestHandler.RestChannelConsumer import org.opensearch.rest.BytesRestResponse import org.opensearch.rest.RestChannel import org.opensearch.rest.RestHandler.ReplacedRoute @@ -43,7 +43,7 @@ import org.opensearch.rest.action.RestResponseListener import org.opensearch.transport.client.node.NodeClient import java.io.IOException import java.time.Instant -import java.util.* +import java.util.Locale private val log = LogManager.getLogger(RestIndexMonitorAction::class.java) @@ -92,6 +92,7 @@ class RestIndexMonitorAction : BaseRestHandler() { val monitor: Monitor val rbacRoles: List? + try { monitor = Monitor.parse(xcp, id).copy(lastUpdateTime = Instant.now()) @@ -135,6 +136,14 @@ class RestIndexMonitorAction : BaseRestHandler() { } } } + + Monitor.MonitorType.PPL_MONITOR -> { + triggers.forEach { + if (it !is PPLSQLTrigger) { + throw IllegalArgumentException("Illegal trigger type, ${it.javaClass.name}, for PPL monitor") + } + } + } } } } catch (e: Exception) { diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestDeleteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestDeleteMonitorV2Action.kt deleted file mode 100644 index 9756dd8b0..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestDeleteMonitorV2Action.kt +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.resthandlerv2 - -import org.apache.logging.log4j.LogManager -import org.apache.logging.log4j.Logger -import org.opensearch.action.support.WriteRequest.RefreshPolicy -import org.opensearch.alerting.AlertingPlugin -import org.opensearch.alerting.actionv2.DeleteMonitorV2Action -import org.opensearch.alerting.actionv2.DeleteMonitorV2Request -import org.opensearch.alerting.util.REFRESH -import org.opensearch.rest.BaseRestHandler -import org.opensearch.rest.RestHandler.Route -import org.opensearch.rest.RestRequest -import org.opensearch.rest.RestRequest.Method.DELETE -import org.opensearch.rest.action.RestToXContentListener -import org.opensearch.transport.client.node.NodeClient -import java.io.IOException - -private val log: Logger = LogManager.getLogger(RestDeleteMonitorV2Action::class.java) - -/** - * This class consists of the REST handler to delete V2 monitors. - * When a monitor is deleted, all alerts are moved to the alert history index if alerting v2 history is enabled, - * or permanently deleted if alerting v2 history is disabled. - * If this process fails the monitor is not deleted. - * - * @opensearch.experimental - */ -class RestDeleteMonitorV2Action : BaseRestHandler() { - - override fun getName(): String { - return "delete_monitor_v2_action" - } - - override fun routes(): List { - return mutableListOf( - Route( - DELETE, - "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitor_id}" - ) - ) - } - - @Throws(IOException::class) - override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { - val monitorV2Id = request.param("monitor_id") - log.info("${request.method()} ${AlertingPlugin.MONITOR_V2_BASE_URI}/$monitorV2Id") - - val refreshPolicy = RefreshPolicy.parse(request.param(REFRESH, RefreshPolicy.IMMEDIATE.value)) - val deleteMonitorV2Request = DeleteMonitorV2Request(monitorV2Id, refreshPolicy) - - return RestChannelConsumer { channel -> - client.execute(DeleteMonitorV2Action.INSTANCE, deleteMonitorV2Request, RestToXContentListener(channel)) - } - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestExecuteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestExecuteMonitorV2Action.kt deleted file mode 100644 index 4c706e747..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestExecuteMonitorV2Action.kt +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.resthandlerv2 - -import org.apache.logging.log4j.LogManager -import org.opensearch.alerting.AlertingPlugin -import org.opensearch.alerting.actionv2.ExecuteMonitorV2Action -import org.opensearch.alerting.actionv2.ExecuteMonitorV2Request -import org.opensearch.alerting.modelv2.MonitorV2 -import org.opensearch.common.unit.TimeValue -import org.opensearch.commons.alerting.util.AlertingException -import org.opensearch.core.xcontent.XContentParser.Token.START_OBJECT -import org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken -import org.opensearch.rest.BaseRestHandler -import org.opensearch.rest.RestHandler.Route -import org.opensearch.rest.RestRequest -import org.opensearch.rest.RestRequest.Method.POST -import org.opensearch.rest.action.RestToXContentListener -import org.opensearch.transport.client.node.NodeClient -import java.time.Instant - -private val log = LogManager.getLogger(RestExecuteMonitorV2Action::class.java) - -/** - * This class consists of the REST handler to execute V2 monitors manually. - * In addition to monitors running on their scheduled jobs, this API allows users - * to execute the monitor themselves to generate alerts and send notifications accordingly - * - * @opensearch.experimental - */ -class RestExecuteMonitorV2Action : BaseRestHandler() { - - override fun getName(): String = "execute_monitor_v2_action" - - override fun routes(): List { - return listOf( - Route( - POST, - "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitor_id}/_execute" - ), - Route( - POST, - "${AlertingPlugin.MONITOR_V2_BASE_URI}/_execute" - ) - ) - } - - override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { - log.debug("${request.method()} ${AlertingPlugin.MONITOR_V2_BASE_URI}/_execute") - - return RestChannelConsumer { channel -> - val dryrun = request.paramAsBoolean("dryrun", false) - val requestEnd = request.paramAsTime("period_end", TimeValue(Instant.now().toEpochMilli())) - - if (request.hasParam("monitor_id")) { - val monitorV2Id = request.param("monitor_id") - val execMonitorV2Request = ExecuteMonitorV2Request(dryrun, true, monitorV2Id, null, requestEnd) - client.execute(ExecuteMonitorV2Action.INSTANCE, execMonitorV2Request, RestToXContentListener(channel)) - } else { - val xcp = request.contentParser() - ensureExpectedToken(START_OBJECT, xcp.nextToken(), xcp) - - val monitorV2: MonitorV2 - try { - monitorV2 = MonitorV2.parse(xcp) - } catch (e: Exception) { - throw AlertingException.wrap(e) - } - - val execMonitorV2Request = ExecuteMonitorV2Request(dryrun, true, null, monitorV2, requestEnd) - client.execute(ExecuteMonitorV2Action.INSTANCE, execMonitorV2Request, RestToXContentListener(channel)) - } - } - } - - override fun responseParams(): Set { - return setOf("dryrun", "period_end", "monitor_id") - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetAlertsV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetAlertsV2Action.kt deleted file mode 100644 index 560d243bc..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetAlertsV2Action.kt +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.resthandlerv2 - -import org.apache.logging.log4j.LogManager -import org.opensearch.alerting.AlertingPlugin -import org.opensearch.alerting.actionv2.GetAlertsV2Action -import org.opensearch.alerting.actionv2.GetAlertsV2Request -import org.opensearch.commons.alerting.model.Table -import org.opensearch.rest.BaseRestHandler -import org.opensearch.rest.RestHandler.Route -import org.opensearch.rest.RestRequest -import org.opensearch.rest.RestRequest.Method.GET -import org.opensearch.rest.action.RestToXContentListener -import org.opensearch.transport.client.node.NodeClient - -/** - * This class consists of the REST handler to retrieve V2 alerts. - * - * @opensearch.experimental - */ -class RestGetAlertsV2Action : BaseRestHandler() { - - private val log = LogManager.getLogger(RestGetAlertsV2Action::class.java) - - override fun getName(): String { - return "get_alerts_v2_action" - } - - override fun routes(): List { - return listOf( - Route( - GET, - "${AlertingPlugin.MONITOR_V2_BASE_URI}/alerts" - ) - ) - } - - override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { - log.debug("${request.method()} ${AlertingPlugin.MONITOR_V2_BASE_URI}/alerts") - - val sortString = request.param("sortString", "monitor_v2_name.keyword") - val sortOrder = request.param("sortOrder", "asc") - val missing: String? = request.param("missing") - val size = request.paramAsInt("size", 20) - val startIndex = request.paramAsInt("startIndex", 0) - val searchString = request.param("searchString", "") - val severityLevel = request.param("severityLevel", "ALL") - val monitorId: String? = request.param("monitorId") - val table = Table( - sortOrder, - sortString, - missing, - size, - startIndex, - searchString - ) - - val getAlertsV2Request = GetAlertsV2Request( - table, - severityLevel, - monitorId?.let { listOf(monitorId) } - ) - return RestChannelConsumer { - channel -> - client.execute(GetAlertsV2Action.INSTANCE, getAlertsV2Request, RestToXContentListener(channel)) - } - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetMonitorV2Action.kt deleted file mode 100644 index 5c471156c..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetMonitorV2Action.kt +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.resthandlerv2 - -import org.apache.logging.log4j.LogManager -import org.opensearch.alerting.AlertingPlugin -import org.opensearch.alerting.actionv2.GetMonitorV2Action -import org.opensearch.alerting.actionv2.GetMonitorV2Request -import org.opensearch.alerting.modelv2.MonitorV2.Companion.UUID_LENGTH -import org.opensearch.alerting.util.context -import org.opensearch.rest.BaseRestHandler -import org.opensearch.rest.RestHandler.Route -import org.opensearch.rest.RestRequest -import org.opensearch.rest.RestRequest.Method.GET -import org.opensearch.rest.RestRequest.Method.HEAD -import org.opensearch.rest.action.RestActions -import org.opensearch.rest.action.RestToXContentListener -import org.opensearch.search.fetch.subphase.FetchSourceContext -import org.opensearch.transport.client.node.NodeClient - -private val log = LogManager.getLogger(RestGetMonitorV2Action::class.java) - -/** - * This class consists of the REST handler to retrieve a V2 monitor by its ID. - * - * @opensearch.experimental - */ -class RestGetMonitorV2Action : BaseRestHandler() { - - override fun getName(): String { - return "get_monitor_v2_action" - } - - override fun routes(): List { - return listOf( - Route( - GET, - "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitor_id}" - ), - Route( - HEAD, - "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitor_id}" - ) - ) - } - - override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { - log.debug("${request.method()} ${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitor_id}") - - val monitorV2Id = request.param("monitor_id") - if (monitorV2Id == null || monitorV2Id.isEmpty()) { - throw IllegalArgumentException("No MonitorV2 ID provided") - } - - if (monitorV2Id.length != UUID_LENGTH) { - throw IllegalArgumentException("MonitorV2 ID provided does not have correct length") - } - - var srcContext = context(request) - if (request.method() == HEAD) { - srcContext = FetchSourceContext.DO_NOT_FETCH_SOURCE - } - - val getMonitorV2Request = GetMonitorV2Request(monitorV2Id, RestActions.parseVersion(request), srcContext) - return RestChannelConsumer { - channel -> - client.execute(GetMonitorV2Action.INSTANCE, getMonitorV2Request, RestToXContentListener(channel)) - } - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestIndexMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestIndexMonitorV2Action.kt deleted file mode 100644 index 8ff03e791..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestIndexMonitorV2Action.kt +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.resthandlerv2 - -import org.apache.logging.log4j.LogManager -import org.opensearch.action.support.WriteRequest -import org.opensearch.alerting.AlertingPlugin -import org.opensearch.alerting.actionv2.IndexMonitorV2Action -import org.opensearch.alerting.actionv2.IndexMonitorV2Request -import org.opensearch.alerting.modelv2.MonitorV2 -import org.opensearch.alerting.util.IF_PRIMARY_TERM -import org.opensearch.alerting.util.IF_SEQ_NO -import org.opensearch.alerting.util.REFRESH -import org.opensearch.commons.alerting.util.AlertingException -import org.opensearch.core.xcontent.XContentParser.Token -import org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken -import org.opensearch.index.seqno.SequenceNumbers -import org.opensearch.rest.BaseRestHandler -import org.opensearch.rest.RestHandler.Route -import org.opensearch.rest.RestRequest -import org.opensearch.rest.RestRequest.Method.POST -import org.opensearch.rest.RestRequest.Method.PUT -import org.opensearch.rest.action.RestToXContentListener -import org.opensearch.transport.client.node.NodeClient -import java.io.IOException - -private val log = LogManager.getLogger(RestIndexMonitorV2Action::class.java) - -/** - * Rest handlers to create and update V2 monitors - * - * @opensearch.experimental - */ -class RestIndexMonitorV2Action : BaseRestHandler() { - override fun getName(): String { - return "index_monitor_v2_action" - } - - override fun routes(): List { - return listOf( - Route( - POST, - AlertingPlugin.MONITOR_V2_BASE_URI - ), - Route( - PUT, - "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitor_id}" - ) - ) - } - - @Throws(IOException::class) - override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { - log.debug("${request.method()} ${request.path()}") - - val xcp = request.contentParser() - ensureExpectedToken(Token.START_OBJECT, xcp.nextToken(), xcp) - - val monitorV2: MonitorV2 - val rbacRoles: List? - try { - monitorV2 = MonitorV2.parse(xcp) - rbacRoles = request.contentParser().map()["rbac_roles"] as List? - } catch (e: Exception) { - throw AlertingException.wrap(IllegalArgumentException(e.localizedMessage)) - } - - val id = request.param("monitor_id", MonitorV2.NO_ID) - val seqNo = request.paramAsLong(IF_SEQ_NO, SequenceNumbers.UNASSIGNED_SEQ_NO) - val primaryTerm = request.paramAsLong(IF_PRIMARY_TERM, SequenceNumbers.UNASSIGNED_PRIMARY_TERM) - val refreshPolicy = if (request.hasParam(REFRESH)) { - WriteRequest.RefreshPolicy.parse(request.param(REFRESH)) - } else { - WriteRequest.RefreshPolicy.IMMEDIATE - } - - val indexMonitorV2Request = IndexMonitorV2Request(id, seqNo, primaryTerm, refreshPolicy, request.method(), monitorV2, rbacRoles) - - return RestChannelConsumer { channel -> - client.execute(IndexMonitorV2Action.INSTANCE, indexMonitorV2Request, RestToXContentListener(channel)) - } - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestSearchMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestSearchMonitorV2Action.kt deleted file mode 100644 index a179078f8..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestSearchMonitorV2Action.kt +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.resthandlerv2 - -import org.apache.logging.log4j.LogManager -import org.opensearch.action.search.SearchRequest -import org.opensearch.action.search.SearchResponse -import org.opensearch.alerting.AlertingPlugin -import org.opensearch.alerting.actionv2.SearchMonitorV2Action -import org.opensearch.alerting.actionv2.SearchMonitorV2Request -import org.opensearch.alerting.settings.AlertingSettings -import org.opensearch.alerting.util.context -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.LoggingDeprecationHandler -import org.opensearch.common.xcontent.XContentFactory.jsonBuilder -import org.opensearch.common.xcontent.XContentType -import org.opensearch.commons.alerting.model.ScheduledJob -import org.opensearch.commons.alerting.model.ScheduledJob.Companion.SCHEDULED_JOBS_INDEX -import org.opensearch.core.common.bytes.BytesReference -import org.opensearch.core.rest.RestStatus -import org.opensearch.core.xcontent.ToXContent.EMPTY_PARAMS -import org.opensearch.rest.BaseRestHandler -import org.opensearch.rest.BytesRestResponse -import org.opensearch.rest.RestChannel -import org.opensearch.rest.RestHandler.Route -import org.opensearch.rest.RestRequest -import org.opensearch.rest.RestRequest.Method.GET -import org.opensearch.rest.RestRequest.Method.POST -import org.opensearch.rest.RestResponse -import org.opensearch.rest.action.RestResponseListener -import org.opensearch.search.builder.SearchSourceBuilder -import org.opensearch.transport.client.node.NodeClient -import java.io.IOException - -private val log = LogManager.getLogger(RestSearchMonitorV2Action::class.java) - -/** - * This class consists of the REST handler to search for v2 monitors with some OpenSearch search query. - * - * @opensearch.experimental - */ -class RestSearchMonitorV2Action( - val settings: Settings, - clusterService: ClusterService, -) : BaseRestHandler() { - - @Volatile private var filterBy = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) - - init { - clusterService.clusterSettings.addSettingsUpdateConsumer(AlertingSettings.FILTER_BY_BACKEND_ROLES) { filterBy = it } - } - - override fun getName(): String { - return "search_monitor_v2_action" - } - - override fun routes(): List { - return listOf( - Route( - POST, - "${AlertingPlugin.MONITOR_V2_BASE_URI}/_search" - ), - Route( - GET, - "${AlertingPlugin.MONITOR_V2_BASE_URI}/_search" - ) - ) - } - - @Throws(IOException::class) - override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { - log.debug("${request.method()} ${AlertingPlugin.MONITOR_V2_BASE_URI}/_search") - - val searchSourceBuilder = SearchSourceBuilder() - searchSourceBuilder.parseXContent(request.contentOrSourceParamParser()) - searchSourceBuilder.fetchSource(context(request)) - - val searchRequest = SearchRequest() - .source(searchSourceBuilder) - .indices(SCHEDULED_JOBS_INDEX) - - val searchMonitorV2Request = SearchMonitorV2Request(searchRequest) - return RestChannelConsumer { channel -> - client.execute(SearchMonitorV2Action.INSTANCE, searchMonitorV2Request, searchMonitorResponse(channel)) - } - } - - // once the search response is received, rewrite the search hits to remove the extra "monitor_v2" JSON object wrapper - // that is used as ScheduledJob metadata - private fun searchMonitorResponse(channel: RestChannel): RestResponseListener { - return object : RestResponseListener(channel) { - @Throws(Exception::class) - override fun buildResponse(response: SearchResponse): RestResponse { - if (response.isTimedOut) { - return BytesRestResponse(RestStatus.REQUEST_TIMEOUT, response.toString()) - } - - try { - for (hit in response.hits) { - XContentType.JSON.xContent().createParser( - channel.request().xContentRegistry, - LoggingDeprecationHandler.INSTANCE, hit.sourceAsString - ).use { hitsParser -> - // when reconstructing XContent, intentionally leave out - // user field in response for security reasons by - // calling ScheduledJob.toXContent instead of - // a MonitorV2's toXContentWithUser - val monitorV2 = ScheduledJob.parse(hitsParser, hit.id, hit.version) - val xcb = monitorV2.toXContent(jsonBuilder(), EMPTY_PARAMS) - - // rewrite the search hit as just the MonitorV2 source, - // without the extra "monitor_v2" JSON object wrapper - hit.sourceRef(BytesReference.bytes(xcb)) - } - } - } catch (e: Exception) { - // Swallow exception and return response as is - log.error("The monitor_v2 parsing failed. Will return response as is.") - } - return BytesRestResponse(RestStatus.OK, response.toXContent(channel.newBuilder(), EMPTY_PARAMS)) - } - } - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/script/PPLTriggerExecutionContext.kt b/alerting/src/main/kotlin/org/opensearch/alerting/script/PPLTriggerExecutionContext.kt index 1b95da951..ced650c73 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/script/PPLTriggerExecutionContext.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/script/PPLTriggerExecutionContext.kt @@ -5,23 +5,30 @@ package org.opensearch.alerting.script -import org.json.JSONObject -import org.opensearch.alerting.modelv2.PPLSQLMonitor -import org.opensearch.alerting.modelv2.PPLSQLMonitorRunResult.Companion.PPL_QUERY_RESULTS_FIELD -import org.opensearch.alerting.modelv2.PPLSQLTrigger -import org.opensearch.alerting.modelv2.PPLSQLTrigger.Companion.PPL_SQL_TRIGGER_FIELD +import org.opensearch.common.settings.ClusterSettings +import org.opensearch.commons.alerting.model.Monitor +import org.opensearch.commons.alerting.model.PPLSQLTrigger +import org.opensearch.commons.alerting.model.PPLSQLTrigger.Companion.PPL_SQL_TRIGGER_FIELD +import java.time.Instant data class PPLTriggerExecutionContext( - override val monitorV2: PPLSQLMonitor, + override val monitor: Monitor, override val error: Exception? = null, + override val results: List>, + override val periodStart: Instant, + override val periodEnd: Instant, + override val clusterSettings: ClusterSettings, val pplTrigger: PPLSQLTrigger, - var pplQueryResults: JSONObject // can be a full set of PPL query results, or an individual result row -) : TriggerV2ExecutionContext(monitorV2, error) { +) : TriggerExecutionContext(monitor, results, periodStart, periodEnd, error, clusterSettings) { override fun asTemplateArg(): Map { val templateArg = super.asTemplateArg().toMutableMap() templateArg[PPL_SQL_TRIGGER_FIELD] = pplTrigger.asTemplateArg() - templateArg[PPL_QUERY_RESULTS_FIELD] = pplQueryResults.toMap() + templateArg[PPL_QUERY_RESULTS_FIELD] = results[0] // PPL/SQL Monitors only ever return one set of results return templateArg.toMap() } + + companion object { + const val PPL_QUERY_RESULTS_FIELD = "ppl_query_results" + } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/script/TriggerV2ExecutionContext.kt b/alerting/src/main/kotlin/org/opensearch/alerting/script/TriggerV2ExecutionContext.kt deleted file mode 100644 index 97384845c..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/script/TriggerV2ExecutionContext.kt +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.script - -import org.opensearch.alerting.modelv2.MonitorV2 - -abstract class TriggerV2ExecutionContext( - open val monitorV2: MonitorV2, - open val error: Exception? = null -) { - - open fun asTemplateArg(): Map { - return mapOf( - "monitorV2" to monitorV2.asTemplateArg(), - "error" to error - ) - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/service/DeleteMonitorService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/service/DeleteMonitorService.kt index c1a56a4c6..00ba144f8 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/service/DeleteMonitorService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/service/DeleteMonitorService.kt @@ -22,7 +22,6 @@ import org.opensearch.action.support.IndicesOptions import org.opensearch.action.support.WriteRequest.RefreshPolicy import org.opensearch.action.support.clustermanager.AcknowledgedResponse import org.opensearch.alerting.MonitorMetadataService -import org.opensearch.alerting.actionv2.DeleteMonitorV2Response import org.opensearch.alerting.core.lock.LockModel import org.opensearch.alerting.core.lock.LockService import org.opensearch.alerting.opensearchapi.suspendUntil @@ -75,19 +74,6 @@ object DeleteMonitorService : return DeleteMonitorResponse(deleteResponse.id, deleteResponse.version) } - /** - * Deletes the monitorV2, which does not come with other metadata and queries - * like doc level monitors - * @param monitorV2Id monitorV2 ID to be deleted - * @param refreshPolicy - */ - suspend fun deleteMonitorV2(monitorV2Id: String, refreshPolicy: RefreshPolicy): DeleteMonitorV2Response { - val deleteResponse = deleteMonitor(monitorV2Id, refreshPolicy) - deleteLock(monitorV2Id) - return DeleteMonitorV2Response(deleteResponse.id, deleteResponse.version) - } - - // both Alerting v1 and v2 workflows flow through this function private suspend fun deleteMonitor(monitorId: String, refreshPolicy: RefreshPolicy): DeleteResponse { val deleteMonitorRequest = DeleteRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, monitorId) .setRefreshPolicy(refreshPolicy) @@ -184,7 +170,6 @@ object DeleteMonitorService : deleteLock(monitor.id) } - // both Alerting v1 and v2 workflows flow through this function private suspend fun deleteLock(monitorId: String) { client.suspendUntil { lockService.deleteLock(LockModel.generateLockId(monitorId), it) } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt index 9fdbb1fec..3c838eae7 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt @@ -345,13 +345,6 @@ class AlertingSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ) - val ALERTING_V2_MAX_MONITORS = Setting.intSetting( - "plugins.alerting.v2.monitor.max_monitors", - 1000, - 1, - Setting.Property.NodeScope, Setting.Property.Dynamic - ) - val ALERTING_V2_MAX_THROTTLE_DURATION = Setting.longSetting( "plugins.alerting.v2.monitor.max_throttle_duration", 7200L, // 5 days, 7200 minutes @@ -366,13 +359,6 @@ class AlertingSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ) - val ALERTING_V2_MAX_LOOK_BACK_WINDOW = Setting.longSetting( - "plugins.alerting.v2.monitor.max_look_back_window", - 10080L, // 7 days, 10080 minutes - 2L, - Setting.Property.NodeScope, Setting.Property.Dynamic - ) - val ALERTING_V2_MAX_QUERY_LENGTH = Setting.longSetting( "plugins.alerting.v2.monitor.max_query_length", 2000L, diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteWorkflowAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteWorkflowAction.kt index 60e018e50..e15262a25 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteWorkflowAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteWorkflowAction.kt @@ -23,7 +23,6 @@ import org.opensearch.action.search.SearchResponse import org.opensearch.action.support.ActionFilters import org.opensearch.action.support.HandledTransportAction import org.opensearch.action.support.WriteRequest.RefreshPolicy -import org.opensearch.alerting.AlertingV2Utils.validateMonitorV1 import org.opensearch.alerting.core.lock.LockModel import org.opensearch.alerting.core.lock.LockService import org.opensearch.alerting.opensearchapi.addFilter @@ -300,13 +299,7 @@ class TransportDeleteWorkflowAction @Inject constructor( xContentRegistry, LoggingDeprecationHandler.INSTANCE, hit.sourceAsString ).use { hitsParser -> - val scheduledJob = ScheduledJob.parse(hitsParser, hit.id, hit.version) - - validateMonitorV1(scheduledJob)?.let { - throw OpenSearchException(it) - } - - val monitor = scheduledJob as Monitor + val monitor = ScheduledJob.parse(hitsParser, hit.id, hit.version) as Monitor deletableMonitors.add(monitor) } } @@ -339,12 +332,7 @@ class TransportDeleteWorkflowAction @Inject constructor( xContentRegistry, LoggingDeprecationHandler.INSTANCE, getResponse.sourceAsBytesRef, XContentType.JSON ) - val scheduledJob = ScheduledJob.parse(xcp, getResponse.id, getResponse.version) - validateMonitorV1(scheduledJob)?.let { - actionListener.onFailure(AlertingException.wrap(it)) - return null - } - return scheduledJob as Workflow + return ScheduledJob.parse(xcp, getResponse.id, getResponse.version) as Workflow } private suspend fun deleteWorkflow(deleteRequest: DeleteRequest): DeleteResponse { diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteMonitorAction.kt index 88e86509a..da775a818 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteMonitorAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteMonitorAction.kt @@ -61,9 +61,16 @@ class TransportExecuteMonitorAction @Inject constructor( private val sdkClient: SdkClient ) : HandledTransportAction ( ExecuteMonitorAction.NAME, transportService, actionFilters, ::ExecuteMonitorRequest -) { +), + SecureTransportAction { @Volatile private var indexTimeout = AlertingSettings.INDEX_TIMEOUT.get(settings) + @Volatile override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) + + init { + listenFilterBySettingChange(clusterService) + } + override fun doExecute(task: Task, execMonitorRequest: ExecuteMonitorRequest, actionListener: ActionListener) { val userStr = client.threadPool().threadContext.getTransient(ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT) @@ -89,7 +96,14 @@ class TransportExecuteMonitorAction @Inject constructor( "Executing monitor from API - id: ${monitor.id}, type: ${monitor.monitorType}, " + "periodStart: $periodStart, periodEnd: $periodEnd, dryrun: ${execMonitorRequest.dryrun}" ) - val monitorRunResult = runner.runJob(monitor, periodStart, periodEnd, execMonitorRequest.dryrun, transportService) + val monitorRunResult = runner.runJob( + monitor, + periodStart, + periodEnd, + execMonitorRequest.dryrun, + execMonitorRequest.manual, + transportService + ) withContext(Dispatchers.IO) { actionListener.onResponse(ExecuteMonitorResponse(monitorRunResult)) } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteWorkflowAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteWorkflowAction.kt index aa3c5af1d..7352cfc53 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteWorkflowAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteWorkflowAction.kt @@ -14,7 +14,6 @@ import org.opensearch.action.get.GetRequest import org.opensearch.action.get.GetResponse import org.opensearch.action.support.ActionFilters import org.opensearch.action.support.HandledTransportAction -import org.opensearch.alerting.AlertingV2Utils.validateMonitorV1 import org.opensearch.alerting.MonitorRunnerService import org.opensearch.alerting.action.ExecuteWorkflowAction import org.opensearch.alerting.action.ExecuteWorkflowRequest @@ -79,6 +78,7 @@ class TransportExecuteWorkflowAction @Inject constructor( periodStart, periodEnd, execWorkflowRequest.dryrun, + execWorkflowRequest.manual, transportService = transportService ) withContext(Dispatchers.IO, { @@ -120,12 +120,7 @@ class TransportExecuteWorkflowAction @Inject constructor( xContentRegistry, LoggingDeprecationHandler.INSTANCE, response.sourceAsBytesRef, XContentType.JSON ).use { xcp -> - val scheduledJob = ScheduledJob.parse(xcp, response.id, response.version) - validateMonitorV1(scheduledJob)?.let { - actionListener.onFailure(AlertingException.wrap(it)) - return - } - val workflow = scheduledJob as Workflow + val workflow = ScheduledJob.parse(xcp, response.id, response.version) as Workflow executeWorkflow(workflow) } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetAlertsAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetAlertsAction.kt index b8e6d31b7..69c43e1e2 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetAlertsAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetAlertsAction.kt @@ -14,6 +14,7 @@ import org.opensearch.action.support.ActionFilters import org.opensearch.action.support.HandledTransportAction import org.opensearch.alerting.AlertingPlugin import org.opensearch.alerting.alerts.AlertIndices +import org.opensearch.alerting.alertsv2.AlertV2Indices.Companion.ALERT_V2_INDEX import org.opensearch.alerting.opensearchapi.addFilter import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.util.use @@ -27,6 +28,8 @@ import org.opensearch.commons.alerting.action.AlertingActions import org.opensearch.commons.alerting.action.GetAlertsRequest import org.opensearch.commons.alerting.action.GetAlertsResponse import org.opensearch.commons.alerting.model.Alert +import org.opensearch.commons.alerting.model.Alert.Companion.MONITOR_NAME_FIELD +import org.opensearch.commons.alerting.model.Alert.Companion.TRIGGER_NAME_FIELD import org.opensearch.commons.alerting.model.Monitor import org.opensearch.commons.alerting.model.ScheduledJob import org.opensearch.commons.alerting.util.AlertingException @@ -136,8 +139,8 @@ class TransportGetAlertsAction @Inject constructor( QueryBuilders .queryStringQuery(tableProp.searchString) .defaultOperator(Operator.AND) - .field("monitor_name") - .field("trigger_name") + .field(MONITOR_NAME_FIELD) + .field(TRIGGER_NAME_FIELD) ) } val searchSourceBuilder = SearchSourceBuilder() @@ -218,11 +221,7 @@ class TransportGetAlertsAction @Inject constructor( xContentRegistry, LoggingDeprecationHandler.INSTANCE, getResponse.sourceAsBytesRef, XContentType.JSON ) - val scheduledJob = ScheduledJob.parse(xcp, getResponse.id, getResponse.version) - validateMonitorV1(scheduledJob)?.let { - throw it - } - return scheduledJob as Monitor + return ScheduledJob.parse(xcp, getResponse.id, getResponse.version) as Monitor } catch (t: Exception) { log.error("Failure in fetching monitor ${getAlertsRequest.monitorId} to resolve alert index in get alerts action", t) return null diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt index a7015c263..c5859be75 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt @@ -8,6 +8,8 @@ package org.opensearch.alerting.transport import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch +import kotlinx.coroutines.newSingleThreadContext +import kotlinx.coroutines.withContext import org.apache.logging.log4j.LogManager import org.opensearch.ExceptionsHelper import org.opensearch.OpenSearchException @@ -27,16 +29,25 @@ import org.opensearch.action.support.HandledTransportAction import org.opensearch.action.support.WriteRequest.RefreshPolicy import org.opensearch.action.support.clustermanager.AcknowledgedResponse import org.opensearch.alerting.AlertingPlugin -import org.opensearch.alerting.AlertingV2Utils.validateMonitorV1 import org.opensearch.alerting.MonitorMetadataService +import org.opensearch.alerting.PPLUtils.appendCustomCondition +import org.opensearch.alerting.PPLUtils.executePplQuery +import org.opensearch.alerting.PPLUtils.findEvalResultVar +import org.opensearch.alerting.PPLUtils.findEvalResultVarIdxInSchema import org.opensearch.alerting.core.ScheduledJobIndices import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.service.DeleteMonitorService import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_MAX_MONITORS +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_EXPIRE_DURATION +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_QUERY_LENGTH +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_THROTTLE_DURATION +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS import org.opensearch.alerting.settings.AlertingSettings.Companion.INDEX_TIMEOUT import org.opensearch.alerting.settings.AlertingSettings.Companion.MAX_ACTION_THROTTLE_VALUE import org.opensearch.alerting.settings.AlertingSettings.Companion.MAX_TRIGGERS_PER_MONITOR +import org.opensearch.alerting.settings.AlertingSettings.Companion.NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH +import org.opensearch.alerting.settings.AlertingSettings.Companion.NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH import org.opensearch.alerting.settings.AlertingSettings.Companion.REQUEST_TIMEOUT import org.opensearch.alerting.settings.DestinationSettings.Companion.ALLOW_LIST import org.opensearch.alerting.util.DocLevelMonitorQueries @@ -59,12 +70,16 @@ import org.opensearch.commons.alerting.action.IndexMonitorResponse import org.opensearch.commons.alerting.model.DocLevelMonitorInput import org.opensearch.commons.alerting.model.DocLevelMonitorInput.Companion.DOC_LEVEL_INPUT_FIELD import org.opensearch.commons.alerting.model.Monitor +import org.opensearch.commons.alerting.model.Monitor.MonitorType import org.opensearch.commons.alerting.model.MonitorMetadata +import org.opensearch.commons.alerting.model.PPLSQLInput +import org.opensearch.commons.alerting.model.PPLSQLTrigger import org.opensearch.commons.alerting.model.ScheduledJob import org.opensearch.commons.alerting.model.ScheduledJob.Companion.SCHEDULED_JOBS_INDEX import org.opensearch.commons.alerting.model.SearchInput import org.opensearch.commons.alerting.model.remote.monitors.RemoteDocLevelMonitorInput import org.opensearch.commons.alerting.model.remote.monitors.RemoteDocLevelMonitorInput.Companion.REMOTE_DOC_LEVEL_MONITOR_INPUT_FIELD +import org.opensearch.commons.alerting.model.userErrorMessage import org.opensearch.commons.alerting.util.AlertingException import org.opensearch.commons.alerting.util.isMonitorOfStandardType import org.opensearch.commons.authuser.User @@ -95,7 +110,7 @@ private val log = LogManager.getLogger(TransportIndexMonitorAction::class.java) private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) class TransportIndexMonitorAction @Inject constructor( - transportService: TransportService, + val transportService: TransportService, val client: Client, actionFilters: ActionFilters, val scheduledJobIndices: ScheduledJobIndices, @@ -116,6 +131,15 @@ class TransportIndexMonitorAction @Inject constructor( @Volatile private var indexTimeout = INDEX_TIMEOUT.get(settings) @Volatile private var maxActionThrottle = MAX_ACTION_THROTTLE_VALUE.get(settings) @Volatile private var allowList = ALLOW_LIST.get(settings) + + // PPL Alerting related settings + @Volatile private var maxThrottleDuration = ALERTING_V2_MAX_THROTTLE_DURATION.get(settings) + @Volatile private var maxExpireDuration = ALERTING_V2_MAX_EXPIRE_DURATION.get(settings) + @Volatile private var maxQueryLength = ALERTING_V2_MAX_QUERY_LENGTH.get(settings) + @Volatile private var maxQueryResults = ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS.get(settings) + @Volatile private var notificationSubjectMaxLength = NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH.get(settings) + @Volatile private var notificationMessageMaxLength = NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH.get(settings) + @Volatile override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) init { @@ -125,6 +149,18 @@ class TransportIndexMonitorAction @Inject constructor( clusterService.clusterSettings.addSettingsUpdateConsumer(INDEX_TIMEOUT) { indexTimeout = it } clusterService.clusterSettings.addSettingsUpdateConsumer(MAX_ACTION_THROTTLE_VALUE) { maxActionThrottle = it } clusterService.clusterSettings.addSettingsUpdateConsumer(ALLOW_LIST) { allowList = it } + + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_MAX_THROTTLE_DURATION) { maxThrottleDuration = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_MAX_EXPIRE_DURATION) { maxExpireDuration = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_MAX_QUERY_LENGTH) { maxQueryLength = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS) { maxQueryResults = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH) { + notificationSubjectMaxLength = it + } + clusterService.clusterSettings.addSettingsUpdateConsumer(NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH) { + notificationMessageMaxLength = it + } + listenFilterBySettingChange(clusterService) } @@ -174,11 +210,13 @@ class TransportIndexMonitorAction @Inject constructor( } } - if (!isADMonitor(transformedRequest.monitor)) { - checkIndicesAndExecute(client, actionListener, transformedRequest, user) - } else { + if (isADMonitor(transformedRequest.monitor)) { // check if user has access to any anomaly detector for AD monitor checkAnomalyDetectorAndExecute(client, actionListener, transformedRequest, user) + } else if (transformedRequest.monitor.monitorType == MonitorType.PPL_MONITOR.value) { + checkPplSqlQueryAndExecute(actionListener, transformedRequest, user) + } else { + checkIndicesAndExecute(client, actionListener, transformedRequest, user) } } @@ -245,6 +283,216 @@ class TransportIndexMonitorAction @Inject constructor( ) } + fun checkPplSqlQueryAndExecute( + actionListener: ActionListener, + indexMonitorRequest: IndexMonitorRequest, + user: User? + ) { + // declare upfront the validation listener that will move on to the next phase + // of monitor indexing after validations are complete + val validationListener = object : ActionListener { // validationListener + override fun onResponse(response: Unit) { + // user permissions to indices have already been checked if we made it to the onResponse(), + // proceed without the context of the user, otherwise, + // we would get permissions errors trying to search the alerting-config + // index as the user. pass the user object itself so backend + // roles can be matched and checked downstream + // roles can be matched and checked downstream + client.threadPool().threadContext.stashContext().use { + val pplSqlMonitor = indexMonitorRequest.monitor + if (user == null) { + indexMonitorRequest.monitor = pplSqlMonitor + .copy(user = User("", listOf(), listOf(), mapOf())) + } else { + indexMonitorRequest.monitor = pplSqlMonitor + .copy(user = User(user.name, user.backendRoles, user.roles, user.customAttributes)) + } + IndexMonitorHandler(client, actionListener, indexMonitorRequest, user).resolveUserAndStart() + } + } + + override fun onFailure(e: Exception) { + actionListener.onFailure(e) + } + } + + client.threadPool().threadContext.stashContext().use { + scope.launch { + val singleThreadContext = newSingleThreadContext("IndexMonitorV2ActionThread") + withContext(singleThreadContext) { + it.restore() + + val pplSqlMonitor = indexMonitorRequest.monitor + + val pplQueryValid = validatePplSqlQuery(pplSqlMonitor, validationListener) + if (!pplQueryValid) { + return@withContext + } + + // run basic validations against the PPL/SQL Monitor + val pplSqlMonitorValid = validatePplSqlMonitor(pplSqlMonitor, validationListener) + if (!pplSqlMonitorValid) { + return@withContext + } + + validationListener.onResponse(Unit) + } + } + } + } + + private suspend fun validatePplSqlQuery( + pplSqlMonitor: Monitor, + validationListener: ActionListener + ): Boolean { + // first attempt to run the monitor query and all possible + // extensions of it (from custom conditions) + try { + val query = (pplSqlMonitor.inputs[0] as PPLSQLInput).query + + // now run the base query as is. + // if there are any PPL syntax, index not found, insufficient index permissions, or other errors, + // this will throw an exception + executePplQuery(query, clusterService.state().nodes.localNode, transportService) + + // scan all the triggers with custom conditions, and ensure each query constructed + // from the base query + custom condition is valid + for (trigger in pplSqlMonitor.triggers) { + val pplTrigger = trigger as PPLSQLTrigger + + if (pplTrigger.conditionType != PPLSQLTrigger.ConditionType.CUSTOM) { + continue + } + + val evalResultVar = findEvalResultVar(pplTrigger.customCondition!!) + + val queryWithCustomCondition = appendCustomCondition(query, pplTrigger.customCondition!!) + + val executePplQueryResponse = executePplQuery( + queryWithCustomCondition, + clusterService.state().nodes.localNode, + transportService + ) + + val evalResultVarIdx = findEvalResultVarIdxInSchema(executePplQueryResponse, evalResultVar) + + val resultVarType = executePplQueryResponse + .getJSONArray("schema") + .getJSONObject(evalResultVarIdx) + .getString("type") + + // custom conditions must evaluate to a boolean result, otherwise it's invalid + if (resultVarType != "boolean") { + validationListener.onFailure( + AlertingException.wrap( + IllegalArgumentException( + "Custom condition in trigger ${pplTrigger.name} is invalid because it does not " + + "evaluate to a boolean, but instead to type: $resultVarType" + ) + ) + ) + return false + } + } + } catch (e: Exception) { + validationListener.onFailure( + AlertingException.wrap( + IllegalArgumentException("Validation error for PPL Query in PPL Monitor: ${e.userErrorMessage()}") + ) + ) + return false + } + + return true + } + + private fun validatePplSqlMonitor(pplSqlMonitor: Monitor, validationListener: ActionListener): Boolean { + // ensure the trigger throttle and expire durations are valid + pplSqlMonitor.triggers.forEach { trigger -> + val pplTrigger = trigger as PPLSQLTrigger + + pplTrigger.throttleDuration?.let { throttleDuration -> + if (throttleDuration > maxThrottleDuration) { + validationListener.onFailure( + AlertingException.wrap( + IllegalArgumentException( + "Throttle duration must be at most $maxThrottleDuration but was $throttleDuration" + ) + ) + ) + return false + } + } + + if (pplTrigger.expireDuration > maxExpireDuration) { + validationListener.onFailure( + AlertingException.wrap( + IllegalArgumentException( + "Expire duration must be at most $maxExpireDuration but was ${trigger.expireDuration}" + ) + ) + ) + return false + } + + if (pplTrigger.conditionType == PPLSQLTrigger.ConditionType.NUMBER_OF_RESULTS && + pplTrigger.numResultsValue!! > maxQueryResults + ) { + validationListener.onFailure( + AlertingException.wrap( + IllegalArgumentException( + "Trigger ${trigger.id} checks for number of results threshold of ${trigger.numResultsValue}, " + + "but Alerting V2 is configured only to retrieve $maxQueryResults query results maximum. " + + "Please lower the number of results value to one below this maximum value, or adjust the cluster " + + "setting: $ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS.key}" + ) + ) + ) + return false + } + + pplTrigger.actions.forEach { action -> + if (action.subjectTemplate?.idOrCode?.length!! > notificationSubjectMaxLength) { + validationListener.onFailure( + AlertingException.wrap( + IllegalArgumentException( + "Notification subject source cannot exceed length: $notificationSubjectMaxLength" + ) + ) + ) + return false + } + + if (action.messageTemplate.idOrCode.length > notificationMessageMaxLength) { + validationListener.onFailure( + AlertingException.wrap( + IllegalArgumentException( + "Notification message source cannot exceed length: $notificationMessageMaxLength" + ) + ) + ) + return false + } + } + } + + val query = (pplSqlMonitor.inputs[0] as PPLSQLInput).query + + // ensure the query length doesn't exceed the limit + if (query.length > maxQueryLength) { + validationListener.onFailure( + AlertingException.wrap( + IllegalArgumentException( + "PPL Query length must be at most $maxQueryLength but was ${query.length}" + ) + ) + ) + return false + } + + return true + } + /** * It's no reasonable to create AD monitor if the user has no access to any detector. Otherwise * the monitor will not get any anomaly result. So we will check user has access to at least 1 @@ -637,14 +885,7 @@ class TransportIndexMonitorAction @Inject constructor( xContentRegistry, LoggingDeprecationHandler.INSTANCE, getResponse.sourceAsBytesRef, XContentType.JSON ) - val scheduledJob = ScheduledJob.parse(xcp, getResponse.id, getResponse.version) - - validateMonitorV1(scheduledJob)?.let { - actionListener.onFailure(AlertingException.wrap(it)) - return - } - - val monitor = scheduledJob as Monitor + val monitor = ScheduledJob.parse(xcp, getResponse.id, getResponse.version) as Monitor onGetResponse(monitor) } catch (t: Exception) { diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexWorkflowAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexWorkflowAction.kt index 6101c66e3..bc91f6de7 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexWorkflowAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexWorkflowAction.kt @@ -27,7 +27,6 @@ import org.opensearch.action.search.SearchResponse import org.opensearch.action.support.ActionFilters import org.opensearch.action.support.HandledTransportAction import org.opensearch.action.support.clustermanager.AcknowledgedResponse -import org.opensearch.alerting.AlertingV2Utils.validateMonitorV1 import org.opensearch.alerting.MonitorMetadataService import org.opensearch.alerting.MonitorRunnerService.monitorCtx import org.opensearch.alerting.WorkflowMetadataService @@ -459,12 +458,7 @@ class TransportIndexWorkflowAction @Inject constructor( xContentRegistry, LoggingDeprecationHandler.INSTANCE, getResponse.sourceAsBytesRef, XContentType.JSON ) - val scheduledJob = ScheduledJob.parse(xcp, getResponse.id, getResponse.version) - validateMonitorV1(scheduledJob)?.let { - actionListener.onFailure(AlertingException.wrap(it)) - return - } - val workflow = scheduledJob as Workflow + val workflow = ScheduledJob.parse(xcp, getResponse.id, getResponse.version) as Workflow onGetResponse(workflow) } catch (t: Exception) { actionListener.onFailure(AlertingException.wrap(t)) @@ -737,11 +731,7 @@ class TransportIndexWorkflowAction @Inject constructor( xContentRegistry, LoggingDeprecationHandler.INSTANCE, hit.sourceAsString ).use { hitsParser -> - val scheduledJob = ScheduledJob.parse(hitsParser, hit.id, hit.version) - validateMonitorV1(scheduledJob)?.let { - throw OpenSearchException(it) - } - val monitor = scheduledJob as Monitor + val monitor = ScheduledJob.parse(hitsParser, hit.id, hit.version) as Monitor monitors.add(monitor) } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportSearchMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportSearchMonitorAction.kt index 8896bff78..77da00f7c 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportSearchMonitorAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportSearchMonitorAction.kt @@ -17,6 +17,7 @@ import org.opensearch.action.support.HandledTransportAction import org.opensearch.alerting.AlertingPlugin import org.opensearch.alerting.opensearchapi.addFilter import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.alerting.util.isIndexNotFoundException import org.opensearch.alerting.util.use import org.opensearch.cluster.service.ClusterService import org.opensearch.common.inject.Inject @@ -114,6 +115,7 @@ class TransportSearchMonitorAction @Inject constructor( } } + // Used in Get and Search monitor functionalities to return a "no results" response fun getEmptySearchResponse(): SearchResponse { val internalSearchResponse = InternalSearchResponse( SearchHits(emptyArray(), TotalHits(0L, Relation.EQUAL_TO), 0.0f), diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportDeleteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportDeleteMonitorV2Action.kt deleted file mode 100644 index b35381474..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportDeleteMonitorV2Action.kt +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.transportv2 - -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch -import org.apache.logging.log4j.LogManager -import org.opensearch.OpenSearchStatusException -import org.opensearch.action.get.GetRequest -import org.opensearch.action.get.GetResponse -import org.opensearch.action.support.ActionFilters -import org.opensearch.action.support.HandledTransportAction -import org.opensearch.alerting.AlertingV2Utils -import org.opensearch.alerting.actionv2.DeleteMonitorV2Action -import org.opensearch.alerting.actionv2.DeleteMonitorV2Request -import org.opensearch.alerting.actionv2.DeleteMonitorV2Response -import org.opensearch.alerting.core.settings.AlertingV2Settings.Companion.ALERTING_V2_ENABLED -import org.opensearch.alerting.modelv2.MonitorV2 -import org.opensearch.alerting.opensearchapi.suspendUntil -import org.opensearch.alerting.service.DeleteMonitorService -import org.opensearch.alerting.settings.AlertingSettings -import org.opensearch.alerting.transport.SecureTransportAction -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.inject.Inject -import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.LoggingDeprecationHandler -import org.opensearch.common.xcontent.XContentHelper -import org.opensearch.common.xcontent.XContentType -import org.opensearch.commons.alerting.model.ScheduledJob -import org.opensearch.commons.alerting.util.AlertingException -import org.opensearch.core.action.ActionListener -import org.opensearch.core.rest.RestStatus -import org.opensearch.core.xcontent.NamedXContentRegistry -import org.opensearch.tasks.Task -import org.opensearch.transport.TransportService -import org.opensearch.transport.client.Client - -private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) -private val log = LogManager.getLogger(TransportDeleteMonitorV2Action::class.java) - -/** - * Transport action that contains the core logic for deleting monitor V2s. - * - * @opensearch.experimental - */ -class TransportDeleteMonitorV2Action @Inject constructor( - transportService: TransportService, - val client: Client, - actionFilters: ActionFilters, - val clusterService: ClusterService, - settings: Settings, - val xContentRegistry: NamedXContentRegistry -) : HandledTransportAction( - DeleteMonitorV2Action.NAME, transportService, actionFilters, ::DeleteMonitorV2Request -), - SecureTransportAction { - - @Volatile private var alertingV2Enabled = ALERTING_V2_ENABLED.get(settings) - - @Volatile override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) - - init { - clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_ENABLED) { alertingV2Enabled = it } - listenFilterBySettingChange(clusterService) - } - - override fun doExecute(task: Task, request: DeleteMonitorV2Request, actionListener: ActionListener) { - if (!alertingV2Enabled) { - actionListener.onFailure( - AlertingException.wrap( - OpenSearchStatusException( - "Alerting V2 is currently disabled, please enable it with the " + - "cluster setting: ${ALERTING_V2_ENABLED.key}.", - RestStatus.FORBIDDEN - ), - ) - ) - return - } - - val user = readUserFromThreadContext(client) - - if (!validateUserBackendRoles(user, actionListener)) { - return - } - - scope.launch { - try { - val monitorV2 = getMonitorV2(request.monitorV2Id, actionListener) ?: return@launch - - val canDelete = user == null || !doFilterForUser(user) || - checkUserPermissionsWithResource(user, monitorV2!!.user, actionListener, "monitor_v2", request.monitorV2Id) - - if (canDelete) { - val deleteResponse = - DeleteMonitorService.deleteMonitorV2(request.monitorV2Id, request.refreshPolicy) - actionListener.onResponse(deleteResponse) - } else { - actionListener.onFailure( - AlertingException( - "Not allowed to delete this Monitor V2", - RestStatus.FORBIDDEN, - IllegalStateException() - ) - ) - } - } catch (e: Exception) { - actionListener.onFailure(e) - } - - // scheduled AlertV2Mover will sweep the alerts and find that this monitor no longer exists, - // and expire this monitor's alerts accordingly - } - } - - private suspend fun getMonitorV2(monitorV2Id: String, actionListener: ActionListener): MonitorV2? { - val getRequest = GetRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, monitorV2Id) - - val getResponse: GetResponse = client.suspendUntil { get(getRequest, it) } - if (!getResponse.isExists) { - actionListener.onFailure( - AlertingException.wrap( - OpenSearchStatusException("Monitor V2 with $monitorV2Id is not found", RestStatus.NOT_FOUND) - ) - ) - return null - } - - val xcp = XContentHelper.createParser( - xContentRegistry, LoggingDeprecationHandler.INSTANCE, - getResponse.sourceAsBytesRef, XContentType.JSON - ) - val scheduledJob = ScheduledJob.parse(xcp, getResponse.id, getResponse.version) - - AlertingV2Utils.validateMonitorV2(scheduledJob)?.let { - actionListener.onFailure(AlertingException.wrap(it)) - return null - } - - val monitorV2 = scheduledJob as MonitorV2 - - return monitorV2 - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportExecuteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportExecuteMonitorV2Action.kt deleted file mode 100644 index 05055afef..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportExecuteMonitorV2Action.kt +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.transportv2 - -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch -import kotlinx.coroutines.withContext -import org.apache.logging.log4j.LogManager -import org.opensearch.OpenSearchStatusException -import org.opensearch.action.get.GetRequest -import org.opensearch.action.get.GetResponse -import org.opensearch.action.support.ActionFilters -import org.opensearch.action.support.HandledTransportAction -import org.opensearch.alerting.AlertingV2Utils.validateMonitorV2 -import org.opensearch.alerting.MonitorRunnerService -import org.opensearch.alerting.actionv2.ExecuteMonitorV2Action -import org.opensearch.alerting.actionv2.ExecuteMonitorV2Request -import org.opensearch.alerting.actionv2.ExecuteMonitorV2Response -import org.opensearch.alerting.core.settings.AlertingV2Settings.Companion.ALERTING_V2_ENABLED -import org.opensearch.alerting.modelv2.MonitorV2 -import org.opensearch.alerting.modelv2.PPLSQLMonitor -import org.opensearch.alerting.modelv2.PPLSQLMonitor.Companion.PPL_SQL_MONITOR_TYPE -import org.opensearch.alerting.settings.AlertingSettings -import org.opensearch.alerting.transport.SecureTransportAction -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.inject.Inject -import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.LoggingDeprecationHandler -import org.opensearch.common.xcontent.XContentHelper -import org.opensearch.common.xcontent.XContentType -import org.opensearch.commons.ConfigConstants -import org.opensearch.commons.alerting.model.ScheduledJob -import org.opensearch.commons.alerting.util.AlertingException -import org.opensearch.commons.authuser.User -import org.opensearch.core.action.ActionListener -import org.opensearch.core.rest.RestStatus -import org.opensearch.core.xcontent.NamedXContentRegistry -import org.opensearch.tasks.Task -import org.opensearch.transport.TransportService -import org.opensearch.transport.client.Client -import java.time.Instant - -private val log = LogManager.getLogger(TransportExecuteMonitorV2Action::class.java) - -/** - * Transport action for executing monitor V2s by calling the respective monitor V2 runners. - * - * @opensearch.experimental - */ -class TransportExecuteMonitorV2Action @Inject constructor( - private val transportService: TransportService, - private val client: Client, - private val clusterService: ClusterService, - private val runner: MonitorRunnerService, - actionFilters: ActionFilters, - val xContentRegistry: NamedXContentRegistry, - private val settings: Settings -) : HandledTransportAction( - ExecuteMonitorV2Action.NAME, transportService, actionFilters, ::ExecuteMonitorV2Request -), - SecureTransportAction { - - @Volatile private var alertingV2Enabled = ALERTING_V2_ENABLED.get(settings) - - @Volatile override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) - - init { - clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_ENABLED) { alertingV2Enabled = it } - listenFilterBySettingChange(clusterService) - } - - override fun doExecute( - task: Task, - execMonitorV2Request: ExecuteMonitorV2Request, - actionListener: ActionListener - ) { - if (!alertingV2Enabled) { - actionListener.onFailure( - AlertingException.wrap( - OpenSearchStatusException( - "Alerting V2 is currently disabled, please enable it with the " + - "cluster setting: ${ALERTING_V2_ENABLED.key}", - RestStatus.FORBIDDEN - ), - ) - ) - return - } - - val userStr = client.threadPool().threadContext.getTransient(ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT) - log.debug("User and roles string from thread context: $userStr") - val user: User? = User.parse(userStr) - - client.threadPool().threadContext.stashContext().use { - /* first define a function that will be used later to run MonitorV2s */ - val executeMonitorV2 = fun (monitorV2: MonitorV2) { - runner.launch { - // get execution end, this will be used to compute the execution interval - // via look back window (if one is supplied) - val periodEnd = Instant.ofEpochMilli(execMonitorV2Request.requestEnd.millis) - - // call the MonitorRunnerService to execute the MonitorV2 - try { - val monitorV2Type = when (monitorV2) { - is PPLSQLMonitor -> PPL_SQL_MONITOR_TYPE - else -> throw IllegalStateException("Unexpected MonitorV2 type: ${monitorV2.javaClass.name}") - } - log.info( - "Executing MonitorV2 from API - id: ${monitorV2.id}, type: $monitorV2Type, " + - "periodEnd: $periodEnd, manual: ${execMonitorV2Request.manual}" - ) - val monitorV2RunResult = runner.runJobV2( - monitorV2, - periodEnd, - execMonitorV2Request.dryrun, - execMonitorV2Request.manual, - transportService - ) - withContext(Dispatchers.IO) { - actionListener.onResponse(ExecuteMonitorV2Response(monitorV2RunResult)) - } - } catch (e: Exception) { - log.error("Unexpected error running monitor", e) - withContext(Dispatchers.IO) { - actionListener.onFailure(AlertingException.wrap(e)) - } - } - } - } - - /* now execute the MonitorV2 */ - - // if both monitor_v2 id and object were passed in, ignore object and proceed with id - if (execMonitorV2Request.monitorV2Id != null && execMonitorV2Request.monitorV2 != null) { - log.info( - "Both a monitor_v2 id and monitor_v2 object were passed in to ExecuteMonitorV2" + - "request. Proceeding to execute by monitor_v2 ID and ignoring monitor_v2 object." - ) - } - - if (execMonitorV2Request.monitorV2Id != null) { // execute with monitor ID case - // search the alerting-config index for the MonitorV2 with this ID - val getMonitorV2Request = GetRequest(ScheduledJob.SCHEDULED_JOBS_INDEX).id(execMonitorV2Request.monitorV2Id) - client.get( - getMonitorV2Request, - object : ActionListener { - override fun onResponse(getMonitorV2Response: GetResponse) { - if (!getMonitorV2Response.isExists) { - actionListener.onFailure( - AlertingException.wrap( - OpenSearchStatusException( - "Can't find monitorV2 with id: ${getMonitorV2Response.id} to execute", - RestStatus.NOT_FOUND - ) - ) - ) - return - } - - if (getMonitorV2Response.isSourceEmpty) { - actionListener.onFailure( - AlertingException.wrap( - OpenSearchStatusException( - "Found monitorV2 with id: ${getMonitorV2Response.id} but it was empty", - RestStatus.NO_CONTENT - ) - ) - ) - return - } - - val xcp = XContentHelper.createParser( - xContentRegistry, - LoggingDeprecationHandler.INSTANCE, - getMonitorV2Response.sourceAsBytesRef, - XContentType.JSON - ) - - val scheduledJob = ScheduledJob.parse(xcp, getMonitorV2Response.id, getMonitorV2Response.version) - - validateMonitorV2(scheduledJob)?.let { - actionListener.onFailure(AlertingException.wrap(it)) - return - } - - val monitorV2 = scheduledJob as MonitorV2 - - // security is enabled and filterby is enabled - // only run this check on manual executions, - // automatic scheduled job executions should - // bypass this check and proceed to execution - if (execMonitorV2Request.manual && - !checkUserPermissionsWithResource( - user, - monitorV2.user, - actionListener, - "monitor", - execMonitorV2Request.monitorV2Id - ) - ) { - return - } - - try { - executeMonitorV2(monitorV2) - } catch (e: Exception) { - actionListener.onFailure(AlertingException.wrap(e)) - } - } - - override fun onFailure(t: Exception) { - actionListener.onFailure(AlertingException.wrap(t)) - } - } - ) - } else { // execute with monitor object case - try { - val monitorV2 = execMonitorV2Request.monitorV2!!.makeCopy(user = user) - executeMonitorV2(monitorV2) - } catch (e: Exception) { - actionListener.onFailure(AlertingException.wrap(e)) - } - } - } - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetAlertsV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetAlertsV2Action.kt deleted file mode 100644 index 4a0cec89d..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetAlertsV2Action.kt +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.transportv2 - -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch -import org.apache.logging.log4j.LogManager -import org.opensearch.OpenSearchStatusException -import org.opensearch.action.search.SearchRequest -import org.opensearch.action.search.SearchResponse -import org.opensearch.action.support.ActionFilters -import org.opensearch.action.support.HandledTransportAction -import org.opensearch.alerting.actionv2.GetAlertsV2Action -import org.opensearch.alerting.actionv2.GetAlertsV2Request -import org.opensearch.alerting.actionv2.GetAlertsV2Response -import org.opensearch.alerting.alertsv2.AlertV2Indices -import org.opensearch.alerting.core.settings.AlertingV2Settings.Companion.ALERTING_V2_ENABLED -import org.opensearch.alerting.modelv2.AlertV2 -import org.opensearch.alerting.modelv2.AlertV2.Companion.MONITOR_V2_ID_FIELD -import org.opensearch.alerting.modelv2.AlertV2.Companion.MONITOR_V2_NAME_FIELD -import org.opensearch.alerting.modelv2.AlertV2.Companion.MONITOR_V2_USER_FIELD -import org.opensearch.alerting.modelv2.AlertV2.Companion.SEVERITY_FIELD -import org.opensearch.alerting.modelv2.AlertV2.Companion.TRIGGER_V2_NAME_FIELD -import org.opensearch.alerting.opensearchapi.addFilter -import org.opensearch.alerting.settings.AlertingSettings -import org.opensearch.alerting.transport.SecureTransportAction -import org.opensearch.alerting.util.use -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.inject.Inject -import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.LoggingDeprecationHandler -import org.opensearch.common.xcontent.XContentHelper -import org.opensearch.common.xcontent.XContentType -import org.opensearch.commons.alerting.util.AlertingException -import org.opensearch.commons.authuser.User -import org.opensearch.commons.authuser.User.BACKEND_ROLES_FIELD -import org.opensearch.core.action.ActionListener -import org.opensearch.core.common.io.stream.NamedWriteableRegistry -import org.opensearch.core.rest.RestStatus -import org.opensearch.core.xcontent.NamedXContentRegistry -import org.opensearch.index.query.Operator -import org.opensearch.index.query.QueryBuilders -import org.opensearch.search.builder.SearchSourceBuilder -import org.opensearch.search.sort.SortBuilders -import org.opensearch.search.sort.SortOrder -import org.opensearch.tasks.Task -import org.opensearch.transport.TransportService -import org.opensearch.transport.client.Client -import java.io.IOException - -private val log = LogManager.getLogger(TransportGetAlertsV2Action::class.java) -private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) - -/** - * Transport action that contains the core logic for retrieving v2 alerts. - * - * @opensearch.experimental - */ -class TransportGetAlertsV2Action @Inject constructor( - transportService: TransportService, - val client: Client, - clusterService: ClusterService, - actionFilters: ActionFilters, - val settings: Settings, - val xContentRegistry: NamedXContentRegistry, - val namedWriteableRegistry: NamedWriteableRegistry -) : HandledTransportAction( - GetAlertsV2Action.NAME, - transportService, - actionFilters, - ::GetAlertsV2Request -), - SecureTransportAction { - - @Volatile private var alertingV2Enabled = ALERTING_V2_ENABLED.get(settings) - - @Volatile - override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) - - init { - clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_ENABLED) { alertingV2Enabled = it } - listenFilterBySettingChange(clusterService) - } - - override fun doExecute( - task: Task, - getAlertsV2Request: GetAlertsV2Request, - actionListener: ActionListener, - ) { - if (!alertingV2Enabled) { - actionListener.onFailure( - AlertingException.wrap( - OpenSearchStatusException( - "Alerting V2 is currently disabled, please enable it with the " + - "cluster setting: ${ALERTING_V2_ENABLED.key}", - RestStatus.FORBIDDEN - ), - ) - ) - return - } - - val user = readUserFromThreadContext(client) - - val tableProp = getAlertsV2Request.table - val sortBuilder = SortBuilders - .fieldSort(tableProp.sortString) - .order(SortOrder.fromString(tableProp.sortOrder)) - if (!tableProp.missing.isNullOrBlank()) { - sortBuilder.missing(tableProp.missing) - } - - val queryBuilder = QueryBuilders.boolQuery() - - if (getAlertsV2Request.severityLevel != "ALL") { - queryBuilder.filter(QueryBuilders.termQuery(SEVERITY_FIELD, getAlertsV2Request.severityLevel)) - } - - if (!getAlertsV2Request.monitorV2Ids.isNullOrEmpty()) { - queryBuilder.filter(QueryBuilders.termsQuery(MONITOR_V2_ID_FIELD, getAlertsV2Request.monitorV2Ids)) - } - - if (!tableProp.searchString.isNullOrBlank()) { - queryBuilder - .must( - QueryBuilders - .queryStringQuery(tableProp.searchString) - .defaultOperator(Operator.AND) - .field(MONITOR_V2_NAME_FIELD) - .field(TRIGGER_V2_NAME_FIELD) - ) - } - val searchSourceBuilder = SearchSourceBuilder() - .version(true) - .seqNoAndPrimaryTerm(true) - .query(queryBuilder) - .sort(sortBuilder) - .size(tableProp.size) - .from(tableProp.startIndex) - - client.threadPool().threadContext.stashContext().use { - scope.launch { - try { - getAlerts(AlertV2Indices.ALERT_V2_INDEX, searchSourceBuilder, actionListener, user) - } catch (t: Exception) { - log.error("Failed to get alerts", t) - if (t is AlertingException) { - actionListener.onFailure(t) - } else { - actionListener.onFailure(AlertingException.wrap(t)) - } - } - } - } - } - - fun getAlerts( - alertIndex: String, - searchSourceBuilder: SearchSourceBuilder, - actionListener: ActionListener, - user: User? - ) { - try { - // if user is null, security plugin is disabled or user is super-admin - // if doFilterForUser() is false, security is enabled but filterby is disabled - if (user != null && doFilterForUser(user)) { - // if security is enabled and filterby is enabled, add search filter - log.info("Filtering result by: ${user.backendRoles}") - addFilter(user, searchSourceBuilder, "$MONITOR_V2_USER_FIELD.$BACKEND_ROLES_FIELD.keyword") - } - - search(alertIndex, searchSourceBuilder, actionListener) - } catch (ex: IOException) { - actionListener.onFailure(AlertingException.wrap(ex)) - } - } - - fun search(alertIndex: String, searchSourceBuilder: SearchSourceBuilder, actionListener: ActionListener) { - val searchRequest = SearchRequest() - .indices(alertIndex) - .source(searchSourceBuilder) - - client.search( - searchRequest, - object : ActionListener { - override fun onResponse(response: SearchResponse) { - val totalAlertCount = response.hits.totalHits?.value?.toInt() - val alerts = response.hits.map { hit -> - val xcp = XContentHelper.createParser( - xContentRegistry, - LoggingDeprecationHandler.INSTANCE, - hit.sourceRef, - XContentType.JSON - ) - val alertV2 = AlertV2.parse(xcp, hit.id, hit.version) - alertV2 - } - actionListener.onResponse(GetAlertsV2Response(alerts, totalAlertCount)) - } - - override fun onFailure(t: Exception) { - actionListener.onFailure(t) - } - } - ) - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetMonitorV2Action.kt deleted file mode 100644 index bc10421c7..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetMonitorV2Action.kt +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.transportv2 - -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import org.apache.logging.log4j.LogManager -import org.opensearch.OpenSearchStatusException -import org.opensearch.action.get.GetRequest -import org.opensearch.action.get.GetResponse -import org.opensearch.action.support.ActionFilters -import org.opensearch.action.support.HandledTransportAction -import org.opensearch.alerting.AlertingV2Utils.isIndexNotFoundException -import org.opensearch.alerting.AlertingV2Utils.validateMonitorV2 -import org.opensearch.alerting.actionv2.GetMonitorV2Action -import org.opensearch.alerting.actionv2.GetMonitorV2Request -import org.opensearch.alerting.actionv2.GetMonitorV2Response -import org.opensearch.alerting.core.settings.AlertingV2Settings.Companion.ALERTING_V2_ENABLED -import org.opensearch.alerting.modelv2.MonitorV2 -import org.opensearch.alerting.settings.AlertingSettings -import org.opensearch.alerting.transport.SecureTransportAction -import org.opensearch.alerting.transport.TransportGetMonitorAction -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.inject.Inject -import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.LoggingDeprecationHandler -import org.opensearch.common.xcontent.XContentHelper -import org.opensearch.common.xcontent.XContentType -import org.opensearch.commons.alerting.model.ScheduledJob -import org.opensearch.commons.alerting.util.AlertingException -import org.opensearch.core.action.ActionListener -import org.opensearch.core.rest.RestStatus -import org.opensearch.core.xcontent.NamedXContentRegistry -import org.opensearch.tasks.Task -import org.opensearch.transport.TransportService -import org.opensearch.transport.client.Client - -private val log = LogManager.getLogger(TransportGetMonitorAction::class.java) -private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) - -/** - * Transport action that contains the core logic for getting a monitor v2 by its ID. - * - * @opensearch.experimental - */ -class TransportGetMonitorV2Action @Inject constructor( - transportService: TransportService, - val client: Client, - actionFilters: ActionFilters, - val xContentRegistry: NamedXContentRegistry, - val clusterService: ClusterService, - settings: Settings, -) : HandledTransportAction( - GetMonitorV2Action.NAME, - transportService, - actionFilters, - ::GetMonitorV2Request -), - SecureTransportAction { - - @Volatile private var alertingV2Enabled = ALERTING_V2_ENABLED.get(settings) - - @Volatile override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) - - init { - clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_ENABLED) { alertingV2Enabled = it } - listenFilterBySettingChange(clusterService) - } - - override fun doExecute(task: Task, request: GetMonitorV2Request, actionListener: ActionListener) { - if (!alertingV2Enabled) { - actionListener.onFailure( - AlertingException.wrap( - OpenSearchStatusException( - "Alerting V2 is currently disabled, please enable it with the " + - "cluster setting: ${ALERTING_V2_ENABLED.key}", - RestStatus.FORBIDDEN - ), - ) - ) - return - } - - val getRequest = GetRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, request.monitorV2Id) - .version(request.version) - .fetchSourceContext(request.srcContext) - - val user = readUserFromThreadContext(client) - - if (!validateUserBackendRoles(user, actionListener)) { - return - } - - client.threadPool().threadContext.stashContext().use { - client.get( - getRequest, - object : ActionListener { - override fun onResponse(response: GetResponse) { - if (!response.isExists) { - actionListener.onFailure( - AlertingException.wrap(OpenSearchStatusException("MonitorV2 not found.", RestStatus.NOT_FOUND)) - ) - return - } - - if (response.isSourceEmpty) { - actionListener.onFailure( - AlertingException.wrap(OpenSearchStatusException("MonitorV2 found but was empty.", RestStatus.NO_CONTENT)) - ) - return - } - - val xcp = XContentHelper.createParser( - xContentRegistry, - LoggingDeprecationHandler.INSTANCE, - response.sourceAsBytesRef, - XContentType.JSON - ) - - val scheduledJob = ScheduledJob.parse(xcp, response.id, response.version) - - validateMonitorV2(scheduledJob)?.let { - actionListener.onFailure(AlertingException.wrap(it)) - return - } - - val monitorV2 = scheduledJob as MonitorV2 - - // security is enabled and filterby is enabled - if (!checkUserPermissionsWithResource( - user, - monitorV2.user, - actionListener, - "monitor", - request.monitorV2Id - ) - ) { - return - } - - actionListener.onResponse( - GetMonitorV2Response( - response.id, - response.version, - response.seqNo, - response.primaryTerm, - monitorV2 - ) - ) - } - - override fun onFailure(e: Exception) { - if (isIndexNotFoundException(e)) { - log.error("Index not found while getting monitor V2", e) - actionListener.onFailure( - AlertingException.wrap( - OpenSearchStatusException("Monitor V2 not found. Backing index is missing.", RestStatus.NOT_FOUND, e) - ) - ) - } else { - log.error("Unexpected error while getting monitor", e) - actionListener.onFailure(AlertingException.wrap(e)) - } - } - } - ) - } - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportIndexMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportIndexMonitorV2Action.kt deleted file mode 100644 index e12637fc6..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportIndexMonitorV2Action.kt +++ /dev/null @@ -1,868 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.transportv2 - -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch -import kotlinx.coroutines.newSingleThreadContext -import kotlinx.coroutines.withContext -import org.apache.logging.log4j.LogManager -import org.opensearch.ExceptionsHelper -import org.opensearch.OpenSearchException -import org.opensearch.OpenSearchStatusException -import org.opensearch.ResourceAlreadyExistsException -import org.opensearch.action.admin.cluster.health.ClusterHealthAction -import org.opensearch.action.admin.cluster.health.ClusterHealthRequest -import org.opensearch.action.admin.cluster.health.ClusterHealthResponse -import org.opensearch.action.admin.indices.create.CreateIndexResponse -import org.opensearch.action.admin.indices.mapping.get.GetMappingsRequest -import org.opensearch.action.get.GetRequest -import org.opensearch.action.get.GetResponse -import org.opensearch.action.index.IndexRequest -import org.opensearch.action.index.IndexResponse -import org.opensearch.action.search.SearchRequest -import org.opensearch.action.search.SearchResponse -import org.opensearch.action.support.ActionFilters -import org.opensearch.action.support.HandledTransportAction -import org.opensearch.action.support.clustermanager.AcknowledgedResponse -import org.opensearch.alerting.AlertingV2Utils.validateMonitorV2 -import org.opensearch.alerting.PPLUtils.appendCustomCondition -import org.opensearch.alerting.PPLUtils.executePplQuery -import org.opensearch.alerting.PPLUtils.findEvalResultVar -import org.opensearch.alerting.PPLUtils.findEvalResultVarIdxInSchema -import org.opensearch.alerting.PPLUtils.getIndicesFromPplQuery -import org.opensearch.alerting.actionv2.IndexMonitorV2Action -import org.opensearch.alerting.actionv2.IndexMonitorV2Request -import org.opensearch.alerting.actionv2.IndexMonitorV2Response -import org.opensearch.alerting.core.ScheduledJobIndices -import org.opensearch.alerting.core.settings.AlertingV2Settings.Companion.ALERTING_V2_ENABLED -import org.opensearch.alerting.modelv2.MonitorV2 -import org.opensearch.alerting.modelv2.MonitorV2.Companion.MONITOR_V2_TYPE -import org.opensearch.alerting.modelv2.PPLSQLMonitor -import org.opensearch.alerting.modelv2.PPLSQLTrigger.ConditionType -import org.opensearch.alerting.opensearchapi.suspendUntil -import org.opensearch.alerting.settings.AlertingSettings -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_EXPIRE_DURATION -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_LOOK_BACK_WINDOW -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_MONITORS -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_QUERY_LENGTH -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_THROTTLE_DURATION -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS -import org.opensearch.alerting.settings.AlertingSettings.Companion.INDEX_TIMEOUT -import org.opensearch.alerting.settings.AlertingSettings.Companion.NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH -import org.opensearch.alerting.settings.AlertingSettings.Companion.NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH -import org.opensearch.alerting.settings.AlertingSettings.Companion.REQUEST_TIMEOUT -import org.opensearch.alerting.transport.SecureTransportAction -import org.opensearch.alerting.util.IndexUtils -import org.opensearch.alerting.util.use -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.inject.Inject -import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.LoggingDeprecationHandler -import org.opensearch.common.xcontent.XContentFactory.jsonBuilder -import org.opensearch.common.xcontent.XContentHelper -import org.opensearch.common.xcontent.XContentType -import org.opensearch.commons.alerting.model.ScheduledJob -import org.opensearch.commons.alerting.model.ScheduledJob.Companion.SCHEDULED_JOBS_INDEX -import org.opensearch.commons.alerting.model.userErrorMessage -import org.opensearch.commons.alerting.util.AlertingException -import org.opensearch.commons.authuser.User -import org.opensearch.core.action.ActionListener -import org.opensearch.core.common.io.stream.NamedWriteableRegistry -import org.opensearch.core.rest.RestStatus -import org.opensearch.core.xcontent.NamedXContentRegistry -import org.opensearch.core.xcontent.ToXContent -import org.opensearch.index.query.QueryBuilders -import org.opensearch.rest.RestRequest -import org.opensearch.search.builder.SearchSourceBuilder -import org.opensearch.tasks.Task -import org.opensearch.transport.TransportService -import org.opensearch.transport.client.Client - -private val log = LogManager.getLogger(TransportIndexMonitorV2Action::class.java) -private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) - -/** - * Transport action that contains the core logic for creating and updating v2 monitors. - * - * @opensearch.experimental - */ -class TransportIndexMonitorV2Action @Inject constructor( - val transportService: TransportService, - val client: Client, - actionFilters: ActionFilters, - val scheduledJobIndices: ScheduledJobIndices, - val clusterService: ClusterService, - val settings: Settings, - val xContentRegistry: NamedXContentRegistry, - val namedWriteableRegistry: NamedWriteableRegistry, -) : HandledTransportAction( - IndexMonitorV2Action.NAME, transportService, actionFilters, ::IndexMonitorV2Request -), - SecureTransportAction { - - // adjustable limits (via settings) - @Volatile private var alertingV2Enabled = ALERTING_V2_ENABLED.get(settings) - @Volatile private var maxMonitors = ALERTING_V2_MAX_MONITORS.get(settings) - @Volatile private var maxThrottleDuration = ALERTING_V2_MAX_THROTTLE_DURATION.get(settings) - @Volatile private var maxExpireDuration = ALERTING_V2_MAX_EXPIRE_DURATION.get(settings) - @Volatile private var maxLookBackWindow = ALERTING_V2_MAX_LOOK_BACK_WINDOW.get(settings) - @Volatile private var maxQueryLength = ALERTING_V2_MAX_QUERY_LENGTH.get(settings) - @Volatile private var maxQueryResults = ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS.get(settings) - @Volatile private var notificationSubjectMaxLength = NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH.get(settings) - @Volatile private var notificationMessageMaxLength = NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH.get(settings) - @Volatile private var requestTimeout = REQUEST_TIMEOUT.get(settings) - @Volatile private var indexTimeout = INDEX_TIMEOUT.get(settings) - @Volatile override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) - - init { - clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_ENABLED) { alertingV2Enabled = it } - clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_MAX_MONITORS) { maxMonitors = it } - clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_MAX_THROTTLE_DURATION) { maxThrottleDuration = it } - clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_MAX_EXPIRE_DURATION) { maxExpireDuration = it } - clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_MAX_LOOK_BACK_WINDOW) { maxLookBackWindow = it } - clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_MAX_QUERY_LENGTH) { maxQueryLength = it } - clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS) { maxQueryResults = it } - clusterService.clusterSettings.addSettingsUpdateConsumer(NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH) { - notificationSubjectMaxLength = it - } - clusterService.clusterSettings.addSettingsUpdateConsumer(NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH) { - notificationMessageMaxLength = it - } - clusterService.clusterSettings.addSettingsUpdateConsumer(REQUEST_TIMEOUT) { requestTimeout = it } - clusterService.clusterSettings.addSettingsUpdateConsumer(INDEX_TIMEOUT) { indexTimeout = it } - listenFilterBySettingChange(clusterService) - } - - override fun doExecute( - task: Task, - indexMonitorV2Request: IndexMonitorV2Request, - actionListener: ActionListener - ) { - if (!alertingV2Enabled) { - actionListener.onFailure( - AlertingException.wrap( - OpenSearchStatusException( - "Alerting V2 is currently disabled, please enable it with the " + - "cluster setting: ${ALERTING_V2_ENABLED.key}", - RestStatus.FORBIDDEN - ), - ) - ) - return - } - - // read the user from thread context immediately, before - // downstream flows spin up new threads with fresh context - val user = readUserFromThreadContext(client) - - // validate the MonitorV2 based on its type - when (indexMonitorV2Request.monitorV2) { - is PPLSQLMonitor -> validatePplSqlMonitorUserPermissionsAndQuery( - indexMonitorV2Request, - user, - object : ActionListener { // validationListener - override fun onResponse(response: Unit) { - // user permissions to indices have already been checked - // proceed without the context of the user, otherwise, - // we would get permissions errors trying to search the alerting-config - // index as the user. pass the user object itself so backend - // roles can be matched and checked downstream - client.threadPool().threadContext.stashContext().use { - val pplSqlMonitor = indexMonitorV2Request.monitorV2 as PPLSQLMonitor - if (user == null) { - indexMonitorV2Request.monitorV2 = pplSqlMonitor - .copy(user = User("", listOf(), listOf(), mapOf())) - } else { - indexMonitorV2Request.monitorV2 = pplSqlMonitor - .copy(user = User(user.name, user.backendRoles, user.roles, user.customAttributes)) - } - checkScheduledJobIndex(indexMonitorV2Request, actionListener, user) - } - } - - override fun onFailure(e: Exception) { - actionListener.onFailure(e) - } - } - ) - else -> actionListener.onFailure( - AlertingException.wrap( - IllegalStateException( - "unexpected MonitorV2 type: ${indexMonitorV2Request.monitorV2.javaClass.name}" - ) - ) - ) - } - } - - // validates the PPL Monitor, its query, and user's permissions to the indices it queries by submitting it to SQL/PPL plugin - private fun validatePplSqlMonitorUserPermissionsAndQuery( - indexMonitorV2Request: IndexMonitorV2Request, - user: User?, - validationListener: ActionListener - ) { - client.threadPool().threadContext.stashContext().use { - scope.launch { - val singleThreadContext = newSingleThreadContext("IndexMonitorV2ActionThread") - withContext(singleThreadContext) { - it.restore() - - val pplSqlMonitor = indexMonitorV2Request.monitorV2 as PPLSQLMonitor - - val pplQueryValid = validatePplSqlQuery(pplSqlMonitor, validationListener) - if (!pplQueryValid) { - return@withContext - } - - // run basic validations against the PPL/SQL Monitor - val pplSqlMonitorValid = validatePplSqlMonitor(pplSqlMonitor, validationListener) - if (!pplSqlMonitorValid) { - return@withContext - } - - // check the user for basic permissions - val userHasPermissions = checkUser(user, indexMonitorV2Request, validationListener) - if (!userHasPermissions) { - return@withContext - } - - // check that given timestamp field is valid - val timestampFieldValid = checkPplQueryIndicesForTimestampField(pplSqlMonitor, validationListener) - if (!timestampFieldValid) { - return@withContext - } - - validationListener.onResponse(Unit) - } - } - } - } - - private suspend fun validatePplSqlQuery( - pplSqlMonitor: PPLSQLMonitor, - validationListener: ActionListener - ): Boolean { - // first attempt to run the monitor query and all possible - // extensions of it (from custom conditions) - try { - // first run the base query as is. - // if there are any PPL syntax or index not found or other errors, - // this will throw an exception - executePplQuery(pplSqlMonitor.query, clusterService.state().nodes.localNode, transportService) - - // now scan all the triggers with custom conditions, and ensure each query constructed - // from the base query + custom condition is valid - for (pplTrigger in pplSqlMonitor.triggers) { - if (pplTrigger.conditionType != ConditionType.CUSTOM) { - continue - } - - val evalResultVar = findEvalResultVar(pplTrigger.customCondition!!) - - val queryWithCustomCondition = appendCustomCondition(pplSqlMonitor.query, pplTrigger.customCondition!!) - - val executePplQueryResponse = executePplQuery( - queryWithCustomCondition, - clusterService.state().nodes.localNode, - transportService - ) - - val evalResultVarIdx = findEvalResultVarIdxInSchema(executePplQueryResponse, evalResultVar) - - val resultVarType = executePplQueryResponse - .getJSONArray("schema") - .getJSONObject(evalResultVarIdx) - .getString("type") - - // custom conditions must evaluate to a boolean result, otherwise it's invalid - if (resultVarType != "boolean") { - validationListener.onFailure( - AlertingException.wrap( - IllegalArgumentException( - "Custom condition in trigger ${pplTrigger.name} is invalid because it does not " + - "evaluate to a boolean, but instead to type: $resultVarType" - ) - ) - ) - return false - } - } - } catch (e: Exception) { - validationListener.onFailure( - AlertingException.wrap( - IllegalArgumentException("Validation error for PPL Query in PPL Monitor: ${e.userErrorMessage()}") - ) - ) - return false - } - - return true - } - - private fun validatePplSqlMonitor(pplSqlMonitor: PPLSQLMonitor, validationListener: ActionListener): Boolean { - // ensure the trigger throttle and expire durations are valid - pplSqlMonitor.triggers.forEach { trigger -> - trigger.throttleDuration?.let { throttleDuration -> - if (throttleDuration > maxThrottleDuration) { - validationListener.onFailure( - AlertingException.wrap( - IllegalArgumentException( - "Throttle duration must be at most $maxThrottleDuration but was $throttleDuration" - ) - ) - ) - return false - } - } - - if (trigger.expireDuration > maxExpireDuration) { - validationListener.onFailure( - AlertingException.wrap( - IllegalArgumentException( - "Expire duration must be at most $maxExpireDuration but was ${trigger.expireDuration}" - ) - ) - ) - return false - } - - if (trigger.conditionType == ConditionType.NUMBER_OF_RESULTS && - trigger.numResultsValue!! > maxQueryResults - ) { - validationListener.onFailure( - AlertingException.wrap( - IllegalArgumentException( - "Trigger ${trigger.id} checks for number of results threshold of ${trigger.numResultsValue}, " + - "but Alerting V2 is configured only to retrieve $maxQueryResults query results maximum. " + - "Please lower the number of results value to one below this maximum value, or adjust the cluster " + - "setting: $ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS.key}" - ) - ) - ) - return false - } - - trigger.actions.forEach { action -> - if (action.subjectTemplate?.idOrCode?.length!! > notificationSubjectMaxLength) { - validationListener.onFailure( - AlertingException.wrap( - IllegalArgumentException( - "Notification subject source cannot exceed length: $notificationSubjectMaxLength" - ) - ) - ) - return false - } - - if (action.messageTemplate.idOrCode.length > notificationMessageMaxLength) { - validationListener.onFailure( - AlertingException.wrap( - IllegalArgumentException( - "Notification message source cannot exceed length: $notificationMessageMaxLength" - ) - ) - ) - return false - } - } - } - - // ensure the query length doesn't exceed the limit - if (pplSqlMonitor.query.length > maxQueryLength) { - validationListener.onFailure( - AlertingException.wrap( - IllegalArgumentException( - "PPL Query length must be at most $maxQueryLength but was ${pplSqlMonitor.query.length}" - ) - ) - ) - return false - } - - // ensure the look back window doesn't exceed the limit - pplSqlMonitor.lookBackWindow?.let { - if (pplSqlMonitor.lookBackWindow > maxLookBackWindow) { - validationListener.onFailure( - AlertingException.wrap( - IllegalArgumentException( - "Look back window must be at most $maxLookBackWindow minutes but was ${pplSqlMonitor.lookBackWindow}" - ) - ) - ) - return false - } - } - - return true - } - - private fun checkUser( - user: User?, - indexMonitorV2Request: IndexMonitorV2Request, - validationListener: ActionListener - ): Boolean { - /* check initial user permissions */ - if (!validateUserBackendRoles(user, validationListener)) { - return false - } - - if ( - user != null && - !isAdmin(user) && - indexMonitorV2Request.rbacRoles != null - ) { - if (indexMonitorV2Request.rbacRoles.stream().anyMatch { !user.backendRoles.contains(it) }) { - log.debug( - "User specified backend roles, ${indexMonitorV2Request.rbacRoles}, " + - "that they don't have access to. User backend roles: ${user.backendRoles}" - ) - validationListener.onFailure( - AlertingException.wrap( - OpenSearchStatusException( - "User specified backend roles that they don't have access to. Contact administrator", RestStatus.FORBIDDEN - ) - ) - ) - return false - } else if (indexMonitorV2Request.rbacRoles.isEmpty()) { - log.debug( - "Non-admin user are not allowed to specify an empty set of backend roles. " + - "Please don't pass in the parameter or pass in at least one backend role." - ) - validationListener.onFailure( - AlertingException.wrap( - OpenSearchStatusException( - "Non-admin user are not allowed to specify an empty set of backend roles.", RestStatus.FORBIDDEN - ) - ) - ) - return false - } - } - - return true - } - - // if look back window is specified, all the indices that the PPL query searches - // must contain the timestamp field specified in the PPL Monitor, and they must - // all be of OpenSearch data type "date" - private suspend fun checkPplQueryIndicesForTimestampField( - pplSqlMonitor: PPLSQLMonitor, - validationListener: ActionListener - ): Boolean { - if (pplSqlMonitor.lookBackWindow == null) { - // if no look back window was specified, no need - // to check for timestamp field in PPL query indices - return true - } - - val pplQuery = pplSqlMonitor.query - val timestampField = pplSqlMonitor.timestampField - - try { - val indices = getIndicesFromPplQuery(pplQuery) - val getMappingsRequest = GetMappingsRequest().indices(*indices.toTypedArray()) - val getMappingsResponse = client.suspendUntil { admin().indices().getMappings(getMappingsRequest, it) } - - val metadataMap = getMappingsResponse.mappings - - for (index in metadataMap.keys) { - val metadata = metadataMap[index]!!.sourceAsMap["properties"] as Map - if (!metadata.keys.contains(timestampField)) { - validationListener.onFailure( - AlertingException.wrap( - IllegalArgumentException("Query index $index don't contain given timestamp field: $timestampField") - ) - ) - return false - } - val typeInfo = metadata[timestampField] as Map - val type = typeInfo["type"] - val dateType = "date" - val dateNanosType = "date_nanos" - if (type != dateType && type != dateNanosType) { - validationListener.onFailure( - AlertingException.wrap( - IllegalArgumentException( - "Timestamp field: $timestampField is present in index $index " + - "but is type $type instead of $dateType or $dateNanosType" - ) - ) - ) - return false - } - } - } catch (e: Exception) { - log.error("failed to read query indices' fields when checking for timestamp field: $timestampField") - validationListener.onFailure( - AlertingException.wrap( - IllegalArgumentException("failed to read query indices' fields when checking for timestamp field: $timestampField", e) - ) - ) - return false - } - - return true - } - - private fun checkScheduledJobIndex( - indexMonitorRequest: IndexMonitorV2Request, - actionListener: ActionListener, - user: User? - ) { - // user permissions to indices have already been checked - // proceed without the context of the user, otherwise, - // we would get permissions errors trying to search the alerting-config - // index as the user - client.threadPool().threadContext.stashContext().use { - /* check to see if alerting-config index (scheduled job index) is created and updated before indexing MonitorV2 into it */ - if (!scheduledJobIndices.scheduledJobIndexExists()) { // if alerting-config index doesn't exist, send request to create it - scheduledJobIndices.initScheduledJobIndex(object : ActionListener { - override fun onResponse(response: CreateIndexResponse) { - onCreateMappingsResponse(response.isAcknowledged, indexMonitorRequest, actionListener, user) - } - - override fun onFailure(e: Exception) { - if (ExceptionsHelper.unwrapCause(e) is ResourceAlreadyExistsException) { - scope.launch { - // Wait for the yellow status - val clusterHealthRequest = ClusterHealthRequest() - .indices(SCHEDULED_JOBS_INDEX) - .waitForYellowStatus() - val response: ClusterHealthResponse = client.suspendUntil { - execute(ClusterHealthAction.INSTANCE, clusterHealthRequest, it) - } - if (response.isTimedOut) { - actionListener.onFailure( - OpenSearchException("Cannot determine that the $SCHEDULED_JOBS_INDEX index is healthy") - ) - } - // Retry mapping of monitor - onCreateMappingsResponse(true, indexMonitorRequest, actionListener, user) - } - } else { - actionListener.onFailure(AlertingException.wrap(e)) - } - } - }) - } else if (!IndexUtils.scheduledJobIndexUpdated) { - IndexUtils.updateIndexMapping( - SCHEDULED_JOBS_INDEX, - ScheduledJobIndices.scheduledJobMappings(), clusterService.state(), client.admin().indices(), - object : ActionListener { - override fun onResponse(response: AcknowledgedResponse) { - onUpdateMappingsResponse(response, indexMonitorRequest, actionListener, user) - } - override fun onFailure(t: Exception) { - actionListener.onFailure(AlertingException.wrap(t)) - } - } - ) - } else { - prepareMonitorIndexing(indexMonitorRequest, actionListener, user) - } - } - } - - private fun onCreateMappingsResponse( - isAcknowledged: Boolean, - request: IndexMonitorV2Request, - actionListener: ActionListener, - user: User? - ) { - if (isAcknowledged) { - log.info("Created $SCHEDULED_JOBS_INDEX with mappings.") - prepareMonitorIndexing(request, actionListener, user) - IndexUtils.scheduledJobIndexUpdated() - } else { - log.info("Create $SCHEDULED_JOBS_INDEX mappings call not acknowledged.") - actionListener.onFailure( - AlertingException.wrap( - OpenSearchStatusException( - "Create $SCHEDULED_JOBS_INDEX mappings call not acknowledged", RestStatus.INTERNAL_SERVER_ERROR - ) - ) - ) - } - } - - private fun onUpdateMappingsResponse( - response: AcknowledgedResponse, - indexMonitorRequest: IndexMonitorV2Request, - actionListener: ActionListener, - user: User? - ) { - if (response.isAcknowledged) { - log.info("Updated $SCHEDULED_JOBS_INDEX with mappings.") - IndexUtils.scheduledJobIndexUpdated() - prepareMonitorIndexing(indexMonitorRequest, actionListener, user) - } else { - log.info("Update $SCHEDULED_JOBS_INDEX mappings call not acknowledged.") - actionListener.onFailure( - AlertingException.wrap( - OpenSearchStatusException( - "Updated $SCHEDULED_JOBS_INDEX mappings call not acknowledged.", - RestStatus.INTERNAL_SERVER_ERROR - ) - ) - ) - } - } - - private fun prepareMonitorIndexing( - indexMonitorRequest: IndexMonitorV2Request, - actionListener: ActionListener, - user: User? - ) { - if (indexMonitorRequest.method == RestRequest.Method.PUT) { // update monitor case - scope.launch { - updateMonitor(indexMonitorRequest, actionListener, user) - } - } else { // create monitor case - val query = QueryBuilders.boolQuery().filter(QueryBuilders.existsQuery(MONITOR_V2_TYPE)) - val searchSource = SearchSourceBuilder().query(query).timeout(requestTimeout) - val searchRequest = SearchRequest(SCHEDULED_JOBS_INDEX).source(searchSource) - - client.search( - searchRequest, - object : ActionListener { - override fun onResponse(searchResponse: SearchResponse) { - onMonitorCountSearchResponse(searchResponse, indexMonitorRequest, actionListener, user) - } - - override fun onFailure(t: Exception) { - actionListener.onFailure(AlertingException.wrap(t)) - } - } - ) - } - } - - /* Functions for Update Monitor flow */ - - private suspend fun updateMonitor( - indexMonitorRequest: IndexMonitorV2Request, - actionListener: ActionListener, - user: User? - ) { - val getRequest = GetRequest(SCHEDULED_JOBS_INDEX, indexMonitorRequest.monitorId) - try { - val getResponse: GetResponse = client.suspendUntil { client.get(getRequest, it) } - if (!getResponse.isExists) { - actionListener.onFailure( - AlertingException.wrap( - OpenSearchStatusException("MonitorV2 with ${indexMonitorRequest.monitorId} is not found", RestStatus.NOT_FOUND) - ) - ) - return - } - val xcp = XContentHelper.createParser( - xContentRegistry, LoggingDeprecationHandler.INSTANCE, - getResponse.sourceAsBytesRef, XContentType.JSON - ) - val scheduledJob = ScheduledJob.parse(xcp, getResponse.id, getResponse.version) - - validateMonitorV2(scheduledJob)?.let { - actionListener.onFailure(AlertingException.wrap(it)) - return - } - - val monitorV2 = scheduledJob as MonitorV2 - - onGetMonitorResponseForUpdate(monitorV2, indexMonitorRequest, actionListener, user) - } catch (e: Exception) { - actionListener.onFailure(AlertingException.wrap(e)) - } - } - - private suspend fun onGetMonitorResponseForUpdate( - existingMonitorV2: MonitorV2, - indexMonitorRequest: IndexMonitorV2Request, - actionListener: ActionListener, - user: User? - ) { - log.info("user: $user") - log.info("monitor user: ${existingMonitorV2.user}") - if ( - !checkUserPermissionsWithResource( - user, - existingMonitorV2.user, - actionListener, - "monitor_v2", - indexMonitorRequest.monitorId - ) - ) { - return - } - - var newMonitorV2 = indexMonitorRequest.monitorV2 - - // If both are enabled, use the current existing monitor enabled time, - // otherwise the next execution will be incorrect. - if (newMonitorV2.enabled && existingMonitorV2.enabled) { - newMonitorV2 = newMonitorV2.makeCopy(enabledTime = existingMonitorV2.enabledTime) - } - - /** - * On update monitor check which backend roles to associate to the monitor. - * Below are 2 examples of how the logic works - * - * Example 1, say we have a Monitor with backend roles [a, b, c, d] associated with it. - * If I'm User A (non-admin user) and I have backend roles [a, b, c] associated with me and I make a request to update - * the Monitor's backend roles to [a, b]. This would mean that the roles to remove are [c] and the roles to add are [a, b]. - * The Monitor's backend roles would then be [a, b, d]. - * - * Example 2, say we have a Monitor with backend roles [a, b, c, d] associated with it. - * If I'm User A (admin user) and I have backend roles [a, b, c] associated with me and I make a request to update - * the Monitor's backend roles to [a, b]. This would mean that the roles to remove are [c, d] and the roles to add are [a, b]. - * The Monitor's backend roles would then be [a, b]. - */ - if (user != null) { - if (indexMonitorRequest.rbacRoles != null) { - if (isAdmin(user)) { - newMonitorV2 = newMonitorV2.makeCopy( - user = User(user.name, indexMonitorRequest.rbacRoles, user.roles, user.customAttributes) - ) - } else { - // rolesToRemove: these are the backend roles to remove from the monitor - val rolesToRemove = user.backendRoles - indexMonitorRequest.rbacRoles - // remove the monitor's roles with rolesToRemove and add any roles passed into the request.rbacRoles - val updatedRbac = existingMonitorV2.user?.backendRoles.orEmpty() - rolesToRemove + indexMonitorRequest.rbacRoles - newMonitorV2 = newMonitorV2.makeCopy( - user = User(user.name, updatedRbac, user.roles, user.customAttributes) - ) - } - } else { - newMonitorV2 = newMonitorV2 - .makeCopy(user = User(user.name, existingMonitorV2.user!!.backendRoles, user.roles, user.customAttributes)) - } - log.info("Update monitor backend roles to: ${newMonitorV2.user?.backendRoles}") - } - - newMonitorV2 = newMonitorV2.makeCopy(schemaVersion = IndexUtils.scheduledJobIndexSchemaVersion) - val indexRequest = IndexRequest(SCHEDULED_JOBS_INDEX) - .setRefreshPolicy(indexMonitorRequest.refreshPolicy) - .source(newMonitorV2.toXContentWithUser(jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - .id(indexMonitorRequest.monitorId) - .routing(indexMonitorRequest.monitorId) - .timeout(indexTimeout) - - log.info( - "Updating monitor, ${existingMonitorV2.id}, from: ${existingMonitorV2.toXContentWithUser( - jsonBuilder(), - ToXContent.MapParams(mapOf("with_type" to "true")) - )} \n to: ${newMonitorV2.toXContentWithUser(jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))}" - ) - - try { - val indexResponse: IndexResponse = client.suspendUntil { client.index(indexRequest, it) } - val failureReasons = IndexUtils.checkShardsFailure(indexResponse) - if (failureReasons != null) { - actionListener.onFailure( - AlertingException.wrap(OpenSearchStatusException(failureReasons.toString(), indexResponse.status())) - ) - return - } - - actionListener.onResponse( - IndexMonitorV2Response( - indexResponse.id, indexResponse.version, indexResponse.seqNo, - indexResponse.primaryTerm, newMonitorV2 - ) - ) - } catch (e: Exception) { - actionListener.onFailure(AlertingException.wrap(e)) - } - } - - /* Functions for Create Monitor flow */ - - /** - * After searching for all existing monitors we validate the system can support another monitor to be created. - */ - private fun onMonitorCountSearchResponse( - monitorCountSearchResponse: SearchResponse, - indexMonitorRequest: IndexMonitorV2Request, - actionListener: ActionListener, - user: User? - ) { - val totalHits = monitorCountSearchResponse.hits.totalHits?.value - if (totalHits != null && totalHits >= maxMonitors) { - log.info("This request would create more than the allowed monitors [$maxMonitors].") - actionListener.onFailure( - AlertingException.wrap( - IllegalArgumentException( - "This request would create more than the allowed monitors [$maxMonitors]." - ) - ) - ) - } else { - scope.launch { - indexMonitor(indexMonitorRequest, actionListener, user) - } - } - } - - private suspend fun indexMonitor( - indexMonitorRequest: IndexMonitorV2Request, - actionListener: ActionListener, - user: User? - ) { - var monitorV2 = indexMonitorRequest.monitorV2 - - if (user != null) { - // Use the backend roles which is an intersection of the requested backend roles and the user's backend roles. - // Admins can pass in any backend role. Also if no backend role is passed in, all the user's backend roles are used. - val rbacRoles = if (indexMonitorRequest.rbacRoles == null) user.backendRoles.toSet() - else if (!isAdmin(user)) indexMonitorRequest.rbacRoles.intersect(user.backendRoles).toSet() - else indexMonitorRequest.rbacRoles - - monitorV2 = monitorV2.makeCopy( - user = User(user.name, rbacRoles.toList(), user.roles, user.customAttributes) - ) - - log.debug("Created monitor's backend roles: $rbacRoles") - } - - val indexRequest = IndexRequest(SCHEDULED_JOBS_INDEX) - .setRefreshPolicy(indexMonitorRequest.refreshPolicy) - .source(monitorV2.toXContentWithUser(jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - .routing(indexMonitorRequest.monitorId) - .setIfSeqNo(indexMonitorRequest.seqNo) - .setIfPrimaryTerm(indexMonitorRequest.primaryTerm) - .timeout(indexTimeout) - - log.info( - "Creating new monitorV2: ${monitorV2.toXContentWithUser( - jsonBuilder(), - ToXContent.MapParams(mapOf("with_type" to "true")) - )}" - ) - - try { - val indexResponse: IndexResponse = client.suspendUntil { client.index(indexRequest, it) } - val failureReasons = IndexUtils.checkShardsFailure(indexResponse) - if (failureReasons != null) { - log.info(failureReasons.toString()) - actionListener.onFailure( - AlertingException.wrap(OpenSearchStatusException(failureReasons.toString(), indexResponse.status())) - ) - return - } - - actionListener.onResponse( - IndexMonitorV2Response( - indexResponse.id, indexResponse.version, indexResponse.seqNo, - indexResponse.primaryTerm, monitorV2 - ) - ) - } catch (t: Exception) { - actionListener.onFailure(AlertingException.wrap(t)) - } - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportSearchMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportSearchMonitorV2Action.kt deleted file mode 100644 index 6a36e2874..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportSearchMonitorV2Action.kt +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.transportv2 - -import org.apache.logging.log4j.LogManager -import org.opensearch.OpenSearchStatusException -import org.opensearch.action.search.SearchResponse -import org.opensearch.action.support.ActionFilters -import org.opensearch.action.support.HandledTransportAction -import org.opensearch.alerting.AlertingV2Utils.getEmptySearchResponse -import org.opensearch.alerting.AlertingV2Utils.isIndexNotFoundException -import org.opensearch.alerting.actionv2.SearchMonitorV2Action -import org.opensearch.alerting.actionv2.SearchMonitorV2Request -import org.opensearch.alerting.core.settings.AlertingV2Settings.Companion.ALERTING_V2_ENABLED -import org.opensearch.alerting.modelv2.MonitorV2.Companion.MONITOR_V2_TYPE -import org.opensearch.alerting.modelv2.PPLSQLMonitor.Companion.PPL_SQL_MONITOR_TYPE -import org.opensearch.alerting.opensearchapi.addFilter -import org.opensearch.alerting.settings.AlertingSettings -import org.opensearch.alerting.transport.SecureTransportAction -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.inject.Inject -import org.opensearch.common.settings.Settings -import org.opensearch.commons.alerting.util.AlertingException -import org.opensearch.core.action.ActionListener -import org.opensearch.core.common.io.stream.NamedWriteableRegistry -import org.opensearch.core.rest.RestStatus -import org.opensearch.index.query.BoolQueryBuilder -import org.opensearch.index.query.QueryBuilders -import org.opensearch.tasks.Task -import org.opensearch.transport.TransportService -import org.opensearch.transport.client.Client - -private val log = LogManager.getLogger(TransportSearchMonitorV2Action::class.java) - -/** - * Transport action that contains the core logic for searching monitor V2s via an OpenSearch search query. - * - * @opensearch.experimental - */ -class TransportSearchMonitorV2Action @Inject constructor( - transportService: TransportService, - val settings: Settings, - val client: Client, - clusterService: ClusterService, - actionFilters: ActionFilters, - val namedWriteableRegistry: NamedWriteableRegistry -) : HandledTransportAction( - SearchMonitorV2Action.NAME, transportService, actionFilters, ::SearchMonitorV2Request -), - SecureTransportAction { - - @Volatile private var alertingV2Enabled = ALERTING_V2_ENABLED.get(settings) - - @Volatile - override var filterByEnabled: Boolean = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) - - init { - clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_ENABLED) { alertingV2Enabled = it } - listenFilterBySettingChange(clusterService) - } - - override fun doExecute(task: Task, request: SearchMonitorV2Request, actionListener: ActionListener) { - if (!alertingV2Enabled) { - actionListener.onFailure( - AlertingException.wrap( - OpenSearchStatusException( - "Alerting V2 is currently disabled, please enable it with the " + - "cluster setting: ${ALERTING_V2_ENABLED.key}", - RestStatus.FORBIDDEN - ), - ) - ) - return - } - - val searchSourceBuilder = request.searchRequest.source() - - val queryBuilder = if (searchSourceBuilder.query() == null) BoolQueryBuilder() - else QueryBuilders.boolQuery().must(searchSourceBuilder.query()) - - // filter out MonitorV1s in the alerting config index - // only return MonitorV2s that match the user-given search query - queryBuilder.filter(QueryBuilders.existsQuery(MONITOR_V2_TYPE)) - - searchSourceBuilder.query(queryBuilder) - .seqNoAndPrimaryTerm(true) - .version(true) - - val user = readUserFromThreadContext(client) - client.threadPool().threadContext.stashContext().use { - // if user is null, security plugin is disabled or user is super-admin - // if doFilterForUser() is false, security is enabled but filterby is disabled - if (user != null && doFilterForUser(user)) { - log.info("Filtering result by: ${user.backendRoles}") - addFilter(user, request.searchRequest.source(), "$MONITOR_V2_TYPE.$PPL_SQL_MONITOR_TYPE.user.backend_roles.keyword") - } - - client.search( - request.searchRequest, - object : ActionListener { - override fun onResponse(response: SearchResponse) { - actionListener.onResponse(response) - } - - override fun onFailure(e: Exception) { - if (isIndexNotFoundException(e)) { - log.error("Index not found while searching monitor", e) - val emptyResponse = getEmptySearchResponse() - actionListener.onResponse(emptyResponse) - } else { - log.error("Unexpected error while searching monitor", e) - actionListener.onFailure(AlertingException.wrap(e)) - } - } - } - ) - } - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/AlertingUtils.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/AlertingUtils.kt index 1bb92e838..83dd8839e 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/AlertingUtils.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/AlertingUtils.kt @@ -6,13 +6,23 @@ package org.opensearch.alerting.util import org.apache.logging.log4j.LogManager +import org.opensearch.OpenSearchSecurityException import org.opensearch.alerting.AlertService +import org.opensearch.alerting.MonitorRunnerExecutionContext import org.opensearch.alerting.MonitorRunnerService +import org.opensearch.alerting.action.GetDestinationsAction +import org.opensearch.alerting.action.GetDestinationsRequest +import org.opensearch.alerting.action.GetDestinationsResponse import org.opensearch.alerting.model.AlertContext import org.opensearch.alerting.model.destination.Destination +import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.script.BucketLevelTriggerExecutionContext import org.opensearch.alerting.script.DocumentLevelTriggerExecutionContext -import org.opensearch.alerting.settings.DestinationSettings +import org.opensearch.alerting.util.destinationmigration.NotificationActionConfigs +import org.opensearch.alerting.util.destinationmigration.NotificationApiUtils.Companion.getNotificationConfigInfo +import org.opensearch.alerting.util.destinationmigration.getTitle +import org.opensearch.alerting.util.destinationmigration.publishLegacyNotification +import org.opensearch.alerting.util.destinationmigration.sendNotification import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.Settings import org.opensearch.common.util.concurrent.ThreadContext @@ -21,13 +31,18 @@ import org.opensearch.commons.alerting.model.BucketLevelTrigger import org.opensearch.commons.alerting.model.BucketLevelTriggerRunResult import org.opensearch.commons.alerting.model.DocumentLevelTrigger import org.opensearch.commons.alerting.model.Monitor +import org.opensearch.commons.alerting.model.Table import org.opensearch.commons.alerting.model.Trigger import org.opensearch.commons.alerting.model.action.Action import org.opensearch.commons.alerting.model.action.ActionExecutionPolicy import org.opensearch.commons.alerting.model.action.ActionExecutionScope import org.opensearch.commons.alerting.util.isBucketLevelMonitor import org.opensearch.commons.alerting.util.isMonitorOfStandardType +import org.opensearch.commons.notifications.model.NotificationConfigInfo +import org.opensearch.index.IndexNotFoundException import org.opensearch.script.Script +import org.opensearch.transport.RemoteTransportException +import org.opensearch.transport.client.node.NodeClient import java.util.Locale import kotlin.math.max @@ -287,3 +302,118 @@ fun printsSampleDocData(trigger: Trigger): Boolean { validTags.any { tag -> action.messageTemplate.idOrCode.contains(tag) } } } + +// Checks if the exception is caused by an IndexNotFoundException (directly or nested). +// Used in Get and Search monitor functionalities to determine whether a "no results" +// response should be returned +fun isIndexNotFoundException(e: Exception): Boolean { + if (e is IndexNotFoundException) { + return true + } + if (e is RemoteTransportException) { + val cause = e.cause + if (cause is IndexNotFoundException) { + return true + } + } + return false +} + +suspend fun getConfigAndSendNotification( + action: Action, + monitorCtx: MonitorRunnerExecutionContext, + subject: String?, + message: String +): String { + val config = getConfigForNotificationAction(action, monitorCtx) + if (config.destination == null && config.channel == null) { + throw IllegalStateException("Unable to find a Notification Channel or Destination config with id [${action.destinationId}]") + } + + // Adding a check on TEST_ACTION Destination type here to avoid supporting it as a LegacyBaseMessage type + // just for Alerting integration tests + if (config.destination?.isTestAction() == true) { + return "test action" + } + + if (config.destination?.isAllowed(monitorCtx.allowList) == false) { + throw IllegalStateException( + "Monitor contains a Destination type that is not allowed: ${config.destination.type}" + ) + } + + var actionResponseContent = "" + actionResponseContent = config.channel + ?.sendNotification( + monitorCtx.client!!, + config.channel.getTitle(subject), + message + ) ?: actionResponseContent + + actionResponseContent = config.destination + ?.buildLegacyBaseMessage(subject, message, monitorCtx.destinationContextFactory!!.getDestinationContext(config.destination)) + ?.publishLegacyNotification(monitorCtx.client!!) + ?: actionResponseContent + + return actionResponseContent +} + +/** + * The "destination" ID referenced in a Monitor Action could either be a Notification config or a Destination config + * depending on whether the background migration process has already migrated it from a Destination to a Notification config. + * + * To cover both of these cases, the Notification config will take precedence and if it is not found, the Destination will be retrieved. + */ +private suspend fun getConfigForNotificationAction( + action: Action, + monitorCtx: MonitorRunnerExecutionContext +): NotificationActionConfigs { + var destination: Destination? = null + var notificationPermissionException: Exception? = null + + var channel: NotificationConfigInfo? = null + try { + channel = getNotificationConfigInfo(monitorCtx.client as NodeClient, action.destinationId) + } catch (e: OpenSearchSecurityException) { + notificationPermissionException = e + } + + // If the channel was not found, try to retrieve the Destination + if (channel == null) { + destination = try { + val table = Table( + "asc", + "destination.name.keyword", + null, + 1, + 0, + null + ) + val getDestinationsRequest = GetDestinationsRequest( + action.destinationId, + 0L, + null, + table, + "ALL" + ) + + val getDestinationsResponse: GetDestinationsResponse = monitorCtx.client!!.suspendUntil { + monitorCtx.client!!.execute(GetDestinationsAction.INSTANCE, getDestinationsRequest, it) + } + getDestinationsResponse.destinations.firstOrNull() + } catch (e: IllegalStateException) { + // Catching the exception thrown when the Destination was not found so the NotificationActionConfigs object can be returned + null + } catch (e: OpenSearchSecurityException) { + if (notificationPermissionException != null) + throw notificationPermissionException + else + throw e + } + + if (destination == null && notificationPermissionException != null) + throw notificationPermissionException + } + + return NotificationActionConfigs(destination, channel) +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/IndexUtils.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/IndexUtils.kt index b388ae757..dac2ff897 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/IndexUtils.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/IndexUtils.kt @@ -64,7 +64,7 @@ class IndexUtils { alertIndexSchemaVersion = getSchemaVersion(AlertIndices.alertMapping()) findingIndexSchemaVersion = getSchemaVersion(AlertIndices.findingMapping()) alertingCommentIndexSchemaVersion = getSchemaVersion(CommentsIndices.commentsMapping()) - alertV2IndexSchemaVersion = getSchemaVersion(AlertV2Indices.alertV2Mapping()) + alertV2IndexSchemaVersion = getSchemaVersion(AlertV2Indices.alertMapping()) } @JvmStatic @@ -89,7 +89,7 @@ class IndexUtils { @JvmStatic fun alertV2IndexUpdated() { - commentsIndexUpdated = true + alertV2IndexUpdated = true } @JvmStatic diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/workflow/CompositeWorkflowRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/workflow/CompositeWorkflowRunner.kt index 1e613bd0f..327de0ff0 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/workflow/CompositeWorkflowRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/workflow/CompositeWorkflowRunner.kt @@ -56,6 +56,7 @@ object CompositeWorkflowRunner : WorkflowRunner() { periodStart: Instant, periodEnd: Instant, dryRun: Boolean, + manual: Boolean, transportService: TransportService ): WorkflowRunResult { val workflowExecutionStartTime = Instant.now() @@ -145,6 +146,7 @@ object CompositeWorkflowRunner : WorkflowRunner() { periodStart, periodEnd, dryRun, + manual, workflowRunContext, executionId, transportService @@ -176,6 +178,11 @@ object CompositeWorkflowRunner : WorkflowRunner() { triggerResults = triggerResults ) val currentAlerts = try { + // create stateless alert indices as well to prevent get alerts from returning error because + // stateless alerts indices couldn't be found + monitorCtx.alertV2Indices!!.createOrUpdateAlertV2Index() + monitorCtx.alertV2Indices!!.createOrUpdateInitialAlertV2HistoryIndex() + monitorCtx.alertIndices!!.createOrUpdateAlertIndex(dataSources!!) monitorCtx.alertIndices!!.createOrUpdateInitialAlertHistoryIndex(dataSources) monitorCtx.alertService!!.loadCurrentAlertsForWorkflow(workflow, dataSources) @@ -249,6 +256,7 @@ object CompositeWorkflowRunner : WorkflowRunner() { periodStart: Instant, periodEnd: Instant, dryRun: Boolean, + manual: Boolean, workflowRunContext: WorkflowRunContext, executionId: String, transportService: TransportService @@ -261,6 +269,7 @@ object CompositeWorkflowRunner : WorkflowRunner() { periodStart, periodEnd, dryRun, + manual, workflowRunContext, executionId, transportService @@ -272,6 +281,7 @@ object CompositeWorkflowRunner : WorkflowRunner() { periodStart, periodEnd, dryRun, + manual, workflowRunContext, executionId, transportService @@ -283,6 +293,7 @@ object CompositeWorkflowRunner : WorkflowRunner() { periodStart, periodEnd, dryRun, + manual, workflowRunContext, executionId, transportService diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/workflow/WorkflowRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/workflow/WorkflowRunner.kt index 56a708444..574b6a670 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/workflow/WorkflowRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/workflow/WorkflowRunner.kt @@ -5,12 +5,12 @@ package org.opensearch.alerting.workflow -import org.opensearch.alerting.AlertingV2Utils.getConfigAndSendNotification import org.opensearch.alerting.MonitorRunnerExecutionContext import org.opensearch.alerting.MonitorRunnerService import org.opensearch.alerting.opensearchapi.InjectorContextElement import org.opensearch.alerting.opensearchapi.withClosableContext import org.opensearch.alerting.script.ChainedAlertTriggerExecutionContext +import org.opensearch.alerting.util.getConfigAndSendNotification import org.opensearch.alerting.util.use import org.opensearch.commons.alerting.model.ActionRunResult import org.opensearch.commons.alerting.model.Workflow @@ -29,6 +29,7 @@ abstract class WorkflowRunner { periodStart: Instant, periodEnd: Instant, dryRun: Boolean, + manual: Boolean, transportService: TransportService ): WorkflowRunResult diff --git a/alerting/src/main/resources/org/opensearch/alerting/alerts/alert_mapping.json b/alerting/src/main/resources/org/opensearch/alerting/alerts/alert_mapping.json index 76e5104cc..f4f25d8df 100644 --- a/alerting/src/main/resources/org/opensearch/alerting/alerts/alert_mapping.json +++ b/alerting/src/main/resources/org/opensearch/alerting/alerts/alert_mapping.json @@ -4,7 +4,7 @@ "required": true }, "_meta" : { - "schema_version": 5 + "schema_version": 6 }, "properties": { "schema_version": { @@ -177,6 +177,28 @@ "type" : "keyword" } } + }, + "ppl_query": { + "type": "text" + }, + "ppl_query_results": { + "type": "nested", + "properties": { + "schema": { + "type": "nested", + "dynamic": true + }, + "datarows": { + "type": "object", + "enabled": false + }, + "total": { + "type": "integer" + }, + "size": { + "type": "integer" + } + } } } } \ No newline at end of file diff --git a/alerting/src/main/resources/org/opensearch/alerting/alertsv2/alert_v2_mapping.json b/alerting/src/main/resources/org/opensearch/alerting/alertsv2/alert_v2_mapping.json deleted file mode 100644 index 5543a289c..000000000 --- a/alerting/src/main/resources/org/opensearch/alerting/alertsv2/alert_v2_mapping.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "dynamic": "strict", - "_routing": { - "required": true - }, - "_meta" : { - "schema_version": 1 - }, - "properties": { - "schema_version": { - "type": "integer" - }, - "monitor_v2_id": { - "type": "keyword" - }, - "monitor_v2_version": { - "type": "long" - }, - "id": { - "type": "keyword" - }, - "version": { - "type": "long" - }, - "severity": { - "type": "keyword" - }, - "monitor_v2_name": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "monitor_v2_user": { - "properties": { - "name": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "backend_roles": { - "type" : "text", - "fields" : { - "keyword" : { - "type" : "keyword" - } - } - }, - "roles": { - "type" : "text", - "fields" : { - "keyword" : { - "type" : "keyword" - } - } - }, - "custom_attribute_names": { - "type" : "text", - "fields" : { - "keyword" : { - "type" : "keyword" - } - } - } - } - }, - "execution_id": { - "type": "keyword" - }, - "trigger_v2_id": { - "type": "keyword" - }, - "trigger_v2_name": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "triggered_time": { - "type": "date" - }, - "error_message": { - "type": "text" - }, - "query": { - "type": "text" - }, - "query_results": { - "type": "nested", - "properties": { - "schema": { - "type": "nested", - "dynamic": true - }, - "datarows": { - "type": "object", - "enabled": false - }, - "total": { - "type": "integer" - }, - "size": { - "type": "integer" - } - } - } - } -} \ No newline at end of file diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt b/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt index 3a5953b00..53575ed1c 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt @@ -17,7 +17,6 @@ import org.opensearch.action.search.SearchResponse import org.opensearch.alerting.AlertingPlugin.Companion.COMMENTS_BASE_URI import org.opensearch.alerting.AlertingPlugin.Companion.EMAIL_ACCOUNT_BASE_URI import org.opensearch.alerting.AlertingPlugin.Companion.EMAIL_GROUP_BASE_URI -import org.opensearch.alerting.AlertingPlugin.Companion.MONITOR_V2_BASE_URI import org.opensearch.alerting.alerts.AlertIndices import org.opensearch.alerting.alerts.AlertIndices.Companion.FINDING_HISTORY_WRITE_INDEX import org.opensearch.alerting.alertsv2.AlertV2Indices @@ -28,9 +27,6 @@ import org.opensearch.alerting.model.destination.Destination import org.opensearch.alerting.model.destination.Slack import org.opensearch.alerting.model.destination.email.EmailAccount import org.opensearch.alerting.model.destination.email.EmailGroup -import org.opensearch.alerting.modelv2.AlertV2 -import org.opensearch.alerting.modelv2.MonitorV2 -import org.opensearch.alerting.modelv2.PPLSQLMonitor import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.settings.DestinationSettings import org.opensearch.alerting.util.DestinationType @@ -47,8 +43,12 @@ import org.opensearch.common.xcontent.XContentFactory import org.opensearch.common.xcontent.XContentFactory.jsonBuilder import org.opensearch.common.xcontent.XContentType import org.opensearch.common.xcontent.json.JsonXContent.jsonXContent +import org.opensearch.commons.alerting.action.GetAlertsResponse.Companion.ALERTS_FIELD +import org.opensearch.commons.alerting.action.GetAlertsResponse.Companion.TOTAL_ALERTS_FIELD import org.opensearch.commons.alerting.action.GetFindingsResponse import org.opensearch.commons.alerting.model.Alert +import org.opensearch.commons.alerting.model.Alert.Companion.ERROR_MESSAGE_FIELD +import org.opensearch.commons.alerting.model.Alert.Companion.STATE_FIELD import org.opensearch.commons.alerting.model.BucketLevelTrigger import org.opensearch.commons.alerting.model.ChainedAlertTrigger import org.opensearch.commons.alerting.model.Comment @@ -59,6 +59,8 @@ import org.opensearch.commons.alerting.model.DocumentLevelTrigger import org.opensearch.commons.alerting.model.Finding import org.opensearch.commons.alerting.model.FindingWithDocs import org.opensearch.commons.alerting.model.Monitor +import org.opensearch.commons.alerting.model.PPLSQLInput +import org.opensearch.commons.alerting.model.PPLSQLTrigger import org.opensearch.commons.alerting.model.QueryLevelTrigger import org.opensearch.commons.alerting.model.ScheduledJob import org.opensearch.commons.alerting.model.SearchInput @@ -110,12 +112,13 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return NamedXContentRegistry( mutableListOf( Monitor.XCONTENT_REGISTRY, - MonitorV2.XCONTENT_REGISTRY, SearchInput.XCONTENT_REGISTRY, DocLevelMonitorInput.XCONTENT_REGISTRY, + PPLSQLInput.XCONTENT_REGISTRY, QueryLevelTrigger.XCONTENT_REGISTRY, BucketLevelTrigger.XCONTENT_REGISTRY, DocumentLevelTrigger.XCONTENT_REGISTRY, + PPLSQLTrigger.XCONTENT_REGISTRY, Workflow.XCONTENT_REGISTRY, ChainedAlertTrigger.XCONTENT_REGISTRY ) + SearchModule(Settings.EMPTY, emptyList()).namedXContents @@ -137,17 +140,6 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return StringEntity(jsonString, APPLICATION_JSON) } - private fun createMonitorV2EntityWithBackendRoles(monitorV2: MonitorV2, rbacRoles: List?): HttpEntity { - if (rbacRoles == null) { - return monitorV2.toHttpEntity() - } - val temp = monitorV2.toJsonString() - val toReplace = temp.lastIndexOf("}") - val rbacString = rbacRoles.joinToString { "\"$it\"" } - val jsonString = temp.substring(0, toReplace) + ", \"rbac_roles\": [$rbacString] }" - return StringEntity(jsonString, APPLICATION_JSON) - } - protected fun createMonitorWithClient( client: RestClient, monitor: Monitor, @@ -169,47 +161,26 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return getMonitor(monitorId = monitorJson["_id"] as String) } - protected fun createMonitorV2WithClient( + // used only for PPL Monitor tests + // a createMonitorWithClient() wrapper that creates an index before proceeding + protected fun createPPLIndexThenMonitorWithClient( client: RestClient, - monitorV2: MonitorV2, - rbacRoles: List? = null - ): MonitorV2 { + monitor: Monitor, + rbacRoles: List? = null, + refresh: Boolean = true + ): Monitor { // every random ppl monitor's query searches index TEST_INDEX_NAME // by default, so create that first before creating the monitor if (!indexExists(TEST_INDEX_NAME)) { createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) } - - // be sure to use the passed in client to send the create monitor request, - // as the user stored in this client is the user whose permissions we want - // to test, not client()'s admin level user - val response = client.makeRequest( - "POST", MONITOR_V2_BASE_URI, emptyMap(), - createMonitorV2EntityWithBackendRoles(monitorV2, rbacRoles) - ) - assertEquals("Unable to create a new monitor v2", RestStatus.OK, response.restStatus()) - - val monitorV2Json = jsonXContent.createParser( - NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, - response.entity.content - ).map() - assertUserNull(monitorV2Json as HashMap) - - return getMonitorV2(monitorV2Id = monitorV2Json["_id"] as String) + return createMonitorWithClient(client, monitor, rbacRoles, refresh) } protected fun createMonitor(monitor: Monitor, refresh: Boolean = true): Monitor { return createMonitorWithClient(client(), monitor, emptyList(), refresh) } - protected fun createMonitorV2(monitorV2: MonitorV2): MonitorV2 { - val client = client() - val response = client.makeRequest("POST", MONITOR_V2_BASE_URI, emptyMap(), monitorV2.toHttpEntity()) - assertEquals("Unable to create a new monitor", RestStatus.OK, response.restStatus()) - - return getMonitorV2(monitorV2Id = response.asMap()["_id"] as String) - } - protected fun deleteMonitor(monitor: Monitor, refresh: Boolean = true): Response { val response = client().makeRequest( "DELETE", "$ALERTING_BASE_URI/${monitor.id}?refresh=$refresh", emptyMap(), @@ -220,15 +191,6 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return response } - protected fun deleteMonitorV2(monitorV2Id: String): Response { - val response = client().makeRequest( - "DELETE", "$MONITOR_V2_BASE_URI/$monitorV2Id?refresh=true", emptyMap() - ) - assertEquals("Unable to delete a monitor", RestStatus.OK, response.restStatus()) - - return response - } - protected fun deleteWorkflow(workflow: Workflow, deleteDelegates: Boolean = false, refresh: Boolean = true): Response { val response = client().makeRequest( "DELETE", @@ -600,7 +562,7 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return getMonitor(monitorId = monitorId) } - protected fun createRandomPPLMonitor(pplMonitorConfig: PPLSQLMonitor = randomPPLMonitor()): PPLSQLMonitor { + protected fun createRandomPPLMonitor(pplMonitorConfig: Monitor = randomPPLMonitor()): Monitor { // every random ppl monitor's query searches index TEST_INDEX_NAME // by default, so create that first before creating the monitor val indexExistsResponse = adminClient().makeRequest("HEAD", TEST_INDEX_NAME) @@ -609,8 +571,8 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { } logger.info("ppl monitor: $pplMonitorConfig") - val pplMonitorId = createMonitorV2(pplMonitorConfig).id - return getMonitorV2(monitorV2Id = pplMonitorId) as PPLSQLMonitor + val pplMonitorId = createMonitor(pplMonitorConfig).id + return getMonitor(monitorId = pplMonitorId) } protected fun createRandomDocumentMonitor(refresh: Boolean = false, withMetadata: Boolean = false): Monitor { @@ -646,16 +608,6 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return getWorkflow(workflowId = workflow.id) } - @Suppress("UNCHECKED_CAST") - protected fun updateMonitorV2(monitorV2: MonitorV2, refresh: Boolean = false): MonitorV2 { - val response = client().makeRequest( - "PUT", "$MONITOR_V2_BASE_URI/${monitorV2.id}?refresh=$refresh", - emptyMap(), monitorV2.toHttpEntity() - ) - assertEquals("Unable to update a monitorV2", RestStatus.OK, response.restStatus()) - return getMonitorV2(monitorV2Id = monitorV2.id) - } - protected fun updateMonitorWithClient( client: RestClient, monitor: Monitor, @@ -723,33 +675,6 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return monitor.copy(id = id, version = version) } - protected fun getMonitorV2( - monitorV2Id: String, - header: BasicHeader = BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") - ): MonitorV2 { - val response = client().makeRequest("GET", "$MONITOR_V2_BASE_URI/$monitorV2Id", null, header) - assertEquals("Unable to get monitorV2 $monitorV2Id", RestStatus.OK, response.restStatus()) - - val parser = createParser(XContentType.JSON.xContent(), response.entity.content) - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser) - - lateinit var id: String - var version: Long = 0 - lateinit var monitorV2: MonitorV2 - - while (parser.nextToken() != XContentParser.Token.END_OBJECT) { - parser.nextToken() - - when (parser.currentName()) { - "_id" -> id = parser.text() - "_version" -> version = parser.longValue() - "monitorV2" -> monitorV2 = MonitorV2.parse(parser) - } - } - - return monitorV2.makeCopy(id = id, version = version) - } - // TODO: understand why doc alerts wont work with the normal search Alerts function protected fun searchAlertsWithFilter( monitor: Monitor, @@ -845,35 +770,6 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { } } - protected fun searchAlertV2s( - monitorV2Id: String, - indices: String = AlertV2Indices.ALERT_V2_INDEX, - refresh: Boolean = true - ): List { - try { - if (refresh) refreshIndex(indices) - } catch (e: Exception) { - logger.warn("Could not refresh index $indices because: ${e.message}") - return emptyList() - } - - // If this is a test monitor (it doesn't have an ID) and no alerts will be saved for it. - val searchParams = if (monitorV2Id != MonitorV2.NO_ID) mapOf("routing" to monitorV2Id) else mapOf() - val request = """ - { "version" : true, - "query" : { "term" : { "${AlertV2.MONITOR_V2_ID_FIELD}" : "$monitorV2Id" } } - } - """.trimIndent() - val httpResponse = adminClient().makeRequest("GET", "/$indices/_search", searchParams, StringEntity(request, APPLICATION_JSON)) - assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus()) - - val searchResponse = SearchResponse.fromXContent(createParser(jsonXContent, httpResponse.entity.content)) - return searchResponse.hits.hits.map { - val xcp = createParser(jsonXContent, it.sourceRef) - AlertV2.parse(xcp, it.id, it.version) - } - } - protected fun acknowledgeAlerts(monitor: Monitor, vararg alerts: Alert): Response { val request = XContentFactory.jsonBuilder().startObject() .array("alerts", *alerts.map { it.id }.toTypedArray()) @@ -926,17 +822,6 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return getAlerts(client(), dataMap, header) } - protected fun getAlertV2s(): Response { - val response = client().makeRequest( - "GET", - "$MONITOR_V2_BASE_URI/alerts", - null, - BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") - ) - assertEquals("Get call failed.", RestStatus.OK, response.restStatus()) - return response - } - protected fun refreshIndex(index: String): Response { val response = client().makeRequest("POST", "/$index/_refresh?expand_wildcards=all") assertEquals("Unable to refresh index", RestStatus.OK, response.restStatus()) @@ -998,9 +883,6 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { protected fun executeMonitor(client: RestClient, monitor: Monitor, params: Map = mapOf()): Response = client.makeRequest("POST", "$ALERTING_BASE_URI/_execute", params, monitor.toHttpEntityWithUser()) - protected fun executeMonitorV2(monitorId: String, params: Map = mutableMapOf()): Response = - client().makeRequest("POST", "$MONITOR_V2_BASE_URI/$monitorId/_execute", params) - protected fun searchFindings(params: Map = mutableMapOf()): GetFindingsResponse { var baseEndpoint = "${AlertingPlugin.FINDING_BASE_URI}/_search?" @@ -1441,7 +1323,7 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { } fun putAlertV2Mappings(mapping: String? = null) { - val mappingHack = if (mapping != null) mapping else AlertV2Indices.alertV2Mapping().trimStart('{').trimEnd('}') + val mappingHack = if (mapping != null) mapping else AlertV2Indices.alertMapping().trimStart('{').trimEnd('}') val encodedHistoryIndex = URLEncoder.encode(AlertV2Indices.ALERT_V2_HISTORY_INDEX_PATTERN, Charsets.UTF_8.toString()) val settings = Settings.builder().put("index.hidden", true).build() createIndex(AlertV2Indices.ALERT_V2_INDEX, settings, mappingHack) @@ -1484,23 +1366,6 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return shuffleXContent(toXContentWithUser(builder, ToXContent.EMPTY_PARAMS)).string() } - protected fun MonitorV2.toHttpEntity(): HttpEntity { - return StringEntity(toJsonString(), APPLICATION_JSON) - } - - private fun MonitorV2.toJsonString(): String { - return shuffleXContent(toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)).string() - } - - protected fun MonitorV2.toHttpEntityWithUser(): HttpEntity { - return StringEntity(toJsonStringWithUser(), APPLICATION_JSON) - } - - private fun MonitorV2.toJsonStringWithUser(): String { - val builder = jsonBuilder() - return shuffleXContent(toXContentWithUser(builder, ToXContent.EMPTY_PARAMS)).string() - } - protected fun Destination.toHttpEntity(): HttpEntity { return StringEntity(toJsonString(), APPLICATION_JSON) } @@ -2197,7 +2062,7 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { indexDoc(TEST_INDEX_NAME, UUID.randomUUID().toString(), testDoc) } - protected fun ensureNumMonitorV2s(expectedNum: Int) { + protected fun ensureNumMonitors(expectedNum: Int) { // if a validation error is thrown but a monitor is still accidentally created, // what happens is that this check runs before the workflows to create // alerting-config index and index the monitor complete, meaning this check gets @@ -2210,7 +2075,7 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { val search = SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).toString() val searchResponse = client().makeRequest( - "POST", "$MONITOR_V2_BASE_URI/_search", + "POST", "${AlertingPlugin.MONITOR_BASE_URI}/_search", StringEntity(search, APPLICATION_JSON) ) @@ -2223,7 +2088,7 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { // takes in an execute monitor API response and returns true if the // trigger condition was met. assumes the monitor executed only had 1 trigger - protected fun isTriggered(pplMonitor: PPLSQLMonitor, executeResponse: Response): Boolean { + protected fun isTriggered(pplMonitor: Monitor, executeResponse: Response): Boolean { val executeResponseMap = entityAsMap(executeResponse) val triggerResultsObj = (executeResponseMap["trigger_results"] as Map)[pplMonitor.triggers[0].id] as Map return triggerResultsObj["triggered"] as Boolean @@ -2232,15 +2097,19 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { // takes in a get alerts API response and returns the current number of active alerts protected fun numAlerts(getAlertsResponse: Response): Int { logger.info("get alerts response: ${entityAsMap(getAlertsResponse)}") - return entityAsMap(getAlertsResponse)["total_alerts_v2"] as Int + return entityAsMap(getAlertsResponse)[TOTAL_ALERTS_FIELD] as Int } protected fun containsErrorAlert(getAlertsResponse: Response): Boolean { val getAlertsMap = entityAsMap(getAlertsResponse) - val alertsList = getAlertsMap["alerts_v2"] as List> + val alertsList = getAlertsMap[ALERTS_FIELD] as List> alertsList.forEach { alert -> - val errorMessage = alert["error_message"] as String? - if (errorMessage != null) return true + val errorMessage = alert[ERROR_MESSAGE_FIELD] as String? + val state = Alert.State.valueOf((alert[STATE_FIELD] as String?)!!.uppercase(Locale.ROOT)) + + if (state == Alert.State.ERROR && errorMessage != null) { + return true + } } return false } diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/DocumentMonitorRunnerIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/DocumentMonitorRunnerIT.kt index 1f0f60bd8..9aa16793e 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/DocumentMonitorRunnerIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/DocumentMonitorRunnerIT.kt @@ -728,7 +728,7 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() { val alerts = searchAlerts(monitor) assertEquals("Alert not saved", 1, alerts.size) - assertEquals("Alert status is incorrect", Alert.State.ERROR, alerts[0].state) + assertEquals("Alert status is incorrect", Alert.State.ERROR, (alerts[0] as Alert).state) } fun `test execute monitor generates alerts and findings with per alert execution for actions`() { diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/MonitorDataSourcesIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/MonitorDataSourcesIT.kt index 1211bca3f..a72f7efde 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/MonitorDataSourcesIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/MonitorDataSourcesIT.kt @@ -590,7 +590,7 @@ class MonitorDataSourcesIT : AlertingSingleNodeTestCase() { .get() Assert.assertTrue(getAlertsResponse != null) Assert.assertTrue(getAlertsResponse.alerts.size == 1) - Assert.assertTrue(getAlertsResponse.alerts[0].state.toString().equals(Alert.State.ERROR.toString())) + Assert.assertTrue((getAlertsResponse.alerts[0] as Alert).state.toString().equals(Alert.State.ERROR.toString())) val findings = searchFindings(id, customFindingsIndex) assertEquals("Findings saved for test monitor", 0, findings.size) } @@ -896,10 +896,10 @@ class MonitorDataSourcesIT : AlertingSingleNodeTestCase() { Assert.assertTrue(getAlertsResponse != null) Assert.assertTrue(getAlertsResponse.alerts.size == 1) Assert.assertTrue( - getAlertsResponse.alerts[0].errorHistory[0].message == + (getAlertsResponse.alerts[0] as Alert).errorHistory[0].message == "AlertingException[closed]; nested: Exception[org.opensearch.indices.IndexClosedException: closed]; " ) - Assert.assertEquals(1, getAlertsResponse.alerts[0].errorHistory.size) + Assert.assertEquals(1, (getAlertsResponse.alerts[0] as Alert).errorHistory.size) Assert.assertTrue(getAlertsResponse.alerts[0].errorMessage!!.contains("Failed to run percolate search")) } diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/PPLSQLMonitorRunnerIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/PPLSQLMonitorRunnerIT.kt index 070073994..33c25afd9 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/PPLSQLMonitorRunnerIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/PPLSQLMonitorRunnerIT.kt @@ -5,15 +5,13 @@ package org.opensearch.alerting -import org.junit.Before -import org.opensearch.alerting.core.settings.AlertingV2Settings -import org.opensearch.alerting.modelv2.PPLSQLTrigger.ConditionType -import org.opensearch.alerting.modelv2.PPLSQLTrigger.NumResultsCondition -import org.opensearch.alerting.modelv2.PPLSQLTrigger.TriggerMode import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.common.settings.Settings import org.opensearch.common.unit.TimeValue import org.opensearch.commons.alerting.model.IntervalSchedule +import org.opensearch.commons.alerting.model.PPLSQLTrigger.ConditionType +import org.opensearch.commons.alerting.model.PPLSQLTrigger.NumResultsCondition +import org.opensearch.commons.alerting.model.PPLSQLTrigger.TriggerMode import org.opensearch.test.OpenSearchTestCase import java.time.temporal.ChronoUnit.MINUTES import java.util.concurrent.TimeUnit @@ -27,11 +25,6 @@ import java.util.concurrent.TimeUnit * --tests "org.opensearch.alerting.PPLMonitorRunnerIT" */ class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { - @Before - fun enableAlertingV2() { - client().updateSettings(AlertingV2Settings.ALERTING_V2_ENABLED.key, "true") - } - fun `test monitor execution timeout generates error alert`() { createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) indexDocFromSomeTimeAgo(2, MINUTES, "abc", 5) @@ -40,7 +33,6 @@ class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { randomPPLMonitor( enabled = true, schedule = IntervalSchedule(interval = 1, unit = MINUTES), - lookBackWindow = null, triggers = listOf( randomPPLTrigger( throttleDuration = null, @@ -59,12 +51,13 @@ class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { // set the monitor execution timebox to 1 nanosecond to guarantee a timeout client().updateSettings(AlertingSettings.ALERT_V2_MONITOR_EXECUTION_MAX_DURATION.key, TimeValue.timeValueNanos(1L)) - val executeMonitorResponse = executeMonitorV2(pplMonitor.id) + val executeMonitorResponse = executeMonitor(pplMonitor.id) - val getAlertsResponse = getAlertV2s() + val getAlertsResponse = getAlerts() val alertsGenerated = numAlerts(getAlertsResponse) > 0 val containsErrorAlert = containsErrorAlert(getAlertsResponse) - val executeResponseContainsError = (entityAsMap(executeMonitorResponse)["error"] as String?) != null + val executeResponseContainsError = + (entityAsMap(executeMonitorResponse).stringMap("input_results")?.get("error") as String).isNotEmpty() assert(alertsGenerated) { "Alerts should have been generated but they weren't" } assert(containsErrorAlert) { "Error alert should have been generated for timeout but wasn't" } @@ -79,7 +72,6 @@ class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { randomPPLMonitor( enabled = true, schedule = IntervalSchedule(interval = 1, unit = MINUTES), - lookBackWindow = null, triggers = listOf( randomPPLTrigger( throttleDuration = null, @@ -97,16 +89,16 @@ class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { val versionBefore = pplMonitor.version - val executeResponse = executeMonitorV2(pplMonitor.id) + val executeResponse = executeMonitor(pplMonitor.id) val triggered = isTriggered(pplMonitor, executeResponse) - val getAlertsResponse = getAlertV2s() + val getAlertsResponse = getAlerts() val alertsGenerated = numAlerts(getAlertsResponse) > 0 assert(triggered) { "Monitor should have triggered but it didn't" } assert(alertsGenerated) { "Alerts should have been generated but they weren't" } - val pplMonitorAfter = getMonitorV2(pplMonitor.id) + val pplMonitorAfter = getMonitor(pplMonitor.id) val versionAfter = pplMonitorAfter.version assert(versionBefore == versionAfter) { "Monitor version changed after monitor execution" } @@ -122,7 +114,6 @@ class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { randomPPLMonitor( enabled = true, schedule = IntervalSchedule(interval = 1, unit = MINUTES), - lookBackWindow = null, triggers = listOf( randomPPLTrigger( throttleDuration = null, @@ -138,10 +129,10 @@ class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { ) ) - val executeResponse = executeMonitorV2(pplMonitor.id) + val executeResponse = executeMonitor(pplMonitor.id) val triggered = isTriggered(pplMonitor, executeResponse) - val getAlertsResponse = getAlertV2s() + val getAlertsResponse = getAlerts() val alertsGenerated = numAlerts(getAlertsResponse) assert(triggered) { "Monitor should have triggered but it didn't" } @@ -167,7 +158,6 @@ class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { randomPPLMonitor( enabled = true, schedule = IntervalSchedule(interval = 1, unit = MINUTES), - lookBackWindow = null, triggers = listOf( randomPPLTrigger( throttleDuration = null, @@ -183,10 +173,10 @@ class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { ) ) - val executeResponse = executeMonitorV2(pplMonitor.id) + val executeResponse = executeMonitor(pplMonitor.id) val triggered = isTriggered(pplMonitor, executeResponse) - val getAlertsResponse = getAlertV2s() + val getAlertsResponse = getAlerts() val alertsGenerated = numAlerts(getAlertsResponse) > 0 assert(triggered) { "Monitor should have triggered but it didn't" } @@ -209,7 +199,6 @@ class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { randomPPLMonitor( enabled = true, schedule = IntervalSchedule(interval = 1, unit = MINUTES), - lookBackWindow = null, triggers = listOf( randomPPLTrigger( throttleDuration = null, @@ -225,10 +214,10 @@ class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { ) ) - val executeResponse = executeMonitorV2(pplMonitor.id) + val executeResponse = executeMonitor(pplMonitor.id) val triggered = isTriggered(pplMonitor, executeResponse) - val getAlertsResponse = getAlertV2s() + val getAlertsResponse = getAlerts() val alertsGenerated = numAlerts(getAlertsResponse) // when the indexed docs above are aggregated by field abc, we have: @@ -245,76 +234,6 @@ class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { ) } - fun `test running ppl monitor with lookback window and doc within lookback window`() { - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - indexDocFromSomeTimeAgo(2, MINUTES, "abc", 5) - - val pplMonitor = createRandomPPLMonitor( - randomPPLMonitor( - enabled = true, - schedule = IntervalSchedule(interval = 1, unit = MINUTES), - lookBackWindow = 5, - timestampField = TIMESTAMP_FIELD, - triggers = listOf( - randomPPLTrigger( - throttleDuration = null, - expireDuration = 5, - mode = TriggerMode.RESULT_SET, - conditionType = ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = NumResultsCondition.GREATER_THAN, - numResultsValue = 0L, - customCondition = null - ) - ), - query = "source = $TEST_INDEX_NAME | head 10" - ) - ) - - val executeResponse = executeMonitorV2(pplMonitor.id) - val triggered = isTriggered(pplMonitor, executeResponse) - - val getAlertsResponse = getAlertV2s() - val alertsGenerated = numAlerts(getAlertsResponse) > 0 - - assert(triggered) { "Monitor should have triggered but it didn't" } - assert(alertsGenerated) { "Alerts should have been generated but they weren't" } - } - - fun `test running ppl monitor with lookback window and doc beyond lookback window`() { - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - indexDocFromSomeTimeAgo(10, MINUTES, "abc", 5) - - val pplMonitor = createRandomPPLMonitor( - randomPPLMonitor( - enabled = true, - schedule = IntervalSchedule(interval = 1, unit = MINUTES), - lookBackWindow = 5, - timestampField = TIMESTAMP_FIELD, - triggers = listOf( - randomPPLTrigger( - throttleDuration = null, - expireDuration = 5, - mode = TriggerMode.RESULT_SET, - conditionType = ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = NumResultsCondition.GREATER_THAN, - numResultsValue = 0L, - customCondition = null - ) - ), - query = "source = $TEST_INDEX_NAME | head 10" - ) - ) - - val executeResponse = executeMonitorV2(pplMonitor.id) - val triggered = isTriggered(pplMonitor, executeResponse) - - val getAlertsResponse = getAlertV2s() - val alertsGenerated = numAlerts(getAlertsResponse) > 0 - - assert(!triggered) { "Monitor should not have triggered but it did" } - assert(!alertsGenerated) { "Alerts should not have been generated but they were" } - } - fun `test execute api generated alert gets expired`() { createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) @@ -338,10 +257,10 @@ class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { ) ) - val executeResponse = executeMonitorV2(pplMonitor.id) + val executeResponse = executeMonitor(pplMonitor.id) val triggered = isTriggered(pplMonitor, executeResponse) - val getAlertsResponsePreExpire = getAlertV2s() + val getAlertsResponsePreExpire = getAlerts() val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 assert(triggered) { "Monitor should have triggered but it didn't" } @@ -352,7 +271,7 @@ class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { return@waitUntil false }, 2, TimeUnit.MINUTES) - val getAlertsResponsePostExpire = getAlertV2s() + val getAlertsResponsePostExpire = getAlerts() val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 assert(!alertsGeneratedPostExpire) } @@ -387,7 +306,7 @@ class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { return@waitUntil false }, 2, TimeUnit.MINUTES) - val getAlertsResponsePreExpire = getAlertV2s() + val getAlertsResponsePreExpire = getAlerts() val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 assert(alertsGeneratedPreExpire) { "Alerts should have been generated but they weren't" } @@ -397,7 +316,7 @@ class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { return@waitUntil false }, 2, TimeUnit.MINUTES) - val getAlertsResponsePostExpire = getAlertV2s() + val getAlertsResponsePostExpire = getAlerts() logger.info("num alerts: ${numAlerts(getAlertsResponsePostExpire)}") val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 assert(!alertsGeneratedPostExpire) @@ -426,10 +345,10 @@ class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { ) ) - val executeResponse = executeMonitorV2(pplMonitor.id) + val executeResponse = executeMonitor(pplMonitor.id) val triggered = isTriggered(pplMonitor, executeResponse) - val getAlertsResponsePreThrottle = getAlertV2s() + val getAlertsResponsePreThrottle = getAlerts() val numAlertsPreThrottle = numAlerts(getAlertsResponsePreThrottle) assert(triggered) { "Monitor should have triggered but it didn't" } @@ -441,7 +360,7 @@ class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { return@waitUntil false }, 2, TimeUnit.MINUTES) - val getAlertsResponsePostThrottled = getAlertV2s() + val getAlertsResponsePostThrottled = getAlerts() val numAlertsPostThrottled = numAlerts(getAlertsResponsePostThrottled) assertEquals("A new alert was generated when it should have been throttled", 1, numAlertsPostThrottled) } @@ -469,10 +388,10 @@ class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { ) ) - val executeResponse = executeMonitorV2(pplMonitor.id) + val executeResponse = executeMonitor(pplMonitor.id) val triggered = isTriggered(pplMonitor, executeResponse) - val getAlertsResponse = getAlertV2s() + val getAlertsResponse = getAlerts() val numAlerts = numAlerts(getAlertsResponse) assert(triggered) { "Monitor should have triggered but it didn't" } @@ -484,10 +403,10 @@ class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { return@waitUntil false }, 10, TimeUnit.SECONDS) - val executeAgainResponse = executeMonitorV2(pplMonitor.id) + val executeAgainResponse = executeMonitor(pplMonitor.id) val triggeredAgain = isTriggered(pplMonitor, executeAgainResponse) - val getAlertsAgainResponse = getAlertV2s() + val getAlertsAgainResponse = getAlerts() val numAlertsAgain = numAlerts(getAlertsAgainResponse) assert(triggeredAgain) { "Monitor should have triggered again but it didn't" } diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt b/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt index 751f69a03..cf54406e1 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt @@ -12,14 +12,6 @@ import org.opensearch.alerting.model.AlertContext import org.opensearch.alerting.model.destination.email.EmailAccount import org.opensearch.alerting.model.destination.email.EmailEntry import org.opensearch.alerting.model.destination.email.EmailGroup -import org.opensearch.alerting.modelv2.AlertV2 -import org.opensearch.alerting.modelv2.PPLSQLMonitor -import org.opensearch.alerting.modelv2.PPLSQLMonitor.QueryLanguage -import org.opensearch.alerting.modelv2.PPLSQLTrigger -import org.opensearch.alerting.modelv2.PPLSQLTrigger.ConditionType -import org.opensearch.alerting.modelv2.PPLSQLTrigger.NumResultsCondition -import org.opensearch.alerting.modelv2.PPLSQLTrigger.TriggerMode -import org.opensearch.alerting.modelv2.TriggerV2.Severity import org.opensearch.alerting.util.getBucketKeysHash import org.opensearch.client.Request import org.opensearch.client.RequestOptions @@ -55,6 +47,13 @@ import org.opensearch.commons.alerting.model.InputRunResults import org.opensearch.commons.alerting.model.IntervalSchedule import org.opensearch.commons.alerting.model.Monitor import org.opensearch.commons.alerting.model.MonitorRunResult +import org.opensearch.commons.alerting.model.PPLSQLInput +import org.opensearch.commons.alerting.model.PPLSQLInput.QueryLanguage +import org.opensearch.commons.alerting.model.PPLSQLTrigger +import org.opensearch.commons.alerting.model.PPLSQLTrigger.ConditionType +import org.opensearch.commons.alerting.model.PPLSQLTrigger.NumResultsCondition +import org.opensearch.commons.alerting.model.PPLSQLTrigger.Severity +import org.opensearch.commons.alerting.model.PPLSQLTrigger.TriggerMode import org.opensearch.commons.alerting.model.QueryLevelTrigger import org.opensearch.commons.alerting.model.QueryLevelTriggerRunResult import org.opensearch.commons.alerting.model.Schedule @@ -243,6 +242,36 @@ fun randomDocumentLevelMonitor( ) } +fun randomPPLMonitor( + name: String = OpenSearchRestTestCase.randomAlphaOfLength(10), + enabled: Boolean = randomBoolean(), + schedule: Schedule = IntervalSchedule(interval = 5, unit = ChronoUnit.MINUTES), + lastUpdateTime: Instant = Instant.now().truncatedTo(ChronoUnit.MILLIS), + enabledTime: Instant? = if (enabled) Instant.now().truncatedTo(ChronoUnit.MILLIS) else null, + triggers: List = List(randomIntBetween(1, 5)) { randomPPLTrigger() }, + user: User? = randomUser(), + queryLanguage: QueryLanguage = QueryLanguage.PPL, + query: String = "source = $TEST_INDEX_NAME | head 10" +): Monitor { + return Monitor( + name = name, + enabled = enabled, + schedule = schedule, + lastUpdateTime = lastUpdateTime, + enabledTime = enabledTime, + monitorType = Monitor.MonitorType.PPL_MONITOR.value, + inputs = listOf( + PPLSQLInput( + query = query, + queryLanguage = queryLanguage + ) + ), + triggers = triggers, + user = user, + uiMetadata = mapOf() + ) +} + fun randomWorkflow( id: String = Workflow.NO_ID, monitorIds: List, @@ -308,36 +337,6 @@ fun randomWorkflowWithDelegates( ) } -fun randomPPLMonitor( - name: String = OpenSearchRestTestCase.randomAlphaOfLength(10), - enabled: Boolean = randomBoolean(), - schedule: Schedule = IntervalSchedule(interval = 5, unit = ChronoUnit.MINUTES), - lookBackWindow: Long? = randomLongBetween(10, 100), - timestampField: String? = lookBackWindow?.let { TIMESTAMP_FIELD }, - lastUpdateTime: Instant = Instant.now().truncatedTo(ChronoUnit.MILLIS), - enabledTime: Instant? = if (enabled) Instant.now().truncatedTo(ChronoUnit.MILLIS) else null, - description: String? = if (randomBoolean()) "some description" else null, - triggers: List = List(randomIntBetween(1, 5)) { randomPPLTrigger() }, - user: User? = randomUser(), - queryLanguage: QueryLanguage = QueryLanguage.PPL, - query: String = "source = $TEST_INDEX_NAME | head 10" -): PPLSQLMonitor { - return PPLSQLMonitor( - name = name, - enabled = enabled, - schedule = schedule, - lookBackWindow = lookBackWindow, - timestampField = timestampField, - lastUpdateTime = lastUpdateTime, - enabledTime = enabledTime, - description = description, - triggers = triggers, - user = user, - queryLanguage = queryLanguage, - query = query - ) -} - fun randomQueryLevelTrigger( id: String = UUIDs.base64UUID(), name: String = OpenSearchRestTestCase.randomAlphaOfLength(10), @@ -397,7 +396,7 @@ fun randomDocumentLevelTrigger( // random PPLTrigger defaults to a number_of_results trigger, because a custom condition // would require knowledge of the PPL Monitor's query // it is on the caller to be explicit and pass in valid arguments that would create either -// a valid PPL Monitor or one that intentionally throws an error. +// a valid PPL Monitor or one that intentionally throws an error during testing. // e.g. to create a valid PPL Monitor, if conditionType is CUSTOM, // numResultsCondition and numResultsValue must be null, while // customCondition must not be null. @@ -417,7 +416,7 @@ fun randomPPLTrigger( return PPLSQLTrigger( id = id, name = name, - severity = severity, + severity = severity.value, throttleDuration = throttleDuration, expireDuration = expireDuration, lastTriggeredTime = null, @@ -555,42 +554,6 @@ fun randomAlert(monitor: Monitor = randomQueryLevelMonitor()): Alert { ) } -fun randomAlertV2( - id: String = UUIDs.base64UUID(), - version: Long = randomLongBetween(1, 10), - schemaVersion: Int = randomIntBetween(1, 10), - monitorId: String = UUIDs.base64UUID(), - monitorName: String = UUIDs.base64UUID(), - monitorVersion: Long = randomLongBetween(1, 10), - monitorUser: User? = randomUser(), - triggerId: String = UUIDs.base64UUID(), - triggerName: String = UUIDs.base64UUID(), - query: String = "source = $TEST_INDEX_NAME | head 10", - queryResults: Map = mapOf(), - triggeredTime: Instant = Instant.now().truncatedTo(ChronoUnit.MILLIS), - errorMessage: String? = "sample error message", - severity: Severity = Severity.entries.random(), - executionId: String? = UUIDs.base64UUID() -): AlertV2 { - return AlertV2( - id = id, - version = version, - schemaVersion = schemaVersion, - monitorId = monitorId, - monitorName = monitorName, - monitorVersion = monitorVersion, - monitorUser = monitorUser, - triggerId = triggerId, - triggerName = triggerName, - query = query, - queryResults = queryResults, - triggeredTime = triggeredTime, - errorMessage = errorMessage, - severity = severity, - executionId = executionId, - ) -} - fun randomDocLevelQuery( id: String = OpenSearchRestTestCase.randomAlphaOfLength(10), query: String = OpenSearchRestTestCase.randomAlphaOfLength(10), @@ -930,7 +893,7 @@ fun Map.objectMap(key: String): Map> { return this[key] as Map> } -fun assertPplMonitorsEqual(pplMonitor1: PPLSQLMonitor, pplMonitor2: PPLSQLMonitor) { +fun assertPplMonitorsEqual(pplMonitor1: Monitor, pplMonitor2: Monitor) { // note: Get and Search Monitor responses do not include User information by // design, so that check is skipped @@ -940,17 +903,22 @@ fun assertPplMonitorsEqual(pplMonitor1: PPLSQLMonitor, pplMonitor2: PPLSQLMonito assertEquals("Monitor enabled fields not equal", pplMonitor1.enabled, pplMonitor2.enabled) assertEquals("Monitor schedules not equal", pplMonitor1.schedule, pplMonitor2.schedule) - assertEquals("Monitor lookback windows not equal", pplMonitor1.lookBackWindow, pplMonitor2.lookBackWindow) - assertEquals("Monitor timestamp fields not equal", pplMonitor1.timestampField, pplMonitor2.timestampField) - assertEquals("Monitor last updated times are not equal", pplMonitor1.lastUpdateTime, pplMonitor2.lastUpdateTime) - assertEquals("Monitor query languages not equal", pplMonitor1.queryLanguage, pplMonitor2.queryLanguage) - assertEquals("Monitor queries not equal", pplMonitor1.query, pplMonitor2.query) + assertEquals( + "Monitor query languages not equal", + (pplMonitor1.inputs[0] as PPLSQLInput).queryLanguage, + (pplMonitor2.inputs[0] as PPLSQLInput).queryLanguage + ) + assertEquals( + "Monitor queries not equal", + (pplMonitor1.inputs[0] as PPLSQLInput).query, + (pplMonitor2.inputs[0] as PPLSQLInput).query + ) assertEquals("Number of triggers in monitor not equal", pplMonitor1.triggers.size, pplMonitor2.triggers.size) val sortedTriggers1 = pplMonitor1.triggers.sortedBy { it.id } val sortedTriggers2 = pplMonitor2.triggers.sortedBy { it.id } for (i in sortedTriggers1.indices) { - assertPplTriggersEqual(sortedTriggers1[i], sortedTriggers2[i]) + assertPplTriggersEqual(sortedTriggers1[i] as PPLSQLTrigger, sortedTriggers2[i] as PPLSQLTrigger) } } @@ -1009,81 +977,3 @@ fun assertPplTriggersEqual(pplTrigger1: PPLSQLTrigger, pplTrigger2: PPLSQLTrigge pplTrigger2.customCondition ) } - -fun assertAlertV2sEqual(alert1: AlertV2, alert2: AlertV2) { - assertEquals( - "AlertV2 IDs are not equal", - alert1.id, - alert2.id - ) - assertEquals( - "AlertV2 versions are not equal", - alert1.version, - alert2.version - ) - assertEquals( - "AlertV2 schema versions are not equal", - alert1.schemaVersion, - alert2.schemaVersion - ) - assertEquals( - "AlertV2 monitor IDs are not equal", - alert1.monitorId, - alert2.monitorId - ) - assertEquals( - "AlertV2 monitor names are not equal", - alert1.monitorName, - alert2.monitorName - ) - assertEquals( - "AlertV2 monitor versions are not equal", - alert1.monitorVersion, - alert2.monitorVersion - ) - assertEquals( - "AlertV2 monitor users are not equal", - alert1.monitorUser.toString(), - alert2.monitorUser.toString() - ) - assertEquals( - "AlertV2 trigger IDs are not equal", - alert1.triggerId, - alert2.triggerId - ) - assertEquals( - "AlertV2 trigger names are not equal", - alert1.triggerName, - alert2.triggerName - ) - assertEquals( - "AlertV2 queries are not equal", - alert1.query, - alert2.query - ) - assertEquals( - "AlertV2 query results are not equal", - alert1.queryResults, - alert2.queryResults - ) - assertEquals( - "AlertV2 triggered times are not equal", - alert1.triggeredTime, - alert2.triggeredTime - ) - assertEquals( - "AlertV2 error messages are not equal", - alert1.errorMessage, - alert2.errorMessage - ) - assertEquals( - "AlertV2 severities are not equal", - alert1.severity, - alert2.severity - ) - assertEquals( - "AlertV2 execution IDs are not equal", - alert1.executionId, - alert2.executionId - ) -} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/action/ExecuteMonitorRequestTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/action/ExecuteMonitorRequestTests.kt index f54b6fea6..45405f8a8 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/action/ExecuteMonitorRequestTests.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/action/ExecuteMonitorRequestTests.kt @@ -17,7 +17,7 @@ class ExecuteMonitorRequestTests : OpenSearchTestCase() { fun `test execute monitor request with id`() { - val req = ExecuteMonitorRequest(false, TimeValue.timeValueSeconds(100L), "1234", null) + val req = ExecuteMonitorRequest(false, false, TimeValue.timeValueSeconds(100L), "1234", null) assertNotNull(req) val out = BytesStreamOutput() @@ -32,7 +32,7 @@ class ExecuteMonitorRequestTests : OpenSearchTestCase() { fun `test execute monitor request with monitor`() { val monitor = randomQueryLevelMonitor().copy(inputs = listOf(SearchInput(emptyList(), SearchSourceBuilder()))) - val req = ExecuteMonitorRequest(false, TimeValue.timeValueSeconds(100L), null, monitor) + val req = ExecuteMonitorRequest(false, false, TimeValue.timeValueSeconds(100L), null, monitor) assertNotNull(req.monitor) val out = BytesStreamOutput() diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2RequestTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2RequestTests.kt deleted file mode 100644 index 727225f93..000000000 --- a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2RequestTests.kt +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.action.support.WriteRequest -import org.opensearch.common.io.stream.BytesStreamOutput -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.test.OpenSearchTestCase - -class DeleteMonitorV2RequestTests : OpenSearchTestCase() { - fun `test get monitor v2 request as stream`() { - val req = DeleteMonitorV2Request( - monitorV2Id = "abc", - refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE - ) - assertNotNull(req) - - val out = BytesStreamOutput() - req.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newReq = DeleteMonitorV2Request(sin) - - assertEquals(req.monitorV2Id, newReq.monitorV2Id) - assertEquals(req.refreshPolicy, newReq.refreshPolicy) - } -} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2ResponseTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2ResponseTests.kt deleted file mode 100644 index f10082d8d..000000000 --- a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2ResponseTests.kt +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.common.io.stream.BytesStreamOutput -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.test.OpenSearchTestCase - -class DeleteMonitorV2ResponseTests : OpenSearchTestCase() { - fun `test get monitor v2 request as stream`() { - val req = DeleteMonitorV2Response( - id = "abc", - version = 3L - ) - assertNotNull(req) - - val out = BytesStreamOutput() - req.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newReq = DeleteMonitorV2Response(sin) - - assertEquals(req.id, newReq.id) - assertEquals(req.version, newReq.version) - } -} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2RequestTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2RequestTests.kt deleted file mode 100644 index b2eb38a93..000000000 --- a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2RequestTests.kt +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.alerting.assertPplMonitorsEqual -import org.opensearch.alerting.modelv2.PPLSQLMonitor -import org.opensearch.alerting.randomPPLMonitor -import org.opensearch.common.io.stream.BytesStreamOutput -import org.opensearch.common.unit.TimeValue -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.test.OpenSearchTestCase - -class ExecuteMonitorV2RequestTests : OpenSearchTestCase() { - fun `test execute monitor v2 request`() { - val req = ExecuteMonitorV2Request( - dryrun = true, - manual = false, - monitorV2Id = "abc", - monitorV2 = randomPPLMonitor(), - requestEnd = TimeValue.timeValueMinutes(30L) - ) - assertNotNull(req) - - val out = BytesStreamOutput() - req.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newReq = ExecuteMonitorV2Request(sin) - - assertEquals(req.dryrun, newReq.dryrun) - assertEquals(req.manual, newReq.manual) - assertEquals(req.monitorV2Id, newReq.monitorV2Id) - assertPplMonitorsEqual(req.monitorV2 as PPLSQLMonitor, newReq.monitorV2 as PPLSQLMonitor) - assertEquals(req.requestEnd, newReq.requestEnd) - } -} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2ResponseTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2ResponseTests.kt deleted file mode 100644 index e4f932170..000000000 --- a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2ResponseTests.kt +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.alerting.modelv2.PPLSQLMonitorRunResult -import org.opensearch.alerting.modelv2.PPLSQLTriggerRunResult -import org.opensearch.common.io.stream.BytesStreamOutput -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.test.OpenSearchTestCase - -class ExecuteMonitorV2ResponseTests : OpenSearchTestCase() { - fun `test execute monitor response`() { - val monitorRunResult = PPLSQLMonitorRunResult( - monitorName = "some-monitor", - error = IllegalArgumentException("some-error"), - triggerResults = mapOf( - "some-trigger-id" to PPLSQLTriggerRunResult( - triggerName = "some-trigger", - triggered = true, - error = IllegalArgumentException("some-error") - ) - ), - pplQueryResults = mapOf("some-result" to mapOf("some-field" to 3)) - ) - val response = ExecuteMonitorV2Response(monitorRunResult) - assertNotNull(response) - - val out = BytesStreamOutput() - response.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newResponse = ExecuteMonitorV2Response(sin) - - assertEquals(response.monitorV2RunResult.monitorName, newResponse.monitorV2RunResult.monitorName) - assertEquals(response.monitorV2RunResult.error?.message, newResponse.monitorV2RunResult.error?.message) - assert(response.monitorV2RunResult.triggerResults.containsKey("some-trigger-id")) - assert(newResponse.monitorV2RunResult.triggerResults.containsKey("some-trigger-id")) - assertEquals( - response.monitorV2RunResult.triggerResults["some-trigger-id"]!!.triggerName, - newResponse.monitorV2RunResult.triggerResults["some-trigger-id"]!!.triggerName - ) - assertEquals( - response.monitorV2RunResult.triggerResults["some-trigger-id"]!!.triggered, - newResponse.monitorV2RunResult.triggerResults["some-trigger-id"]!!.triggered - ) - assertEquals( - response.monitorV2RunResult.triggerResults["some-trigger-id"]!!.error?.message, - newResponse.monitorV2RunResult.triggerResults["some-trigger-id"]!!.error?.message - ) - assertEquals( - (response.monitorV2RunResult as PPLSQLMonitorRunResult).pplQueryResults, - (newResponse.monitorV2RunResult as PPLSQLMonitorRunResult).pplQueryResults - ) - } -} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2RequestTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2RequestTests.kt deleted file mode 100644 index aa963c8a0..000000000 --- a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2RequestTests.kt +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.common.io.stream.BytesStreamOutput -import org.opensearch.commons.alerting.model.Table -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.test.OpenSearchTestCase - -class GetAlertsV2RequestTests : OpenSearchTestCase() { - fun `test get alerts request as stream`() { - val table = Table("asc", "sortString", null, 1, 0, "") - - val req = GetAlertsV2Request( - table = table, - severityLevel = "1", - monitorV2Ids = listOf("1", "2"), - ) - assertNotNull(req) - - val out = BytesStreamOutput() - req.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newReq = GetAlertsV2Request(sin) - - assertEquals("1", newReq.severityLevel) - assertEquals(table, newReq.table) - assertTrue(newReq.monitorV2Ids!!.contains("1")) - assertTrue(newReq.monitorV2Ids!!.contains("2")) - } - - fun `test get alerts request with filter as stream`() { - val table = Table("asc", "sortString", null, 1, 0, "") - val req = GetAlertsV2Request(table, "1", null) - assertNotNull(req) - - val out = BytesStreamOutput() - req.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newReq = GetAlertsV2Request(sin) - - assertEquals("1", newReq.severityLevel) - assertNull(newReq.monitorV2Ids) - assertEquals(table, newReq.table) - } -} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2ResponseTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2ResponseTests.kt deleted file mode 100644 index 8f803bc00..000000000 --- a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2ResponseTests.kt +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.alerting.modelv2.AlertV2 -import org.opensearch.alerting.modelv2.TriggerV2 -import org.opensearch.alerting.randomUser -import org.opensearch.common.io.stream.BytesStreamOutput -import org.opensearch.common.xcontent.XContentType -import org.opensearch.commons.alerting.util.string -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.core.xcontent.ToXContent -import org.opensearch.core.xcontent.XContentBuilder -import org.opensearch.test.OpenSearchTestCase -import java.time.Instant -import java.util.Collections - -class GetAlertsV2ResponseTests : OpenSearchTestCase() { - fun `test get alerts response with no alerts`() { - val req = GetAlertsV2Response(Collections.emptyList(), 0) - assertNotNull(req) - - val out = BytesStreamOutput() - req.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newReq = GetAlertsV2Response(sin) - assertTrue(newReq.alertV2s.isEmpty()) - assertEquals(0, newReq.totalAlertV2s) - } - - fun `test get alerts response with alerts`() { - val alert = AlertV2( - monitorId = "id", - monitorName = "name", - monitorVersion = AlertV2.NO_VERSION, - monitorUser = randomUser(), - triggerId = "triggerId", - triggerName = "triggerNamer", - query = "source = some_index", - queryResults = mapOf(), - triggeredTime = Instant.now(), - errorMessage = null, - severity = TriggerV2.Severity.LOW, - executionId = "executionId" - ) - val res = GetAlertsV2Response(listOf(alert), 1) - assertNotNull(res) - - val out = BytesStreamOutput() - res.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newRes = GetAlertsV2Response(sin) - assertEquals(1, newRes.alertV2s.size) - assertEquals(alert, newRes.alertV2s[0]) - assertEquals(1, newRes.totalAlertV2s) - } - - fun `test toXContent for get alerts response`() { - val now = Instant.now() - val alert = AlertV2( - monitorId = "id", - monitorName = "name", - monitorVersion = AlertV2.NO_VERSION, - monitorUser = randomUser(), - triggerId = "triggerId", - triggerName = "triggerName", - query = "source = some_index", - queryResults = mapOf(), - triggeredTime = now, - errorMessage = null, - severity = TriggerV2.Severity.LOW, - executionId = "executionId" - ) - - val req = GetAlertsV2Response(listOf(alert), 1) - var actualXContentString = req.toXContent( - XContentBuilder.builder(XContentType.JSON.xContent()), - ToXContent.EMPTY_PARAMS - ).string() - val expectedXContentString = "{\"alerts_v2\":[{\"id\":\"\",\"version\":-1,\"monitor_v2_id\":\"id\",\"schema_version\":0," + - "\"monitor_v2_version\":-1,\"monitor_v2_name\":\"name\",\"execution_id\":\"executionId\",\"trigger_v2_id\":\"triggerId\"," + - "\"trigger_v2_name\":\"triggerName\",\"query\":\"source = some_index\",\"query_results\":{},\"error_message\":null," + - "\"severity\":\"low\",\"triggered_time\":${now.toEpochMilli()}}],\"total_alerts_v2\":1}" - - logger.info("expected: $expectedXContentString") - logger.info("actual: $actualXContentString") - - assertEquals(expectedXContentString, actualXContentString) - } -} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2RequestTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2RequestTests.kt deleted file mode 100644 index a4ede1ad2..000000000 --- a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2RequestTests.kt +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.common.io.stream.BytesStreamOutput -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.search.fetch.subphase.FetchSourceContext -import org.opensearch.test.OpenSearchTestCase - -class GetMonitorV2RequestTests : OpenSearchTestCase() { - fun `test get monitor v2 request as stream`() { - val req = GetMonitorV2Request( - monitorV2Id = "abc", - version = 2L, - srcContext = FetchSourceContext.FETCH_SOURCE - ) - assertNotNull(req) - - val out = BytesStreamOutput() - req.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newReq = GetMonitorV2Request(sin) - - assertEquals(req.monitorV2Id, newReq.monitorV2Id) - assertEquals(req.version, newReq.version) - assertEquals(req.srcContext, newReq.srcContext) - } -} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2ResponseTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2ResponseTests.kt deleted file mode 100644 index 11a8c218b..000000000 --- a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2ResponseTests.kt +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.alerting.modelv2.MonitorV2 -import org.opensearch.alerting.randomPPLMonitor -import org.opensearch.common.io.stream.BytesStreamOutput -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.test.OpenSearchTestCase - -class GetMonitorV2ResponseTests : OpenSearchTestCase() { - fun `test get monitor v2 response as stream`() { - val req = GetMonitorV2Response( - id = "abc", - version = 2L, - seqNo = 1L, - primaryTerm = 2L, - monitorV2 = randomPPLMonitor() as MonitorV2 - ) - assertNotNull(req) - - val out = BytesStreamOutput() - req.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newReq = GetMonitorV2Response(sin) - - assertEquals(req.id, newReq.id) - assertEquals(req.version, newReq.version) - assertEquals(req.seqNo, newReq.seqNo) - assertEquals(req.primaryTerm, newReq.primaryTerm) - assertEquals(req.monitorV2, newReq.monitorV2) - } -} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2RequestTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2RequestTests.kt deleted file mode 100644 index 1e7c7cd08..000000000 --- a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2RequestTests.kt +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.action.support.WriteRequest -import org.opensearch.alerting.modelv2.MonitorV2 -import org.opensearch.alerting.randomPPLMonitor -import org.opensearch.common.io.stream.BytesStreamOutput -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.rest.RestRequest -import org.opensearch.test.OpenSearchTestCase - -class IndexMonitorV2RequestTests : OpenSearchTestCase() { - fun `test index monitor v2 request as stream`() { - val req = IndexMonitorV2Request( - monitorId = "abc", - seqNo = 1L, - primaryTerm = 1L, - refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, - method = RestRequest.Method.POST, - monitorV2 = randomPPLMonitor() as MonitorV2, - rbacRoles = listOf("role-a", "role-b") - ) - assertNotNull(req) - - val out = BytesStreamOutput() - req.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newReq = IndexMonitorV2Request(sin) - - assertEquals(req.monitorId, newReq.monitorId) - assertEquals(req.seqNo, newReq.seqNo) - assertEquals(req.primaryTerm, newReq.primaryTerm) - assertEquals(req.refreshPolicy, newReq.refreshPolicy) - assertEquals(req.method, newReq.method) - assertEquals(req.monitorV2, newReq.monitorV2) - assertEquals(req.rbacRoles, newReq.rbacRoles) - } -} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2ResponseTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2ResponseTests.kt deleted file mode 100644 index 2de07698e..000000000 --- a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2ResponseTests.kt +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.alerting.modelv2.MonitorV2 -import org.opensearch.alerting.randomPPLMonitor -import org.opensearch.common.io.stream.BytesStreamOutput -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.test.OpenSearchTestCase - -class IndexMonitorV2ResponseTests : OpenSearchTestCase() { - fun `test index monitor v2 response as stream`() { - val req = IndexMonitorV2Response( - id = "abc", - version = 2L, - seqNo = 1L, - primaryTerm = 1L, - monitorV2 = randomPPLMonitor() as MonitorV2 - ) - assertNotNull(req) - - val out = BytesStreamOutput() - req.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newReq = IndexMonitorV2Response(sin) - - assertEquals(req.id, newReq.id) - assertEquals(req.version, newReq.version) - assertEquals(req.seqNo, newReq.seqNo) - assertEquals(req.primaryTerm, newReq.primaryTerm) - assertEquals(req.monitorV2, newReq.monitorV2) - } -} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2RequestTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2RequestTests.kt deleted file mode 100644 index 987a738a2..000000000 --- a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2RequestTests.kt +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.actionv2 - -import org.opensearch.action.search.SearchRequest -import org.opensearch.common.io.stream.BytesStreamOutput -import org.opensearch.common.unit.TimeValue -import org.opensearch.commons.alerting.action.SearchMonitorRequest -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.search.builder.SearchSourceBuilder -import org.opensearch.test.OpenSearchTestCase -import org.opensearch.test.rest.OpenSearchRestTestCase -import java.util.concurrent.TimeUnit - -class SearchMonitorV2RequestTests : OpenSearchTestCase() { - fun `test search monitors request`() { - val searchSourceBuilder = SearchSourceBuilder().from(0).size(100).timeout(TimeValue(60, TimeUnit.SECONDS)) - val searchRequest = SearchRequest().indices(OpenSearchRestTestCase.randomAlphaOfLength(10)).source(searchSourceBuilder) - val req = SearchMonitorV2Request(searchRequest) - assertNotNull(req) - - val out = BytesStreamOutput() - req.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newReq = SearchMonitorRequest(sin) - - assertNotNull(newReq.searchRequest) - assertEquals(1, newReq.searchRequest.indices().size) - assertEquals(req.searchRequest, newReq.searchRequest) - } -} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/alerts/AlertIndicesIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/alerts/AlertIndicesIT.kt index 9e1e2437f..5f19e27d7 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/alerts/AlertIndicesIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/alerts/AlertIndicesIT.kt @@ -54,7 +54,7 @@ class AlertIndicesIT : AlertingRestTestCase() { putAlertMappings( AlertIndices.alertMapping().trimStart('{').trimEnd('}') - .replace("\"schema_version\": 5", "\"schema_version\": 0") + .replace("\"schema_version\": 6", "\"schema_version\": 0") ) assertIndexExists(AlertIndices.ALERT_INDEX) assertIndexExists(AlertIndices.ALERT_HISTORY_WRITE_INDEX) @@ -65,8 +65,8 @@ class AlertIndicesIT : AlertingRestTestCase() { assertIndexExists(AlertIndices.ALERT_INDEX) assertIndexExists(AlertIndices.ALERT_HISTORY_WRITE_INDEX) verifyIndexSchemaVersion(ScheduledJob.SCHEDULED_JOBS_INDEX, 9) - verifyIndexSchemaVersion(AlertIndices.ALERT_INDEX, 5) - verifyIndexSchemaVersion(AlertIndices.ALERT_HISTORY_WRITE_INDEX, 5) + verifyIndexSchemaVersion(AlertIndices.ALERT_INDEX, 6) + verifyIndexSchemaVersion(AlertIndices.ALERT_HISTORY_WRITE_INDEX, 6) } fun `test update finding index mapping with new schema version`() { diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/alertsv2/AlertV2IndicesIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/alertsv2/AlertV2IndicesIT.kt index 672bc3902..7c206016b 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/alertsv2/AlertV2IndicesIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/alertsv2/AlertV2IndicesIT.kt @@ -5,23 +5,20 @@ package org.opensearch.alerting.alertsv2 -import org.junit.Before import org.opensearch.alerting.AlertingRestTestCase import org.opensearch.alerting.TEST_INDEX_MAPPINGS import org.opensearch.alerting.TEST_INDEX_NAME -import org.opensearch.alerting.core.settings.AlertingV2Settings import org.opensearch.alerting.makeRequest -import org.opensearch.alerting.modelv2.PPLSQLMonitor -import org.opensearch.alerting.modelv2.PPLSQLTrigger -import org.opensearch.alerting.modelv2.PPLSQLTrigger.ConditionType -import org.opensearch.alerting.modelv2.PPLSQLTrigger.NumResultsCondition -import org.opensearch.alerting.modelv2.PPLSQLTrigger.TriggerMode import org.opensearch.alerting.randomPPLMonitor import org.opensearch.alerting.randomPPLTrigger import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.common.settings.Settings import org.opensearch.common.xcontent.XContentType import org.opensearch.commons.alerting.model.IntervalSchedule +import org.opensearch.commons.alerting.model.Monitor +import org.opensearch.commons.alerting.model.PPLSQLTrigger.ConditionType +import org.opensearch.commons.alerting.model.PPLSQLTrigger.NumResultsCondition +import org.opensearch.commons.alerting.model.PPLSQLTrigger.TriggerMode import org.opensearch.commons.alerting.model.ScheduledJob import org.opensearch.core.rest.RestStatus import org.opensearch.test.OpenSearchTestCase @@ -36,11 +33,6 @@ import java.util.concurrent.TimeUnit * --tests "org.opensearch.alerting.alertsv2.AlertV2IndicesIT" */ class AlertV2IndicesIT : AlertingRestTestCase() { - @Before - fun enableAlertingV2() { - client().updateSettings(AlertingV2Settings.ALERTING_V2_ENABLED.key, "true") - } - fun `test create alert v2 index`() { generateAlertV2s() @@ -54,8 +46,8 @@ class AlertV2IndicesIT : AlertingRestTestCase() { assertIndexDoesNotExist(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) putAlertV2Mappings( - AlertV2Indices.alertV2Mapping().trimStart('{').trimEnd('}') - .replace("\"schema_version\": 1", "\"schema_version\": 0") + AlertV2Indices.alertMapping().trimStart('{').trimEnd('}') + .replace("\"schema_version\": 6", "\"schema_version\": 0") ) assertIndexExists(AlertV2Indices.ALERT_V2_INDEX) assertIndexExists(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) @@ -68,8 +60,8 @@ class AlertV2IndicesIT : AlertingRestTestCase() { assertIndexExists(AlertV2Indices.ALERT_V2_INDEX) assertIndexExists(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) verifyIndexSchemaVersion(ScheduledJob.SCHEDULED_JOBS_INDEX, 9) - verifyIndexSchemaVersion(AlertV2Indices.ALERT_V2_INDEX, 1) - verifyIndexSchemaVersion(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX, 1) + verifyIndexSchemaVersion(AlertV2Indices.ALERT_V2_INDEX, 6) + verifyIndexSchemaVersion(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX, 6) } fun `test alert v2 index gets recreated automatically if deleted`() { @@ -109,13 +101,13 @@ class AlertV2IndicesIT : AlertingRestTestCase() { // Disable alert history client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_ENABLED.key, "false") - val pplMonitorId = generateAlertV2s( + val pplMonitor = generateAlertV2s( randomPPLMonitor( schedule = IntervalSchedule(interval = 30, unit = MINUTES), query = "source = $TEST_INDEX_NAME | head 3", triggers = listOf( randomPPLTrigger( - mode = PPLSQLTrigger.TriggerMode.RESULT_SET, + mode = TriggerMode.RESULT_SET, conditionType = ConditionType.NUMBER_OF_RESULTS, numResultsCondition = NumResultsCondition.GREATER_THAN, numResultsValue = 0L, @@ -125,7 +117,7 @@ class AlertV2IndicesIT : AlertingRestTestCase() { ) ) - val alerts1 = searchAlertV2s(pplMonitorId) + val alerts1 = searchAlerts(pplMonitor, AlertV2Indices.ALERT_V2_INDEX) assertEquals("1 alert should be present", 1, alerts1.size) // wait for alert to expire. @@ -136,20 +128,20 @@ class AlertV2IndicesIT : AlertingRestTestCase() { }, 2, TimeUnit.MINUTES) // Since history is disabled, the alert should be hard deleted by now - val alerts2 = searchAlertV2s(pplMonitorId, AlertV2Indices.ALL_ALERT_V2_INDEX_PATTERN) + val alerts2 = searchAlerts(pplMonitor, AlertV2Indices.ALL_ALERT_V2_INDEX_PATTERN) assertTrue("There should be no alerts, but alerts were found", alerts2.isEmpty()) } fun `test short retention period`() { resetHistorySettings() - val pplMonitorId = generateAlertV2s( + val pplMonitor = generateAlertV2s( randomPPLMonitor( schedule = IntervalSchedule(interval = 30, unit = MINUTES), query = "source = $TEST_INDEX_NAME | head 3", triggers = listOf( randomPPLTrigger( - mode = PPLSQLTrigger.TriggerMode.RESULT_SET, + mode = TriggerMode.RESULT_SET, conditionType = ConditionType.NUMBER_OF_RESULTS, numResultsCondition = NumResultsCondition.GREATER_THAN, numResultsValue = 0L, @@ -159,7 +151,7 @@ class AlertV2IndicesIT : AlertingRestTestCase() { ) ) - val alerts1 = searchAlertV2s(pplMonitorId) + val alerts1 = searchAlerts(pplMonitor, AlertV2Indices.ALERT_V2_INDEX) assertEquals("1 alert should be present", 1, alerts1.size) // history index should be created but empty @@ -172,7 +164,7 @@ class AlertV2IndicesIT : AlertingRestTestCase() { return@waitUntil false }, 2, TimeUnit.MINUTES) - assertTrue(searchAlertV2s(pplMonitorId).isEmpty()) + assertTrue(searchAlerts(pplMonitor, AlertV2Indices.ALERT_V2_INDEX).isEmpty()) assertEquals(1, getAlertV2HistoryDocCount()) // update rollover check and max docs as well as decreasing the retention period @@ -215,24 +207,24 @@ class AlertV2IndicesIT : AlertingRestTestCase() { ) ) - val executeResponse = executeMonitorV2(pplMonitor.id) + val executeResponse = executeMonitor(pplMonitor.id) val triggered = isTriggered(pplMonitor, executeResponse) - val getAlertsResponsePreExpire = getAlertV2s() + val getAlertsResponsePreExpire = getAlerts() val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 assert(triggered) { "Monitor should have triggered but it didn't" } assert(alertsGeneratedPreExpire) { "Alerts should have been generated but they weren't" } // delete the monitor - deleteMonitorV2(pplMonitor.id) + deleteMonitor(pplMonitor) // sleep so postDelete can expire the generated alert OpenSearchTestCase.waitUntil({ return@waitUntil false }, 5, TimeUnit.SECONDS) - val getAlertsResponsePostExpire = getAlertV2s() + val getAlertsResponsePostExpire = getAlerts() val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 assert(!alertsGeneratedPostExpire) @@ -265,10 +257,10 @@ class AlertV2IndicesIT : AlertingRestTestCase() { val pplMonitor = createRandomPPLMonitor(initialPplMonitorConfig) - val executeResponse = executeMonitorV2(pplMonitor.id) + val executeResponse = executeMonitor(pplMonitor.id) val triggered = isTriggered(pplMonitor, executeResponse) - val getAlertsResponsePreExpire = getAlertV2s() + val getAlertsResponsePreExpire = getAlerts() val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 assert(triggered) { "Monitor should have triggered but it didn't" } @@ -276,14 +268,14 @@ class AlertV2IndicesIT : AlertingRestTestCase() { // update the monitor to any new config, // and more importantly, updated triggers - updateMonitorV2(randomPPLMonitor().makeCopy(pplMonitor.id, pplMonitor.version)) + updateMonitor(randomPPLMonitor().copy(id = pplMonitor.id, version = pplMonitor.version)) // sleep so postIndex can expire the generated alert OpenSearchTestCase.waitUntil({ return@waitUntil false }, 5, TimeUnit.SECONDS) - val getAlertsResponsePostExpire = getAlertV2s() + val getAlertsResponsePostExpire = getAlerts() val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 assert(!alertsGeneratedPostExpire) @@ -316,24 +308,24 @@ class AlertV2IndicesIT : AlertingRestTestCase() { ) ) - val executeResponse = executeMonitorV2(pplMonitor.id) + val executeResponse = executeMonitor(pplMonitor.id) val triggered = isTriggered(pplMonitor, executeResponse) - val getAlertsResponsePreExpire = getAlertV2s() + val getAlertsResponsePreExpire = getAlerts() val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 assert(triggered) { "Monitor should have triggered but it didn't" } assert(alertsGeneratedPreExpire) { "Alerts should have been generated but they weren't" } // delete the monitor - deleteMonitorV2(pplMonitor.id) + deleteMonitor(pplMonitor) // sleep so postDelete can expire the generated alert OpenSearchTestCase.waitUntil({ return@waitUntil false }, 5, TimeUnit.SECONDS) - val getAlertsResponsePostExpire = getAlertV2s() + val getAlertsResponsePostExpire = getAlerts() val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 assert(!alertsGeneratedPostExpire) @@ -368,24 +360,24 @@ class AlertV2IndicesIT : AlertingRestTestCase() { val pplMonitor = createRandomPPLMonitor(initialPplMonitorConfig) - val executeResponse = executeMonitorV2(pplMonitor.id) + val executeResponse = executeMonitor(pplMonitor.id) val triggered = isTriggered(pplMonitor, executeResponse) - val getAlertsResponsePreExpire = getAlertV2s() + val getAlertsResponsePreExpire = getAlerts() val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 assert(triggered) { "Monitor should have triggered but it didn't" } assert(alertsGeneratedPreExpire) { "Alerts should have been generated but they weren't" } // update the monitor to any new config - updateMonitorV2(randomPPLMonitor().makeCopy(pplMonitor.id, pplMonitor.version)) + updateMonitor(randomPPLMonitor().copy(id = pplMonitor.id, version = pplMonitor.version)) // sleep so postIndex can expire the generated alert OpenSearchTestCase.waitUntil({ return@waitUntil false }, 5, TimeUnit.SECONDS) - val getAlertsResponsePostExpire = getAlertV2s() + val getAlertsResponsePostExpire = getAlerts() val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 assert(!alertsGeneratedPostExpire) @@ -420,7 +412,7 @@ class AlertV2IndicesIT : AlertingRestTestCase() { // generates alerts by creating then executing a monitor private fun generateAlertV2s( - pplMonitorConfig: PPLSQLMonitor = randomPPLMonitor( + pplMonitorConfig: Monitor = randomPPLMonitor( query = "source = $TEST_INDEX_NAME | head 3", triggers = listOf( randomPPLTrigger( @@ -430,7 +422,7 @@ class AlertV2IndicesIT : AlertingRestTestCase() { ) ) ) - ): String { + ): Monitor { createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) indexDocFromSomeTimeAgo(2, MINUTES, "def", 10) @@ -438,13 +430,13 @@ class AlertV2IndicesIT : AlertingRestTestCase() { val pplMonitor = createRandomPPLMonitor(pplMonitorConfig) - val executeResponse = executeMonitorV2(pplMonitor.id) + val executeResponse = executeMonitor(pplMonitor.id) // ensure execute call succeeded val xcp = createParser(XContentType.JSON.xContent(), executeResponse.entity.content) val output = xcp.map() assertNull("Error running monitor v2", output["error"]) - return pplMonitor.id + return pplMonitor } } diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/AlertV2Tests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/AlertV2Tests.kt deleted file mode 100644 index fade152bc..000000000 --- a/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/AlertV2Tests.kt +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.modelv2 - -import org.opensearch.alerting.assertAlertV2sEqual -import org.opensearch.alerting.modelv2.AlertV2.Companion.ALERT_V2_ID_FIELD -import org.opensearch.alerting.modelv2.AlertV2.Companion.ALERT_V2_VERSION_FIELD -import org.opensearch.alerting.modelv2.AlertV2.Companion.ERROR_MESSAGE_FIELD -import org.opensearch.alerting.modelv2.AlertV2.Companion.EXECUTION_ID_FIELD -import org.opensearch.alerting.modelv2.AlertV2.Companion.SEVERITY_FIELD -import org.opensearch.alerting.randomAlertV2 -import org.opensearch.common.io.stream.BytesStreamOutput -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.test.OpenSearchTestCase - -class AlertV2Tests : OpenSearchTestCase() { - fun `test alertv2 as stream`() { - val alertV2 = randomAlertV2() - val out = BytesStreamOutput() - alertV2.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newAlertV2 = AlertV2(sin) - assertAlertV2sEqual(alertV2, newAlertV2) - } - - fun `test alertv2 asTemplateArgs`() { - val alertV2 = randomAlertV2() - val templateArgs = alertV2.asTemplateArg() - - assertEquals( - "Template args field $ALERT_V2_ID_FIELD doesn't match", - alertV2.id, - templateArgs[ALERT_V2_ID_FIELD] - ) - assertEquals( - "Template args field $ALERT_V2_VERSION_FIELD doesn't match", - alertV2.version, - templateArgs[ALERT_V2_VERSION_FIELD] - ) - assertEquals( - "Template args field $ERROR_MESSAGE_FIELD doesn't match", - alertV2.errorMessage, - templateArgs[ERROR_MESSAGE_FIELD] - ) - assertEquals( - "Template args field $EXECUTION_ID_FIELD doesn't match", - alertV2.executionId, - templateArgs[EXECUTION_ID_FIELD] - ) - assertEquals( - "Template args field $SEVERITY_FIELD doesn't match", - alertV2.severity.value, - templateArgs[SEVERITY_FIELD] - ) - } -} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/MonitorV2Tests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/MonitorV2Tests.kt deleted file mode 100644 index 998131ad2..000000000 --- a/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/MonitorV2Tests.kt +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.modelv2 - -import org.opensearch.alerting.assertPplMonitorsEqual -import org.opensearch.alerting.modelv2.MonitorV2.Companion.ALERTING_V2_MAX_NAME_LENGTH -import org.opensearch.alerting.modelv2.MonitorV2.Companion.ENABLED_FIELD -import org.opensearch.alerting.modelv2.MonitorV2.Companion.ENABLED_TIME_FIELD -import org.opensearch.alerting.modelv2.MonitorV2.Companion.LAST_UPDATE_TIME_FIELD -import org.opensearch.alerting.modelv2.MonitorV2.Companion.LOOK_BACK_WINDOW_FIELD -import org.opensearch.alerting.modelv2.MonitorV2.Companion.MONITOR_V2_MIN_LOOK_BACK_WINDOW -import org.opensearch.alerting.modelv2.MonitorV2.Companion.NAME_FIELD -import org.opensearch.alerting.modelv2.MonitorV2.Companion.SCHEDULE_FIELD -import org.opensearch.alerting.modelv2.PPLSQLMonitor.Companion.QUERY_FIELD -import org.opensearch.alerting.randomPPLMonitor -import org.opensearch.alerting.randomPPLTrigger -import org.opensearch.common.io.stream.BytesStreamOutput -import org.opensearch.commons.alerting.util.IndexUtils.Companion._ID -import org.opensearch.commons.alerting.util.IndexUtils.Companion._VERSION -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.test.OpenSearchTestCase -import java.lang.IllegalArgumentException -import java.time.Instant - -class MonitorV2Tests : OpenSearchTestCase() { - fun `test enabled time`() { - val pplMonitor = randomPPLMonitor(enabled = true, enabledTime = Instant.now()) - try { - pplMonitor.makeCopy(enabled = false) - fail("Disabling monitor with enabled time set should fail.") - } catch (_: IllegalArgumentException) {} - - val disabledMonitor = pplMonitor.copy(enabled = false, enabledTime = null) - - try { - disabledMonitor.makeCopy(enabled = true) - fail("Enabling monitor without enabled time should fail") - } catch (_: IllegalArgumentException) {} - } - - fun `test max triggers`() { - val tooManyTriggers = mutableListOf() - for (i in 0..10) { // 11 times - tooManyTriggers.add(randomPPLTrigger()) - } - - try { - randomPPLMonitor(triggers = tooManyTriggers) - fail("Monitor with too many triggers should be rejected.") - } catch (_: IllegalArgumentException) {} - } - - fun `test monitor name too long`() { - var monitorName = "" - for (i in 0 until ALERTING_V2_MAX_NAME_LENGTH + 1) { - monitorName += "a" - } - - try { - randomPPLMonitor(name = monitorName) - fail("Monitor with too long a name should be rejected.") - } catch (_: IllegalArgumentException) {} - } - - fun `test monitor min look back window`() { - try { - randomPPLMonitor( - lookBackWindow = MONITOR_V2_MIN_LOOK_BACK_WINDOW - 1 - ) - fail("Monitor with too long a name should be rejected.") - } catch (_: IllegalArgumentException) {} - } - - fun `test monitor no triggers`() { - try { - randomPPLMonitor( - triggers = listOf() - ) - fail("Monitor without triggers be rejected.") - } catch (_: IllegalArgumentException) {} - } - - fun `test monitor with look back window without timestamp field`() { - try { - randomPPLMonitor( - lookBackWindow = randomLongBetween(1, 10), - timestampField = null - ) - fail("Monitor with look back window but without timestamp field be rejected.") - } catch (_: IllegalArgumentException) {} - } - - fun `test monitor without look back window with timestamp field`() { - try { - randomPPLMonitor( - lookBackWindow = null, - timestampField = "some_timestamp_field" - ) - fail("Monitor without look back window but with timestamp field be rejected.") - } catch (_: IllegalArgumentException) {} - } - - fun `test monitor v2 as stream`() { - val pplMonitor = randomPPLMonitor() - val monitorV2 = pplMonitor as MonitorV2 - val out = BytesStreamOutput() - MonitorV2.writeTo(out, monitorV2) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newMonitorV2 = MonitorV2.readFrom(sin) - val newPplMonitor = newMonitorV2 as PPLSQLMonitor - assertPplMonitorsEqual(pplMonitor, newPplMonitor) - } - - fun `test ppl monitor as stream`() { - val pplMonitor = randomPPLMonitor() - val out = BytesStreamOutput() - pplMonitor.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newPplMonitor = PPLSQLMonitor(sin) - assertPplMonitorsEqual(pplMonitor, newPplMonitor) - } - - fun `test ppl monitor asTemplateArgs`() { - val pplMonitor = randomPPLMonitor() - val templateArgs = pplMonitor.asTemplateArg() - - assertEquals( - "Template args field $_ID doesn't match", - pplMonitor.id, - templateArgs[_ID] - ) - assertEquals( - "Template args field $_VERSION doesn't match", - pplMonitor.version, - templateArgs[_VERSION] - ) - assertEquals( - "Template args field $NAME_FIELD doesn't match", - pplMonitor.name, - templateArgs[NAME_FIELD] - ) - assertEquals( - "Template args field $ENABLED_FIELD doesn't match", - pplMonitor.enabled, - templateArgs[ENABLED_FIELD] - ) - assertNotNull(templateArgs[SCHEDULE_FIELD]) - assertEquals( - "Template args field $LOOK_BACK_WINDOW_FIELD doesn't match", - pplMonitor.lookBackWindow, - templateArgs[LOOK_BACK_WINDOW_FIELD] - ) - assertEquals( - "Template args field $LAST_UPDATE_TIME_FIELD doesn't match", - pplMonitor.lastUpdateTime.toEpochMilli(), - templateArgs[LAST_UPDATE_TIME_FIELD] - ) - assertEquals( - "Template args field $ENABLED_TIME_FIELD doesn't match", - pplMonitor.enabledTime?.toEpochMilli(), - templateArgs[ENABLED_TIME_FIELD] - ) - assertEquals( - "Template args field $QUERY_FIELD doesn't match", - pplMonitor.query, - templateArgs[QUERY_FIELD] - ) - } -} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/RunResultV2Tests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/RunResultV2Tests.kt deleted file mode 100644 index 4290af27c..000000000 --- a/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/RunResultV2Tests.kt +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.modelv2 - -import org.opensearch.common.io.stream.BytesStreamOutput -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.test.OpenSearchTestCase - -class RunResultV2Tests : OpenSearchTestCase() { - fun `test ppl sql trigger run result as stream`() { - val runResult = PPLSQLTriggerRunResult( - triggerName = "some-trigger", - triggered = true, - error = IllegalArgumentException("some-error") - ) - val out = BytesStreamOutput() - runResult.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newRunResult = PPLSQLTriggerRunResult(sin) - assertEquals(runResult.triggerName, newRunResult.triggerName) - } - - fun `test ppl sql monitor run result as monitor v2 run result as stream`() { - val monitorRunResult = PPLSQLMonitorRunResult( - monitorName = "some-monitor", - error = IllegalArgumentException("some-error"), - triggerResults = mapOf( - "some-trigger-id" to PPLSQLTriggerRunResult( - triggerName = "some-trigger", - triggered = true, - error = IllegalArgumentException("some-error") - ) - ), - pplQueryResults = mapOf("some-result" to mapOf("some-field" to 2)) - ) as MonitorV2RunResult - - val out = BytesStreamOutput() - MonitorV2RunResult.writeTo(out, monitorRunResult) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newMonitorRunResult = MonitorV2RunResult.readFrom(sin) - assertEquals(monitorRunResult.monitorName, newMonitorRunResult.monitorName) - assertEquals(monitorRunResult.error?.message, newMonitorRunResult.error?.message) - assert(monitorRunResult.triggerResults.containsKey("some-trigger-id")) - assert(newMonitorRunResult.triggerResults.containsKey("some-trigger-id")) - assertEquals( - monitorRunResult.triggerResults["some-trigger-id"]!!.triggerName, - newMonitorRunResult.triggerResults["some-trigger-id"]!!.triggerName - ) - assertEquals( - monitorRunResult.triggerResults["some-trigger-id"]!!.triggered, - newMonitorRunResult.triggerResults["some-trigger-id"]!!.triggered - ) - assertEquals( - monitorRunResult.triggerResults["some-trigger-id"]!!.error?.message, - newMonitorRunResult.triggerResults["some-trigger-id"]!!.error?.message - ) - } -} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/TriggerV2Tests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/TriggerV2Tests.kt deleted file mode 100644 index 430572254..000000000 --- a/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/TriggerV2Tests.kt +++ /dev/null @@ -1,254 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.modelv2 - -import org.opensearch.alerting.assertPplTriggersEqual -import org.opensearch.alerting.modelv2.MonitorV2.Companion.ALERTING_V2_MAX_NAME_LENGTH -import org.opensearch.alerting.modelv2.PPLSQLTrigger.Companion.CONDITION_TYPE_FIELD -import org.opensearch.alerting.modelv2.PPLSQLTrigger.Companion.CUSTOM_CONDITION_FIELD -import org.opensearch.alerting.modelv2.PPLSQLTrigger.Companion.MODE_FIELD -import org.opensearch.alerting.modelv2.PPLSQLTrigger.Companion.NUM_RESULTS_CONDITION_FIELD -import org.opensearch.alerting.modelv2.PPLSQLTrigger.Companion.NUM_RESULTS_VALUE_FIELD -import org.opensearch.alerting.modelv2.PPLSQLTrigger.ConditionType -import org.opensearch.alerting.modelv2.PPLSQLTrigger.NumResultsCondition -import org.opensearch.alerting.modelv2.TriggerV2.Companion.ACTIONS_FIELD -import org.opensearch.alerting.modelv2.TriggerV2.Companion.EXPIRE_FIELD -import org.opensearch.alerting.modelv2.TriggerV2.Companion.ID_FIELD -import org.opensearch.alerting.modelv2.TriggerV2.Companion.MONITOR_V2_MIN_EXPIRE_DURATION_MINUTES -import org.opensearch.alerting.modelv2.TriggerV2.Companion.MONITOR_V2_MIN_THROTTLE_DURATION_MINUTES -import org.opensearch.alerting.modelv2.TriggerV2.Companion.NAME_FIELD -import org.opensearch.alerting.modelv2.TriggerV2.Companion.NOTIFICATIONS_ID_MAX_LENGTH -import org.opensearch.alerting.modelv2.TriggerV2.Companion.SEVERITY_FIELD -import org.opensearch.alerting.modelv2.TriggerV2.Companion.THROTTLE_FIELD -import org.opensearch.alerting.randomAction -import org.opensearch.alerting.randomPPLTrigger -import org.opensearch.common.io.stream.BytesStreamOutput -import org.opensearch.core.common.io.stream.StreamInput -import org.opensearch.test.OpenSearchTestCase -import java.lang.IllegalArgumentException - -class TriggerV2Tests : OpenSearchTestCase() { - fun `test min throttle duration`() { - try { - randomPPLTrigger( - throttleDuration = MONITOR_V2_MIN_THROTTLE_DURATION_MINUTES - 1 - ) - fail("Trigger with throttle duration less than 1 should be rejected") - } catch (_: IllegalArgumentException) {} - } - - fun `test min expire duration`() { - try { - randomPPLTrigger( - expireDuration = MONITOR_V2_MIN_EXPIRE_DURATION_MINUTES - 1 - ) - fail("Trigger with expire duration less than 1 should be rejected") - } catch (_: IllegalArgumentException) {} - } - - fun `test trigger name too long`() { - var triggerName = "" - for (i in 0 until ALERTING_V2_MAX_NAME_LENGTH + 1) { - triggerName += "a" - } - - try { - randomPPLTrigger(name = triggerName) - fail("Trigger with too long a name should be rejected.") - } catch (_: IllegalArgumentException) {} - } - - fun `test number of results trigger with negative number of results value`() { - try { - randomPPLTrigger( - conditionType = ConditionType.NUMBER_OF_RESULTS, - numResultsValue = -1L, - numResultsCondition = NumResultsCondition.GREATER_THAN - ) - fail("Number of results trigger with negative number of results value should be rejected.") - } catch (_: IllegalArgumentException) {} - } - - fun `test trigger action name too long`() { - var actionName = "" - for (i in 0 until ALERTING_V2_MAX_NAME_LENGTH + 1) { - actionName += "a" - } - - try { - randomPPLTrigger( - actions = listOf( - randomAction( - name = actionName - ) - ) - ) - fail("Trigger action with too long a name should be rejected.") - } catch (_: IllegalArgumentException) {} - } - - fun `test trigger action channel ID too long`() { - var channelId = "" - for (i in 0 until NOTIFICATIONS_ID_MAX_LENGTH + 1) { - channelId += "a" - } - - try { - randomPPLTrigger( - actions = listOf( - randomAction( - destinationId = channelId - ) - ) - ) - fail("Trigger action with too long a channel ID should be rejected.") - } catch (_: IllegalArgumentException) {} - } - - fun `test number_of_results trigger has no number_of_results value field`() { - try { - randomPPLTrigger( - conditionType = ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = NumResultsCondition.entries.random(), - numResultsValue = null, - customCondition = null - ) - fail("Number of results trigger that has no number of results value should be rejected.") - } catch (_: IllegalArgumentException) {} - } - - fun `test number_of_results trigger has no number_of_results condition field`() { - try { - randomPPLTrigger( - conditionType = ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = null, - numResultsValue = randomLongBetween(1, 10), - customCondition = null - ) - fail("Number of results trigger that has no number of results condition should be rejected.") - } catch (_: IllegalArgumentException) {} - } - - fun `test number_of_results trigger has custom_condition value field`() { - try { - randomPPLTrigger( - conditionType = ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = null, - numResultsValue = null, - customCondition = "eval result = something > 5" - ) - fail("Number of results trigger that has custom condition should be rejected.") - } catch (_: IllegalArgumentException) {} - } - - fun `test custom trigger has number_of_results value field`() { - try { - randomPPLTrigger( - conditionType = ConditionType.CUSTOM, - numResultsCondition = NumResultsCondition.entries.random(), - numResultsValue = null, - customCondition = null - ) - fail("Number of results trigger that has no number of results value should be rejected.") - } catch (_: IllegalArgumentException) {} - } - - fun `test custom trigger has number_of_results condition field`() { - try { - randomPPLTrigger( - conditionType = ConditionType.CUSTOM, - numResultsCondition = null, - numResultsValue = randomLongBetween(1, 10), - customCondition = null - ) - fail("Number of results trigger that has no number of results condition should be rejected.") - } catch (_: IllegalArgumentException) {} - } - - fun `test custom trigger has no custom_condition value field`() { - try { - randomPPLTrigger( - conditionType = ConditionType.CUSTOM, - numResultsCondition = null, - numResultsValue = null, - customCondition = null - ) - fail("Number of results trigger that has custom condition should be rejected.") - } catch (_: IllegalArgumentException) {} - } - - fun `test ppl trigger as stream`() { - val pplTrigger = randomPPLTrigger() - val out = BytesStreamOutput() - pplTrigger.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newPplTrigger = PPLSQLTrigger(sin) - assertPplTriggersEqual(pplTrigger, newPplTrigger) - } - - fun `test ppl trigger asTemplateArgs`() { - val pplTrigger = randomPPLTrigger() - val templateArgs = pplTrigger.asTemplateArg() - - assertEquals( - "Template args field $ID_FIELD doesn't match", - pplTrigger.id, - templateArgs[ID_FIELD] - ) - assertEquals( - "Template args field $NAME_FIELD doesn't match", - pplTrigger.name, - templateArgs[NAME_FIELD] - ) - assertEquals( - "Template args field $SEVERITY_FIELD doesn't match", - pplTrigger.severity.value, - templateArgs[SEVERITY_FIELD] - ) - assertEquals( - "Template args field $THROTTLE_FIELD doesn't match", - pplTrigger.throttleDuration, - templateArgs[THROTTLE_FIELD] - ) - assertEquals( - "Template args field $EXPIRE_FIELD doesn't match", - pplTrigger.expireDuration, - templateArgs[EXPIRE_FIELD] - ) - assertEquals( - "Template args field $EXPIRE_FIELD doesn't match", - pplTrigger.expireDuration, - templateArgs[EXPIRE_FIELD] - ) - val actions = templateArgs[ACTIONS_FIELD] as List<*> - assertEquals("number of trigger actions doesn't match", pplTrigger.actions.size, actions.size) - assertEquals( - "Template args field $MODE_FIELD doesn't match", - pplTrigger.mode.value, - templateArgs[MODE_FIELD] - ) - assertEquals( - "Template args field $CONDITION_TYPE_FIELD doesn't match", - pplTrigger.conditionType.value, - templateArgs[CONDITION_TYPE_FIELD] - ) - assertEquals( - "Template args field $NUM_RESULTS_CONDITION_FIELD doesn't match", - pplTrigger.numResultsCondition?.value, - templateArgs[NUM_RESULTS_CONDITION_FIELD] - ) - assertEquals( - "Template args field $NUM_RESULTS_VALUE_FIELD doesn't match", - pplTrigger.numResultsValue, - templateArgs[NUM_RESULTS_VALUE_FIELD] - ) - assertEquals( - "Template args field $CUSTOM_CONDITION_FIELD doesn't match", - pplTrigger.customCondition, - templateArgs[CUSTOM_CONDITION_FIELD] - ) - } -} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorV2RestApiIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorV2RestApiIT.kt index dbf379e91..3c5c5920e 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorV2RestApiIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorV2RestApiIT.kt @@ -7,26 +7,19 @@ package org.opensearch.alerting.resthandler import org.apache.hc.core5.http.ContentType import org.apache.hc.core5.http.io.entity.StringEntity -import org.junit.Before -import org.opensearch.alerting.AlertingPlugin.Companion.MONITOR_V2_BASE_URI +import org.opensearch.alerting.AlertingPlugin.Companion.MONITOR_BASE_URI import org.opensearch.alerting.AlertingRestTestCase import org.opensearch.alerting.TEST_INDEX_MAPPINGS import org.opensearch.alerting.TEST_INDEX_NAME import org.opensearch.alerting.assertPplMonitorsEqual -import org.opensearch.alerting.core.settings.AlertingV2Settings import org.opensearch.alerting.makeRequest -import org.opensearch.alerting.modelv2.MonitorV2 -import org.opensearch.alerting.modelv2.PPLSQLMonitor -import org.opensearch.alerting.modelv2.PPLSQLTrigger.ConditionType import org.opensearch.alerting.randomAction import org.opensearch.alerting.randomPPLMonitor import org.opensearch.alerting.randomPPLTrigger -import org.opensearch.alerting.randomQueryLevelMonitor import org.opensearch.alerting.randomTemplateScript import org.opensearch.alerting.resthandler.MonitorRestApiIT.Companion.USE_TYPED_KEYS +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_MAX_MONITORS import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_EXPIRE_DURATION -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_LOOK_BACK_WINDOW -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_MONITORS import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_QUERY_LENGTH import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_THROTTLE_DURATION import org.opensearch.alerting.settings.AlertingSettings.Companion.NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH @@ -35,6 +28,8 @@ import org.opensearch.client.ResponseException import org.opensearch.common.UUIDs import org.opensearch.common.settings.Settings import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.alerting.model.Monitor +import org.opensearch.commons.alerting.model.PPLSQLTrigger.ConditionType import org.opensearch.commons.alerting.model.ScheduledJob import org.opensearch.core.common.bytes.BytesReference import org.opensearch.core.rest.RestStatus @@ -56,23 +51,18 @@ import java.time.temporal.ChronoUnit.MINUTES @TestLogging("level:DEBUG", reason = "Debug for tests.") @Suppress("UNCHECKED_CAST") class MonitorV2RestApiIT : AlertingRestTestCase() { - @Before - fun enableAlertingV2() { - client().updateSettings(AlertingV2Settings.ALERTING_V2_ENABLED.key, "true") - } - /* Simple Case Tests */ fun `test create ppl monitor`() { createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) val pplMonitor = randomPPLMonitor() - val response = client().makeRequest("POST", MONITOR_V2_BASE_URI, emptyMap(), pplMonitor.toHttpEntity()) - assertEquals("Unable to create a new monitor v2", RestStatus.OK, response.restStatus()) + val response = client().makeRequest("POST", MONITOR_BASE_URI, emptyMap(), pplMonitor.toHttpEntity()) + assertEquals("Unable to create a new monitor v2", RestStatus.CREATED, response.restStatus()) val responseBody = response.asMap() val createdId = responseBody["_id"] as String val createdVersion = responseBody["_version"] as Int - assertNotEquals("response is missing Id", MonitorV2.NO_ID, createdId) + assertNotEquals("response is missing Id", Monitor.NO_ID, createdId) assertEquals("incorrect version", 1, createdVersion) } @@ -83,7 +73,7 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { val updateResponse = client().makeRequest( "PUT", - "$MONITOR_V2_BASE_URI/${originalMonitor.id}", + "$MONITOR_BASE_URI/${originalMonitor.id}", emptyMap(), newMonitorConfig.toHttpEntity() ) @@ -92,7 +82,7 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { assertEquals("Updated monitor id doesn't match", originalMonitor.id, responseBody["_id"] as String) assertEquals("Version not incremented", (originalMonitor.version + 1).toInt(), responseBody["_version"] as Int) - val updatedMonitor = getMonitorV2(originalMonitor.id) as PPLSQLMonitor + val updatedMonitor = getMonitor(originalMonitor.id) assertPplMonitorsEqual(newMonitorConfig, updatedMonitor) } @@ -101,15 +91,15 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) val pplMonitor = randomPPLMonitor() - val createResponse = client().makeRequest("POST", MONITOR_V2_BASE_URI, emptyMap(), pplMonitor.toHttpEntity()) - assertEquals("Unable to create a new monitor v2", RestStatus.OK, createResponse.restStatus()) + val createResponse = client().makeRequest("POST", MONITOR_BASE_URI, emptyMap(), pplMonitor.toHttpEntity()) + assertEquals("Unable to create a new monitor v2", RestStatus.CREATED, createResponse.restStatus()) val responseBody = createResponse.asMap() val pplMonitorId = responseBody["_id"] as String val pplMonitorVersion = (responseBody["_version"] as Int).toLong() // then attempt to get it - val response = client().makeRequest("GET", "$MONITOR_V2_BASE_URI/$pplMonitorId") + val response = client().makeRequest("GET", "$MONITOR_BASE_URI/$pplMonitorId") assertEquals("Unable to get monitorV2 $pplMonitorId", RestStatus.OK, response.restStatus()) val parser = createParser(XContentType.JSON.xContent(), response.entity.content) @@ -117,7 +107,7 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { lateinit var id: String var version: Long = 0 - lateinit var storedPplMonitor: PPLSQLMonitor + lateinit var storedPplMonitor: Monitor while (parser.nextToken() != XContentParser.Token.END_OBJECT) { parser.nextToken() @@ -125,7 +115,17 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { when (parser.currentName()) { "_id" -> id = parser.text() "_version" -> version = parser.longValue() - "monitorV2" -> storedPplMonitor = MonitorV2.parse(parser) as PPLSQLMonitor + "monitor" -> storedPplMonitor = Monitor.parse(parser) + "associated_workflows" -> { + XContentParserUtils.ensureExpectedToken( + XContentParser.Token.START_ARRAY, + parser.currentToken(), + parser + ) + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + // do nothing + } + } } } @@ -142,8 +142,8 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { fun `test head ppl monitor`() { val submittedPplMonitor = createRandomPPLMonitor() - val response = client().makeRequest("HEAD", "$MONITOR_V2_BASE_URI/${submittedPplMonitor.id}") - assertEquals("Unable to get monitorV2 ${submittedPplMonitor.id}", RestStatus.NO_CONTENT, response.restStatus()) + val response = client().makeRequest("HEAD", "$MONITOR_BASE_URI/${submittedPplMonitor.id}") + assertEquals("Unable to get monitorV2 ${submittedPplMonitor.id}", RestStatus.OK, response.restStatus()) } fun `test search ppl monitor with GET and match_all`() { @@ -151,7 +151,7 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { val search = SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).toString() val searchResponse = client().makeRequest( - "GET", "$MONITOR_V2_BASE_URI/_search", + "GET", "$MONITOR_BASE_URI/_search", emptyMap(), StringEntity(search, ContentType.APPLICATION_JSON) ) @@ -167,7 +167,7 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { val search = SearchSourceBuilder().query(QueryBuilders.termQuery("_id", pplMonitor.id)).toString() val searchResponse = client().makeRequest( - "POST", "$MONITOR_V2_BASE_URI/_search", + "POST", "$MONITOR_BASE_URI/_search", emptyMap(), StringEntity(search, ContentType.APPLICATION_JSON) ) @@ -181,10 +181,10 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { fun `test delete ppl monitor`() { val pplMonitor = createRandomPPLMonitor() - val deleteResponse = client().makeRequest("DELETE", "$MONITOR_V2_BASE_URI/${pplMonitor.id}") + val deleteResponse = client().makeRequest("DELETE", "$MONITOR_BASE_URI/${pplMonitor.id}") assertEquals("Delete failed", RestStatus.OK, deleteResponse.restStatus()) - val getResponse = client().makeRequest("HEAD", "$MONITOR_V2_BASE_URI/${pplMonitor.id}") + val getResponse = client().makeRequest("HEAD", "$MONITOR_BASE_URI/${pplMonitor.id}") assertEquals("Deleted monitor still exists", RestStatus.NOT_FOUND, getResponse.restStatus()) } @@ -198,30 +198,6 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { assertEquals(monitorV2, scheduledJob) } - fun `test monitor stats v1 and v2 only return stats for their respective monitors`() { - enableScheduledJob() - - val monitorV1Id = createMonitor(randomQueryLevelMonitor(enabled = true)).id - val monitorV2Id = createRandomPPLMonitor(randomPPLMonitor(enabled = true)).id - - val statsAllResponse = getAlertingStats(alertingVersion = null) - val statsV1Response = getAlertingStats(alertingVersion = "v1") - val statsV2Response = getAlertingStats(alertingVersion = "v2") - - logger.info("all stats: $statsAllResponse") - logger.info("v1 stats: $statsV1Response") - logger.info("v2 stats: $statsV2Response") - - assertTrue("All stats does not contain V1 Monitor", isMonitorScheduled(monitorV1Id, statsAllResponse)) - assertTrue("All stats does not contain V2 Monitor", isMonitorScheduled(monitorV2Id, statsAllResponse)) - - assertTrue("V1 stats does not contain V1 Monitor", isMonitorScheduled(monitorV1Id, statsV1Response)) - assertFalse("V1 stats contains V2 Monitor", isMonitorScheduled(monitorV2Id, statsV1Response)) - - assertTrue("V2 stats does not contain V2 Monitor", isMonitorScheduled(monitorV2Id, statsV2Response)) - assertFalse("V2 stats contains V1 Monitor", isMonitorScheduled(monitorV1Id, statsV2Response)) - } - /* Validation Tests */ fun `test create ppl monitor that queries nonexistent index fails`() { val pplMonitorConfig = randomPPLMonitor( @@ -237,11 +213,11 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { } // ensure no monitor was created - ensureNumMonitorV2s(0) + ensureNumMonitors(0) } fun `test create ppl monitor with more than max allowed monitors fails`() { - adminClient().updateSettings(ALERTING_V2_MAX_MONITORS.key, 1) + adminClient().updateSettings(ALERTING_MAX_MONITORS.key, 1) createRandomPPLMonitor() @@ -254,7 +230,7 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { } // ensure no monitor was created - ensureNumMonitorV2s(1) + ensureNumMonitors(1) } fun `test create ppl monitor with throttle greater than max fails`() { @@ -276,7 +252,7 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { } // ensure no monitor was created - ensureNumMonitorV2s(0) + ensureNumMonitors(0) } fun `test create ppl monitor with expire greater than max fails`() { @@ -298,27 +274,7 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { } // ensure no monitor was created - ensureNumMonitorV2s(0) - } - - fun `test create ppl monitor with look back window greater than max fails`() { - val maxLookBackWindow = 60L - client().updateSettings(ALERTING_V2_MAX_LOOK_BACK_WINDOW.key, maxLookBackWindow) - - // ensure the request fails - try { - createRandomPPLMonitor( - randomPPLMonitor( - lookBackWindow = maxLookBackWindow + 10 - ) - ) - fail("Expected request to fail with BAD_REQUEST but it succeeded") - } catch (e: ResponseException) { - assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) - } - - // ensure no monitor was created - ensureNumMonitorV2s(0) + ensureNumMonitors(0) } fun `test create ppl monitor with invalid query fails`() { @@ -335,7 +291,7 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { } // ensure no monitor was created - ensureNumMonitorV2s(0) + ensureNumMonitors(0) } fun `test create ppl monitor with query that's too long fails`() { @@ -354,7 +310,7 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { } // ensure no monitor was created - ensureNumMonitorV2s(0) + ensureNumMonitors(0) } fun `test create ppl monitor with invalid custom condition fails`() { @@ -379,7 +335,7 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { } // ensure no monitor was created - ensureNumMonitorV2s(0) + ensureNumMonitors(0) } fun `test create ppl monitor with custom condition that evals to num not bool fails`() { @@ -408,7 +364,7 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { } // ensure no monitor was created - ensureNumMonitorV2s(0) + ensureNumMonitors(0) } fun `test create ppl monitor with notification subject source too long fails`() { @@ -445,7 +401,7 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { } // ensure no monitor was created - ensureNumMonitorV2s(0) + ensureNumMonitors(0) } fun `test create ppl monitor with notification message source too long fails`() { @@ -482,17 +438,7 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { } // ensure no monitor was created - ensureNumMonitorV2s(0) - } - - fun `test get ppl monitor with invalid monitor ID length`() { - val badId = UUIDs.base64UUID() + "extra" - try { - client().makeRequest("GET", "$MONITOR_V2_BASE_URI/$badId") - fail("Expected request to fail with BAD_REQUEST but it succeeded") - } catch (e: ResponseException) { - assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) - } + ensureNumMonitors(0) } fun `test update nonexistent ppl monitor fails`() { @@ -504,7 +450,7 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { val randomId = UUIDs.base64UUID() try { - client().makeRequest("PUT", "$MONITOR_V2_BASE_URI/$randomId", emptyMap(), monitorV2.toHttpEntity()) + client().makeRequest("PUT", "$MONITOR_BASE_URI/$randomId", emptyMap(), monitorV2.toHttpEntity()) fail("Expected request to fail with NOT_FOUND but it succeeded") } catch (e: ResponseException) { assertEquals("Unexpected status", RestStatus.NOT_FOUND, e.response.restStatus()) @@ -515,7 +461,7 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { val randomId = UUIDs.base64UUID() try { - client().makeRequest("DELETE", "$MONITOR_V2_BASE_URI/$randomId") + client().makeRequest("DELETE", "$MONITOR_BASE_URI/$randomId") fail("Expected request to fail with NOT_FOUND but it succeeded") } catch (e: ResponseException) { assertEquals("Unexpected status", RestStatus.NOT_FOUND, e.response.restStatus()) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureMonitorV2RestApiIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureMonitorV2RestApiIT.kt index ef82094bb..d937eeae8 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureMonitorV2RestApiIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureMonitorV2RestApiIT.kt @@ -14,18 +14,13 @@ import org.junit.Before import org.junit.BeforeClass import org.opensearch.alerting.ALERTING_FULL_ACCESS_ROLE import org.opensearch.alerting.ALL_ACCESS_ROLE -import org.opensearch.alerting.AlertingPlugin.Companion.MONITOR_V2_BASE_URI +import org.opensearch.alerting.AlertingPlugin.Companion.MONITOR_BASE_URI import org.opensearch.alerting.AlertingRestTestCase import org.opensearch.alerting.PPL_FULL_ACCESS_ROLE import org.opensearch.alerting.ROLE_TO_PERMISSION_MAPPING import org.opensearch.alerting.TEST_INDEX_MAPPINGS import org.opensearch.alerting.TEST_INDEX_NAME -import org.opensearch.alerting.core.settings.AlertingV2Settings import org.opensearch.alerting.makeRequest -import org.opensearch.alerting.modelv2.PPLSQLMonitor -import org.opensearch.alerting.modelv2.PPLSQLTrigger.ConditionType -import org.opensearch.alerting.modelv2.PPLSQLTrigger.NumResultsCondition -import org.opensearch.alerting.modelv2.PPLSQLTrigger.TriggerMode import org.opensearch.alerting.randomPPLMonitor import org.opensearch.alerting.randomPPLTrigger import org.opensearch.client.ResponseException @@ -33,6 +28,9 @@ import org.opensearch.client.RestClient import org.opensearch.common.settings.Settings import org.opensearch.common.xcontent.XContentType import org.opensearch.commons.alerting.model.IntervalSchedule +import org.opensearch.commons.alerting.model.PPLSQLTrigger.ConditionType +import org.opensearch.commons.alerting.model.PPLSQLTrigger.NumResultsCondition +import org.opensearch.commons.alerting.model.PPLSQLTrigger.TriggerMode import org.opensearch.commons.rest.SecureRestClientBuilder import org.opensearch.core.rest.RestStatus import org.opensearch.index.query.QueryBuilders @@ -61,7 +59,6 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { @Before fun create() { - client().updateSettings(AlertingV2Settings.ALERTING_V2_ENABLED.key, "true") if (userClient == null) { createUser(user, arrayOf()) userClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), user, password) @@ -93,16 +90,16 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { ) try { - createMonitorV2WithClient( + createPPLIndexThenMonitorWithClient( userClient!!, - monitorV2 = pplMonitorConfig + monitor = pplMonitorConfig ) fail("Expected create monitor to fail as user does not have permissions to call alerting APIs") } catch (e: ResponseException) { assertEquals("Unexpected error status", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) } - ensureNumMonitorV2s(0) + ensureNumMonitors(0) } fun `test create monitor that queries index user doesn't have access to fails`() { @@ -125,16 +122,16 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { ) try { - createMonitorV2WithClient( + createPPLIndexThenMonitorWithClient( userClient!!, - monitorV2 = pplMonitorConfig + monitor = pplMonitorConfig ) fail("Expected create monitor to fail as user does not have permissions to index that monitor queries") } catch (e: ResponseException) { assertEquals("Unexpected error status", RestStatus.BAD_REQUEST.status, e.response.statusLine.statusCode) } - ensureNumMonitorV2s(0) + ensureNumMonitors(0) } fun `test update monitor that queries index user doesn't have access to fails`() { @@ -157,7 +154,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { ) // this function automatically creates index TEST_INDEX_NAME, then a monitor that queries it - val pplMonitor = createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf(backendRole)) + val pplMonitor = createPPLIndexThenMonitorWithClient(userClient!!, pplMonitorConfig, listOf(backendRole)) /* user: String, @@ -184,7 +181,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { .build() // update some field that isn't the PPL query and the index it's querying - val newMonitor = pplMonitorConfig.makeCopy(name = "some_random_name") + val newMonitor = pplMonitorConfig.copy(name = "some_random_name") try { // noIndicesUser, who only has access to index unrelated_index, should be blocked @@ -192,7 +189,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { // has no access to TEST_INDEX_NAME noIndicesUserClient!!.makeRequest( "PUT", - "$MONITOR_V2_BASE_URI/${pplMonitor.id}", + "$MONITOR_BASE_URI/${pplMonitor.id}", newMonitor.toHttpEntity(), BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") ) @@ -220,9 +217,9 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { false ) - createMonitorV2WithClient(userClient!!, monitorV2 = pplMonitorConfig, listOf("backend_role_a")) + createPPLIndexThenMonitorWithClient(userClient!!, monitor = pplMonitorConfig, listOf("backend_role_a")) - ensureNumMonitorV2s(1) + ensureNumMonitors(1) } fun `test RBAC create monitor with backend roles user has no access to fails`() { @@ -241,9 +238,9 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { ) try { - createMonitorV2WithClient( + createPPLIndexThenMonitorWithClient( userClient!!, - monitorV2 = pplMonitorConfig, + monitor = pplMonitorConfig, listOf("backend_role_a", "backend_role_b", "backend_role_c") ) fail("Expected create monitor to fail as user does not have backend_role_c backend role") @@ -251,7 +248,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { assertEquals("Unexpected error status", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) } - ensureNumMonitorV2s(0) + ensureNumMonitors(0) } fun `test RBAC update monitorV2 as user with correct backend roles succeeds`() { @@ -268,7 +265,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { false ) - val pplMonitor = createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) + val pplMonitor = createPPLIndexThenMonitorWithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) // getUser should have access to the monitor above created by user val updateUser = "updateUser" @@ -288,7 +285,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { val newMonitor = randomPPLMonitor() val updateMonitorResponse = getUserClient!!.makeRequest( "PUT", - "$MONITOR_V2_BASE_URI/${pplMonitor.id}", + "$MONITOR_BASE_URI/${pplMonitor.id}", newMonitor.toHttpEntity(), BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") ) @@ -312,7 +309,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { false ) - val pplMonitor = createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) + val pplMonitor = createPPLIndexThenMonitorWithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) // updateUser should have access to the monitor above created by user val updateUser = "updateUser" @@ -334,7 +331,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { try { getUserClient!!.makeRequest( "PUT", - "$MONITOR_V2_BASE_URI/${pplMonitor.id}", + "$MONITOR_BASE_URI/${pplMonitor.id}", newMonitor.toHttpEntity(), BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") ) @@ -361,7 +358,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { false ) - val pplMonitor = createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) + val pplMonitor = createPPLIndexThenMonitorWithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) // getUser should have access to the monitor above created by user val getUser = "getUser" @@ -380,7 +377,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { val getMonitorResponse = getUserClient!!.makeRequest( "GET", - "$MONITOR_V2_BASE_URI/${pplMonitor.id}", + "$MONITOR_BASE_URI/${pplMonitor.id}", null, BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") ) @@ -404,7 +401,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { false ) - val pplMonitor = createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) + val pplMonitor = createPPLIndexThenMonitorWithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) // getUser should not have access to the monitor above created by user val getUser = "getUser" @@ -424,7 +421,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { try { getUserClient!!.makeRequest( "GET", - "$MONITOR_V2_BASE_URI/${pplMonitor.id}", + "$MONITOR_BASE_URI/${pplMonitor.id}", null, BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") ) @@ -450,7 +447,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { false ) - createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) + createPPLIndexThenMonitorWithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) // getUser should have access to the monitor above created by user val searchUser = "searchUser" @@ -470,7 +467,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { val search = SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).toString() val searchMonitorResponse = searchUserClient!!.makeRequest( "POST", - "$MONITOR_V2_BASE_URI/_search", + "$MONITOR_BASE_URI/_search", StringEntity(search, ContentType.APPLICATION_JSON), BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") ) @@ -501,7 +498,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { false ) - createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) + createPPLIndexThenMonitorWithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) // getUser should have access to the monitor above created by user val searchUser = "searchUser" @@ -521,7 +518,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { val search = SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).toString() val searchMonitorResponse = searchUserClient!!.makeRequest( "POST", - "$MONITOR_V2_BASE_URI/_search", + "$MONITOR_BASE_URI/_search", StringEntity(search, ContentType.APPLICATION_JSON), BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") ) @@ -551,7 +548,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { false ) - val pplMonitor = createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) + val pplMonitor = createPPLIndexThenMonitorWithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) // getUser should have access to the monitor above created by user val executeUser = "executeUser" @@ -570,7 +567,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { val getMonitorResponse = getUserClient!!.makeRequest( "POST", - "$MONITOR_V2_BASE_URI/${pplMonitor.id}/_execute", + "$MONITOR_BASE_URI/${pplMonitor.id}/_execute", null, BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") ) @@ -594,7 +591,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { false ) - val pplMonitor = createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) + val pplMonitor = createPPLIndexThenMonitorWithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) // getUser should not have access to the monitor above created by user val executeUser = "executeUser" @@ -614,7 +611,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { try { getUserClient!!.makeRequest( "POST", - "$MONITOR_V2_BASE_URI/${pplMonitor.id}/_execute", + "$MONITOR_BASE_URI/${pplMonitor.id}/_execute", null, BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") ) @@ -639,7 +636,6 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { randomPPLMonitor( enabled = true, schedule = IntervalSchedule(interval = 1, unit = MINUTES), - lookBackWindow = null, triggers = listOf( randomPPLTrigger( throttleDuration = null, @@ -662,13 +658,13 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { false ) - val pplMonitor = createMonitorV2WithClient( + val pplMonitor = createPPLIndexThenMonitorWithClient( userClient!!, pplMonitorConfig, null - ) as PPLSQLMonitor + ) - val executeResponse = executeMonitorV2(pplMonitor.id) + val executeResponse = executeMonitor(pplMonitor.id) val triggered = isTriggered(pplMonitor, executeResponse) assertTrue(triggered) @@ -692,7 +688,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { val getAlertsResponse = getAlertsUserClient!!.makeRequest( "GET", - "$MONITOR_V2_BASE_URI/alerts", + "$MONITOR_BASE_URI/alerts", null, BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") ) @@ -718,7 +714,6 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { randomPPLMonitor( enabled = true, schedule = IntervalSchedule(interval = 1, unit = MINUTES), - lookBackWindow = null, triggers = listOf( randomPPLTrigger( throttleDuration = null, @@ -741,13 +736,13 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { false ) - val pplMonitor = createMonitorV2WithClient( + val pplMonitor = createPPLIndexThenMonitorWithClient( userClient!!, pplMonitorConfig, null - ) as PPLSQLMonitor + ) - val executeResponse = executeMonitorV2(pplMonitor.id) + val executeResponse = executeMonitor(pplMonitor.id) val triggered = isTriggered(pplMonitor, executeResponse) assertTrue(triggered) @@ -767,7 +762,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { val getAlertsResponse = getAlertsUserClient!!.makeRequest( "GET", - "$MONITOR_V2_BASE_URI/alerts", + "$MONITOR_BASE_URI/alerts", null, BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") ) @@ -794,7 +789,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { false ) - val pplMonitor = createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) + val pplMonitor = createPPLIndexThenMonitorWithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) // getUser should have access to the monitor above created by user val deleteUser = "deleteUser" @@ -813,13 +808,13 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { val getMonitorResponse = deleteUserClient!!.makeRequest( "DELETE", - "$MONITOR_V2_BASE_URI/${pplMonitor.id}", + "$MONITOR_BASE_URI/${pplMonitor.id}", null, BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") ) assertEquals("Get monitorV2 failed", RestStatus.OK, getMonitorResponse.restStatus()) - ensureNumMonitorV2s(0) + ensureNumMonitors(0) // cleanup deleteUserClient.close() @@ -839,7 +834,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { false ) - val pplMonitor = createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) + val pplMonitor = createPPLIndexThenMonitorWithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) // getUser should not have access to the monitor above created by user val deleteUser = "deleteUser" @@ -859,7 +854,7 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { try { deleteUserClient!!.makeRequest( "DELETE", - "$MONITOR_V2_BASE_URI/${pplMonitor.id}", + "$MONITOR_BASE_URI/${pplMonitor.id}", null, BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") ) @@ -870,6 +865,6 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { deleteUserClient?.close() } - ensureNumMonitorV2s(1) + ensureNumMonitors(1) } } diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/transport/AlertingSingleNodeTestCase.kt b/alerting/src/test/kotlin/org/opensearch/alerting/transport/AlertingSingleNodeTestCase.kt index 06af4c3d3..ea4b8ed3f 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/transport/AlertingSingleNodeTestCase.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/transport/AlertingSingleNodeTestCase.kt @@ -98,7 +98,7 @@ abstract class AlertingSingleNodeTestCase : OpenSearchSingleNodeTestCase() { } protected fun executeMonitor(monitor: Monitor, id: String?, dryRun: Boolean = true): ExecuteMonitorResponse? { - val request = ExecuteMonitorRequest(dryRun, TimeValue(Instant.now().toEpochMilli()), id, monitor) + val request = ExecuteMonitorRequest(dryRun, false, TimeValue(Instant.now().toEpochMilli()), id, monitor) return client().execute(ExecuteMonitorAction.INSTANCE, request).get() } @@ -491,7 +491,7 @@ abstract class AlertingSingleNodeTestCase : OpenSearchSingleNodeTestCase() { } protected fun executeWorkflow(workflow: Workflow? = null, id: String? = null, dryRun: Boolean = true): ExecuteWorkflowResponse? { - val request = ExecuteWorkflowRequest(dryRun, TimeValue(Instant.now().toEpochMilli()), id, workflow) + val request = ExecuteWorkflowRequest(dryRun, false, TimeValue(Instant.now().toEpochMilli()), id, workflow) return client().execute(ExecuteWorkflowAction.INSTANCE, request).get() } diff --git a/build.gradle b/build.gradle index 261213c07..ca13dade9 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ buildscript { apply from: 'build-tools/repositories.gradle' ext { - opensearch_version = System.getProperty("opensearch.version", "3.6.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "3.5.0-SNAPSHOT") buildVersionQualifier = System.getProperty("build.version_qualifier", "") isSnapshot = "true" == System.getProperty("build.snapshot", "true") // 3.0.0-SNAPSHOT -> 3.0.0.0-SNAPSHOT diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobStats.kt b/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobStats.kt index 0e02817fc..4be049334 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobStats.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobStats.kt @@ -7,8 +7,8 @@ package org.opensearch.alerting.core.action.node import org.opensearch.action.support.nodes.BaseNodeResponse import org.opensearch.alerting.core.JobSweeperMetrics -import org.opensearch.alerting.core.resthandler.StatsRequestUtils.JOBS_INFO -import org.opensearch.alerting.core.resthandler.StatsRequestUtils.JOB_SCHEDULING_METRICS +import org.opensearch.alerting.core.resthandler.RestScheduledJobStatsHandler.Companion.JOBS_INFO +import org.opensearch.alerting.core.resthandler.RestScheduledJobStatsHandler.Companion.JOB_SCHEDULING_METRICS import org.opensearch.alerting.core.schedule.JobSchedulerMetrics import org.opensearch.cluster.node.DiscoveryNode import org.opensearch.core.common.io.stream.StreamInput diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsRequest.kt b/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsRequest.kt index 1d9bd0578..6a82e8204 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsRequest.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsRequest.kt @@ -17,25 +17,18 @@ import java.io.IOException class ScheduledJobsStatsRequest : BaseNodesRequest { var jobSchedulingMetrics: Boolean = true var jobsInfo: Boolean = true - // show Alerting V2 scheduled jobs if true, Alerting V1 scheduled jobs if false, all scheduled jobs if null - var showAlertingV2ScheduledJobs: Boolean? = null constructor(si: StreamInput) : super(si) { jobSchedulingMetrics = si.readBoolean() jobsInfo = si.readBoolean() - showAlertingV2ScheduledJobs = si.readOptionalBoolean() - } - - constructor(nodeIds: Array, showAlertingV2ScheduledJobs: Boolean?) : super(*nodeIds) { - this.showAlertingV2ScheduledJobs = showAlertingV2ScheduledJobs } + constructor(nodeIds: Array) : super(*nodeIds) @Throws(IOException::class) override fun writeTo(out: StreamOutput) { super.writeTo(out) out.writeBoolean(jobSchedulingMetrics) out.writeBoolean(jobsInfo) - out.writeOptionalBoolean(showAlertingV2ScheduledJobs) } fun all(): ScheduledJobsStatsRequest { diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsTransportAction.kt b/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsTransportAction.kt index 398f5634a..f2ed94623 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsTransportAction.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsTransportAction.kt @@ -93,8 +93,7 @@ class ScheduledJobsStatsTransportAction : TransportNodesAction Unit>( + JOB_SCHEDULING_METRICS to { it -> it.jobSchedulingMetrics = true }, + JOBS_INFO to { it -> it.jobsInfo = true } + ) + } + override fun getName(): String { return "${path}_jobs_stats" } @@ -58,14 +70,7 @@ class RestScheduledJobStatsHandler(private val path: String) : BaseRestHandler() } override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { - val alertingVersion = request.param("version") - if (alertingVersion != null && alertingVersion !in listOf("v1", "v2")) { - throw IllegalArgumentException("Version parameter must be one of v1 or v2") - } - - val showV2ScheduledJobs: Boolean? = alertingVersion?.let { it == "v2" } - - val scheduledJobNodesStatsRequest = StatsRequestUtils.getStatsRequest(request, showV2ScheduledJobs, this::unrecognized) + val scheduledJobNodesStatsRequest = getRequest(request) return RestChannelConsumer { channel -> client.execute( ScheduledJobsStatsAction.INSTANCE, @@ -74,4 +79,43 @@ class RestScheduledJobStatsHandler(private val path: String) : BaseRestHandler() ) } } + + private fun getRequest(request: RestRequest): ScheduledJobsStatsRequest { + val nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId")) + val metrics = Strings.tokenizeByCommaToSet(request.param("metric")) + val scheduledJobsStatsRequest = ScheduledJobsStatsRequest(nodesIds) + scheduledJobsStatsRequest.timeout(request.param("timeout")) + + if (metrics.isEmpty()) { + return scheduledJobsStatsRequest + } else if (metrics.size == 1 && metrics.contains("_all")) { + scheduledJobsStatsRequest.all() + } else if (metrics.contains("_all")) { + throw IllegalArgumentException( + String.format( + Locale.ROOT, + "request [%s] contains _all and individual metrics [%s]", + request.path(), + request.param("metric") + ) + ) + } else { + // use a sorted set so the unrecognized parameters appear in a reliable sorted order + scheduledJobsStatsRequest.clear() + val invalidMetrics = TreeSet() + for (metric in metrics) { + val handler = METRICS[metric] + if (handler != null) { + handler.invoke(scheduledJobsStatsRequest) + } else { + invalidMetrics.add(metric) + } + } + + if (!invalidMetrics.isEmpty()) { + throw IllegalArgumentException(unrecognized(request, invalidMetrics, METRICS.keys, "metric")) + } + } + return scheduledJobsStatsRequest + } } diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/StatsRequestUtils.kt b/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/StatsRequestUtils.kt deleted file mode 100644 index 58b33d709..000000000 --- a/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/StatsRequestUtils.kt +++ /dev/null @@ -1,63 +0,0 @@ -package org.opensearch.alerting.core.resthandler - -import org.opensearch.alerting.core.action.node.ScheduledJobsStatsRequest -import org.opensearch.core.common.Strings -import org.opensearch.rest.RestRequest -import java.util.Locale -import java.util.TreeSet - -internal object StatsRequestUtils { - - const val JOB_SCHEDULING_METRICS: String = "job_scheduling_metrics" - const val JOBS_INFO: String = "jobs_info" - val METRICS = mapOf Unit>( - JOB_SCHEDULING_METRICS to { it.jobSchedulingMetrics = true }, - JOBS_INFO to { it.jobsInfo = true } - ) - - fun getStatsRequest( - request: RestRequest, - showAlertingV2ScheduledJobs: Boolean?, - unrecognizedFn: (RestRequest, Set, Set, String) -> String - ): ScheduledJobsStatsRequest { - val nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId")) - val metrics = Strings.tokenizeByCommaToSet(request.param("metric")) - val scheduledJobsStatsRequest = ScheduledJobsStatsRequest( - nodeIds = nodesIds, - showAlertingV2ScheduledJobs = showAlertingV2ScheduledJobs - ) - scheduledJobsStatsRequest.timeout(request.param("timeout")) - - if (metrics.isEmpty()) { - return scheduledJobsStatsRequest - } else if (metrics.size == 1 && metrics.contains("_all")) { - scheduledJobsStatsRequest.all() - } else if (metrics.contains("_all")) { - throw IllegalArgumentException( - String.format( - Locale.ROOT, - "request [%s] contains _all and individual metrics [%s]", - request.path(), - request.param("metric") - ) - ) - } else { - // use a sorted set so the unrecognized parameters appear in a reliable sorted order - scheduledJobsStatsRequest.clear() - val invalidMetrics = TreeSet() - for (metric in metrics) { - val handler = METRICS[metric] - if (handler != null) { - handler.invoke(scheduledJobsStatsRequest) - } else { - invalidMetrics.add(metric) - } - } - - if (!invalidMetrics.isEmpty()) { - throw IllegalArgumentException(unrecognizedFn(request, invalidMetrics, METRICS.keys, "metric")) - } - } - return scheduledJobsStatsRequest - } -} diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/schedule/JobScheduler.kt b/core/src/main/kotlin/org/opensearch/alerting/core/schedule/JobScheduler.kt index 8245e1a78..a4a729121 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/core/schedule/JobScheduler.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/core/schedule/JobScheduler.kt @@ -30,12 +30,6 @@ class JobScheduler(private val threadPool: ThreadPool, private val jobRunner: Jo */ private val scheduledJobIdToInfo = ConcurrentHashMap() - /** - * The scheduled job type of Monitor V2s, for filtering - * out V1 vs V2 Monitors when collecting Monitor Stats - */ - private val monitorV2Type = "monitor_v2" - /** * Schedules the jobs in [jobsToSchedule] for execution. * @@ -197,19 +191,8 @@ class JobScheduler(private val threadPool: ThreadPool, private val jobRunner: Jo return true } - fun getJobSchedulerMetric(showAlertingV2ScheduledJobs: Boolean?): List { - val scheduledJobEntries = scheduledJobIdToInfo.entries - - val filteredScheduledJobEntries = if (showAlertingV2ScheduledJobs == null) { - // if no alerting version was specified, do not filter - scheduledJobEntries - } else if (showAlertingV2ScheduledJobs) { - scheduledJobEntries.filter { it.value.scheduledJob.type == monitorV2Type } - } else { - scheduledJobEntries.filter { it.value.scheduledJob.type != monitorV2Type } - } - - return filteredScheduledJobEntries.stream() + fun getJobSchedulerMetric(): List { + return scheduledJobIdToInfo.entries.stream() .map { entry -> JobSchedulerMetrics( entry.value.scheduledJobId, diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/settings/AlertingV2Settings.kt b/core/src/main/kotlin/org/opensearch/alerting/core/settings/AlertingV2Settings.kt deleted file mode 100644 index fd92dd5e9..000000000 --- a/core/src/main/kotlin/org/opensearch/alerting/core/settings/AlertingV2Settings.kt +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.core.settings - -import org.opensearch.common.settings.Setting - -/** - * This class exclusively houses the Alerting V2 enabled setting, so that both Monitor V2 Stats - * and the rest of the CRUD APIs can read it - */ -class AlertingV2Settings { - companion object { - val ALERTING_V2_ENABLED = Setting.boolSetting( - "plugins.alerting.v2.enabled", - true, - Setting.Property.NodeScope, Setting.Property.Dynamic - ) - } -} diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/util/XContentExtensions.kt b/core/src/main/kotlin/org/opensearch/alerting/core/util/XContentExtensions.kt deleted file mode 100644 index 9ca03ed6b..000000000 --- a/core/src/main/kotlin/org/opensearch/alerting/core/util/XContentExtensions.kt +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.core.util - -import org.opensearch.core.xcontent.XContentBuilder -import java.time.Instant - -fun XContentBuilder.nonOptionalTimeField(name: String, instant: Instant): XContentBuilder { - return this.timeField(name, "${name}_in_millis", instant.toEpochMilli()) -} From d0d3d842b09f788848a558ed9afa2c72b899583e Mon Sep 17 00:00:00 2001 From: Dennis Toepker Date: Wed, 11 Mar 2026 14:49:36 -0700 Subject: [PATCH 3/9] PPL Alerting: Removing Stateless Alerts and Supporting Only Stateful Alerts --- .../org/opensearch/alerting/AlertService.kt | 21 +- .../org/opensearch/alerting/AlertingPlugin.kt | 14 +- .../alerting/BucketLevelMonitorRunner.kt | 5 - .../alerting/DocumentLevelMonitorRunner.kt | 5 - .../alerting/MonitorRunnerExecutionContext.kt | 2 - .../alerting/MonitorRunnerService.kt | 23 +- .../alerting/PPLSQLMonitorRunner.kt | 652 ------------- .../org/opensearch/alerting/PPLUtils.kt | 148 +-- .../alerting/QueryLevelMonitorRunner.kt | 83 +- .../org/opensearch/alerting/TriggerService.kt | 144 +++ .../alerting/alertsv2/AlertV2Indices.kt | 419 --------- .../alerting/alertsv2/AlertV2Mover.kt | 482 ---------- .../QueryLevelTriggerExecutionContext.kt | 20 +- .../alerting/settings/AlertingSettings.kt | 53 +- .../transport/TransportGetAlertsAction.kt | 1 - .../transport/TransportIndexMonitorAction.kt | 47 +- .../opensearch/alerting/util/IndexUtils.kt | 12 - .../workflow/CompositeWorkflowRunner.kt | 5 - .../alerting/alerts/alert_mapping.json | 17 +- .../alerting/org.opensearch.alerting.txt | 4 +- .../alerting/AlertingRestTestCase.kt | 35 +- .../alerting/MonitorRunnerServiceIT.kt | 549 +++++++++++ .../alerting/PPLSQLMonitorRunnerIT.kt | 415 --------- .../org/opensearch/alerting/PPLUtilsTests.kt | 472 ++++++++++ .../org/opensearch/alerting/TestHelpers.kt | 28 +- .../alerting/alertsv2/AlertV2IndicesIT.kt | 442 --------- .../alerting/resthandler/MonitorRestApiIT.kt | 390 ++++++++ .../resthandler/MonitorV2RestApiIT.kt | 470 ---------- .../resthandler/SecureMonitorV2RestApiIT.kt | 870 ------------------ 29 files changed, 1770 insertions(+), 4058 deletions(-) delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/PPLSQLMonitorRunner.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Indices.kt delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Mover.kt delete mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/PPLSQLMonitorRunnerIT.kt create mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/PPLUtilsTests.kt delete mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/alertsv2/AlertV2IndicesIT.kt delete mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorV2RestApiIT.kt delete mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureMonitorV2RestApiIT.kt diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertService.kt index 38706ac71..990b88ef9 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertService.kt @@ -35,11 +35,13 @@ import org.opensearch.commons.alerting.model.ClusterMetricsTriggerRunResult import org.opensearch.commons.alerting.model.DataSources import org.opensearch.commons.alerting.model.Monitor import org.opensearch.commons.alerting.model.NoOpTrigger +import org.opensearch.commons.alerting.model.PPLSQLInput import org.opensearch.commons.alerting.model.QueryLevelTriggerRunResult import org.opensearch.commons.alerting.model.Trigger import org.opensearch.commons.alerting.model.Workflow import org.opensearch.commons.alerting.model.WorkflowRunContext import org.opensearch.commons.alerting.model.action.AlertCategory +import org.opensearch.commons.alerting.util.isPplSqlMonitor import org.opensearch.core.action.ActionListener import org.opensearch.core.common.bytes.BytesReference import org.opensearch.core.rest.RestStatus @@ -151,7 +153,9 @@ class AlertService( result: QueryLevelTriggerRunResult, alertError: AlertError?, executionId: String, - workflorwRunContext: WorkflowRunContext? + workflorwRunContext: WorkflowRunContext?, + monitorCtx: MonitorRunnerExecutionContext, + pplSqlQueryResult: List>? ): Alert? { val currentTime = Instant.now() val currentAlert = ctx.alert?.alert @@ -201,6 +205,10 @@ class AlertService( } } + // populate PPL Monitor specific fields + val query = if (ctx.monitor.isPplSqlMonitor()) (ctx.monitor.inputs[0] as PPLSQLInput).query else null + val queryResults = if (ctx.monitor.isPplSqlMonitor()) pplSqlQueryResult.orEmpty() else emptyList() + // Merge the alert's error message to the current alert's history val updatedHistory = currentAlert?.errorHistory.update(alertError) return if (alertError == null && !result.triggered) { @@ -211,7 +219,9 @@ class AlertService( errorHistory = updatedHistory, actionExecutionResults = updatedActionExecutionResults, schemaVersion = IndexUtils.alertIndexSchemaVersion, - clusters = triggeredClusters + clusters = triggeredClusters, + pplQuery = query, + pplQueryResults = queryResults ) } else if (alertError == null && currentAlert?.isAcknowledged() == true) { null @@ -224,20 +234,23 @@ class AlertService( errorHistory = updatedHistory, actionExecutionResults = updatedActionExecutionResults, schemaVersion = IndexUtils.alertIndexSchemaVersion, - clusters = triggeredClusters + clusters = triggeredClusters, + pplQuery = query, + pplQueryResults = queryResults ) } else { val alertState = if (workflorwRunContext?.auditDelegateMonitorAlerts == true) { Alert.State.AUDIT } else if (alertError == null) Alert.State.ACTIVE else Alert.State.ERROR + // TODO: does the trigger field get populated as QueryLevel or PPLSQL Trigger correctly? Alert( monitor = ctx.monitor, trigger = ctx.trigger, startTime = currentTime, lastNotificationTime = currentTime, state = alertState, errorMessage = alertError?.message, errorHistory = updatedHistory, actionExecutionResults = updatedActionExecutionResults, schemaVersion = IndexUtils.alertIndexSchemaVersion, executionId = executionId, workflowId = workflorwRunContext?.workflowId ?: "", - clusters = triggeredClusters + clusters = triggeredClusters, pplQuery = query, pplQueryResults = queryResults ) } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index 24e8943da..5e32d967c 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -16,9 +16,6 @@ import org.opensearch.alerting.action.SearchEmailAccountAction import org.opensearch.alerting.action.SearchEmailGroupAction import org.opensearch.alerting.alerts.AlertIndices import org.opensearch.alerting.alerts.AlertIndices.Companion.ALL_ALERT_INDEX_PATTERN -import org.opensearch.alerting.alertsv2.AlertV2Indices -import org.opensearch.alerting.alertsv2.AlertV2Indices.Companion.ALL_ALERT_V2_INDEX_PATTERN -import org.opensearch.alerting.alertsv2.AlertV2Mover import org.opensearch.alerting.comments.CommentsIndices import org.opensearch.alerting.comments.CommentsIndices.Companion.ALL_COMMENTS_INDEX_PATTERN import org.opensearch.alerting.core.JobSweeper @@ -198,10 +195,8 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R lateinit var docLevelMonitorQueries: DocLevelMonitorQueries lateinit var threadPool: ThreadPool lateinit var alertIndices: AlertIndices - lateinit var alertV2Indices: AlertV2Indices lateinit var clusterService: ClusterService lateinit var destinationMigrationCoordinator: DestinationMigrationCoordinator - lateinit var alertV2Mover: AlertV2Mover var monitorTypeToMonitorRunners: MutableMap = mutableMapOf() override fun getRestHandlers( @@ -335,7 +330,6 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R .registerSettings(settings) .registerThreadPool(threadPool) .registerAlertIndices(alertIndices) - .registerAlertV2Indices(alertV2Indices) .registerInputService( InputService( client, @@ -362,7 +356,6 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R scheduler = JobScheduler(threadPool, runner) sweeper = JobSweeper(environment.settings(), client, clusterService, threadPool, xContentRegistry, scheduler, ALERTING_JOB_TYPES) destinationMigrationCoordinator = DestinationMigrationCoordinator(client, clusterService, threadPool, scheduledJobIndices) - alertV2Mover = AlertV2Mover(environment.settings(), client, threadPool, clusterService, xContentRegistry) this.threadPool = threadPool this.clusterService = clusterService @@ -391,7 +384,6 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R commentsIndices, docLevelMonitorQueries, destinationMigrationCoordinator, - alertV2Mover, lockService, alertService, triggerService, @@ -490,12 +482,9 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R AlertingSettings.ALERT_V2_HISTORY_MAX_DOCS, AlertingSettings.ALERT_V2_HISTORY_RETENTION_PERIOD, AlertingSettings.ALERT_V2_MONITOR_EXECUTION_MAX_DURATION, - AlertingSettings.ALERTING_V2_MAX_THROTTLE_DURATION, - AlertingSettings.ALERTING_V2_MAX_EXPIRE_DURATION, AlertingSettings.ALERTING_V2_MAX_QUERY_LENGTH, AlertingSettings.ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS, AlertingSettings.ALERT_V2_QUERY_RESULTS_MAX_SIZE, - AlertingSettings.ALERT_V2_PER_RESULT_TRIGGER_MAX_ALERTS, AlertingSettings.NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH, AlertingSettings.NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH, ) @@ -515,8 +504,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R return listOf( SystemIndexDescriptor(ALL_ALERT_INDEX_PATTERN, "Alerting Plugin system index pattern"), SystemIndexDescriptor(SCHEDULED_JOBS_INDEX, "Alerting Plugin Configuration index"), - SystemIndexDescriptor(ALL_COMMENTS_INDEX_PATTERN, "Alerting Comments system index pattern"), - SystemIndexDescriptor(ALL_ALERT_V2_INDEX_PATTERN, "Alerting V2 Alerts index pattern") + SystemIndexDescriptor(ALL_COMMENTS_INDEX_PATTERN, "Alerting Comments system index pattern") ) } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/BucketLevelMonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/BucketLevelMonitorRunner.kt index 11e963e63..24b523e27 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/BucketLevelMonitorRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/BucketLevelMonitorRunner.kt @@ -86,11 +86,6 @@ object BucketLevelMonitorRunner : MonitorRunner() { var monitorResult = MonitorRunResult(monitor.name, periodStart, periodEnd) val currentAlerts = try { - // create stateless alert indices as well to prevent get alerts from returning error because - // stateless alerts indices couldn't be found - monitorCtx.alertV2Indices!!.createOrUpdateAlertV2Index() - monitorCtx.alertV2Indices!!.createOrUpdateInitialAlertV2HistoryIndex() - monitorCtx.alertIndices!!.createOrUpdateAlertIndex(monitor.dataSources) monitorCtx.alertIndices!!.createOrUpdateInitialAlertHistoryIndex(monitor.dataSources) if (monitor.dataSources.findingsEnabled == true) { diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt index 873184d6e..67c567e46 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt @@ -70,11 +70,6 @@ class DocumentLevelMonitorRunner : MonitorRunner() { monitorCtx.findingsToTriggeredQueries = mutableMapOf() try { - // create stateless alert indices as well to prevent get alerts from returning error because - // stateless alerts indices couldn't be found - monitorCtx.alertV2Indices!!.createOrUpdateAlertV2Index() - monitorCtx.alertV2Indices!!.createOrUpdateInitialAlertV2HistoryIndex() - monitorCtx.alertIndices!!.createOrUpdateAlertIndex(monitor.dataSources) monitorCtx.alertIndices!!.createOrUpdateInitialAlertHistoryIndex(monitor.dataSources) monitorCtx.alertIndices!!.createOrUpdateInitialFindingHistoryIndex(monitor.dataSources) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerExecutionContext.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerExecutionContext.kt index 38659e835..ba12100ba 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerExecutionContext.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerExecutionContext.kt @@ -7,7 +7,6 @@ package org.opensearch.alerting import org.opensearch.action.bulk.BackoffPolicy import org.opensearch.alerting.alerts.AlertIndices -import org.opensearch.alerting.alertsv2.AlertV2Indices import org.opensearch.alerting.core.lock.LockService import org.opensearch.alerting.model.destination.DestinationContextFactory import org.opensearch.alerting.remote.monitors.RemoteMonitorRegistry @@ -36,7 +35,6 @@ data class MonitorRunnerExecutionContext( var settings: Settings? = null, var threadPool: ThreadPool? = null, var alertIndices: AlertIndices? = null, - var alertV2Indices: AlertV2Indices? = null, var inputService: InputService? = null, var triggerService: TriggerService? = null, var alertService: AlertService? = null, diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt index 561a36c92..9a7582d9b 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt @@ -23,8 +23,6 @@ import org.opensearch.alerting.action.ExecuteWorkflowRequest import org.opensearch.alerting.action.ExecuteWorkflowResponse import org.opensearch.alerting.alerts.AlertIndices import org.opensearch.alerting.alerts.AlertMover.Companion.moveAlerts -import org.opensearch.alerting.alertsv2.AlertV2Indices -import org.opensearch.alerting.alertsv2.AlertV2Mover.Companion.moveAlertV2s import org.opensearch.alerting.core.JobRunner import org.opensearch.alerting.core.ScheduledJobIndices import org.opensearch.alerting.core.lock.LockModel @@ -140,11 +138,6 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon return this } - fun registerAlertV2Indices(alertV2Indices: AlertV2Indices): MonitorRunnerService { - this.monitorCtx.alertV2Indices = alertV2Indices - return this - } - fun registerInputService(inputService: InputService): MonitorRunnerService { this.monitorCtx.inputService = inputService return this @@ -314,9 +307,6 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon if (monitorCtx.alertIndices!!.isAlertInitialized(job.dataSources)) { moveAlerts(monitorCtx.client!!, job.id, job) } - if (monitorCtx.alertV2Indices!!.isAlertV2Initialized()) { - moveAlertV2s(job.id, job, monitorCtx) - } } } catch (e: Exception) { logger.error("Failed to move active alerts for monitor [${job.id}].", e) @@ -355,15 +345,6 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon } catch (e: Exception) { logger.error("Failed to move active alerts for monitor [$jobId].", e) } - try { - monitorCtx.moveAlertsRetryPolicy!!.retry(logger) { - if (monitorCtx.alertV2Indices!!.isAlertV2Initialized()) { - moveAlertV2s(jobId, null, monitorCtx) - } - } - } catch (e: Exception) { - logger.error("Failed to move active alertV2s for monitorV2 [$jobId].", e) - } } } @@ -489,7 +470,9 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon "periodEnd: $periodEnd, dryrun: $dryrun, executionId: $executionId" ) val runResult = if (monitor.isPplSqlMonitor()) { - PPLSQLMonitorRunner.runMonitor( + // PPL/SQL Monitor runs with QueryLevelMonitorRunner + // as PPL/SQL Monitors are ultimately query-based + QueryLevelMonitorRunner.runMonitor( monitor, monitorCtx, periodStart, diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/PPLSQLMonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/PPLSQLMonitorRunner.kt deleted file mode 100644 index 18cbc2ef1..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/PPLSQLMonitorRunner.kt +++ /dev/null @@ -1,652 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting - -import kotlinx.coroutines.TimeoutCancellationException -import kotlinx.coroutines.withTimeout -import org.apache.logging.log4j.LogManager -import org.json.JSONArray -import org.json.JSONObject -import org.opensearch.ExceptionsHelper -import org.opensearch.action.DocWriteRequest -import org.opensearch.action.bulk.BackoffPolicy -import org.opensearch.action.bulk.BulkRequest -import org.opensearch.action.bulk.BulkResponse -import org.opensearch.action.index.IndexRequest -import org.opensearch.action.support.WriteRequest -import org.opensearch.alerting.PPLUtils.appendCustomCondition -import org.opensearch.alerting.PPLUtils.appendDataRowsLimit -import org.opensearch.alerting.PPLUtils.capPPLQueryResultsSize -import org.opensearch.alerting.PPLUtils.executePplQuery -import org.opensearch.alerting.PPLUtils.findEvalResultVar -import org.opensearch.alerting.PPLUtils.findEvalResultVarIdxInSchema -import org.opensearch.alerting.alertsv2.AlertV2Indices -import org.opensearch.alerting.opensearchapi.InjectorContextElement -import org.opensearch.alerting.opensearchapi.retry -import org.opensearch.alerting.opensearchapi.suspendUntil -import org.opensearch.alerting.opensearchapi.withClosableContext -import org.opensearch.alerting.script.PPLTriggerExecutionContext -import org.opensearch.alerting.settings.AlertingSettings -import org.opensearch.alerting.util.getConfigAndSendNotification -import org.opensearch.common.xcontent.XContentFactory -import org.opensearch.commons.alerting.alerts.AlertError -import org.opensearch.commons.alerting.model.Alert -import org.opensearch.commons.alerting.model.InputRunResults -import org.opensearch.commons.alerting.model.Monitor -import org.opensearch.commons.alerting.model.Monitor.MonitorType -import org.opensearch.commons.alerting.model.MonitorRunResult -import org.opensearch.commons.alerting.model.PPLSQLInput -import org.opensearch.commons.alerting.model.PPLSQLTrigger -import org.opensearch.commons.alerting.model.PPLSQLTrigger.ConditionType -import org.opensearch.commons.alerting.model.PPLSQLTrigger.NumResultsCondition -import org.opensearch.commons.alerting.model.PPLSQLTrigger.Severity -import org.opensearch.commons.alerting.model.PPLSQLTrigger.TriggerMode -import org.opensearch.commons.alerting.model.PPLSQLTriggerRunResult -import org.opensearch.commons.alerting.model.ScheduledJob.Companion.SCHEDULED_JOBS_INDEX -import org.opensearch.commons.alerting.model.WorkflowRunContext -import org.opensearch.commons.alerting.model.action.Action -import org.opensearch.commons.alerting.model.userErrorMessage -import org.opensearch.core.common.Strings -import org.opensearch.core.rest.RestStatus -import org.opensearch.core.xcontent.ToXContent -import org.opensearch.index.VersionType -import org.opensearch.transport.TransportService -import org.opensearch.transport.client.node.NodeClient -import java.time.Duration -import java.time.Instant -import java.time.temporal.ChronoUnit -import kotlin.math.min -import kotlin.time.measureTimedValue - -/** - * This class contains the core logic for running a PPLSQLMonitor. - * The logic for checking throttles, executing the PPL Query, evaluating - * the results against the trigger condition, generating alerts, sending - * notifications, and updating the monitor document with last triggered - * time, are all here. - * - * @opensearch.experimental - */ - -object PPLSQLMonitorRunner : MonitorRunner() { - private val logger = LogManager.getLogger(javaClass) - - override suspend fun runMonitor( - monitor: Monitor, - monitorCtx: MonitorRunnerExecutionContext, - periodStart: Instant, - periodEnd: Instant, - dryRun: Boolean, - manual: Boolean, - workflowRunContext: WorkflowRunContext?, - executionId: String, - transportService: TransportService, - ): MonitorRunResult { - if (monitor.monitorType != MonitorType.PPL_MONITOR.value) { - throw IllegalStateException("Unexpected monitor type: ${monitor.javaClass.name}") - } - - if (monitor.id == Monitor.NO_ID) { - throw IllegalStateException("Received PPL Monitor to execute that unexpectedly has no ID") - } - - logger.debug("Running PPL Monitor: ${monitor.id}. Thread: ${Thread.currentThread().name}") - - var monitorResult = MonitorRunResult(monitor.name, periodStart, periodEnd) - val pplSqlInput = monitor.inputs[0] as PPLSQLInput - - // time the monitor execution run for informational logging - val monitorRunStart = Instant.now() - - val nodeClient = monitorCtx.client as NodeClient - - // create some objects that will be used later - val triggerResults = mutableMapOf() - val pplSqlQueryResults = mutableMapOf>() - - // set the current execution time - // use threadpool time for cross node consistency - val timeOfCurrentExecution = Instant.ofEpochMilli(MonitorRunnerService.monitorCtx.threadPool!!.absoluteTimeInMillis()) - - // check for and create the active alerts and alert history indices - // so we have indices to write alerts to - try { - monitorCtx.alertV2Indices!!.createOrUpdateAlertV2Index() - monitorCtx.alertV2Indices!!.createOrUpdateInitialAlertV2HistoryIndex() - - // create stateful alert indices as well to prevent get alerts from returning error because - // stateful alerts indices couldn't be found - monitorCtx.alertIndices!!.createOrUpdateAlertIndex(monitor.dataSources) - monitorCtx.alertIndices!!.createOrUpdateInitialAlertHistoryIndex(monitor.dataSources) - } catch (e: Exception) { - val id = if (monitor.id.trim().isEmpty()) "_na_" else monitor.id - logger.error("Error loading alerts for monitorV2: $id", e) - return monitorResult.copy(error = e) - } - - val monitorExecutionDuration = monitorCtx - .clusterService!! - .clusterSettings - .get(AlertingSettings.ALERT_V2_MONITOR_EXECUTION_MAX_DURATION) - - // for storing any exception that may or may not happen - // while executing monitor - var exception: Exception? = null - - // run each trigger - try { - withTimeout(monitorExecutionDuration.millis) { - runTriggers( - monitor, - pplSqlInput.query, - timeOfCurrentExecution, - manual, - dryRun, - periodStart, - periodEnd, - triggerResults, - pplSqlQueryResults, - executionId, - monitorCtx, - nodeClient, - transportService - ) - } - } catch (e: TimeoutCancellationException) { - // generate an alert that the monitor's triggers took - // too long to run. this error alert is generated - // even if some triggers managed to run successfully within - // the above time frame and generate their own alerts - monitorCtx.retryPolicy?.let { - saveAlertsV2( - generateErrorAlert(null, monitor, e, executionId, timeOfCurrentExecution), - monitor, - it, - nodeClient - ) - } - - exception = e - } - - // for throttle checking purposes, reindex the PPL Monitor into the alerting-config index - // with updated last triggered times for each of its triggers - if (triggerResults.any { it.value.triggered }) { - updateMonitorWithLastTriggeredTimes(monitor, nodeClient) - } - - val monitorRunEnd = Instant.now() - - val monitorRunTime = Duration.between(monitorRunStart, monitorRunEnd) - - logger.info("monitor ${monitor.id} execution $executionId run time: $monitorRunTime") - - return monitorResult.copy( - inputResults = InputRunResults( - results = listOf(pplSqlQueryResults), - error = exception, - aggTriggersAfterKey = null // aggTriggersAfterKey not relevant for PPL/SQL Monitors - ), - triggerResults = triggerResults - ) - } - - suspend fun runTriggers( - pplSqlMonitor: Monitor, - timeFilteredQuery: String, - timeOfCurrentExecution: Instant, - manual: Boolean, - dryRun: Boolean, - periodStart: Instant, - periodEnd: Instant, - triggerResults: MutableMap, - pplSqlQueryResults: MutableMap>, - executionId: String, - monitorCtx: MonitorRunnerExecutionContext, - nodeClient: NodeClient, - transportService: TransportService - ) { - for (trigger in pplSqlMonitor.triggers) { - val pplSqlTrigger = trigger as PPLSQLTrigger - - try { - // check for throttle and skip execution - // before even running the trigger itself - val throttled = checkForThrottle(pplSqlTrigger, timeOfCurrentExecution, manual) - if (throttled) { - logger.info("throttling trigger ${pplSqlTrigger.id} from monitor ${pplSqlMonitor.id}") - - // automatically return that this trigger is untriggered - triggerResults[pplSqlTrigger.id] = PPLSQLTriggerRunResult(pplSqlTrigger.name, null, false) - - continue - } - logger.debug("throttle check passed, executing trigger ${pplSqlTrigger.id} from monitor ${pplSqlMonitor.id}") - - logger.debug("checking if custom condition is used and appending to base query") - // if trigger uses custom condition, append the custom condition to query, otherwise simply proceed - val queryToExecute = if (pplSqlTrigger.conditionType == ConditionType.NUMBER_OF_RESULTS) { // number of results trigger - timeFilteredQuery - } else { // custom condition trigger - appendCustomCondition(timeFilteredQuery, pplSqlTrigger.customCondition!!) - } - - // limit the number of PPL query result data rows returned - val dataRowsLimit = monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS) - val limitedQueryToExecute = appendDataRowsLimit(queryToExecute, dataRowsLimit) - - // TODO: after getting ppl query results, see if the number of results - // retrieved equals the max allowed number of query results. this implies - // query results might have been excluded, in which case a warning message - // in the alert and notification must be added that results were excluded - // and an alert that should have been generated might not have been - - logger.debug("executing the PPL query of monitor: ${pplSqlMonitor.id}") - // execute the PPL query - val (queryResponseJson, timeTaken) = measureTimedValue { - withClosableContext( - InjectorContextElement( - pplSqlMonitor.id, - monitorCtx.settings!!, - monitorCtx.threadPool!!.threadContext, - pplSqlMonitor.user?.roles, - pplSqlMonitor.user - ) - ) { - executePplQuery( - limitedQueryToExecute, - monitorCtx.clusterService!!.state().nodes.localNode, - transportService - ) - } - } - logger.debug("query results for trigger ${pplSqlTrigger.id}: $queryResponseJson") - logger.debug("time taken to execute query against sql/ppl plugin: $timeTaken") - - // store the query results for Execute Monitor API response - // unlike the query results stored in alerts and notifications, which must be size capped - // (because they will be stored in the OpenSearch cluster or sent as notification) and must be based - // on only the query results that met the trigger condition (because alerts should generate - // on query results that met trigger condition, not those that didn't), the pplQueryResults - // here will be returned as part of the Execute Monitor API response. This will return the original, - // untouched set of query results, and whether this causes size exceed errors is deferred - // to HTTP's response size limits - pplSqlQueryResults[pplSqlTrigger.id] = queryResponseJson.toMap() - - // determine if the trigger condition has been met - val triggered = if (pplSqlTrigger.conditionType == ConditionType.NUMBER_OF_RESULTS) { // number of results trigger - evaluateNumResultsTrigger(queryResponseJson, pplSqlTrigger.numResultsCondition!!, pplSqlTrigger.numResultsValue!!) - } else { // custom condition trigger - evaluateCustomTrigger(queryResponseJson, pplSqlTrigger.customCondition!!) - } - - logger.debug("PPLTrigger ${pplSqlTrigger.name} with ID ${pplSqlTrigger.id} triggered: $triggered") - - // store the trigger execution results for Execute Monitor API response - triggerResults[pplSqlTrigger.id] = PPLSQLTriggerRunResult(pplSqlTrigger.name, null, triggered) - - if (triggered) { - logger.debug("generating alerts for PPLTrigger ${pplSqlTrigger.name} with ID ${pplSqlTrigger.id}") - // retrieve some limits from settings - val maxQueryResultsSize = - monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.ALERT_V2_QUERY_RESULTS_MAX_SIZE) - val maxAlerts = - monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.ALERT_V2_PER_RESULT_TRIGGER_MAX_ALERTS) - - // if trigger is on result set mode, this list will have exactly 1 element - // if trigger is on per result mode, this list will have as many elements as the query results had - // trigger condition-meeting rows, up to the max number of alerts a per result trigger can generate - val preparedQueryResults = splitUpQueryResults(pplSqlTrigger, queryResponseJson, maxQueryResultsSize, maxAlerts) - - // generate alerts based on trigger mode - // if this trigger is on result_set mode, this list contains exactly 1 alert - // if this trigger is on per_result mode, this list has as many alerts as there are - // trigger condition-meeting query results - val thisTriggersGeneratedAlerts = generateAlerts( - pplSqlTrigger, - pplSqlMonitor, - preparedQueryResults, - executionId, - timeOfCurrentExecution - ) - - // for future throttle checks, update the trigger's last execution time - // in the monitor object stored in memory - pplSqlTrigger.lastTriggeredTime = timeOfCurrentExecution - - // send alert notifications - for (action in pplSqlTrigger.actions) { - for (queryResult in preparedQueryResults) { - val pplTriggerExecutionContext = PPLTriggerExecutionContext( - pplSqlMonitor, - null, - listOf(queryResult.toMap()), - periodStart, - periodEnd, - monitorCtx.clusterService!!.clusterSettings, - pplSqlTrigger - ) - - runAction( - action, - pplTriggerExecutionContext, - monitorCtx, - pplSqlMonitor, - dryRun - ) - } - } - - // write the alerts to the alerts index - monitorCtx.retryPolicy?.let { - saveAlertsV2(thisTriggersGeneratedAlerts, pplSqlMonitor, it, nodeClient) - } - - logger.debug("PPL Trigger ${pplSqlTrigger.id} executed successfully") - } - } catch (e: Exception) { - logger.error( - "failed to run PPL Trigger ${pplSqlTrigger.name} (id: ${pplSqlTrigger.id} " + - "from PPL Monitor ${pplSqlMonitor.name} (id: ${pplSqlMonitor.id}", - e - ) - - // generate an alert with an error message - monitorCtx.retryPolicy?.let { - saveAlertsV2( - generateErrorAlert(pplSqlTrigger, pplSqlMonitor, e, executionId, timeOfCurrentExecution), - pplSqlMonitor, - it, - nodeClient - ) - } - } - } - } - - // returns true if the pplTrigger should be throttled - private fun checkForThrottle(pplTrigger: PPLSQLTrigger, timeOfCurrentExecution: Instant, manual: Boolean): Boolean { - // manual calls from the user to execute a monitor should never be throttled - if (manual) { - return false - } - - // the interval between throttledTimeBound and now is the throttle window - // i.e. any PPLTrigger whose last trigger time is in this window must be throttled - val throttleTimeBound = pplTrigger.throttleDuration?.let { - timeOfCurrentExecution.minus(pplTrigger.throttleDuration!!, ChronoUnit.MINUTES) - } - - // the trigger must be throttled if... - return pplTrigger.throttleDuration != null && // throttling is enabled on the PPLTrigger - pplTrigger.lastTriggeredTime != null && // and it has triggered before at least once - pplTrigger.lastTriggeredTime!!.isAfter(throttleTimeBound!!) // and it's not yet out of its throttle window - } - - private fun evaluateNumResultsTrigger( - pplQueryResponse: JSONObject, - numResultsCondition: NumResultsCondition, - numResultsValue: Long - ): Boolean { - val numResults = pplQueryResponse.getLong("total") - return when (numResultsCondition) { - NumResultsCondition.GREATER_THAN -> numResults > numResultsValue - NumResultsCondition.GREATER_THAN_EQUAL -> numResults >= numResultsValue - NumResultsCondition.LESS_THAN -> numResults < numResultsValue - NumResultsCondition.LESS_THAN_EQUAL -> numResults <= numResultsValue - NumResultsCondition.EQUAL -> numResults == numResultsValue - NumResultsCondition.NOT_EQUAL -> numResults != numResultsValue - } - } - - private fun evaluateCustomTrigger(pplQueryResponse: JSONObject, customCondition: String): Boolean { - // find the name of the eval result variable defined in custom condition - val evalResultVarName = findEvalResultVar(customCondition) - - // find the index eval statement result variable in the PPL query response schema - val evalResultVarIdx = findEvalResultVarIdxInSchema(pplQueryResponse, evalResultVarName) - - val dataRowList = pplQueryResponse.getJSONArray("datarows") - for (i in 0 until dataRowList.length()) { - val dataRow = dataRowList.getJSONArray(i) - val evalResult = dataRow.getBoolean(evalResultVarIdx) - if (evalResult) { - return true - } - } - - return false - } - - // prepares the query results to be passed into alerts and notifications based on trigger mode - // if result set, alert and notification simply stores all query results. - // if per result, each alert and notification stores a single row of the query results. - // this function then ensures that only a capped number of results are returned to generate alerts - // and notifications based on. it also caps the size of the query results themselves. - private fun splitUpQueryResults( - pplTrigger: PPLSQLTrigger, - pplQueryResults: JSONObject, - maxQueryResultsSize: Long, - maxAlerts: Int - ): List { - // case: result set - // return the results as a single set of all the results - if (pplTrigger.mode == TriggerMode.RESULT_SET) { - val sizeCappedRelevantQueryResultRows = capPPLQueryResultsSize(pplQueryResults, maxQueryResultsSize) - return listOf(sizeCappedRelevantQueryResultRows) - } - - // case: per result - // prepare to generate an alert for each relevant query result row, - // up to the maxAlerts limit - val individualRows = mutableListOf() - if (pplTrigger.conditionType == ConditionType.NUMBER_OF_RESULTS) { - // nested case: number_of_results - val numAlertsToGenerate = min(maxAlerts, pplQueryResults.getInt("total")) - for (i in 0 until numAlertsToGenerate) { - addRowToList(individualRows, pplQueryResults, i, maxQueryResultsSize) - } - } else { - // nested case: custom - val evalResultVarName = findEvalResultVar(pplTrigger.customCondition!!) - val evalResultVarIdx = findEvalResultVarIdxInSchema(pplQueryResults, evalResultVarName) - val dataRowList = pplQueryResults.getJSONArray("datarows") - for (i in 0 until dataRowList.length()) { - val dataRow = dataRowList.getJSONArray(i) - val evalResult = dataRow.getBoolean(evalResultVarIdx) - if (evalResult) { - addRowToList(individualRows, pplQueryResults, i, maxQueryResultsSize) - } - if (individualRows.size >= maxAlerts) { - break - } - } - } - - logger.debug("individualRows: $individualRows") - - return individualRows - } - - private fun addRowToList( - individualRows: MutableList, - pplQueryResults: JSONObject, - i: Int, - maxQueryResultsSize: Long - ) { - val individualRow = JSONObject() - individualRow.put("total", 1) // set the size explicitly to 1 for consistency - individualRow.put("size", 1) - individualRow.put("schema", JSONArray(pplQueryResults.getJSONArray("schema").toList())) - individualRow.put( - "datarows", - JSONArray().put( - JSONArray(pplQueryResults.getJSONArray("datarows").getJSONArray(i).toList()) - ) - ) - val sizeCappedIndividualRow = capPPLQueryResultsSize(individualRow, maxQueryResultsSize) - individualRows.add(sizeCappedIndividualRow) - } - - private fun generateAlerts( - pplSqlTrigger: PPLSQLTrigger, - pplSqlMonitor: Monitor, - preparedQueryResults: List, - executionId: String, - timeOfCurrentExecution: Instant - ): List { - val alertV2s = mutableListOf() - for (queryResult in preparedQueryResults) { - val alertV2 = Alert( - monitorId = pplSqlMonitor.id, - monitorName = pplSqlMonitor.name, - monitorVersion = pplSqlMonitor.version, - monitorUser = pplSqlMonitor.user, - triggerId = pplSqlTrigger.id, - triggerName = pplSqlTrigger.name, - pplQuery = (pplSqlMonitor.inputs[0] as PPLSQLInput).query, - pplQueryResults = queryResult.toMap(), - state = Alert.State.ACTIVE, - triggeredTime = timeOfCurrentExecution, - severity = pplSqlTrigger.severity, - executionId = executionId - ) - alertV2s.add(alertV2) - } - - return alertV2s.toList() // return as immutable list - } - - private fun generateErrorAlert( - pplSqlTrigger: PPLSQLTrigger?, - pplSqlMonitor: Monitor, - exception: Exception, - executionId: String, - timeOfCurrentExecution: Instant - ): List { - val errorMessage = "Failed to run PPL Monitor ${pplSqlMonitor.id}, PPL Trigger ${pplSqlTrigger?.id}: " + - exception.userErrorMessage() - val obfuscatedErrorMessage = AlertError.obfuscateIPAddresses(errorMessage) - - val alertV2 = Alert( - monitorId = pplSqlMonitor.id, - monitorName = pplSqlMonitor.name, - monitorVersion = pplSqlMonitor.version, - monitorUser = pplSqlMonitor.user, - triggerId = pplSqlTrigger?.id ?: "", - triggerName = pplSqlTrigger?.name ?: "", - pplQuery = (pplSqlMonitor.inputs[0] as PPLSQLInput).query, - pplQueryResults = mapOf(), - state = Alert.State.ERROR, - triggeredTime = timeOfCurrentExecution, - errorMessage = obfuscatedErrorMessage, - severity = Severity.ERROR.value, - executionId = executionId - ) - - return listOf(alertV2) - } - - private suspend fun saveAlertsV2( - alerts: List, - pplSqlMonitor: Monitor, - retryPolicy: BackoffPolicy, - client: NodeClient - ) { - logger.debug("received alerts: $alerts") - - var requestsToRetry = alerts.flatMap { alert -> - listOf>( - IndexRequest(AlertV2Indices.ALERT_V2_INDEX) - .routing(pplSqlMonitor.id) // set routing ID to PPL Monitor ID - .source(alert.toXContentWithUser(XContentFactory.jsonBuilder())) - .id(if (alert.id != Alert.NO_ID) alert.id else null) - ) - } - - if (requestsToRetry.isEmpty()) return - // Retry Bulk requests if there was any 429 response - retryPolicy.retry(logger, listOf(RestStatus.TOO_MANY_REQUESTS)) { - val bulkRequest = BulkRequest().add(requestsToRetry).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - val bulkResponse: BulkResponse = client.suspendUntil { client.bulk(bulkRequest, it) } - val failedResponses = (bulkResponse.items ?: arrayOf()).filter { it.isFailed } - failedResponses.forEach { - logger.info("write alerts failed responses: ${it.failureMessage}") - } - requestsToRetry = failedResponses.filter { it.status() == RestStatus.TOO_MANY_REQUESTS } - .map { bulkRequest.requests()[it.itemId] as IndexRequest } - - if (requestsToRetry.isNotEmpty()) { - val retryCause = failedResponses.first { it.status() == RestStatus.TOO_MANY_REQUESTS }.failure.cause - throw ExceptionsHelper.convertToOpenSearchException(retryCause) - } - } - } - - // during monitor execution, the ppl sql monitor object stored in memory had its triggers updated - // with their last trigger times. this function simply indexes those updated triggers into the - // alerting-config index - private suspend fun updateMonitorWithLastTriggeredTimes(pplSqlMonitor: Monitor, client: NodeClient) { - val indexRequest = IndexRequest(SCHEDULED_JOBS_INDEX) - .id(pplSqlMonitor.id) - .source( - pplSqlMonitor.toXContentWithUser( - XContentFactory.jsonBuilder(), - ToXContent.MapParams( - mapOf("with_type" to "true") - ) - ) - ) - .routing(pplSqlMonitor.id) - .version(pplSqlMonitor.version) - .versionType(VersionType.EXTERNAL_GTE) - - val indexResponse = client.suspendUntil { index(indexRequest, it) } - - logger.debug("PPLSQLMonitor update with last execution times index response: ${indexResponse.result}") - } - - suspend fun runAction( - action: Action, - triggerCtx: PPLTriggerExecutionContext, - monitorCtx: MonitorRunnerExecutionContext, - pplSqlMonitor: Monitor, - dryrun: Boolean - ) { - // this function can throw an exception, which is caught by the try - // catch in runMonitor() to generate an error alert - - val notifSubject = if (action.subjectTemplate != null) - MonitorRunnerService.compileTemplate(action.subjectTemplate!!, triggerCtx) - else "" - - var notifMessage = MonitorRunnerService.compileTemplate(action.messageTemplate, triggerCtx) - if (Strings.isNullOrEmpty(notifMessage)) { - throw IllegalStateException("Message content missing in the Destination with id: ${action.destinationId}") - } - - if (!dryrun) { - monitorCtx.client!!.threadPool().threadContext.stashContext().use { - withClosableContext( - InjectorContextElement( - pplSqlMonitor.id, - monitorCtx.settings!!, - monitorCtx.threadPool!!.threadContext, - pplSqlMonitor.user?.roles, - pplSqlMonitor.user - ) - ) { - getConfigAndSendNotification( - action, - monitorCtx, - notifSubject, - notifMessage - ) - } - } - } - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt b/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt index 0bb4babc8..f1021e2e3 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt @@ -20,13 +20,9 @@ object PPLUtils { // captures the name of the result variable in a PPL monitor's custom condition // e.g. custom condition: `eval apple = avg_latency > 100` // captures: "apple" - private val evalResultVarRegex = """\beval\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*=""".toRegex() + private val evalResultVarRegex = """^(?!.*\|)\s*(?i:eval)\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*=""".toRegex() - // captures the list of indices and index patterns that a given PPL query searches - // e.g. PPL query: search source = index_1,index_pattern*,index_3 | where responseCode = 500 | head 10 - // captures: index_1,index_pattern*,index_3 - private val indicesListRegex = - """(?i)source(?:\s*)=(?:\s*)((?:`[^`]+`|[-\w.*'+]+(?:\*)?)(?:\s*,\s*(?:`[^`]+`|[-\w.*'+]+\*?))*)\s*\|*""".toRegex() + const val PPL_RESULTS_SIZE_EXCEEDED_MESSAGE = "The PPL Query results were too large and thus excluded." /** * Appends a user-defined custom condition to a PPL query. @@ -195,63 +191,12 @@ object PPLUtils { return evalResultVarIdx } - /** - * Extracts the list of indices from a PPL query's source statement. - * - * Parses the PPL `source=` clause to identify which indices, index patterns, or index - * aliases are being queried. This information is primarily used for permission checks. - * Supports comma-separated lists of indices and wildcard patterns. - * - * @param pplQuery The complete PPL query string containing a source statement - * @return A list of index names, patterns, or aliases (e.g., ["logs-*", "metrics-2024"]) - * @throws IllegalStateException if no valid source statement is found, even after - * the query has been validated by the SQL/PPL plugin - * - * @example - * ``` - * val query = "source=logs-* | where level='ERROR'" - * val indices = getIndicesFromPplQuery(query) - * // Returns: ["logs-*"] - * - * val multiQuery = "source=logs-*, metrics-2024, .kibana | stats count()" - * val multiIndices = getIndicesFromPplQuery(multiQuery) - * // Returns: ["logs-*", "metrics-2024", ".kibana"] - * ``` - * - * @note Supports concrete indices, wildcard patterns (*), dot-prefixed system indices, - * and index aliases - * @note PPL queries contain exactly one source statement, so only the first match is used - * @note The regex pattern handles optional whitespace around `=` and commas - * - */ - fun getIndicesFromPplQuery(pplQuery: String): List { - // use find() instead of findAll() because a PPL query only ever has one source statement - // the only capture group specified in the regex captures the comma separated string of indices/index patterns - val indices = indicesListRegex.find(pplQuery)?.groupValues?.get(1)?.split(",")?.map { it.trim() } - ?: throw IllegalStateException( - "Could not find indices that PPL Monitor query searches even " + - "after validating the query through SQL/PPL plugin." - ) - - // remove any backticks that might have been read in - val unBackTickedIndices = mutableListOf() - indices.forEach { - if (it.startsWith("`") && it.endsWith("`")) { - unBackTickedIndices.add(it.substring(1, it.length - 1)) - } else { - unBackTickedIndices.add(it) - } - } - - return unBackTickedIndices.toList() - } - /** * Caps the size of PPL query results to prevent memory issues and oversized alert payloads. * * Checks if the serialized query results exceed a specified size limit. If the results - * are within the limit, they are returned unchanged. If they exceed the limit, the datarows - * are replaced with an informational message while preserving the schema and metadata fields. + * are within the limit, they are returned unchanged. If they exceed the limit, the whole response + * is replaced with an informational message while preserving the original structure of the response. * This ensures alerts can still be created even when query results are too large. * * @param pplQueryResults The PPL query response JSONObject @@ -290,16 +235,89 @@ object PPLUtils { // ppl query response fields like schema, total, and size val limitExceedMessageQueryResults = JSONObject() - val schema = JSONArray(pplQueryResults.getJSONArray("schema").toList()) - val datarows = JSONArray().put(JSONArray(listOf("The PPL Query results were too large and thus excluded"))) - val total = pplQueryResults.getInt("total") - val size = pplQueryResults.getInt("size") + val schema = JSONArray().put(JSONObject(mapOf("name" to "message", "type" to "string"))) +// val schema = JSONArray(pplQueryResults.getJSONArray("schema").toList()) + val datarows = JSONArray().put(JSONArray(listOf(PPL_RESULTS_SIZE_EXCEEDED_MESSAGE))) limitExceedMessageQueryResults.put("schema", schema) limitExceedMessageQueryResults.put("datarows", datarows) - limitExceedMessageQueryResults.put("total", total) - limitExceedMessageQueryResults.put("size", size) + limitExceedMessageQueryResults.put("total", 1) + limitExceedMessageQueryResults.put("size", 1) return limitExceedMessageQueryResults } + + /** + * Transforms PPL query results from array-based format (that SQL Plugin Execute API response uses) + * to map-based format for easier template access. + * + * PPL query responses contain a `schema` array that defines field names and types, and a `datarows` array + * that contains the actual data values in positional format. This function combines them into a list of maps + * where each list element represents a row with field names as keys and corresponding values from the datarows. + * + * ### Input Format + * The input should be a PPL query result with this structure: + * ```json + * { + * "schema": [ + * {"name": "abc", "type": "string"}, + * {"name": "number", "type": "integer"} + * ], + * "datarows": [ + * ["xyz", 3], + * ["def", 5] + * ] + * } + * ``` + * + * ### Output Format + * The function returns a list where each element is a map representing a data row: + * ```json + * [ + * {"abc": "xyz", "number": 3}, + * {"abc": "def", "number": 5} + * ] + * ``` + * + * ### Edge Cases + * - If `schema` is missing or empty, returns an empty list + * - If `datarows` is missing or empty, returns an empty list + * - If a schema entry is malformed (not a map or missing "name" field), it is skipped + * - If a datarow has fewer values than schema fields, missing values are set to `null` + * - If a datarow has more values than schema fields, extra values are ignored + * - If a datarow is not a list, that row is skipped + * + * @param rawQueryResults The PPL query results map from SQL Plugin Execute API response + * containing "schema" and "datarows" fields. + * @return A list of maps where each map represents a data row with field names as keys and + * corresponding values from datarows. Returns an empty list if schema or datarows + * are missing, empty, or malformed. + * + * @see org.opensearch.alerting.script.QueryLevelTriggerExecutionContext.asTemplateArg + * @see org.opensearch.alerting.PPLUtils.executePplQuery + */ + fun constructPPLQueryResultsMap(rawQueryResults: Map): List> { + // Extract schema array + val schema = rawQueryResults["schema"] as? List<*> ?: return emptyList() + + // Extract field names from schema + val fieldNames = schema.mapNotNull { schemaEntry -> + (schemaEntry as? Map<*, *>)?.get("name") as? String + } + + if (fieldNames.isEmpty()) return emptyList() + + // Extract datarows array + val datarows = rawQueryResults["datarows"] as? List<*> ?: return emptyList() + + // Transform each row into a map + return datarows.mapNotNull { row -> + val rowList = row as? List<*> ?: return@mapNotNull null + + // Create a map from field names to values + fieldNames.mapIndexed { index, fieldName -> + fieldName to rowList.getOrNull(index) + }.toMap() + } + } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/QueryLevelMonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/QueryLevelMonitorRunner.kt index 44370228e..a45eb82a7 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/QueryLevelMonitorRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/QueryLevelMonitorRunner.kt @@ -6,6 +6,9 @@ package org.opensearch.alerting import org.apache.logging.log4j.LogManager +import org.json.JSONObject +import org.opensearch.alerting.PPLUtils.capPPLQueryResultsSize +import org.opensearch.alerting.PPLUtils.constructPPLQueryResultsMap import org.opensearch.alerting.model.AlertContext import org.opensearch.alerting.opensearchapi.InjectorContextElement import org.opensearch.alerting.opensearchapi.withClosableContext @@ -15,15 +18,20 @@ import org.opensearch.alerting.trigger.RemoteQueryLevelTriggerEvaluator import org.opensearch.alerting.util.CommentsUtils import org.opensearch.alerting.util.isADMonitor import org.opensearch.commons.alerting.model.Alert +import org.opensearch.commons.alerting.model.InputRunResults import org.opensearch.commons.alerting.model.Monitor import org.opensearch.commons.alerting.model.MonitorRunResult +import org.opensearch.commons.alerting.model.PPLSQLInput +import org.opensearch.commons.alerting.model.PPLSQLTrigger import org.opensearch.commons.alerting.model.QueryLevelTrigger import org.opensearch.commons.alerting.model.QueryLevelTriggerRunResult import org.opensearch.commons.alerting.model.SearchInput import org.opensearch.commons.alerting.model.WorkflowRunContext +import org.opensearch.commons.alerting.util.isPplSqlMonitor import org.opensearch.transport.TransportService import java.time.Instant import java.util.Locale +import kotlin.collections.set object QueryLevelMonitorRunner : MonitorRunner() { private val logger = LogManager.getLogger(javaClass) @@ -48,11 +56,6 @@ object QueryLevelMonitorRunner : MonitorRunner() { var monitorResult = MonitorRunResult(monitor.name, periodStart, periodEnd) val currentAlerts = try { - // create stateless alert indices as well to prevent get alerts from returning error because - // stateless alerts indices couldn't be found - monitorCtx.alertV2Indices!!.createOrUpdateAlertV2Index() - monitorCtx.alertV2Indices!!.createOrUpdateInitialAlertV2HistoryIndex() - monitorCtx.alertIndices!!.createOrUpdateAlertIndex(monitor.dataSources) monitorCtx.alertIndices!!.createOrUpdateInitialAlertHistoryIndex(monitor.dataSources) monitorCtx.alertService!!.loadCurrentAlertsForQueryLevelMonitor(monitor, workflowRunContext) @@ -72,9 +75,21 @@ object QueryLevelMonitorRunner : MonitorRunner() { monitor.user ) ) { - monitorResult = monitorResult.copy( - inputResults = monitorCtx.inputService!!.collectInputResults(monitor, periodStart, periodEnd, null, workflowRunContext) - ) + // If monitor is Query-level, proceed with collecting input results as usual. + // If monitor is PPLSQL Monitor, input results have to be collected during + // trigger execution because the PPL query run can be different per trigger. + // In the PPLSQL case, delay storing the input results until after trigger execution. + if (!monitor.isPplSqlMonitor()) { + monitorResult = monitorResult.copy( + inputResults = monitorCtx.inputService!!.collectInputResults( + monitor, + periodStart, + periodEnd, + null, + workflowRunContext + ) + ) + } } } else { monitorResult = monitorResult.copy( @@ -111,6 +126,11 @@ object QueryLevelMonitorRunner : MonitorRunner() { alertsToExecuteActionsForIds, maxComments ) + + // if this is a PPL/SQL Monitor run, this will be populated with + // the query results for each trigger + val pplSqlQueryResults = mutableMapOf>() + for (trigger in monitor.triggers) { val currentAlert = currentAlerts[trigger] val currentAlertContext = currentAlert?.let { @@ -119,7 +139,7 @@ object QueryLevelMonitorRunner : MonitorRunner() { val triggerCtx = QueryLevelTriggerExecutionContext( monitor, - trigger as QueryLevelTrigger, + trigger, monitorResult, currentAlertContext, monitorCtx.clusterService!!.clusterSettings @@ -131,16 +151,26 @@ object QueryLevelMonitorRunner : MonitorRunner() { } else { when (Monitor.MonitorType.valueOf(monitor.monitorType.uppercase(Locale.ROOT))) { Monitor.MonitorType.QUERY_LEVEL_MONITOR -> - monitorCtx.triggerService!!.runQueryLevelTrigger(monitor, trigger, triggerCtx) + monitorCtx.triggerService!!.runQueryLevelTrigger(monitor, trigger as QueryLevelTrigger, triggerCtx) Monitor.MonitorType.CLUSTER_METRICS_MONITOR -> { val remoteMonitoringEnabled = monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.CROSS_CLUSTER_MONITORING_ENABLED) logger.debug("Remote monitoring enabled: {}", remoteMonitoringEnabled) if (remoteMonitoringEnabled) monitorCtx.triggerService!!.runClusterMetricsTrigger( - monitor, trigger, triggerCtx, monitorCtx.clusterService!! + monitor, trigger as QueryLevelTrigger, triggerCtx, monitorCtx.clusterService!! ) - else monitorCtx.triggerService!!.runQueryLevelTrigger(monitor, trigger, triggerCtx) + else monitorCtx.triggerService!!.runQueryLevelTrigger(monitor, trigger as QueryLevelTrigger, triggerCtx) + } + Monitor.MonitorType.PPL_MONITOR -> { + monitorCtx.triggerService!!.runPplSqlTrigger( + monitor, + trigger as PPLSQLTrigger, + (monitor.inputs[0] as PPLSQLInput).query, + pplSqlQueryResults, + monitorCtx, + transportService + ) } else -> throw IllegalArgumentException("Unsupported monitor type: ${monitor.monitorType}.") @@ -149,23 +179,46 @@ object QueryLevelMonitorRunner : MonitorRunner() { triggerResults[trigger.id] = triggerResult + // the query results passed into notifications and alerts must be size-capped and reformatted + val sizeCappedFormattedPPLQueryResults = pplSqlQueryResults[trigger.id]?.let { + val sizeCappedRows = capPPLQueryResultsSize( + JSONObject(it), + monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.ALERT_V2_QUERY_RESULTS_MAX_SIZE) + ) + constructPPLQueryResultsMap(sizeCappedRows.toMap()) + } + + val postRunTriggerCtx = triggerCtx.copy( + pplSqlQueryResult = sizeCappedFormattedPPLQueryResults + ) + if (monitorCtx.triggerService!!.isQueryLevelTriggerActionable(triggerCtx, triggerResult, workflowRunContext)) { - val actionCtx = triggerCtx.copy(error = monitorResult.error ?: triggerResult.error) + val actionCtx = postRunTriggerCtx.copy(error = monitorResult.error ?: triggerResult.error) for (action in trigger.actions) { triggerResult.actionResults[action.id] = this.runAction(action, actionCtx, monitorCtx, monitor, dryrun) } } val updatedAlert = monitorCtx.alertService!!.composeQueryLevelAlert( - triggerCtx, + postRunTriggerCtx, triggerResult, monitorResult.alertError() ?: triggerResult.alertError(), executionId, - workflowRunContext + workflowRunContext, + monitorCtx, + sizeCappedFormattedPPLQueryResults ) if (updatedAlert != null) updatedAlerts += updatedAlert } + // store input results after trigger runs, as each trigger could have + // run a different query + if (monitor.isPplSqlMonitor()) { + monitorResult = monitorResult.copy( + inputResults = InputRunResults(results = pplSqlQueryResults.values.toList()) + ) + } + // Don't save alerts if this is a test monitor if (!dryrun && monitor.id != Monitor.NO_ID) { monitorCtx.retryPolicy?.let { diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt index d4b3d9ded..1d8e27c52 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt @@ -5,12 +5,22 @@ package org.opensearch.alerting +import kotlinx.coroutines.withTimeout import org.apache.logging.log4j.LogManager +import org.json.JSONObject +import org.opensearch.alerting.PPLUtils.appendCustomCondition +import org.opensearch.alerting.PPLUtils.appendDataRowsLimit +import org.opensearch.alerting.PPLUtils.executePplQuery +import org.opensearch.alerting.PPLUtils.findEvalResultVar +import org.opensearch.alerting.PPLUtils.findEvalResultVarIdxInSchema import org.opensearch.alerting.chainedAlertCondition.parsers.ChainedAlertExpressionParser +import org.opensearch.alerting.opensearchapi.InjectorContextElement +import org.opensearch.alerting.opensearchapi.withClosableContext import org.opensearch.alerting.script.BucketLevelTriggerExecutionContext import org.opensearch.alerting.script.ChainedAlertTriggerExecutionContext import org.opensearch.alerting.script.QueryLevelTriggerExecutionContext import org.opensearch.alerting.script.TriggerScript +import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.triggercondition.parsers.TriggerExpressionParser import org.opensearch.alerting.util.CrossClusterMonitorUtils import org.opensearch.alerting.util.getBucketKeysHash @@ -29,6 +39,9 @@ import org.opensearch.commons.alerting.model.DocLevelQuery import org.opensearch.commons.alerting.model.DocumentLevelTrigger import org.opensearch.commons.alerting.model.DocumentLevelTriggerRunResult import org.opensearch.commons.alerting.model.Monitor +import org.opensearch.commons.alerting.model.PPLSQLTrigger +import org.opensearch.commons.alerting.model.PPLSQLTrigger.ConditionType +import org.opensearch.commons.alerting.model.PPLSQLTrigger.NumResultsCondition import org.opensearch.commons.alerting.model.QueryLevelTrigger import org.opensearch.commons.alerting.model.QueryLevelTriggerRunResult import org.opensearch.commons.alerting.model.Workflow @@ -38,6 +51,8 @@ import org.opensearch.script.ScriptService import org.opensearch.search.aggregations.Aggregation import org.opensearch.search.aggregations.Aggregations import org.opensearch.search.aggregations.support.AggregationPath +import org.opensearch.transport.TransportService +import kotlin.time.measureTimedValue /** Service that handles executing Triggers */ class TriggerService(val scriptService: ScriptService) { @@ -235,4 +250,133 @@ class TriggerService(val scriptService: ScriptService) { return keyValuesList } + + suspend fun runPplSqlTrigger( + pplSqlMonitor: Monitor, + pplSqlTrigger: PPLSQLTrigger, + query: String, + pplSqlQueryResults: MutableMap>, + monitorCtx: MonitorRunnerExecutionContext, + transportService: TransportService + ): QueryLevelTriggerRunResult { + // TODO: change name to trigger max duration + val monitorExecutionDuration = monitorCtx + .clusterService!! + .clusterSettings + .get(AlertingSettings.ALERT_V2_MONITOR_EXECUTION_MAX_DURATION) + + var triggered: Boolean? = null + + try { + withTimeout(monitorExecutionDuration.millis) { + logger.debug("checking if custom condition is used and appending to base query") + // if trigger uses custom condition, append the custom condition to query, otherwise simply proceed + val queryToExecute = if (pplSqlTrigger.conditionType == ConditionType.NUMBER_OF_RESULTS) { // number of results trigger + query + } else { // custom condition trigger + appendCustomCondition(query, pplSqlTrigger.customCondition!!) + } + + // limit the number of PPL query result data rows returned + val dataRowsLimit = monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS) + val limitedQueryToExecute = appendDataRowsLimit(queryToExecute, dataRowsLimit) + + // TODO: after getting ppl query results, see if the number of results + // retrieved equals the max allowed number of query results. this implies + // query results might have been excluded, in which case a warning message + // in the alert and notification must be added that results were excluded + // and an alert that should have been generated might not have been + + logger.debug("executing the PPL query of monitor: ${pplSqlMonitor.id}") + // execute the PPL query + val (queryResponseJson, timeTaken) = measureTimedValue { + withClosableContext( + InjectorContextElement( + pplSqlMonitor.id, + monitorCtx.settings!!, + monitorCtx.threadPool!!.threadContext, + pplSqlMonitor.user?.roles, + pplSqlMonitor.user + ) + ) { + executePplQuery( + limitedQueryToExecute, + monitorCtx.clusterService!!.state().nodes.localNode, + transportService + ) + } + } + logger.debug("query results for trigger ${pplSqlTrigger.id}: $queryResponseJson") + logger.debug("time taken to execute query against sql/ppl plugin: $timeTaken") + + // store the query results for Execute Monitor API response + // unlike the query results stored in alerts and notifications, which must be size capped + // (because they will be stored in the OpenSearch cluster or sent as notification) and must be based + // on only the query results that met the trigger condition (because alerts should generate + // on query results that met trigger condition, not those that didn't), the pplQueryResults + // here will be returned as part of the Execute Monitor API response. This will return the original, + // untouched set of query results, and whether this causes size exceed errors is deferred + // to HTTP's response size limits + pplSqlQueryResults[pplSqlTrigger.id] = queryResponseJson.toMap() + + // determine if the trigger condition has been met + triggered = if (pplSqlTrigger.conditionType == ConditionType.NUMBER_OF_RESULTS) { // number of results trigger + evaluateNumResultsTrigger( + queryResponseJson, + pplSqlTrigger.numResultsCondition!!, + pplSqlTrigger.numResultsValue!! + ) + } else { // custom condition trigger + evaluateCustomTrigger(queryResponseJson, pplSqlTrigger.customCondition!!) + } + + logger.debug("PPLTrigger ${pplSqlTrigger.name} with ID ${pplSqlTrigger.id} triggered: $triggered") + } + + return QueryLevelTriggerRunResult(pplSqlTrigger.name, triggered!!, null) + } catch (e: Exception) { + logger.error( + "failed to run PPL Trigger ${pplSqlTrigger.name} (id: ${pplSqlTrigger.id} " + + "from PPL Monitor ${pplSqlMonitor.name} (id: ${pplSqlMonitor.id}", + e + ) + + return QueryLevelTriggerRunResult(pplSqlTrigger.name, true, e) + } + } + + private fun evaluateNumResultsTrigger( + pplQueryResponse: JSONObject, + numResultsCondition: NumResultsCondition, + numResultsValue: Long + ): Boolean { + val numResults = pplQueryResponse.getLong("total") + return when (numResultsCondition) { + NumResultsCondition.GREATER_THAN -> numResults > numResultsValue + NumResultsCondition.GREATER_THAN_EQUAL -> numResults >= numResultsValue + NumResultsCondition.LESS_THAN -> numResults < numResultsValue + NumResultsCondition.LESS_THAN_EQUAL -> numResults <= numResultsValue + NumResultsCondition.EQUAL -> numResults == numResultsValue + NumResultsCondition.NOT_EQUAL -> numResults != numResultsValue + } + } + + private fun evaluateCustomTrigger(pplQueryResponse: JSONObject, customCondition: String): Boolean { + // find the name of the eval result variable defined in custom condition + val evalResultVarName = findEvalResultVar(customCondition) + + // find the index eval statement result variable in the PPL query response schema + val evalResultVarIdx = findEvalResultVarIdxInSchema(pplQueryResponse, evalResultVarName) + + val dataRowList = pplQueryResponse.getJSONArray("datarows") + for (i in 0 until dataRowList.length()) { + val dataRow = dataRowList.getJSONArray(i) + val evalResult = dataRow.getBoolean(evalResultVarIdx) + if (evalResult) { + return true + } + } + + return false + } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Indices.kt b/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Indices.kt deleted file mode 100644 index 5e3d93fc3..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Indices.kt +++ /dev/null @@ -1,419 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.alertsv2 - -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch -import org.apache.logging.log4j.LogManager -import org.opensearch.ExceptionsHelper -import org.opensearch.ResourceAlreadyExistsException -import org.opensearch.action.admin.cluster.state.ClusterStateRequest -import org.opensearch.action.admin.cluster.state.ClusterStateResponse -import org.opensearch.action.admin.indices.alias.Alias -import org.opensearch.action.admin.indices.create.CreateIndexRequest -import org.opensearch.action.admin.indices.create.CreateIndexResponse -import org.opensearch.action.admin.indices.delete.DeleteIndexRequest -import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest -import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse -import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest -import org.opensearch.action.admin.indices.rollover.RolloverRequest -import org.opensearch.action.admin.indices.rollover.RolloverResponse -import org.opensearch.action.support.IndicesOptions -import org.opensearch.action.support.clustermanager.AcknowledgedResponse -import org.opensearch.alerting.alerts.AlertIndices -import org.opensearch.alerting.opensearchapi.suspendUntil -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_V2_HISTORY_ENABLED -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_V2_HISTORY_INDEX_MAX_AGE -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_V2_HISTORY_MAX_DOCS -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_V2_HISTORY_RETENTION_PERIOD -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_V2_HISTORY_ROLLOVER_PERIOD -import org.opensearch.alerting.settings.AlertingSettings.Companion.REQUEST_TIMEOUT -import org.opensearch.alerting.util.IndexUtils -import org.opensearch.cluster.ClusterChangedEvent -import org.opensearch.cluster.ClusterStateListener -import org.opensearch.cluster.metadata.IndexMetadata -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.settings.Settings -import org.opensearch.common.unit.TimeValue -import org.opensearch.common.xcontent.XContentType -import org.opensearch.commons.alerting.util.AlertingException -import org.opensearch.core.action.ActionListener -import org.opensearch.threadpool.Scheduler.Cancellable -import org.opensearch.threadpool.ThreadPool -import org.opensearch.transport.client.Client -import java.time.Instant - -private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) -private val logger = LogManager.getLogger(AlertV2Indices::class.java) - -/** - * This class handles the rollover and management of v2 alerts history indices - * - * @opensearch.experimental - */ -class AlertV2Indices( - settings: Settings, - private val client: Client, - private val threadPool: ThreadPool, - private val clusterService: ClusterService -) : ClusterStateListener { - - init { - clusterService.addListener(this) - clusterService.clusterSettings.addSettingsUpdateConsumer(ALERT_V2_HISTORY_ENABLED) { alertV2HistoryEnabled = it } - clusterService.clusterSettings.addSettingsUpdateConsumer(ALERT_V2_HISTORY_MAX_DOCS) { alertV2HistoryMaxDocs = it } - clusterService.clusterSettings.addSettingsUpdateConsumer(ALERT_V2_HISTORY_INDEX_MAX_AGE) { alertV2HistoryMaxAge = it } - clusterService.clusterSettings.addSettingsUpdateConsumer(ALERT_V2_HISTORY_ROLLOVER_PERIOD) { - alertV2HistoryRolloverPeriod = it - rescheduleAlertRollover() - } - clusterService.clusterSettings.addSettingsUpdateConsumer(ALERT_V2_HISTORY_RETENTION_PERIOD) { - alertV2HistoryRetentionPeriod = it - } - clusterService.clusterSettings.addSettingsUpdateConsumer(REQUEST_TIMEOUT) { requestTimeout = it } - } - - companion object { - - /** The in progress alert history index. */ - const val ALERT_V2_INDEX = ".opensearch-alerting-v2-alerts" - - /** The alias of the index in which to write alert history */ - const val ALERT_V2_HISTORY_WRITE_INDEX = ".opensearch-alerting-v2-alert-history-write" - - /** The index name pattern referring to all alert history indices */ - const val ALERT_V2_HISTORY_ALL = ".opensearch-alerting-v2-alert-history*" - - /** The index name pattern to create alert history indices */ - const val ALERT_V2_HISTORY_INDEX_PATTERN = "<.opensearch-alerting-v2-alert-history-{now/d}-1>" - - /** The index name pattern to query all alerts, history and current alerts. */ - const val ALL_ALERT_V2_INDEX_PATTERN = ".opensearch-alerting-v2-alert*" - - @JvmStatic - fun alertMapping() = - AlertIndices::class.java.getResource("alert_mapping.json").readText() - } - - @Volatile private var alertV2HistoryEnabled = ALERT_V2_HISTORY_ENABLED.get(settings) - - @Volatile private var alertV2HistoryMaxDocs = ALERT_V2_HISTORY_MAX_DOCS.get(settings) - - @Volatile private var alertV2HistoryMaxAge = ALERT_V2_HISTORY_INDEX_MAX_AGE.get(settings) - - @Volatile private var alertV2HistoryRolloverPeriod = ALERT_V2_HISTORY_ROLLOVER_PERIOD.get(settings) - - @Volatile private var alertV2HistoryRetentionPeriod = ALERT_V2_HISTORY_RETENTION_PERIOD.get(settings) - - @Volatile private var requestTimeout = REQUEST_TIMEOUT.get(settings) - - @Volatile private var isClusterManager = false - - // for JobsMonitor to report - var lastRolloverTime: TimeValue? = null - - private var alertV2HistoryIndexInitialized: Boolean = false - - private var alertV2IndexInitialized: Boolean = false - - private var scheduledAlertV2Rollover: Cancellable? = null - - fun onClusterManager() { - try { - // try to rollover immediately as we might be restarting the cluster - rolloverAlertV2HistoryIndex() - - // schedule the next rollover for approx MAX_AGE later - scheduledAlertV2Rollover = threadPool - .scheduleWithFixedDelay({ rolloverAndDeleteAlertV2HistoryIndices() }, alertV2HistoryRolloverPeriod, executorName()) - } catch (e: Exception) { - logger.error("Error rolling over alerts v2 history index.", e) - } - } - - fun offClusterManager() { - scheduledAlertV2Rollover?.cancel() - } - - private fun executorName(): String { - return ThreadPool.Names.MANAGEMENT - } - - override fun clusterChanged(event: ClusterChangedEvent) { - // Instead of using a LocalNodeClusterManagerListener to track clustermanager changes, this service will - // track them here to avoid conditions where clustermanager listener events run after other - // listeners that depend on what happened in the clustermanager listener - if (this.isClusterManager != event.localNodeClusterManager()) { - this.isClusterManager = event.localNodeClusterManager() - if (this.isClusterManager) { - onClusterManager() - } else { - offClusterManager() - } - } - - // if the indexes have been deleted they need to be reinitialized - alertV2IndexInitialized = event.state().routingTable().hasIndex(ALERT_V2_INDEX) - alertV2HistoryIndexInitialized = event.state().metadata().hasAlias(ALERT_V2_HISTORY_WRITE_INDEX) - } - - private fun rescheduleAlertRollover() { - if (clusterService.state().nodes.isLocalNodeElectedClusterManager) { - scheduledAlertV2Rollover?.cancel() - scheduledAlertV2Rollover = threadPool - .scheduleWithFixedDelay({ rolloverAndDeleteAlertV2HistoryIndices() }, alertV2HistoryRolloverPeriod, executorName()) - } - } - - suspend fun createOrUpdateAlertV2Index() { - if (!alertV2IndexInitialized) { - alertV2IndexInitialized = createIndex(ALERT_V2_INDEX, alertMapping()) - if (alertV2IndexInitialized) IndexUtils.alertIndexUpdated() - } else { - if (!IndexUtils.alertIndexUpdated) updateIndexMapping(ALERT_V2_INDEX, alertMapping()) - } - alertV2IndexInitialized - } - - suspend fun createOrUpdateInitialAlertV2HistoryIndex() { - if (!alertV2HistoryIndexInitialized) { - alertV2HistoryIndexInitialized = createIndex(ALERT_V2_HISTORY_INDEX_PATTERN, alertMapping(), ALERT_V2_HISTORY_WRITE_INDEX) - if (alertV2HistoryIndexInitialized) - IndexUtils.lastUpdatedAlertV2HistoryIndex = IndexUtils.getIndexNameWithAlias( - clusterService.state(), - ALERT_V2_HISTORY_WRITE_INDEX - ) - } else { - updateIndexMapping(ALERT_V2_HISTORY_WRITE_INDEX, alertMapping(), true) - } - alertV2HistoryIndexInitialized - } - - fun isAlertV2Initialized(): Boolean { - return alertV2IndexInitialized && alertV2HistoryIndexInitialized - } - - private fun rolloverAndDeleteAlertV2HistoryIndices() { - if (alertV2HistoryEnabled) rolloverAlertV2HistoryIndex() - deleteOldIndices("History", ALERT_V2_HISTORY_ALL) - } - - private suspend fun createIndex(index: String, schemaMapping: String, alias: String? = null): Boolean { - // This should be a fast check of local cluster state. Should be exceedingly rare that the local cluster - // state does not contain the index and multiple nodes concurrently try to create the index. - // If it does happen that error is handled we catch the ResourceAlreadyExistsException - val existsResponse: IndicesExistsResponse = client.admin().indices().suspendUntil { - exists(IndicesExistsRequest(index).local(true), it) - } - if (existsResponse.isExists) return true - - logger.debug("index: [$index] schema mappings: [$schemaMapping]") - val request = CreateIndexRequest(index) - .mapping(schemaMapping) - .settings(Settings.builder().put("index.hidden", true).build()) - - if (alias != null) request.alias(Alias(alias)) - return try { - val createIndexResponse: CreateIndexResponse = client.admin().indices().suspendUntil { create(request, it) } - createIndexResponse.isAcknowledged - } catch (t: Exception) { - if (ExceptionsHelper.unwrapCause(t) is ResourceAlreadyExistsException) { - true - } else { - throw AlertingException.wrap(t) - } - } - } - - private suspend fun updateIndexMapping(index: String, mapping: String, alias: Boolean = false) { - val clusterState = clusterService.state() - var targetIndex = index - if (alias) { - targetIndex = IndexUtils.getIndexNameWithAlias(clusterState, index) - } - - if (targetIndex == IndexUtils.lastUpdatedAlertV2HistoryIndex) { - return - } - - val putMappingRequest: PutMappingRequest = PutMappingRequest(targetIndex) - .source(mapping, XContentType.JSON) - val updateResponse: AcknowledgedResponse = client.admin().indices().suspendUntil { putMapping(putMappingRequest, it) } - if (updateResponse.isAcknowledged) { - logger.info("Index mapping of $targetIndex is updated") - setIndexUpdateFlag(index, targetIndex) - } else { - logger.info("Failed to update index mapping of $targetIndex") - } - } - - private fun setIndexUpdateFlag(index: String, targetIndex: String) { - when (index) { - ALERT_V2_INDEX -> IndexUtils.alertV2IndexUpdated() - ALERT_V2_HISTORY_WRITE_INDEX -> IndexUtils.lastUpdatedAlertV2HistoryIndex = targetIndex - } - } - - private fun rolloverIndex( - initialized: Boolean, - index: String, - pattern: String, - map: String, - docsCondition: Long, - ageCondition: TimeValue, - writeIndex: String - ) { - if (!initialized) { - return - } - - // We have to pass null for newIndexName in order to get Elastic to increment the index count. - val request = RolloverRequest(index, null) - request.createIndexRequest.index(pattern) - .mapping(map) - .settings(Settings.builder().put("index.hidden", true).build()) - request.addMaxIndexDocsCondition(docsCondition) - request.addMaxIndexAgeCondition(ageCondition) - client.admin().indices().rolloverIndex( - request, - object : ActionListener { - override fun onResponse(response: RolloverResponse) { - if (!response.isRolledOver) { - logger.info("$writeIndex not rolled over. Conditions were: ${response.conditionStatus}") - } else { - lastRolloverTime = TimeValue.timeValueMillis(threadPool.absoluteTimeInMillis()) - } - } - override fun onFailure(e: Exception) { - logger.error("$writeIndex not roll over failed.") - } - } - ) - } - - private fun rolloverAlertV2HistoryIndex() { - rolloverIndex( - alertV2HistoryIndexInitialized, - ALERT_V2_HISTORY_WRITE_INDEX, - ALERT_V2_HISTORY_INDEX_PATTERN, - alertMapping(), - alertV2HistoryMaxDocs, - alertV2HistoryMaxAge, - ALERT_V2_HISTORY_WRITE_INDEX - ) - } - - private fun deleteOldIndices(tag: String, indices: String) { - val clusterStateRequest = ClusterStateRequest() - .clear() - .indices(indices) - .metadata(true) - .local(true) - .indicesOptions(IndicesOptions.strictExpand()) - client.admin().cluster().state( - clusterStateRequest, - object : ActionListener { - override fun onResponse(clusterStateResponse: ClusterStateResponse) { - if (clusterStateResponse.state.metadata.indices.isNotEmpty()) { - scope.launch { - val indicesToDelete = getIndicesToDelete(clusterStateResponse) - logger.info("Deleting old $tag indices viz $indicesToDelete") - deleteAllOldHistoryIndices(indicesToDelete) - } - } else { - logger.info("No Old $tag Indices to delete") - } - } - override fun onFailure(e: Exception) { - logger.error("Error fetching cluster state") - } - } - ) - } - - private fun getIndicesToDelete(clusterStateResponse: ClusterStateResponse): List { - val indicesToDelete = mutableListOf() - for (entry in clusterStateResponse.state.metadata.indices) { - val indexMetaData = entry.value - getHistoryIndexToDelete( - indexMetaData, - alertV2HistoryRetentionPeriod.millis, - ALERT_V2_HISTORY_WRITE_INDEX, - alertV2HistoryEnabled - )?.let { indicesToDelete.add(it) } - } - return indicesToDelete - } - - private fun getHistoryIndexToDelete( - indexMetadata: IndexMetadata, - retentionPeriodMillis: Long, - writeIndex: String, - historyEnabled: Boolean - ): String? { - val creationTime = indexMetadata.creationDate - if ((Instant.now().toEpochMilli() - creationTime) > retentionPeriodMillis) { - val alias = indexMetadata.aliases.entries.firstOrNull { writeIndex == it.value.alias } - if (alias != null) { - if (historyEnabled) { - // If the index has the write alias and history is enabled, don't delete the index - return null - } else if (writeIndex == ALERT_V2_HISTORY_WRITE_INDEX) { - // Otherwise reset alertHistoryIndexInitialized since index will be deleted - alertV2HistoryIndexInitialized = false - } - } - - return indexMetadata.index.name - } - return null - } - - private fun deleteAllOldHistoryIndices(indicesToDelete: List) { - if (indicesToDelete.isNotEmpty()) { - val deleteIndexRequest = DeleteIndexRequest(*indicesToDelete.toTypedArray()) - client.admin().indices().delete( - deleteIndexRequest, - object : ActionListener { - override fun onResponse(deleteIndicesResponse: AcknowledgedResponse) { - if (!deleteIndicesResponse.isAcknowledged) { - logger.error( - "Could not delete one or more Alerting V2 history indices: $indicesToDelete. Retrying one by one." - ) - deleteOldHistoryIndex(indicesToDelete) - } - } - override fun onFailure(e: Exception) { - logger.error("Delete for Alerting V2 History Indices $indicesToDelete Failed. Retrying one by one.") - deleteOldHistoryIndex(indicesToDelete) - } - } - ) - } - } - - private fun deleteOldHistoryIndex(indicesToDelete: List) { - for (index in indicesToDelete) { - val singleDeleteRequest = DeleteIndexRequest(*indicesToDelete.toTypedArray()) - client.admin().indices().delete( - singleDeleteRequest, - object : ActionListener { - override fun onResponse(acknowledgedResponse: AcknowledgedResponse?) { - if (acknowledgedResponse != null) { - if (!acknowledgedResponse.isAcknowledged) { - logger.error("Could not delete one or more Alerting V2 history indices: $index") - } - } - } - override fun onFailure(e: Exception) { - logger.error("Exception ${e.message} while deleting the index $index") - } - } - ) - } - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Mover.kt b/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Mover.kt deleted file mode 100644 index 7f768d310..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Mover.kt +++ /dev/null @@ -1,482 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.alertsv2 - -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch -import org.apache.logging.log4j.LogManager -import org.opensearch.action.bulk.BulkRequest -import org.opensearch.action.bulk.BulkResponse -import org.opensearch.action.delete.DeleteRequest -import org.opensearch.action.index.IndexRequest -import org.opensearch.action.search.SearchRequest -import org.opensearch.action.search.SearchResponse -import org.opensearch.alerting.MonitorRunnerExecutionContext -import org.opensearch.alerting.alertsv2.AlertV2Indices.Companion.ALERT_V2_HISTORY_WRITE_INDEX -import org.opensearch.alerting.alertsv2.AlertV2Indices.Companion.ALERT_V2_INDEX -import org.opensearch.alerting.opensearchapi.suspendUntil -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_V2_HISTORY_ENABLED -import org.opensearch.alerting.util.MAX_SEARCH_SIZE -import org.opensearch.cluster.ClusterChangedEvent -import org.opensearch.cluster.ClusterStateListener -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.settings.Settings -import org.opensearch.common.unit.TimeValue -import org.opensearch.common.xcontent.LoggingDeprecationHandler -import org.opensearch.common.xcontent.XContentFactory -import org.opensearch.common.xcontent.XContentHelper -import org.opensearch.common.xcontent.XContentType -import org.opensearch.commons.alerting.model.Alert -import org.opensearch.commons.alerting.model.Alert.Companion.MONITOR_ID_FIELD -import org.opensearch.commons.alerting.model.Alert.Companion.START_TIME_FIELD -import org.opensearch.commons.alerting.model.Alert.Companion.TRIGGER_ID_FIELD -import org.opensearch.commons.alerting.model.Monitor -import org.opensearch.commons.alerting.model.Monitor.Companion.MONITOR_TYPE -import org.opensearch.commons.alerting.model.Monitor.Companion.MONITOR_TYPE_FIELD -import org.opensearch.commons.alerting.model.Monitor.Companion.TRIGGERS_FIELD -import org.opensearch.commons.alerting.model.Monitor.MonitorType -import org.opensearch.commons.alerting.model.PPLSQLTrigger.Companion.EXPIRE_FIELD -import org.opensearch.commons.alerting.model.PPLSQLTrigger.Companion.PPL_SQL_TRIGGER_FIELD -import org.opensearch.commons.alerting.model.ScheduledJob.Companion.SCHEDULED_JOBS_INDEX -import org.opensearch.commons.alerting.model.Trigger.Companion.ID_FIELD -import org.opensearch.core.common.bytes.BytesReference -import org.opensearch.core.rest.RestStatus -import org.opensearch.core.xcontent.NamedXContentRegistry -import org.opensearch.core.xcontent.ToXContent -import org.opensearch.core.xcontent.XContentParser -import org.opensearch.core.xcontent.XContentParserUtils -import org.opensearch.index.VersionType -import org.opensearch.index.query.QueryBuilders -import org.opensearch.search.builder.SearchSourceBuilder -import org.opensearch.threadpool.Scheduler -import org.opensearch.threadpool.ThreadPool -import org.opensearch.transport.client.Client -import java.time.Instant -import java.util.concurrent.TimeUnit - -private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) -private val logger = LogManager.getLogger(AlertV2Mover::class.java) - -/** - * This class handles sweeping the active v2 alerts index for expired alerts, and - * either moving them to v2 alerts history index (if alert v2 history enabled) or - * permanently deleting them (if alert v2 history disabled). It also contains the - * logic for moving alerts in response to a monitor update or deletion. - * - * Lifecycle: - * * 1. AlertV2 is generated when a TriggerV2's condition is met. The TriggerV2 fires and forgets the AlertV2. - * * 2. AlertV2 is stored in the alerts index. AlertV2s are stateless. (e.g. they are never ACTIVE or COMPLETED) - * * 3. AlertV2 is soft deleted after its expire duration (determined by its trigger), and archived in an alert history index - * * 4. Based on the alert v2 history retention period, the AlertV2 is permanently deleted - * - * @opensearch.experimental - */ -class AlertV2Mover( - settings: Settings, - private val client: Client, - private val threadPool: ThreadPool, - private val clusterService: ClusterService, - private val xContentRegistry: NamedXContentRegistry, -) : ClusterStateListener { - init { - clusterService.addListener(this) - clusterService.clusterSettings.addSettingsUpdateConsumer(ALERT_V2_HISTORY_ENABLED) { alertV2HistoryEnabled = it } - } - - @Volatile private var isClusterManager = false - - private var alertV2IndexInitialized = false - - private var alertV2HistoryIndexInitialized = false - - private var alertV2HistoryEnabled = ALERT_V2_HISTORY_ENABLED.get(settings) - - private var scheduledAlertsV2CheckAndExpire: Scheduler.Cancellable? = null - - private val executorName = ThreadPool.Names.MANAGEMENT - - private val checkForExpirationInterval = TimeValue(1L, TimeUnit.MINUTES) - - override fun clusterChanged(event: ClusterChangedEvent) { - if (this.isClusterManager != event.localNodeClusterManager()) { - this.isClusterManager = event.localNodeClusterManager() - if (this.isClusterManager) { - onManager() - } else { - offManager() - } - } - - alertV2IndexInitialized = event.state().routingTable().hasIndex(ALERT_V2_INDEX) - alertV2HistoryIndexInitialized = event.state().metadata().hasAlias(ALERT_V2_HISTORY_WRITE_INDEX) - } - - fun onManager() { - try { - // try to sweep current AlertV2s for expiration immediately as we might be restarting the cluster - moveOrDeleteAlertV2s() - // schedule expiration checks and expirations to happen repeatedly at some interval - scheduledAlertsV2CheckAndExpire = threadPool - .scheduleWithFixedDelay({ moveOrDeleteAlertV2s() }, checkForExpirationInterval, executorName) - } catch (e: Exception) { - // This should be run on cluster startup - logger.error( - "Error sweeping AlertV2s for expiration. This cannot be done until clustermanager node is restarted.", - e - ) - } - } - - fun offManager() { - scheduledAlertsV2CheckAndExpire?.cancel() - } - - // if alertV2 history is enabled, move expired alerts to alertV2 history indices - // if alertV2 history is disabled, permanently delete expired alerts - private fun moveOrDeleteAlertV2s() { - if (!areAlertV2IndicesPresent()) { - return - } - - scope.launch { - val expiredAlerts = searchForExpiredAlerts() - - var copyResponse: BulkResponse? = null - val deleteResponse: BulkResponse? - if (!alertV2HistoryEnabled) { - deleteResponse = deleteExpiredAlerts(expiredAlerts) - } else { - copyResponse = copyExpiredAlerts(expiredAlerts) - deleteResponse = deleteExpiredAlertsThatWereCopied(copyResponse, expiredAlerts) - } - checkForFailures(copyResponse) - checkForFailures(deleteResponse) - } - } - - private suspend fun searchForExpiredAlerts(): List { - logger.debug("beginning search for expired alerts") - /* first collect all triggers and their expire durations */ - // when searching the alerting-config index, only trigger IDs and their expire durations are needed - val monitorV2sSearchQuery = SearchSourceBuilder.searchSource() - .query(QueryBuilders.termQuery("$MONITOR_TYPE.$MONITOR_TYPE_FIELD", MonitorType.PPL_MONITOR.value)) - .fetchSource( - arrayOf( - "$MONITOR_TYPE.$TRIGGERS_FIELD.$PPL_SQL_TRIGGER_FIELD.$ID_FIELD", - "$MONITOR_TYPE.$TRIGGERS_FIELD.$PPL_SQL_TRIGGER_FIELD.$EXPIRE_FIELD" - ), - null - ) - .size(MAX_SEARCH_SIZE) - .version(true) - val monitorV2sRequest = SearchRequest(SCHEDULED_JOBS_INDEX) - .source(monitorV2sSearchQuery) - val searchMonitorV2sResponse: SearchResponse = client.suspendUntil { search(monitorV2sRequest, it) } - logger.debug("search monitor response num hits: ${searchMonitorV2sResponse.hits.totalHits.value}") - - logger.debug("searching triggers for their expire durations") - // construct a map that stores each trigger's expiration time - // TODO: create XContent parser specifically for responses to the above search to avoid casting - val triggerToExpireDuration = mutableMapOf() - searchMonitorV2sResponse.hits.forEach { hit -> - val monitorObj = hit.sourceAsMap[MONITOR_TYPE] as Map - val triggers = monitorObj[TRIGGERS_FIELD] as List> - triggers.forEach { trigger -> - val pplSqlTrigger = trigger[PPL_SQL_TRIGGER_FIELD] as Map - val pplSqlTriggerId = pplSqlTrigger[ID_FIELD] as String - val expireDuration = (pplSqlTrigger[EXPIRE_FIELD] as Int).toLong() - logger.debug("triggerId: $pplSqlTriggerId") - logger.debug("triggerExpires: $expireDuration") - triggerToExpireDuration[pplSqlTriggerId] = expireDuration - } - } - - logger.debug("trigger to expire duration map: $triggerToExpireDuration") - - /* now collect all expired alerts */ - logger.debug("searching active alerts index for expired alerts") - - val now = Instant.now().toEpochMilli() - - val expiredAlertsBoolQuery = QueryBuilders.boolQuery() - - // collect, in an overarching should clause, each trigger and its expiration time. - // any alert that matches both the trigger ID and the expiration time check should - // be returned by the search query - triggerToExpireDuration.forEach { (triggerId, expireDuration) -> - val expireDurationMillis = expireDuration * 60 * 1000 - val maxValidTime = now - expireDurationMillis - - expiredAlertsBoolQuery.should( - QueryBuilders.boolQuery() - .must(QueryBuilders.termQuery(TRIGGER_ID_FIELD, triggerId)) - .must(QueryBuilders.rangeQuery(START_TIME_FIELD).lte(maxValidTime)) - ) - } - - // add orphaned alerts to should clause as well (i.e. alerts whose trigger IDs cannot - // be found in the list of currently existent triggers), since orphaned alerts should be expired. - // note: this is a redundancy with MonitorRunnerService's - // postIndex and postDelete, which handles moving alerts in response - // to a monitor update or delete event. this cleanly handles the case - // that even with those measures in place, an alert that came from a - // now nonexistent trigger was somehow found - expiredAlertsBoolQuery.should( - QueryBuilders.boolQuery() - .mustNot(QueryBuilders.termsQuery(TRIGGER_ID_FIELD, triggerToExpireDuration.keys.toList())) - ) - - // Explicitly specify that at least one should clause must match - expiredAlertsBoolQuery.minimumShouldMatch(1) - - // search for the expired alerts - val expiredAlertsSearchQuery = SearchSourceBuilder.searchSource() - .query(expiredAlertsBoolQuery) - .size(MAX_SEARCH_SIZE) - .version(true) - val expiredAlertsRequest = SearchRequest(ALERT_V2_INDEX) - .source(expiredAlertsSearchQuery) - val expiredAlertsResponse: SearchResponse = client.suspendUntil { search(expiredAlertsRequest, it) } - - // parse the search results into full alert docs, as they will need to be - // indexed into alert history indices - val expiredAlertV2s = mutableListOf() - expiredAlertsResponse.hits.forEach { hit -> - expiredAlertV2s.add( - Alert.parse(alertContentParser(hit.sourceRef), hit.id, hit.version) - ) - } - - logger.debug("expired alerts: $expiredAlertV2s") - - return expiredAlertV2s - } - - private suspend fun deleteExpiredAlerts(expiredAlerts: List): BulkResponse? { - logger.debug("beginning to hard delete expired alerts permanently") - // If no expired alerts are found, simply return - if (expiredAlerts.isEmpty()) { - return null - } - - val deleteRequests = expiredAlerts.map { - DeleteRequest(ALERT_V2_INDEX, it.id) - .routing(it.monitorId) - .version(it.version) - .versionType(VersionType.EXTERNAL_GTE) - } - - val deleteRequest = BulkRequest().add(deleteRequests) - val deleteResponse: BulkResponse = client.suspendUntil { bulk(deleteRequest, it) } - - return deleteResponse - } - - private suspend fun copyExpiredAlerts(expiredAlerts: List): BulkResponse? { - logger.debug("beginning to copy expired alerts to history write index") - // If no expired alerts are found, simply return - if (expiredAlerts.isEmpty()) { - return null - } - - val indexRequests = expiredAlerts.map { - IndexRequest(ALERT_V2_HISTORY_WRITE_INDEX) - .routing(it.monitorId) - .source(it.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) - .version(it.version) - .versionType(VersionType.EXTERNAL_GTE) - .id(it.id) - } - - val copyRequest = BulkRequest().add(indexRequests) - val copyResponse: BulkResponse = client.suspendUntil { bulk(copyRequest, it) } - - return copyResponse - } - - private suspend fun deleteExpiredAlertsThatWereCopied(copyResponse: BulkResponse?, expiredAlerts: List): BulkResponse? { - logger.debug("beginning to delete expired alerts that were copied to history write index") - // if there were no expired alerts to copy, skip deleting anything - if (copyResponse == null) { - return null - } - - // pre-index the alerts so retrieving their - // monitor IDs for routing is easier - val alertsById: Map = expiredAlerts.associateBy { it.id } - - val deleteRequests = copyResponse.items.filterNot { it.isFailed }.map { - DeleteRequest(ALERT_V2_INDEX, it.id) - .routing(alertsById[it.id]!!.monitorId) - .version(it.version) - .versionType(VersionType.EXTERNAL_GTE) - } - val deleteRequest = BulkRequest().add(deleteRequests) - val deleteResponse: BulkResponse = client.suspendUntil { bulk(deleteRequest, it) } - - return deleteResponse - } - - private fun checkForFailures(bulkResponse: BulkResponse?) { - bulkResponse?.let { - if (bulkResponse.hasFailures()) { - val retryCause = bulkResponse.items.filter { it.isFailed } - .firstOrNull { it.status() == RestStatus.TOO_MANY_REQUESTS } - ?.failure?.cause - logger.error( - "Failed to move or delete alert v2s: ${bulkResponse.buildFailureMessage()}", - retryCause - ) - } - } - } - - private fun areAlertV2IndicesPresent(): Boolean { - return alertV2IndexInitialized && alertV2HistoryIndexInitialized - } - - companion object { - private fun alertContentParser(bytesReference: BytesReference): XContentParser { - val xcp = XContentHelper.createParser( - NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, - bytesReference, XContentType.JSON - ) - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp) - return xcp - } - - // this method is used by MonitorRunnerService's postIndex and postDelete - // functions to move (in the case of alert v2 history enabled) or delete - // (in the case of alert v2 history disabled) the alerts generated by - // a monitor in response to the event that the monitor gets updated - // or deleted - suspend fun moveAlertV2s(monitorV2Id: String, monitorV2: Monitor?, monitorCtx: MonitorRunnerExecutionContext) { - logger.debug("beginning to move alerts for postIndex or postDelete of monitor: $monitorV2Id") - val client = monitorCtx.client!! - - // first collect all alerts that came from this updated or deleted monitor - val boolQuery = QueryBuilders.boolQuery() - .filter(QueryBuilders.termQuery(MONITOR_ID_FIELD, monitorV2Id)) - - /* - this monitorV2 != null case happens when this function is called by postIndex. if the monitor is updated, - we don't want to expire alerts that were generated by triggers that still exist - in the updated monitor, so filter those out. only expire alerts from triggers in - this monitor that may no longer exist in the updated version of the monitor. - edge case: user can edit the trigger itself while explicitly keeping the ID the same, - which means alerts generated by that trigger will (incorrectly) not be filtered out by this logic - even though it was edited. to mitigate this, recall that callers of the update monitor API - must supply the full MonitorV2 object of the updated monitor config. this is important - because it means they don't have to reference the triggers by ID when updating the triggers, - they simply declare a whole new monitor with whatever new triggers they want it to have, and when doing this, - likely won't explicitly pass in trigger IDs for their updated triggers that exactly match - the IDs of the old triggers. this means Alerting will generate a new ID for the updated triggers by default, - meaning this logic will pick up those updated triggers and correctly move/delete the alerts - */ - if (monitorV2 != null) { - boolQuery.mustNot(QueryBuilders.termsQuery(TRIGGER_ID_FIELD, monitorV2.triggers.map { it.id })) - } - - val alertsSearchQuery = SearchSourceBuilder.searchSource() - .query(boolQuery) - .size(MAX_SEARCH_SIZE) - .version(true) - val activeAlertsRequest = SearchRequest(ALERT_V2_INDEX) - .source(alertsSearchQuery) - val searchAlertsResponse: SearchResponse = client.suspendUntil { search(activeAlertsRequest, it) } - - // If no alerts are found, simply return - if (searchAlertsResponse.hits.totalHits?.value == 0L) return - - val activeAlerts = mutableListOf() - searchAlertsResponse.hits.forEach { hit -> - activeAlerts.add( - Alert.parse( - alertContentParser(hit.sourceRef), - hit.id, - hit.version - ) - ) - } - - // pre-index the alerts so retrieving their - // monitor IDs for routing is easier - val alertsById: Map = activeAlerts.associateBy { it.id } - - val alertV2HistoryEnabled = monitorCtx.clusterService!!.clusterSettings.get(ALERT_V2_HISTORY_ENABLED) - - // if alert v2 history is enabled, migrate the relevant alerts - // to the alert v2 history index pattern instead of hard deleting them - var copyResponse: BulkResponse? = null - if (alertV2HistoryEnabled) { - logger.debug("alert v2 history enabled, copying alerts to history write index") - val indexRequests = searchAlertsResponse.hits.map { hit -> - IndexRequest(ALERT_V2_HISTORY_WRITE_INDEX) - .routing(monitorV2Id) - .source( - Alert.parse(alertContentParser(hit.sourceRef), hit.id, hit.version) - .toXContentWithUser(XContentFactory.jsonBuilder()) - ) - .version(hit.version) - .versionType(VersionType.EXTERNAL_GTE) - .id(hit.id) - } - val copyRequest = BulkRequest().add(indexRequests) - copyResponse = client.suspendUntil { bulk(copyRequest, it) } - - if (copyResponse!!.hasFailures()) { - val retryCause = copyResponse.items.filter { it.isFailed } - .firstOrNull { it.status() == RestStatus.TOO_MANY_REQUESTS } - ?.failure?.cause - throw RuntimeException( - "Failed to copy alertV2s for [$monitorV2Id, ${monitorV2?.triggers?.map { it.id }}]: " + - copyResponse.buildFailureMessage(), - retryCause - ) - } - } - - logger.debug("deleting alerts related to monitor: $monitorV2Id") - - // prepare deletion request - val deleteRequests = if (alertV2HistoryEnabled) { - // if alerts were to be migrated, delete only the ones - // that were successfully copied over - copyResponse!!.items.filterNot { it.isFailed }.map { - DeleteRequest(ALERT_V2_INDEX, it.id) - .routing(alertsById[it.id]!!.monitorId) - .version(it.version) - .versionType(VersionType.EXTERNAL_GTE) - } - } else { - // otherwise just directly get the original - // set of alerts - searchAlertsResponse.hits.map { hit -> - DeleteRequest(ALERT_V2_INDEX, hit.id) - .routing(alertsById[hit.id]!!.monitorId) - .version(hit.version) - .versionType(VersionType.EXTERNAL_GTE) - } - } - - // execute delete request - val deleteRequest = BulkRequest().add(deleteRequests) - val deleteResponse: BulkResponse = client.suspendUntil { bulk(deleteRequest, it) } - - if (deleteResponse.hasFailures()) { - val retryCause = deleteResponse.items.filter { it.isFailed } - .firstOrNull { it.status() == RestStatus.TOO_MANY_REQUESTS } - ?.failure?.cause - throw RuntimeException( - "Failed to delete alertV2s for [$monitorV2Id, ${monitorV2?.triggers?.map { it.id }}]: " + - deleteResponse.buildFailureMessage(), - retryCause - ) - } - } - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/script/QueryLevelTriggerExecutionContext.kt b/alerting/src/main/kotlin/org/opensearch/alerting/script/QueryLevelTriggerExecutionContext.kt index 2fc5b402b..097564abf 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/script/QueryLevelTriggerExecutionContext.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/script/QueryLevelTriggerExecutionContext.kt @@ -9,24 +9,34 @@ import org.opensearch.alerting.model.AlertContext import org.opensearch.common.settings.ClusterSettings import org.opensearch.commons.alerting.model.Monitor import org.opensearch.commons.alerting.model.MonitorRunResult +import org.opensearch.commons.alerting.model.PPLSQLTrigger import org.opensearch.commons.alerting.model.QueryLevelTrigger import org.opensearch.commons.alerting.model.QueryLevelTriggerRunResult +import org.opensearch.commons.alerting.model.Trigger import java.time.Instant data class QueryLevelTriggerExecutionContext( override val monitor: Monitor, - val trigger: QueryLevelTrigger, + val trigger: Trigger, override val results: List>, override val periodStart: Instant, override val periodEnd: Instant, val alert: AlertContext? = null, override val error: Exception? = null, - override val clusterSettings: ClusterSettings + override val clusterSettings: ClusterSettings, + val pplSqlQueryResult: List>? = null, // each list element is a result row ) : TriggerExecutionContext(monitor, results, periodStart, periodEnd, error, clusterSettings) { + init { + require(trigger is QueryLevelTrigger || trigger is PPLSQLTrigger) { + "QueryLevelTriggerExecutionContext must only store Triggers for per-query style monitoring, " + + "like QueryLevelTrigger or PPLSQLTrigger" + } + } + constructor( monitor: Monitor, - trigger: QueryLevelTrigger, + trigger: Trigger, monitorRunResult: MonitorRunResult, alertContext: AlertContext? = null, clusterSettings: ClusterSettings @@ -38,7 +48,8 @@ data class QueryLevelTriggerExecutionContext( monitorRunResult.periodEnd, alertContext, monitorRunResult.scriptContextError(trigger), - clusterSettings + clusterSettings, + null ) /** @@ -49,6 +60,7 @@ data class QueryLevelTriggerExecutionContext( val tempArg = super.asTemplateArg().toMutableMap() tempArg["trigger"] = trigger.asTemplateArg() tempArg["alert"] = alert?.asTemplateArg() // map "alert" templateArg field to AlertContext wrapper instead of Alert object + tempArg["ppl_query_results"] = pplSqlQueryResult return tempArg } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt index 3c838eae7..ad888c11e 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt @@ -309,53 +309,9 @@ class AlertingSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ) - val ALERT_V2_HISTORY_ENABLED = Setting.boolSetting( - "plugins.alerting.v2.alert_history_enabled", - true, - Setting.Property.NodeScope, Setting.Property.Dynamic - ) - - val ALERT_V2_HISTORY_ROLLOVER_PERIOD = Setting.positiveTimeSetting( - "plugins.alerting.v2.alert_history_rollover_period", - TimeValue(12, TimeUnit.HOURS), - Setting.Property.NodeScope, Setting.Property.Dynamic - ) - - val ALERT_V2_HISTORY_INDEX_MAX_AGE = Setting.positiveTimeSetting( - "plugins.alerting.v2.alert_history_max_age", - TimeValue(30, TimeUnit.DAYS), - Setting.Property.NodeScope, Setting.Property.Dynamic - ) - - val ALERT_V2_HISTORY_MAX_DOCS = Setting.longSetting( - "plugins.alerting.v2.alert_history_max_docs", - 1000L, 0L, - Setting.Property.NodeScope, Setting.Property.Dynamic - ) - - val ALERT_V2_HISTORY_RETENTION_PERIOD = Setting.positiveTimeSetting( - "plugins.alerting.v2.alert_history_retention_period", - TimeValue(60, TimeUnit.DAYS), - Setting.Property.NodeScope, Setting.Property.Dynamic - ) - val ALERT_V2_MONITOR_EXECUTION_MAX_DURATION = Setting.positiveTimeSetting( "plugins.alerting.v2.alert_monitor_execution_max_duration", - TimeValue(4, TimeUnit.MINUTES), - Setting.Property.NodeScope, Setting.Property.Dynamic - ) - - val ALERTING_V2_MAX_THROTTLE_DURATION = Setting.longSetting( - "plugins.alerting.v2.monitor.max_throttle_duration", - 7200L, // 5 days, 7200 minutes - 2L, - Setting.Property.NodeScope, Setting.Property.Dynamic - ) - - val ALERTING_V2_MAX_EXPIRE_DURATION = Setting.longSetting( - "plugins.alerting.v2.monitor.max_expire_duration", - 43200L, // 30 days, 43200 minutes - 2L, + TimeValue(1, TimeUnit.MINUTES), Setting.Property.NodeScope, Setting.Property.Dynamic ) @@ -383,13 +339,6 @@ class AlertingSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ) - val ALERT_V2_PER_RESULT_TRIGGER_MAX_ALERTS = Setting.intSetting( - "plugins.alerting.v2.per_result_trigger_max_alerts", - 10, - 1, - Setting.Property.NodeScope, Setting.Property.Dynamic - ) - val NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH = Setting.intSetting( "plugins.alerting.v2.notification_subject_source_max_length", 1000, diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetAlertsAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetAlertsAction.kt index 69c43e1e2..ea9338d4c 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetAlertsAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetAlertsAction.kt @@ -14,7 +14,6 @@ import org.opensearch.action.support.ActionFilters import org.opensearch.action.support.HandledTransportAction import org.opensearch.alerting.AlertingPlugin import org.opensearch.alerting.alerts.AlertIndices -import org.opensearch.alerting.alertsv2.AlertV2Indices.Companion.ALERT_V2_INDEX import org.opensearch.alerting.opensearchapi.addFilter import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.util.use diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt index c5859be75..cfb7e6051 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt @@ -31,6 +31,7 @@ import org.opensearch.action.support.clustermanager.AcknowledgedResponse import org.opensearch.alerting.AlertingPlugin import org.opensearch.alerting.MonitorMetadataService import org.opensearch.alerting.PPLUtils.appendCustomCondition +import org.opensearch.alerting.PPLUtils.appendDataRowsLimit import org.opensearch.alerting.PPLUtils.executePplQuery import org.opensearch.alerting.PPLUtils.findEvalResultVar import org.opensearch.alerting.PPLUtils.findEvalResultVarIdxInSchema @@ -39,9 +40,7 @@ import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.service.DeleteMonitorService import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_MAX_MONITORS -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_EXPIRE_DURATION import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_QUERY_LENGTH -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_THROTTLE_DURATION import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS import org.opensearch.alerting.settings.AlertingSettings.Companion.INDEX_TIMEOUT import org.opensearch.alerting.settings.AlertingSettings.Companion.MAX_ACTION_THROTTLE_VALUE @@ -133,8 +132,6 @@ class TransportIndexMonitorAction @Inject constructor( @Volatile private var allowList = ALLOW_LIST.get(settings) // PPL Alerting related settings - @Volatile private var maxThrottleDuration = ALERTING_V2_MAX_THROTTLE_DURATION.get(settings) - @Volatile private var maxExpireDuration = ALERTING_V2_MAX_EXPIRE_DURATION.get(settings) @Volatile private var maxQueryLength = ALERTING_V2_MAX_QUERY_LENGTH.get(settings) @Volatile private var maxQueryResults = ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS.get(settings) @Volatile private var notificationSubjectMaxLength = NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH.get(settings) @@ -150,8 +147,6 @@ class TransportIndexMonitorAction @Inject constructor( clusterService.clusterSettings.addSettingsUpdateConsumer(MAX_ACTION_THROTTLE_VALUE) { maxActionThrottle = it } clusterService.clusterSettings.addSettingsUpdateConsumer(ALLOW_LIST) { allowList = it } - clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_MAX_THROTTLE_DURATION) { maxThrottleDuration = it } - clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_MAX_EXPIRE_DURATION) { maxExpireDuration = it } clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_MAX_QUERY_LENGTH) { maxQueryLength = it } clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS) { maxQueryResults = it } clusterService.clusterSettings.addSettingsUpdateConsumer(NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH) { @@ -283,7 +278,7 @@ class TransportIndexMonitorAction @Inject constructor( ) } - fun checkPplSqlQueryAndExecute( + private fun checkPplSqlQueryAndExecute( actionListener: ActionListener, indexMonitorRequest: IndexMonitorRequest, user: User? @@ -316,6 +311,7 @@ class TransportIndexMonitorAction @Inject constructor( } } + // initiate the PPL monitor and PPL query validations client.threadPool().threadContext.stashContext().use { scope.launch { val singleThreadContext = newSingleThreadContext("IndexMonitorV2ActionThread") @@ -324,6 +320,8 @@ class TransportIndexMonitorAction @Inject constructor( val pplSqlMonitor = indexMonitorRequest.monitor + // validate the PPL query syntax and that user has permissions to + // the indices being queried val pplQueryValid = validatePplSqlQuery(pplSqlMonitor, validationListener) if (!pplQueryValid) { return@withContext @@ -350,10 +348,12 @@ class TransportIndexMonitorAction @Inject constructor( try { val query = (pplSqlMonitor.inputs[0] as PPLSQLInput).query + val limitedQueryToExecute = appendDataRowsLimit(query, maxQueryResults) + // now run the base query as is. // if there are any PPL syntax, index not found, insufficient index permissions, or other errors, - // this will throw an exception - executePplQuery(query, clusterService.state().nodes.localNode, transportService) + // this will throw an exception from the SQL/PPL plugin + executePplQuery(limitedQueryToExecute, clusterService.state().nodes.localNode, transportService) // scan all the triggers with custom conditions, and ensure each query constructed // from the base query + custom condition is valid @@ -367,9 +367,12 @@ class TransportIndexMonitorAction @Inject constructor( val evalResultVar = findEvalResultVar(pplTrigger.customCondition!!) val queryWithCustomCondition = appendCustomCondition(query, pplTrigger.customCondition!!) + val limitedQueryWithCustomCondition = appendDataRowsLimit(queryWithCustomCondition, maxQueryResults) + // if the custom condition is invalid, this will throw an exception + // from the SQL/PPL plugin val executePplQueryResponse = executePplQuery( - queryWithCustomCondition, + limitedQueryWithCustomCondition, clusterService.state().nodes.localNode, transportService ) @@ -411,30 +414,6 @@ class TransportIndexMonitorAction @Inject constructor( pplSqlMonitor.triggers.forEach { trigger -> val pplTrigger = trigger as PPLSQLTrigger - pplTrigger.throttleDuration?.let { throttleDuration -> - if (throttleDuration > maxThrottleDuration) { - validationListener.onFailure( - AlertingException.wrap( - IllegalArgumentException( - "Throttle duration must be at most $maxThrottleDuration but was $throttleDuration" - ) - ) - ) - return false - } - } - - if (pplTrigger.expireDuration > maxExpireDuration) { - validationListener.onFailure( - AlertingException.wrap( - IllegalArgumentException( - "Expire duration must be at most $maxExpireDuration but was ${trigger.expireDuration}" - ) - ) - ) - return false - } - if (pplTrigger.conditionType == PPLSQLTrigger.ConditionType.NUMBER_OF_RESULTS && pplTrigger.numResultsValue!! > maxQueryResults ) { diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/IndexUtils.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/IndexUtils.kt index dac2ff897..994293f1d 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/IndexUtils.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/IndexUtils.kt @@ -10,7 +10,6 @@ import org.opensearch.action.index.IndexResponse import org.opensearch.action.support.IndicesOptions import org.opensearch.action.support.clustermanager.AcknowledgedResponse import org.opensearch.alerting.alerts.AlertIndices -import org.opensearch.alerting.alertsv2.AlertV2Indices import org.opensearch.alerting.comments.CommentsIndices import org.opensearch.alerting.core.ScheduledJobIndices import org.opensearch.cluster.ClusterState @@ -40,8 +39,6 @@ class IndexUtils { private set var alertingCommentIndexSchemaVersion: Int private set - var alertV2IndexSchemaVersion: Int - private set var scheduledJobIndexUpdated: Boolean = false private set @@ -51,20 +48,16 @@ class IndexUtils { private set var commentsIndexUpdated: Boolean = false private set - var alertV2IndexUpdated: Boolean = false - private set var lastUpdatedAlertHistoryIndex: String? = null var lastUpdatedFindingHistoryIndex: String? = null var lastUpdatedCommentsHistoryIndex: String? = null - var lastUpdatedAlertV2HistoryIndex: String? = null init { scheduledJobIndexSchemaVersion = getSchemaVersion(ScheduledJobIndices.scheduledJobMappings()) alertIndexSchemaVersion = getSchemaVersion(AlertIndices.alertMapping()) findingIndexSchemaVersion = getSchemaVersion(AlertIndices.findingMapping()) alertingCommentIndexSchemaVersion = getSchemaVersion(CommentsIndices.commentsMapping()) - alertV2IndexSchemaVersion = getSchemaVersion(AlertV2Indices.alertMapping()) } @JvmStatic @@ -87,11 +80,6 @@ class IndexUtils { commentsIndexUpdated = true } - @JvmStatic - fun alertV2IndexUpdated() { - alertV2IndexUpdated = true - } - @JvmStatic fun getSchemaVersion(mapping: String): Int { val xcp = XContentType.JSON.xContent().createParser( diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/workflow/CompositeWorkflowRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/workflow/CompositeWorkflowRunner.kt index 327de0ff0..5f47fe360 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/workflow/CompositeWorkflowRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/workflow/CompositeWorkflowRunner.kt @@ -178,11 +178,6 @@ object CompositeWorkflowRunner : WorkflowRunner() { triggerResults = triggerResults ) val currentAlerts = try { - // create stateless alert indices as well to prevent get alerts from returning error because - // stateless alerts indices couldn't be found - monitorCtx.alertV2Indices!!.createOrUpdateAlertV2Index() - monitorCtx.alertV2Indices!!.createOrUpdateInitialAlertV2HistoryIndex() - monitorCtx.alertIndices!!.createOrUpdateAlertIndex(dataSources!!) monitorCtx.alertIndices!!.createOrUpdateInitialAlertHistoryIndex(dataSources) monitorCtx.alertService!!.loadCurrentAlertsForWorkflow(workflow, dataSources) diff --git a/alerting/src/main/resources/org/opensearch/alerting/alerts/alert_mapping.json b/alerting/src/main/resources/org/opensearch/alerting/alerts/alert_mapping.json index f4f25d8df..57dfc0fa4 100644 --- a/alerting/src/main/resources/org/opensearch/alerting/alerts/alert_mapping.json +++ b/alerting/src/main/resources/org/opensearch/alerting/alerts/alert_mapping.json @@ -183,22 +183,7 @@ }, "ppl_query_results": { "type": "nested", - "properties": { - "schema": { - "type": "nested", - "dynamic": true - }, - "datarows": { - "type": "object", - "enabled": false - }, - "total": { - "type": "integer" - }, - "size": { - "type": "integer" - } - } + "dynamic": true } } } \ No newline at end of file diff --git a/alerting/src/main/resources/org/opensearch/alerting/org.opensearch.alerting.txt b/alerting/src/main/resources/org/opensearch/alerting/org.opensearch.alerting.txt index c8ac9e253..47d8eda2b 100644 --- a/alerting/src/main/resources/org/opensearch/alerting/org.opensearch.alerting.txt +++ b/alerting/src/main/resources/org/opensearch/alerting/org.opensearch.alerting.txt @@ -23,7 +23,7 @@ class org.opensearch.alerting.script.TriggerExecutionContext { class org.opensearch.alerting.script.QueryLevelTriggerExecutionContext { Monitor getMonitor() - QueryLevelTrigger getTrigger() + Trigger getTrigger() List getResults() java.time.Instant getPeriodStart() java.time.Instant getPeriodEnd() @@ -38,7 +38,7 @@ class org.opensearch.commons.alerting.model.Monitor { boolean getEnabled() } -class org.opensearch.commons.alerting.model.QueryLevelTrigger { +class org.opensearch.commons.alerting.model.Trigger { String getId() String getName() String getSeverity() diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt b/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt index 53575ed1c..9305ac25e 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt @@ -19,7 +19,6 @@ import org.opensearch.alerting.AlertingPlugin.Companion.EMAIL_ACCOUNT_BASE_URI import org.opensearch.alerting.AlertingPlugin.Companion.EMAIL_GROUP_BASE_URI import org.opensearch.alerting.alerts.AlertIndices import org.opensearch.alerting.alerts.AlertIndices.Companion.FINDING_HISTORY_WRITE_INDEX -import org.opensearch.alerting.alertsv2.AlertV2Indices import org.opensearch.alerting.core.settings.ScheduledJobSettings import org.opensearch.alerting.model.destination.Chime import org.opensearch.alerting.model.destination.CustomWebhook @@ -1322,14 +1321,6 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { createIndex(encodedHistoryIndex, settings, mappingHack, "\"${AlertIndices.FINDING_HISTORY_WRITE_INDEX}\" : {}") } - fun putAlertV2Mappings(mapping: String? = null) { - val mappingHack = if (mapping != null) mapping else AlertV2Indices.alertMapping().trimStart('{').trimEnd('}') - val encodedHistoryIndex = URLEncoder.encode(AlertV2Indices.ALERT_V2_HISTORY_INDEX_PATTERN, Charsets.UTF_8.toString()) - val settings = Settings.builder().put("index.hidden", true).build() - createIndex(AlertV2Indices.ALERT_V2_INDEX, settings, mappingHack) - createIndex(encodedHistoryIndex, settings, mappingHack, "\"${AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX}\" : {}") - } - fun scheduledJobMappings(): String { return javaClass.classLoader.getResource("mappings/scheduled-jobs.json").readText() } @@ -2055,11 +2046,17 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { // this function only works on the TEST_INDEX_NAME index created // specifically for this IT suite. It has fields // "timestamp" (date), "abc" (string), "number" (integer) - protected fun indexDocFromSomeTimeAgo(timeValue: Long, timeUnit: ChronoUnit, abc: String, number: Int) { + protected fun indexDocFromSomeTimeAgo( + timeValue: Long, + timeUnit: ChronoUnit, + abc: String, + number: Int, + id: String = UUID.randomUUID().toString() + ) { val someTimeAgo = ZonedDateTime.now().minus(timeValue, timeUnit).truncatedTo(MILLIS) val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(someTimeAgo) // the timestamp string is given a random timezone offset val testDoc = """{ "timestamp" : "$testTime", "abc": "$abc", "number" : "$number" }""" - indexDoc(TEST_INDEX_NAME, UUID.randomUUID().toString(), testDoc) + indexDoc(TEST_INDEX_NAME, id, testDoc) } protected fun ensureNumMonitors(expectedNum: Int) { @@ -2113,20 +2110,4 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { } return false } - - protected fun getAlertV2HistoryDocCount(): Long { - val request = """ - { - "query": { - "match_all": {} - } - } - """.trimIndent() - val response = adminClient().makeRequest( - "POST", "${AlertV2Indices.ALERT_V2_HISTORY_ALL}/_search", emptyMap(), - StringEntity(request, APPLICATION_JSON) - ) - assertEquals("Request to get alert v2 history failed", RestStatus.OK, response.restStatus()) - return SearchResponse.fromXContent(createParser(jsonXContent, response.entity.content)).hits.totalHits!!.value - } } diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt index 8e8d71fe6..a9b7b97d6 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt @@ -6,17 +6,20 @@ package org.opensearch.alerting import org.junit.Assert +import org.opensearch.alerting.PPLUtils.PPL_RESULTS_SIZE_EXCEEDED_MESSAGE import org.opensearch.alerting.alerts.AlertIndices import org.opensearch.alerting.model.destination.CustomWebhook import org.opensearch.alerting.model.destination.Destination import org.opensearch.alerting.model.destination.email.Email import org.opensearch.alerting.model.destination.email.Recipient +import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.util.DestinationType import org.opensearch.alerting.util.getBucketKeysHash import org.opensearch.client.Request import org.opensearch.client.ResponseException import org.opensearch.client.WarningFailureException import org.opensearch.common.settings.Settings +import org.opensearch.common.unit.TimeValue import org.opensearch.commons.alerting.aggregation.bucketselectorext.BucketSelectorExtAggregationBuilder import org.opensearch.commons.alerting.alerts.AlertError import org.opensearch.commons.alerting.model.ActionExecutionResult @@ -31,6 +34,7 @@ import org.opensearch.commons.alerting.model.DocLevelMonitorInput import org.opensearch.commons.alerting.model.DocLevelQuery import org.opensearch.commons.alerting.model.IntervalSchedule import org.opensearch.commons.alerting.model.Monitor +import org.opensearch.commons.alerting.model.PPLSQLTrigger import org.opensearch.commons.alerting.model.SearchInput import org.opensearch.commons.alerting.model.action.ActionExecutionPolicy import org.opensearch.commons.alerting.model.action.AlertCategory @@ -2228,6 +2232,551 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() { } } + fun `test execute PPL monitor execution timeout generates error alert`() { + // Create test index with PPL-compatible mappings + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(2, MINUTES, "abc", 5) + + // Create PPL monitor with NUMBER_OF_RESULTS trigger condition + val monitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + triggers = listOf( + randomPPLTrigger( + conditionType = PPLSQLTrigger.ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = PPLSQLTrigger.NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + // Set monitor execution timeout to 1 nanosecond to force a timeout + adminClient().updateSettings(AlertingSettings.ALERT_V2_MONITOR_EXECUTION_MAX_DURATION.key, TimeValue.timeValueNanos(1)) + + val response = executeMonitor(monitor.id) + + val output = entityAsMap(response) + assertEquals(monitor.name, output["monitor_name"]) + + // Verify execute response contains error + @Suppress("UNCHECKED_CAST") + for (triggerResult in output.objectMap("trigger_results").values) { + assertTrue("Missing trigger error message", (triggerResult?.get("error") as? String)?.isNotEmpty() == true) + } + + // Verify ERROR alert was created + val alerts = searchAlerts(monitor) + assertEquals("Alert not saved", 1, alerts.size) + verifyAlert(alerts.single(), monitor, ERROR) + } + + fun `test execute PPL monitor with size exceeded query results`() { + // Create test index with PPL-compatible mappings + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(2, MINUTES, "abc", 5) + + // Create PPL monitor with NUMBER_OF_RESULTS trigger condition + val monitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + triggers = listOf( + randomPPLTrigger( + conditionType = PPLSQLTrigger.ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = PPLSQLTrigger.NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + // Set max PPL query results size to guarantee it will be exceeded + adminClient().updateSettings(AlertingSettings.ALERT_V2_QUERY_RESULTS_MAX_SIZE.key, 5L) + + executeMonitor(monitor.id) + + val alerts = searchAlerts(monitor) + assertEquals(1, alerts.size) + + val alert = alerts[0] + assertEquals(ACTIVE, alert.state) + assertEquals(monitor.id, alert.monitorId) + assertEquals(monitor.triggers[0].id, alert.triggerId) + + // Verify ppl_query_results contains the size exceeded message + assertNotNull(alert.pplQueryResults) + assertEquals(1, alert.pplQueryResults.size) + + val firstResultRow = alert.pplQueryResults[0] + assertTrue(firstResultRow.containsKey("message")) + assertEquals(PPL_RESULTS_SIZE_EXCEEDED_MESSAGE, firstResultRow["message"]) + } + + fun `test execute PPL monitor with number of results condition basic case`() { + // Setup test index and data + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + val docId = "test-doc-1" + indexDocFromSomeTimeAgo(2, MINUTES, "abc", 5, docId) + + // Create PPL monitor with NUMBER_OF_RESULTS condition + val monitor = createMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + triggers = listOf( + randomPPLTrigger( + conditionType = PPLSQLTrigger.ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = PPLSQLTrigger.NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + val versionBefore = monitor.version + + // Execute monitor - should trigger because document exists + val response = executeMonitor(monitor.id) + + // Verify monitor execution response + val output = entityAsMap(response) + assertEquals(monitor.name, output["monitor_name"]) + @Suppress("UNCHECKED_CAST") + val triggerResults = output.objectMap("trigger_results") + assertTrue("Trigger should have run", triggerResults.isNotEmpty()) + + // Verify alert was generated with ACTIVE state + val alerts = searchAlerts(monitor) + assertEquals("Alert should have been generated", 1, alerts.size) + + val activeAlert = alerts.single() + verifyAlert(activeAlert, monitor, ACTIVE) + + // Verify alert contains PPL query and results + assertNotNull("PPL query should be stored in alert", activeAlert.pplQuery) + assertEquals("source = $TEST_INDEX_NAME | head 10", activeAlert.pplQuery) + assertTrue("PPL query results should not be empty", activeAlert.pplQueryResults.isNotEmpty()) + + // Verify the query results are in the new transformed format (list of maps) + assertEquals("Should have exactly 1 result row from the indexed document", 1, activeAlert.pplQueryResults.size) + + // Get the first (and only) result row + val resultRow = activeAlert.pplQueryResults[0] + + // Verify the result row contains the expected fields from the indexed document + assertTrue("Result should contain 'timestamp' field", resultRow.containsKey("timestamp")) + assertTrue("Result should contain 'abc' field", resultRow.containsKey("abc")) + assertTrue("Result should contain 'number' field", resultRow.containsKey("number")) + + // Verify the field values match what we indexed + assertEquals("Field 'abc' should match indexed value", "abc", resultRow["abc"]) + assertEquals("Field 'number' should match indexed value", 5, resultRow["number"]) + + // Verify timestamp field exists and is not null (exact value varies due to time calculation) + assertNotNull("Field 'timestamp' should not be null", resultRow["timestamp"]) + + // Verify monitor version didn't change after execution + val monitorAfter = getMonitor(monitor.id) + assertEquals("Monitor version should not change after execution", versionBefore, monitorAfter.version) + + // Delete the document to make the trigger condition no longer true + val deleteRequest = Request("DELETE", "/$TEST_INDEX_NAME/_doc/$docId?refresh=true") + client().performRequest(deleteRequest) + + // Execute monitor again - should NOT trigger because document is deleted + val responseAfterDelete = executeMonitor(monitor.id) + + val outputAfterDelete = entityAsMap(responseAfterDelete) + assertEquals(monitor.name, outputAfterDelete["monitor_name"]) + + // Verify there are no ACTIVE alerts (moved to history) + assertTrue("Active alert should have moved to history", searchAlerts(monitor, AlertIndices.ALERT_INDEX).isEmpty()) + + // Verify the same alert is now in COMPLETED state (search ALL alert indices including history) + val alertsAfterDelete = searchAlerts(monitor, AlertIndices.ALL_ALERT_INDEX_PATTERN) + assertEquals("Should still have exactly one alert", 1, alertsAfterDelete.size) + + val completedAlert = alertsAfterDelete.single() + + // Verify it's the same alert (same ID) + assertEquals("Alert ID should be the same", activeAlert.id, completedAlert.id) + assertEquals("Alert should transition to COMPLETED state", COMPLETED, completedAlert.state) + verifyAlert(completedAlert, monitor, COMPLETED) + + // Verify the alert has an end time now + assertNotNull("Completed alert should have an end time", completedAlert.endTime) + + // Verify completed alert has no error message + assertNull("Completed alert should not have an error message", completedAlert.errorMessage) + } + + fun `test execute PPL monitor with custom condition basic case`() { + // Setup test index and data with multiple groups + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + + // Index documents for group "abc" - max number = 3 (won't satisfy condition) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 1, "abc-doc-1") + indexDocFromSomeTimeAgo(2, MINUTES, "abc", 2, "abc-doc-2") + indexDocFromSomeTimeAgo(3, MINUTES, "abc", 3, "abc-doc-3") + + // Index documents for group "def" - max number = 6 (will satisfy condition max_num > 5) + indexDocFromSomeTimeAgo(4, MINUTES, "def", 4, "def-doc-1") + indexDocFromSomeTimeAgo(5, MINUTES, "def", 5, "def-doc-2") + indexDocFromSomeTimeAgo(6, MINUTES, "def", 6, "def-doc-3") + + // Index documents for group "ghi" - max number = 9 (will satisfy condition max_num > 5) + indexDocFromSomeTimeAgo(7, MINUTES, "ghi", 7, "ghi-doc-1") + indexDocFromSomeTimeAgo(8, MINUTES, "ghi", 8, "ghi-doc-2") + indexDocFromSomeTimeAgo(9, MINUTES, "ghi", 9, "ghi-doc-3") + + // Create PPL monitor with CUSTOM condition + // Query aggregates by "abc" field and computes max(number) + // Custom condition evaluates: max_num > 5 + val monitor = createMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + triggers = listOf( + randomPPLTrigger( + conditionType = PPLSQLTrigger.ConditionType.CUSTOM, + customCondition = "eval result = max_num > 5", + numResultsCondition = null, + numResultsValue = null + ) + ), + query = "source = $TEST_INDEX_NAME | stats max(number) as max_num by abc" + ) + ) + + // Execute monitor - should trigger because def and ghi groups have max > 5 + val response = executeMonitor(monitor.id) + + // Verify monitor execution response + val output = entityAsMap(response) + assertEquals(monitor.name, output["monitor_name"]) + @Suppress("UNCHECKED_CAST") + val triggerResults = output.objectMap("trigger_results") + assertTrue("Trigger should have run", triggerResults.isNotEmpty()) + + // Verify alert was generated with ACTIVE state + val alerts = searchAlerts(monitor) + assertEquals("Alert should have been generated", 1, alerts.size) + + val activeAlert = alerts.single() + verifyAlert(activeAlert, monitor, ACTIVE) + + // Verify alert contains PPL query and aggregated results + assertNotNull("PPL query should be stored in alert", activeAlert.pplQuery) + assertEquals("source = $TEST_INDEX_NAME | stats max(number) as max_num by abc", activeAlert.pplQuery) + assertTrue("PPL query results should not be empty", activeAlert.pplQueryResults.isNotEmpty()) + + // Verify the query results are in the new transformed format (list of maps) + assertEquals("Should have 3 aggregation result rows", 3, activeAlert.pplQueryResults.size) + + // Convert results to a map of group name -> result row for easier validation + val groupResults = activeAlert.pplQueryResults.associateBy { it["abc"] as String } + + // Verify all expected groups are present + assertTrue("Should contain group 'abc'", groupResults.containsKey("abc")) + assertTrue("Should contain group 'def'", groupResults.containsKey("def")) + assertTrue("Should contain group 'ghi'", groupResults.containsKey("ghi")) + + // Verify each result row has the expected structure and values + val abcResult = groupResults["abc"]!! + assertTrue("abc result should contain 'max_num' field", abcResult.containsKey("max_num")) + assertTrue("abc result should contain 'abc' field", abcResult.containsKey("abc")) + assertTrue("abc result should contain 'result' field", abcResult.containsKey("result")) + assertEquals("Group 'abc' max should be 3", 3, (abcResult["max_num"] as Number).toInt()) + assertEquals("Group 'abc' field should be 'abc'", "abc", abcResult["abc"]) + assertEquals("Group 'abc' eval result should be false (3 > 5 is false)", false, abcResult["result"]) + + val defResult = groupResults["def"]!! + assertTrue("def result should contain 'max_num' field", defResult.containsKey("max_num")) + assertTrue("def result should contain 'abc' field", defResult.containsKey("abc")) + assertTrue("def result should contain 'result' field", defResult.containsKey("result")) + assertEquals("Group 'def' max should be 6", 6, (defResult["max_num"] as Number).toInt()) + assertEquals("Group 'def' field should be 'def'", "def", defResult["abc"]) + assertEquals("Group 'def' eval result should be true (6 > 5 is true)", true, defResult["result"]) + + val ghiResult = groupResults["ghi"]!! + assertTrue("ghi result should contain 'max_num' field", ghiResult.containsKey("max_num")) + assertTrue("ghi result should contain 'abc' field", ghiResult.containsKey("abc")) + assertTrue("ghi result should contain 'result' field", ghiResult.containsKey("result")) + assertEquals("Group 'ghi' max should be 9", 9, (ghiResult["max_num"] as Number).toInt()) + assertEquals("Group 'ghi' field should be 'ghi'", "ghi", ghiResult["abc"]) + assertEquals("Group 'ghi' eval result should be true (9 > 5 is true)", true, ghiResult["result"]) + + // Custom condition "eval result = max_num > 5" should evaluate to true for def (6 > 5) and ghi (9 > 5) + // This caused the trigger to fire and create an ACTIVE alert + + // Delete documents from "def" and "ghi" groups to make condition no longer true + // After deletion, only "abc" group remains with max=3, which doesn't satisfy max_num > 5 + val deleteDefDocs = """ + { + "query": { + "term": { + "abc": "def" + } + } + } + """.trimIndent() + val deleteGhiDocs = """ + { + "query": { + "term": { + "abc": "ghi" + } + } + } + """.trimIndent() + + val deleteDefRequest = Request("POST", "/$TEST_INDEX_NAME/_delete_by_query?refresh=true") + deleteDefRequest.setJsonEntity(deleteDefDocs) + client().performRequest(deleteDefRequest) + + val deleteGhiRequest = Request("POST", "/$TEST_INDEX_NAME/_delete_by_query?refresh=true") + deleteGhiRequest.setJsonEntity(deleteGhiDocs) + client().performRequest(deleteGhiRequest) + + // Execute monitor again - should NOT trigger because only abc group remains (max=3 <= 5) + val responseAfterDelete = executeMonitor(monitor.id) + + val outputAfterDelete = entityAsMap(responseAfterDelete) + assertEquals(monitor.name, outputAfterDelete["monitor_name"]) + + // Verify there are no ACTIVE alerts (moved to history) + assertTrue("Active alert should have moved to history", searchAlerts(monitor, AlertIndices.ALERT_INDEX).isEmpty()) + + // Verify the same alert is now in COMPLETED state (search ALL alert indices including history) + val alertsAfterDelete = searchAlerts(monitor, AlertIndices.ALL_ALERT_INDEX_PATTERN) + assertEquals("Should still have exactly one alert", 1, alertsAfterDelete.size) + + val completedAlert = alertsAfterDelete.single() + + // Verify it's the same alert (same ID) + assertEquals("Alert ID should be the same", activeAlert.id, completedAlert.id) + assertEquals("Alert should transition to COMPLETED state", COMPLETED, completedAlert.state) + verifyAlert(completedAlert, monitor, COMPLETED) + + // Verify the alert has an end time now + assertNotNull("Completed alert should have an end time", completedAlert.endTime) + } + + fun `test execute PPL monitor updates query results on subsequent runs`() { + // Setup test index and data + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + val docId1 = "test-doc-1" + indexDocFromSomeTimeAgo(2, MINUTES, "test-value", 5, docId1) + + // Create PPL monitor with NUMBER_OF_RESULTS condition + val monitor = createMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + triggers = listOf( + randomPPLTrigger( + conditionType = PPLSQLTrigger.ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = PPLSQLTrigger.NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + // Execute monitor - should trigger because document exists + val response = executeMonitor(monitor.id) + + // Verify monitor execution response + val output = entityAsMap(response) + assertEquals(monitor.name, output["monitor_name"]) + + // Verify alert was generated with ACTIVE state + val alerts = searchAlerts(monitor) + assertEquals("Alert should have been generated", 1, alerts.size) + + val firstAlert = alerts.single() + verifyAlert(firstAlert, monitor, ACTIVE) + + // Verify alert contains PPL query and results with 1 document + assertNotNull("PPL query should be stored in alert", firstAlert.pplQuery) + assertEquals("source = $TEST_INDEX_NAME | head 10", firstAlert.pplQuery) + assertTrue("PPL query results should not be empty", firstAlert.pplQueryResults.isNotEmpty()) + + // Verify the query results are in the new transformed format (list of maps) + assertEquals("Should have 1 result row from the first indexed document", 1, firstAlert.pplQueryResults.size) + + // Get the first result row and verify it has the expected structure + val firstResultRow = firstAlert.pplQueryResults[0] + assertTrue("Result should contain 'timestamp' field", firstResultRow.containsKey("timestamp")) + assertTrue("Result should contain 'abc' field", firstResultRow.containsKey("abc")) + assertTrue("Result should contain 'number' field", firstResultRow.containsKey("number")) + + // Verify the field values match the first document we indexed + assertEquals("Field 'abc' should match first document", "test-value", firstResultRow["abc"]) + assertEquals("Field 'number' should match first document", 5, (firstResultRow["number"] as Number).toInt()) + + // Add a second document to the index + val docId2 = "test-doc-2" + indexDocFromSomeTimeAgo(3, MINUTES, "test-value-2", 10, docId2) + + // Execute monitor again - should still trigger because documents still exist (now 2 docs) + val responseAfterSecondDoc = executeMonitor(monitor.id) + + val outputAfterSecondDoc = entityAsMap(responseAfterSecondDoc) + assertEquals(monitor.name, outputAfterSecondDoc["monitor_name"]) + + // Verify the same alert still exists + val alertsAfterSecondDoc = searchAlerts(monitor) + assertEquals("Should still have exactly one alert", 1, alertsAfterSecondDoc.size) + + val updatedAlert = alertsAfterSecondDoc.single() + + // Verify it's the same alert (same ID) + assertEquals("Alert ID should be the same", firstAlert.id, updatedAlert.id) + + // Verify the alert's query results have been updated to include both documents + assertNotNull("PPL query should still be stored in alert", updatedAlert.pplQuery) + assertTrue("PPL query results should not be empty", updatedAlert.pplQueryResults.isNotEmpty()) + + // Verify the query results now contain 2 rows (updated from previous 1 row) + assertEquals("Should now have 2 result rows from both indexed documents", 2, updatedAlert.pplQueryResults.size) + + // Verify both result rows have the expected structure + updatedAlert.pplQueryResults.forEach { row -> + assertTrue("Each result row should contain 'timestamp' field", row.containsKey("timestamp")) + assertTrue("Each result row should contain 'abc' field", row.containsKey("abc")) + assertTrue("Each result row should contain 'number' field", row.containsKey("number")) + } + + // Verify we can find both documents in the results by their unique 'abc' values + val abcValues = updatedAlert.pplQueryResults.map { it["abc"] }.toSet() + assertTrue("Results should contain first document with abc='test-value'", abcValues.contains("test-value")) + assertTrue("Results should contain second document with abc='test-value-2'", abcValues.contains("test-value-2")) + + // Verify we can find both documents by their number values + val numberValues = updatedAlert.pplQueryResults.map { (it["number"] as Number).toInt() }.toSet() + assertTrue("Results should contain first document with number=5", numberValues.contains(5)) + assertTrue("Results should contain second document with number=10", numberValues.contains(10)) + + // Verify lastNotificationTime was updated + assertNotEquals( + "Last notification time should be updated", + firstAlert.lastNotificationTime, + updatedAlert.lastNotificationTime + ) + + // Verify startTime remains the same (alert not recreated) + assertEquals( + "Start time should remain the same", + firstAlert.startTime, + updatedAlert.startTime + ) + } + + fun `test PPL monitor action template renders query results correctly`() { + // Setup test index and data + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "group-a", 10) + indexDocFromSomeTimeAgo(2, MINUTES, "group-b", 20) + indexDocFromSomeTimeAgo(3, MINUTES, "group-a", 30) + + // Create complex template that accesses PPL query results at multiple levels + val messageTemplate = """ + Monitor: {{ctx.monitor.name}} + Trigger: {{ctx.trigger.name}} + + === PPL Query Results === + Query: {{ctx.monitor.inputs.0.ppl_input.query}} + + --- Fine-Grained Access to First Row --- + {{#ctx.ppl_query_results}} + Row: + Field 1 (number): {{number}} + Field 2 (abc): {{abc}} + Field 3 (timestamp): {{timestamp}} + + {{/ctx.ppl_query_results}} + """.trimIndent() + + val destination = createDestination() + val action = randomAction( + template = randomTemplateScript(messageTemplate), + subjectTemplate = randomTemplateScript("A Subject"), + destinationId = destination.id + ) + + val query = "source = $TEST_INDEX_NAME | head 10" + + // Create PPL monitor with action + val monitor = createMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + triggers = listOf( + randomPPLTrigger( + conditionType = PPLSQLTrigger.ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = PPLSQLTrigger.NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null, + actions = listOf(action) + ) + ), + query = query + ) + ) + + // Execute monitor to trigger action + val response = executeMonitor(monitor.id, params = DRYRUN_MONITOR) + val output = entityAsMap(response) + + // Verify action was executed + assertEquals(monitor.name, output["monitor_name"]) + val triggerResults = output.objectMap("trigger_results") + assertTrue("Trigger should have executed", triggerResults.isNotEmpty()) + + // Extract and verify action output + val triggerResult = triggerResults.values.first() + val actionResults = triggerResult.objectMap("action_results") + assertTrue("Action results should exist", actionResults.isNotEmpty()) + + val actionResult = actionResults.values.first() + + @Suppress("UNCHECKED_CAST") + val actionOutputMap = actionResult["output"] as? Map + assertNotNull("Action output should not be null", actionOutputMap) + + val message = actionOutputMap?.get("message") + + logger.info("Rendered message: $message") + + assertNotNull("Action message should not be null", message) + assertNull("Action should not have error", actionResult["error"]) + + // Verify basic context variables + assertTrue("Message should contain monitor name", message!!.contains("Monitor: ${monitor.name}")) + assertTrue("Message should contain trigger name", message.contains("Trigger: ${triggerResult["name"]}")) + + // Verify PPL query is accessible + assertTrue("Message should contain the PPL query", message.contains(query)) + + // Verify fine-grained access to each row's fields (order may vary) + assertTrue("Message should contain group-a", message.contains("group-a")) + assertTrue("Message should contain group-b", message.contains("group-b")) + assertTrue("Message should contain value 10", message.contains("10")) + assertTrue("Message should contain value 20", message.contains("20")) + assertTrue("Message should contain value 30", message.contains("30")) + } + private fun prepareTestAnomalyResult(detectorId: String, user: User) { val adResultIndex = ".opendistro-anomaly-results-history-2020.10.17" try { diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/PPLSQLMonitorRunnerIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/PPLSQLMonitorRunnerIT.kt deleted file mode 100644 index 33c25afd9..000000000 --- a/alerting/src/test/kotlin/org/opensearch/alerting/PPLSQLMonitorRunnerIT.kt +++ /dev/null @@ -1,415 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting - -import org.opensearch.alerting.settings.AlertingSettings -import org.opensearch.common.settings.Settings -import org.opensearch.common.unit.TimeValue -import org.opensearch.commons.alerting.model.IntervalSchedule -import org.opensearch.commons.alerting.model.PPLSQLTrigger.ConditionType -import org.opensearch.commons.alerting.model.PPLSQLTrigger.NumResultsCondition -import org.opensearch.commons.alerting.model.PPLSQLTrigger.TriggerMode -import org.opensearch.test.OpenSearchTestCase -import java.time.temporal.ChronoUnit.MINUTES -import java.util.concurrent.TimeUnit - -/*** - * Create various kinds of monitors and ensures they all generate alerts - * under the expected circumstances - * - * Gradle command to run this suite: - * ./gradlew :alerting:integTest -Dhttps=true -Dsecurity=true -Duser=admin -Dpassword=admin \ - * --tests "org.opensearch.alerting.PPLMonitorRunnerIT" - */ -class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { - fun `test monitor execution timeout generates error alert`() { - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - indexDocFromSomeTimeAgo(2, MINUTES, "abc", 5) - - val pplMonitor = createRandomPPLMonitor( - randomPPLMonitor( - enabled = true, - schedule = IntervalSchedule(interval = 1, unit = MINUTES), - triggers = listOf( - randomPPLTrigger( - throttleDuration = null, - expireDuration = 5, - mode = TriggerMode.RESULT_SET, - conditionType = ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = NumResultsCondition.GREATER_THAN, - numResultsValue = 0L, - customCondition = null - ) - ), - query = "source = $TEST_INDEX_NAME | head 10" - ) - ) - - // set the monitor execution timebox to 1 nanosecond to guarantee a timeout - client().updateSettings(AlertingSettings.ALERT_V2_MONITOR_EXECUTION_MAX_DURATION.key, TimeValue.timeValueNanos(1L)) - - val executeMonitorResponse = executeMonitor(pplMonitor.id) - - val getAlertsResponse = getAlerts() - val alertsGenerated = numAlerts(getAlertsResponse) > 0 - val containsErrorAlert = containsErrorAlert(getAlertsResponse) - val executeResponseContainsError = - (entityAsMap(executeMonitorResponse).stringMap("input_results")?.get("error") as String).isNotEmpty() - - assert(alertsGenerated) { "Alerts should have been generated but they weren't" } - assert(containsErrorAlert) { "Error alert should have been generated for timeout but wasn't" } - assert(executeResponseContainsError) { "Execute monitor response should've included an error message but didn't" } - } - - fun `test running number of results condition and result set mode ppl monitor`() { - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - indexDocFromSomeTimeAgo(2, MINUTES, "abc", 5) - - val pplMonitor = createRandomPPLMonitor( - randomPPLMonitor( - enabled = true, - schedule = IntervalSchedule(interval = 1, unit = MINUTES), - triggers = listOf( - randomPPLTrigger( - throttleDuration = null, - expireDuration = 5, - mode = TriggerMode.RESULT_SET, - conditionType = ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = NumResultsCondition.GREATER_THAN, - numResultsValue = 0L, - customCondition = null - ) - ), - query = "source = $TEST_INDEX_NAME | head 10" - ) - ) - - val versionBefore = pplMonitor.version - - val executeResponse = executeMonitor(pplMonitor.id) - val triggered = isTriggered(pplMonitor, executeResponse) - - val getAlertsResponse = getAlerts() - val alertsGenerated = numAlerts(getAlertsResponse) > 0 - - assert(triggered) { "Monitor should have triggered but it didn't" } - assert(alertsGenerated) { "Alerts should have been generated but they weren't" } - - val pplMonitorAfter = getMonitor(pplMonitor.id) - val versionAfter = pplMonitorAfter.version - - assert(versionBefore == versionAfter) { "Monitor version changed after monitor execution" } - } - - fun `test running number of results condition and per result mode ppl monitor`() { - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) - indexDocFromSomeTimeAgo(2, MINUTES, "def", 10) - indexDocFromSomeTimeAgo(3, MINUTES, "ghi", 7) - - val pplMonitor = createRandomPPLMonitor( - randomPPLMonitor( - enabled = true, - schedule = IntervalSchedule(interval = 1, unit = MINUTES), - triggers = listOf( - randomPPLTrigger( - throttleDuration = null, - expireDuration = 5, - mode = TriggerMode.PER_RESULT, - conditionType = ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = NumResultsCondition.GREATER_THAN, - numResultsValue = 0L, - customCondition = null - ) - ), - query = "source = $TEST_INDEX_NAME | head 10" - ) - ) - - val executeResponse = executeMonitor(pplMonitor.id) - val triggered = isTriggered(pplMonitor, executeResponse) - - val getAlertsResponse = getAlerts() - val alertsGenerated = numAlerts(getAlertsResponse) - - assert(triggered) { "Monitor should have triggered but it didn't" } - assertEquals( - "A number of alerts matching the number of docs ingested (3) should have been generated", - 3, alertsGenerated - ) - } - - fun `test running custom condition and result set mode ppl monitor`() { - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - indexDocFromSomeTimeAgo(1, MINUTES, "abc", 1) - indexDocFromSomeTimeAgo(2, MINUTES, "abc", 2) - indexDocFromSomeTimeAgo(3, MINUTES, "abc", 3) - indexDocFromSomeTimeAgo(4, MINUTES, "def", 4) - indexDocFromSomeTimeAgo(5, MINUTES, "def", 5) - indexDocFromSomeTimeAgo(6, MINUTES, "def", 6) - indexDocFromSomeTimeAgo(7, MINUTES, "ghi", 7) - indexDocFromSomeTimeAgo(8, MINUTES, "ghi", 8) - indexDocFromSomeTimeAgo(9, MINUTES, "ghi", 9) - - val pplMonitor = createRandomPPLMonitor( - randomPPLMonitor( - enabled = true, - schedule = IntervalSchedule(interval = 1, unit = MINUTES), - triggers = listOf( - randomPPLTrigger( - throttleDuration = null, - expireDuration = 5, - mode = TriggerMode.RESULT_SET, - conditionType = ConditionType.CUSTOM, - customCondition = "eval result = max_num > 5", - numResultsCondition = null, - numResultsValue = null - ) - ), - query = "source = $TEST_INDEX_NAME | stats max(number) as max_num by abc" - ) - ) - - val executeResponse = executeMonitor(pplMonitor.id) - val triggered = isTriggered(pplMonitor, executeResponse) - - val getAlertsResponse = getAlerts() - val alertsGenerated = numAlerts(getAlertsResponse) > 0 - - assert(triggered) { "Monitor should have triggered but it didn't" } - assert(alertsGenerated) { "Alerts should have been generated but they weren't" } - } - - fun `test running custom condition and per result mode ppl monitor`() { - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - indexDocFromSomeTimeAgo(1, MINUTES, "abc", 1) - indexDocFromSomeTimeAgo(2, MINUTES, "abc", 2) - indexDocFromSomeTimeAgo(3, MINUTES, "abc", 3) - indexDocFromSomeTimeAgo(4, MINUTES, "def", 4) - indexDocFromSomeTimeAgo(5, MINUTES, "def", 5) - indexDocFromSomeTimeAgo(6, MINUTES, "def", 6) - indexDocFromSomeTimeAgo(7, MINUTES, "ghi", 7) - indexDocFromSomeTimeAgo(8, MINUTES, "ghi", 8) - indexDocFromSomeTimeAgo(9, MINUTES, "ghi", 9) - - val pplMonitor = createRandomPPLMonitor( - randomPPLMonitor( - enabled = true, - schedule = IntervalSchedule(interval = 1, unit = MINUTES), - triggers = listOf( - randomPPLTrigger( - throttleDuration = null, - expireDuration = 5, - mode = TriggerMode.PER_RESULT, - conditionType = ConditionType.CUSTOM, - customCondition = "eval evaluation = max_num > 5", - numResultsCondition = null, - numResultsValue = null - ) - ), - query = "source = $TEST_INDEX_NAME | stats max(number) as max_num by abc" - ) - ) - - val executeResponse = executeMonitor(pplMonitor.id) - val triggered = isTriggered(pplMonitor, executeResponse) - - val getAlertsResponse = getAlerts() - val alertsGenerated = numAlerts(getAlertsResponse) - - // when the indexed docs above are aggregated by field abc, we have: - // max("abc") = 3 - // max("def") = 6 - // max("ghi") = 9 - // only 2 of these buckets satisfy the custom condition max_num > 5, so - // only 2 alerts should be generated - - assert(triggered) { "Monitor should have triggered but it didn't" } - assertEquals( - "A number of alerts matching the number of docs ingested (2) should have been generated", - 2, alertsGenerated - ) - } - - fun `test execute api generated alert gets expired`() { - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) - - val pplMonitor = createRandomPPLMonitor( - randomPPLMonitor( - enabled = true, - schedule = IntervalSchedule(interval = 20, unit = MINUTES), - triggers = listOf( - randomPPLTrigger( - throttleDuration = null, - expireDuration = 1L, - mode = TriggerMode.RESULT_SET, - conditionType = ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = NumResultsCondition.GREATER_THAN, - numResultsValue = 0L, - customCondition = null - ) - ), - query = "source = $TEST_INDEX_NAME | head 10" - ) - ) - - val executeResponse = executeMonitor(pplMonitor.id) - val triggered = isTriggered(pplMonitor, executeResponse) - - val getAlertsResponsePreExpire = getAlerts() - val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 - - assert(triggered) { "Monitor should have triggered but it didn't" } - assert(alertsGeneratedPreExpire) { "Alerts should have been generated but they weren't" } - - // sleep briefly so alert mover can expire the alert - OpenSearchTestCase.waitUntil({ - return@waitUntil false - }, 2, TimeUnit.MINUTES) - - val getAlertsResponsePostExpire = getAlerts() - val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 - assert(!alertsGeneratedPostExpire) - } - - fun `test scheduled job generated alert gets expired`() { - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) - - // the monitor should generate 1 alert, then not generate - // any alerts for the rest of the test - createRandomPPLMonitor( - randomPPLMonitor( - enabled = true, - schedule = IntervalSchedule(interval = 1, unit = MINUTES), - triggers = listOf( - randomPPLTrigger( - throttleDuration = 100L, - expireDuration = 1L, - mode = TriggerMode.RESULT_SET, - conditionType = ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = NumResultsCondition.GREATER_THAN, - numResultsValue = 0L, - customCondition = null - ) - ), - query = "source = $TEST_INDEX_NAME | head 10" - ) - ) - - // sleep briefly so scheduled job can generate the alert - OpenSearchTestCase.waitUntil({ - return@waitUntil false - }, 2, TimeUnit.MINUTES) - - val getAlertsResponsePreExpire = getAlerts() - val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 - - assert(alertsGeneratedPreExpire) { "Alerts should have been generated but they weren't" } - - // sleep briefly so alert mover can expire the alert - OpenSearchTestCase.waitUntil({ - return@waitUntil false - }, 2, TimeUnit.MINUTES) - - val getAlertsResponsePostExpire = getAlerts() - logger.info("num alerts: ${numAlerts(getAlertsResponsePostExpire)}") - val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 - assert(!alertsGeneratedPostExpire) - } - - fun `test scheduled job monitor execution gets throttled`() { - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) - - val pplMonitor = createRandomPPLMonitor( - randomPPLMonitor( - enabled = true, - schedule = IntervalSchedule(interval = 1, unit = MINUTES), - triggers = listOf( - randomPPLTrigger( - throttleDuration = 10, - expireDuration = 5, - mode = TriggerMode.RESULT_SET, - conditionType = ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = NumResultsCondition.GREATER_THAN, - numResultsValue = 0L, - customCondition = null - ) - ), - query = "source = $TEST_INDEX_NAME | head 10" - ) - ) - - val executeResponse = executeMonitor(pplMonitor.id) - val triggered = isTriggered(pplMonitor, executeResponse) - - val getAlertsResponsePreThrottle = getAlerts() - val numAlertsPreThrottle = numAlerts(getAlertsResponsePreThrottle) - - assert(triggered) { "Monitor should have triggered but it didn't" } - assertEquals("Alerts should have been generated but they weren't", 1, numAlertsPreThrottle) - - // sleep briefly to give the monitor to execute again - // automatically and get throttled - OpenSearchTestCase.waitUntil({ - return@waitUntil false - }, 2, TimeUnit.MINUTES) - - val getAlertsResponsePostThrottled = getAlerts() - val numAlertsPostThrottled = numAlerts(getAlertsResponsePostThrottled) - assertEquals("A new alert was generated when it should have been throttled", 1, numAlertsPostThrottled) - } - - fun `test manual monitor execution bypasses throttle`() { - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) - - val pplMonitor = createRandomPPLMonitor( - randomPPLMonitor( - enabled = true, - schedule = IntervalSchedule(interval = 30, unit = MINUTES), - triggers = listOf( - randomPPLTrigger( - throttleDuration = 20, - expireDuration = 5, - mode = TriggerMode.RESULT_SET, - conditionType = ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = NumResultsCondition.GREATER_THAN, - numResultsValue = 0L, - customCondition = null - ) - ), - query = "source = $TEST_INDEX_NAME | head 10" - ) - ) - - val executeResponse = executeMonitor(pplMonitor.id) - val triggered = isTriggered(pplMonitor, executeResponse) - - val getAlertsResponse = getAlerts() - val numAlerts = numAlerts(getAlertsResponse) - - assert(triggered) { "Monitor should have triggered but it didn't" } - assertEquals("Alerts should have been generated but they weren't", 1, numAlerts) - - // sleep briefly to get comfortable inside - // the throttle window - OpenSearchTestCase.waitUntil({ - return@waitUntil false - }, 10, TimeUnit.SECONDS) - - val executeAgainResponse = executeMonitor(pplMonitor.id) - val triggeredAgain = isTriggered(pplMonitor, executeAgainResponse) - - val getAlertsAgainResponse = getAlerts() - val numAlertsAgain = numAlerts(getAlertsAgainResponse) - - assert(triggeredAgain) { "Monitor should have triggered again but it didn't" } - assertEquals("A new alert should have been generated but was instead throttled", 2, numAlertsAgain) - } -} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/PPLUtilsTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/PPLUtilsTests.kt new file mode 100644 index 000000000..b66fd1494 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/PPLUtilsTests.kt @@ -0,0 +1,472 @@ +package org.opensearch.alerting + +import org.json.JSONArray +import org.json.JSONObject +import org.opensearch.alerting.PPLUtils.PPL_RESULTS_SIZE_EXCEEDED_MESSAGE +import org.opensearch.test.OpenSearchTestCase + +class PPLUtilsTests : OpenSearchTestCase() { + + fun `test findEvalResultVar with simple variable name`() { + val condition = "eval _some_result1 = avg_latency > 100" + assertEquals("_some_result1", PPLUtils.findEvalResultVar(condition)) + } + + fun `test findEvalResultVar with extra whitespace`() { + val condition = " eval myVar = value > 100" + assertEquals("myVar", PPLUtils.findEvalResultVar(condition)) + } + + fun `test findEvalResultVar with no whitespace after equals`() { + val condition = "eval apple=avg_latency > 100" + assertEquals("apple", PPLUtils.findEvalResultVar(condition)) + } + + fun `test findEvalResultVar with complex expression`() { + val condition = "eval is_high_latency = (avg_latency > 100 AND error_count > 5) OR max_latency > 500" + assertEquals("is_high_latency", PPLUtils.findEvalResultVar(condition)) + } + + fun `test findEvalResultVar is case insensitive for eval keyword`() { + val condition = "EVAL result = value > 100" + assertEquals("result", PPLUtils.findEvalResultVar(condition)) + } + + fun `test findEvalResultVar with nested parentheses`() { + val condition = "eval complex = ((a + b) * (c - d)) > threshold" + assertEquals("complex", PPLUtils.findEvalResultVar(condition)) + } + + fun `test findEvalResultVar with eval in middle of string`() { + val condition = "stats count() | eval threshold_met = count > 10" + assertThrows(IllegalArgumentException::class.java) { + PPLUtils.findEvalResultVar(condition) + } + } + + fun `test findEvalResultVar with multiple eval statements`() { + val condition = "eval first = a > 1 | eval second = b > 2" + assertThrows(IllegalArgumentException::class.java) { + PPLUtils.findEvalResultVar(condition) + } + } + + fun `test findEvalResultVar throws exception when eval keyword missing`() { + val condition = "result = avg_latency > 100" + assertThrows(IllegalArgumentException::class.java) { + PPLUtils.findEvalResultVar(condition) + } + } + + fun `test findEvalResultVar throws exception when variable name invalid`() { + val condition = "eval 123invalid = value > 100" + assertThrows(IllegalArgumentException::class.java) { + PPLUtils.findEvalResultVar(condition) + } + } + + fun `test findEvalResultVar throws exception when equals sign missing`() { + val condition = "eval result value > 100" + assertThrows(IllegalArgumentException::class.java) { + PPLUtils.findEvalResultVar(condition) + } + } + + fun `test findEvalResultVar throws exception with empty string`() { + assertThrows(IllegalArgumentException::class.java) { + PPLUtils.findEvalResultVar("") + } + } + + fun `test findEvalResultVar throws exception when eval not followed by whitespace`() { + val condition = "evalresult = value > 100" + assertThrows(IllegalArgumentException::class.java) { + PPLUtils.findEvalResultVar(condition) + } + } + + fun `test findEvalResultVarIdxInSchema finds variable at first position`() { + val queryResponse = JSONObject( + """ + { + "schema": [ + {"name": "count", "type": "integer"}, + {"name": "result", "type": "boolean"}, + {"name": "avg", "type": "double"} + ] + } + """.trimIndent() + ) + + val index = PPLUtils.findEvalResultVarIdxInSchema(queryResponse, "result") + assertEquals(1, index) + } + + fun `test findEvalResultVarIdxInSchema throws exception when variable not found`() { + val queryResponse = JSONObject( + """ + { + "schema": [ + {"name": "field1", "type": "string"}, + {"name": "field2", "type": "integer"} + ] + } + """.trimIndent() + ) + + val exception = assertThrows(IllegalStateException::class.java) { + PPLUtils.findEvalResultVarIdxInSchema(queryResponse, "nonexistent") + } + assertTrue(exception.message!!.contains("Expected to find eval statement results variable")) + assertTrue(exception.message!!.contains("nonexistent")) + } + + fun `test findEvalResultVarIdxInSchema with variable names containing special characters`() { + val queryResponse = JSONObject( + """ + { + "schema": [ + {"name": "field_with_underscores", "type": "boolean"}, + {"name": "field-with-dashes", "type": "boolean"}, + {"name": "field.with.dots", "type": "boolean"} + ] + } + """.trimIndent() + ) + + assertEquals(0, PPLUtils.findEvalResultVarIdxInSchema(queryResponse, "field_with_underscores")) + assertEquals(1, PPLUtils.findEvalResultVarIdxInSchema(queryResponse, "field-with-dashes")) + assertEquals(2, PPLUtils.findEvalResultVarIdxInSchema(queryResponse, "field.with.dots")) + } + + fun `test findEvalResultVarIdxInSchema with large schema`() { + val schemaBuilder = StringBuilder("[") + for (i in 0 until 50) { + if (i > 0) schemaBuilder.append(",") + schemaBuilder.append("""{"name": "field$i", "type": "string"}""") + } + schemaBuilder.append("]") + + val queryResponse = JSONObject() + queryResponse.put("schema", JSONArray(schemaBuilder.toString())) + + val index = PPLUtils.findEvalResultVarIdxInSchema(queryResponse, "field25") + assertEquals(25, index) + } + + fun `test constructPPLQueryResultsMap with simple types`() { + // Arrange: Simple query result with basic types + val rawResults = mapOf( + "schema" to listOf( + mapOf("name" to "username", "type" to "string"), + mapOf("name" to "count", "type" to "integer"), + mapOf("name" to "active", "type" to "boolean") + ), + "datarows" to listOf( + listOf("alice", 42, true), + listOf("bob", 17, false), + listOf("charlie", 99, true) + ), + "total" to 3, + "size" to 3 + ) + + // Act + val result = PPLUtils.constructPPLQueryResultsMap(rawResults) + + // Assert + assertEquals(3, result.size) + + // First row + assertEquals("alice", result[0]["username"]) + assertEquals(42, result[0]["count"]) + assertEquals(true, result[0]["active"]) + + // Second row + assertEquals("bob", result[1]["username"]) + assertEquals(17, result[1]["count"]) + assertEquals(false, result[1]["active"]) + + // Third row + assertEquals("charlie", result[2]["username"]) + assertEquals(99, result[2]["count"]) + assertEquals(true, result[2]["active"]) + } + + fun `test constructPPLQueryResultsMap with size exceeded message`() { + // Arrange: Simple query result with basic types + val rawResults = mapOf( + "schema" to listOf( + mapOf("name" to "message", "type" to "string") + ), + "datarows" to listOf( + listOf(PPL_RESULTS_SIZE_EXCEEDED_MESSAGE) + ), + "total" to 3, + "size" to 3 + ) + + // Act + val result = PPLUtils.constructPPLQueryResultsMap(rawResults) + + // Assert + assertEquals(1, result.size) + + // First row + assertEquals(PPL_RESULTS_SIZE_EXCEEDED_MESSAGE, result[0]["message"]) + } + + fun `test constructPPLQueryResultsMap with nested objects and nulls`() { + // Arrange: Complex query result with nested arrays, objects, and nulls + val rawResults = mapOf( + "schema" to listOf( + mapOf("name" to "list", "type" to "bigint"), + mapOf("name" to "user", "type" to "struct"), + mapOf("name" to "abc", "type" to "string") + ), + "datarows" to listOf( + // Row 1: All fields populated with complex types + listOf( + listOf(1, 2, 3), // Nested array + mapOf("name" to "bob", "age" to 32), // Nested object + "abc" // Simple string + ), + // Row 2: First field is null, rest populated + listOf( + null, // Null array + mapOf("name" to "bob", "age" to 32), // Nested object + "abc" // Simple string + ), + // Row 3: Multiple null fields + listOf( + null, // Null array + null, // Null object + "abc" // Simple string + ) + ), + "total" to 3, + "size" to 3 + ) + + // Act + val result = PPLUtils.constructPPLQueryResultsMap(rawResults) + + // Assert + assertEquals(3, result.size) + + // Row 1: Nested array and nested object + val row1 = result[0] + val list1 = row1["list"] as? List<*> + assertNotNull(list1) + assertEquals(3, list1?.size) + assertEquals(1, list1?.get(0)) + assertEquals(2, list1?.get(1)) + assertEquals(3, list1?.get(2)) + + val user1 = row1["user"] as? Map<*, *> + assertNotNull(user1) + assertEquals("bob", user1?.get("name")) + assertEquals(32, user1?.get("age")) + + assertEquals("abc", row1["abc"]) + + // Row 2: Null list, populated user + val row2 = result[1] + assertNull(row2["list"]) + + val user2 = row2["user"] as? Map<*, *> + assertNotNull(user2) + assertEquals("bob", user2?.get("name")) + assertEquals(32, user2?.get("age")) + + assertEquals("abc", row2["abc"]) + + // Row 3: Multiple nulls + val row3 = result[2] + assertNull(row3["list"]) + assertNull(row3["user"]) + assertEquals("abc", row3["abc"]) + } + + fun `test constructPPLQueryResultsMap with empty schema`() { + // Arrange: Empty schema + val rawResults = mapOf( + "schema" to emptyList>(), + "datarows" to listOf( + listOf("value1", "value2") + ), + "total" to 1, + "size" to 1 + ) + + // Act + val result = PPLUtils.constructPPLQueryResultsMap(rawResults) + + // Assert + assertTrue(result.isEmpty()) + } + + fun `test constructPPLQueryResultsMap with empty datarows`() { + // Arrange: Empty datarows + val rawResults = mapOf( + "schema" to listOf( + mapOf("name" to "field1", "type" to "string"), + mapOf("name" to "field2", "type" to "integer") + ), + "datarows" to emptyList>(), + "total" to 0, + "size" to 0 + ) + + // Act + val result = PPLUtils.constructPPLQueryResultsMap(rawResults) + + // Assert + assertTrue(result.isEmpty()) + } + + fun `test constructPPLQueryResultsMap with missing schema`() { + // Arrange: Missing schema field + val rawResults = mapOf( + "datarows" to listOf( + listOf("value1", "value2") + ), + "total" to 1, + "size" to 1 + ) + + // Act + val result = PPLUtils.constructPPLQueryResultsMap(rawResults) + + // Assert + assertTrue(result.isEmpty()) + } + + fun `test constructPPLQueryResultsMap with missing datarows`() { + // Arrange: Missing datarows field + val rawResults = mapOf( + "schema" to listOf( + mapOf("name" to "field1", "type" to "string") + ), + "total" to 0, + "size" to 0 + ) + + // Act + val result = PPLUtils.constructPPLQueryResultsMap(rawResults) + + // Assert + assertTrue(result.isEmpty()) + } + + fun `test constructPPLQueryResultsMap with mismatched row lengths`() { + // Arrange: More schema fields than datarow values + val rawResults = mapOf( + "schema" to listOf( + mapOf("name" to "field1", "type" to "string"), + mapOf("name" to "field2", "type" to "integer"), + mapOf("name" to "field3", "type" to "boolean") + ), + "datarows" to listOf( + listOf("value1", 42), // Missing third field + listOf("value2") // Missing second and third fields + ), + "total" to 2, + "size" to 2 + ) + + // Act + val result = PPLUtils.constructPPLQueryResultsMap(rawResults) + + // Assert + assertEquals(2, result.size) + + // First row: third field should be null + assertEquals("value1", result[0]["field1"]) + assertEquals(42, result[0]["field2"]) + assertNull(result[0]["field3"]) + + // Second row: second and third fields should be null + assertEquals("value2", result[1]["field1"]) + assertNull(result[1]["field2"]) + assertNull(result[1]["field3"]) + } + + fun `test constructPPLQueryResultsMap with extra datarow values`() { + // Arrange: More datarow values than schema fields + val rawResults = mapOf( + "schema" to listOf( + mapOf("name" to "field1", "type" to "string"), + mapOf("name" to "field2", "type" to "integer") + ), + "datarows" to listOf( + listOf("value1", 42, true, "extra") // Extra values ignored + ), + "total" to 1, + "size" to 1 + ) + + // Act + val result = PPLUtils.constructPPLQueryResultsMap(rawResults) + + // Assert + assertEquals(1, result.size) + assertEquals(2, result[0].size) + assertEquals("value1", result[0]["field1"]) + assertEquals(42, result[0]["field2"]) + // Extra values are ignored + } + + fun `test constructPPLQueryResultsMap with deeply nested structures`() { + // Arrange: Deeply nested objects and arrays + val rawResults = mapOf( + "schema" to listOf( + mapOf("name" to "nested_data", "type" to "struct") + ), + "datarows" to listOf( + listOf( + mapOf( + "level1" to mapOf( + "level2" to mapOf( + "level3" to listOf(1, 2, 3) + ) + ), + "array_of_objects" to listOf( + mapOf("id" to 1, "name" to "item1"), + mapOf("id" to 2, "name" to "item2") + ) + ) + ) + ), + "total" to 1, + "size" to 1 + ) + + // Act + val result = PPLUtils.constructPPLQueryResultsMap(rawResults) + + // Assert + assertEquals(1, result.size) + + val nestedData = result[0]["nested_data"] as? Map<*, *> + assertNotNull(nestedData) + + val level1 = nestedData?.get("level1") as? Map<*, *> + assertNotNull(level1) + + val level2 = level1?.get("level2") as? Map<*, *> + assertNotNull(level2) + + val level3 = level2?.get("level3") as? List<*> + assertNotNull(level3) + assertEquals(3, level3?.size) + + val arrayOfObjects = nestedData?.get("array_of_objects") as? List<*> + assertNotNull(arrayOfObjects) + assertEquals(2, arrayOfObjects?.size) + + val firstObject = arrayOfObjects?.get(0) as? Map<*, *> + assertEquals(1, firstObject?.get("id")) + assertEquals("item1", firstObject?.get("name")) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt b/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt index cf54406e1..331eb96c3 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt @@ -52,8 +52,6 @@ import org.opensearch.commons.alerting.model.PPLSQLInput.QueryLanguage import org.opensearch.commons.alerting.model.PPLSQLTrigger import org.opensearch.commons.alerting.model.PPLSQLTrigger.ConditionType import org.opensearch.commons.alerting.model.PPLSQLTrigger.NumResultsCondition -import org.opensearch.commons.alerting.model.PPLSQLTrigger.Severity -import org.opensearch.commons.alerting.model.PPLSQLTrigger.TriggerMode import org.opensearch.commons.alerting.model.QueryLevelTrigger import org.opensearch.commons.alerting.model.QueryLevelTriggerRunResult import org.opensearch.commons.alerting.model.Schedule @@ -403,11 +401,8 @@ fun randomDocumentLevelTrigger( fun randomPPLTrigger( id: String = UUIDs.base64UUID(), name: String = OpenSearchRestTestCase.randomAlphaOfLength(10), - severity: Severity = Severity.entries.random(), - throttleDuration: Long? = randomLongBetween(1, 100), - expireDuration: Long = randomLongBetween(1, 100), + severity: String = "1", actions: List = mutableListOf(), - mode: TriggerMode = TriggerMode.entries.random(), conditionType: ConditionType = ConditionType.NUMBER_OF_RESULTS, numResultsCondition: NumResultsCondition? = NumResultsCondition.entries.random(), numResultsValue: Long? = randomLongBetween(1L, 50L), @@ -416,12 +411,8 @@ fun randomPPLTrigger( return PPLSQLTrigger( id = id, name = name, - severity = severity.value, - throttleDuration = throttleDuration, - expireDuration = expireDuration, - lastTriggeredTime = null, + severity = severity, actions = actions, - mode = mode, conditionType = conditionType, numResultsCondition = numResultsCondition, numResultsValue = numResultsValue, @@ -941,21 +932,6 @@ fun assertPplTriggersEqual(pplTrigger1: PPLSQLTrigger, pplTrigger2: PPLSQLTrigge pplTrigger1.severity, pplTrigger2.severity ) - assertEquals( - "Monitor trigger $id throttle durations not equal", - pplTrigger1.throttleDuration, - pplTrigger2.throttleDuration - ) - assertEquals( - "Monitor trigger $id expire durations not equal", - pplTrigger1.expireDuration, - pplTrigger2.expireDuration - ) - assertEquals( - "Monitor trigger $id modes not equal", - pplTrigger1.mode, - pplTrigger2.mode - ) assertEquals( "Monitor trigger $id condition types not equal", pplTrigger1.conditionType, diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/alertsv2/AlertV2IndicesIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/alertsv2/AlertV2IndicesIT.kt deleted file mode 100644 index 7c206016b..000000000 --- a/alerting/src/test/kotlin/org/opensearch/alerting/alertsv2/AlertV2IndicesIT.kt +++ /dev/null @@ -1,442 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.alertsv2 - -import org.opensearch.alerting.AlertingRestTestCase -import org.opensearch.alerting.TEST_INDEX_MAPPINGS -import org.opensearch.alerting.TEST_INDEX_NAME -import org.opensearch.alerting.makeRequest -import org.opensearch.alerting.randomPPLMonitor -import org.opensearch.alerting.randomPPLTrigger -import org.opensearch.alerting.settings.AlertingSettings -import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.XContentType -import org.opensearch.commons.alerting.model.IntervalSchedule -import org.opensearch.commons.alerting.model.Monitor -import org.opensearch.commons.alerting.model.PPLSQLTrigger.ConditionType -import org.opensearch.commons.alerting.model.PPLSQLTrigger.NumResultsCondition -import org.opensearch.commons.alerting.model.PPLSQLTrigger.TriggerMode -import org.opensearch.commons.alerting.model.ScheduledJob -import org.opensearch.core.rest.RestStatus -import org.opensearch.test.OpenSearchTestCase -import java.time.temporal.ChronoUnit.MINUTES -import java.util.concurrent.TimeUnit - -/** - * Tests AlertV2 history migration, AlertV2 deletion, and AlertV2 expiration functionality - * - * Gradle command to run this suite: - * ./gradlew :alerting:integTest -Dhttps=true -Dsecurity=true -Duser=admin -Dpassword=admin \ - * --tests "org.opensearch.alerting.alertsv2.AlertV2IndicesIT" - */ -class AlertV2IndicesIT : AlertingRestTestCase() { - fun `test create alert v2 index`() { - generateAlertV2s() - - assertIndexExists(AlertV2Indices.ALERT_V2_INDEX) - assertIndexExists(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) - } - - fun `test update alert v2 index mapping with new schema version`() { - wipeAllODFEIndices() - assertIndexDoesNotExist(AlertV2Indices.ALERT_V2_INDEX) - assertIndexDoesNotExist(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) - - putAlertV2Mappings( - AlertV2Indices.alertMapping().trimStart('{').trimEnd('}') - .replace("\"schema_version\": 6", "\"schema_version\": 0") - ) - assertIndexExists(AlertV2Indices.ALERT_V2_INDEX) - assertIndexExists(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) - verifyIndexSchemaVersion(AlertV2Indices.ALERT_V2_INDEX, 0) - verifyIndexSchemaVersion(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX, 0) - - wipeAllODFEIndices() - - generateAlertV2s() - assertIndexExists(AlertV2Indices.ALERT_V2_INDEX) - assertIndexExists(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) - verifyIndexSchemaVersion(ScheduledJob.SCHEDULED_JOBS_INDEX, 9) - verifyIndexSchemaVersion(AlertV2Indices.ALERT_V2_INDEX, 6) - verifyIndexSchemaVersion(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX, 6) - } - - fun `test alert v2 index gets recreated automatically if deleted`() { - wipeAllODFEIndices() - assertIndexDoesNotExist(AlertV2Indices.ALERT_V2_INDEX) - - generateAlertV2s() - - assertIndexExists(AlertV2Indices.ALERT_V2_INDEX) - assertIndexExists(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) - wipeAllODFEIndices() - assertIndexDoesNotExist(AlertV2Indices.ALERT_V2_INDEX) - assertIndexDoesNotExist(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) - - // ensure execute monitor succeeds even after alert indices are deleted - generateAlertV2s() - } - - fun `test rollover alert v2 history index`() { - // Update the rollover check to be every 1 second and the index max age to be 1 second - client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_ROLLOVER_PERIOD.key, "1s") - client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_INDEX_MAX_AGE.key, "1s") - - generateAlertV2s() - - // Allow for a rollover index. - OpenSearchTestCase.waitUntil({ - return@waitUntil (getAlertV2Indices().size >= 3) - }, 2, TimeUnit.SECONDS) - - assertTrue("Did not find 3 alert v2 indices", getAlertV2Indices().size >= 3) - } - - fun `test alert v2 history disabled`() { - resetHistorySettings() - - // Disable alert history - client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_ENABLED.key, "false") - - val pplMonitor = generateAlertV2s( - randomPPLMonitor( - schedule = IntervalSchedule(interval = 30, unit = MINUTES), - query = "source = $TEST_INDEX_NAME | head 3", - triggers = listOf( - randomPPLTrigger( - mode = TriggerMode.RESULT_SET, - conditionType = ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = NumResultsCondition.GREATER_THAN, - numResultsValue = 0L, - expireDuration = 1L - ) - ) - ) - ) - - val alerts1 = searchAlerts(pplMonitor, AlertV2Indices.ALERT_V2_INDEX) - assertEquals("1 alert should be present", 1, alerts1.size) - - // wait for alert to expire. - // since alert history is disabled, this should result - // in hard deletion - OpenSearchTestCase.waitUntil({ - return@waitUntil false - }, 2, TimeUnit.MINUTES) - - // Since history is disabled, the alert should be hard deleted by now - val alerts2 = searchAlerts(pplMonitor, AlertV2Indices.ALL_ALERT_V2_INDEX_PATTERN) - assertTrue("There should be no alerts, but alerts were found", alerts2.isEmpty()) - } - - fun `test short retention period`() { - resetHistorySettings() - - val pplMonitor = generateAlertV2s( - randomPPLMonitor( - schedule = IntervalSchedule(interval = 30, unit = MINUTES), - query = "source = $TEST_INDEX_NAME | head 3", - triggers = listOf( - randomPPLTrigger( - mode = TriggerMode.RESULT_SET, - conditionType = ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = NumResultsCondition.GREATER_THAN, - numResultsValue = 0L, - expireDuration = 1L - ) - ) - ) - ) - - val alerts1 = searchAlerts(pplMonitor, AlertV2Indices.ALERT_V2_INDEX) - assertEquals("1 alert should be present", 1, alerts1.size) - - // history index should be created but empty - assertEquals(0, getAlertV2HistoryDocCount()) - - // wait for alert to expire. - // since alert history is enabled, this should result - // in the alert being archived in history index - OpenSearchTestCase.waitUntil({ - return@waitUntil false - }, 2, TimeUnit.MINUTES) - - assertTrue(searchAlerts(pplMonitor, AlertV2Indices.ALERT_V2_INDEX).isEmpty()) - assertEquals(1, getAlertV2HistoryDocCount()) - - // update rollover check and max docs as well as decreasing the retention period - client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_ROLLOVER_PERIOD.key, "3s") - client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_MAX_DOCS.key, 1) - client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_RETENTION_PERIOD.key, "1s") - - // give some time for newly updated settings to take effect - OpenSearchTestCase.waitUntil({ - return@waitUntil getAlertV2HistoryDocCount() == 0L - }, 40, TimeUnit.SECONDS) - - // Given the max_docs and retention settings above, the history index will rollover and the non-write index will be deleted. - // This leaves two indices: active alerts index and an empty history write index - assertEquals("Did not find 2 alert v2 indices", 2, getAlertV2Indices().size) - assertEquals(0, getAlertV2HistoryDocCount()) - } - - fun `test generated alert gets expired because monitor was deleted with alert history enabled`() { - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) - - val pplMonitor = createRandomPPLMonitor( - randomPPLMonitor( - enabled = true, - schedule = IntervalSchedule(interval = 20, unit = MINUTES), - triggers = listOf( - randomPPLTrigger( - throttleDuration = null, - // for this test, configured expire can't be the reason for alert expiration - expireDuration = 1000L, - mode = TriggerMode.RESULT_SET, - conditionType = ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = NumResultsCondition.GREATER_THAN, - numResultsValue = 0L, - customCondition = null - ) - ), - query = "source = $TEST_INDEX_NAME | head 10" - ) - ) - - val executeResponse = executeMonitor(pplMonitor.id) - val triggered = isTriggered(pplMonitor, executeResponse) - - val getAlertsResponsePreExpire = getAlerts() - val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 - - assert(triggered) { "Monitor should have triggered but it didn't" } - assert(alertsGeneratedPreExpire) { "Alerts should have been generated but they weren't" } - - // delete the monitor - deleteMonitor(pplMonitor) - - // sleep so postDelete can expire the generated alert - OpenSearchTestCase.waitUntil({ - return@waitUntil false - }, 5, TimeUnit.SECONDS) - - val getAlertsResponsePostExpire = getAlerts() - val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 - assert(!alertsGeneratedPostExpire) - - assertEquals(1, getAlertV2HistoryDocCount()) - } - - fun `test generated alert gets expired because monitor was edited with alert history enabled`() { - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) - - // first create a ppl monitor that's guaranteed to generate an alert - val initialPplTrigger = randomPPLTrigger( - id = "initialID", - throttleDuration = null, - // for this test, configured expire can't be the reason for alert expiration - expireDuration = 1000L, - mode = TriggerMode.RESULT_SET, - conditionType = ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = NumResultsCondition.GREATER_THAN, - numResultsValue = 0L, - customCondition = null - ) - - val initialPplMonitorConfig = randomPPLMonitor( - enabled = true, - schedule = IntervalSchedule(interval = 20, unit = MINUTES), - triggers = listOf(initialPplTrigger), - query = "source = $TEST_INDEX_NAME | head 10" - ) - - val pplMonitor = createRandomPPLMonitor(initialPplMonitorConfig) - - val executeResponse = executeMonitor(pplMonitor.id) - val triggered = isTriggered(pplMonitor, executeResponse) - - val getAlertsResponsePreExpire = getAlerts() - val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 - - assert(triggered) { "Monitor should have triggered but it didn't" } - assert(alertsGeneratedPreExpire) { "Alerts should have been generated but they weren't" } - - // update the monitor to any new config, - // and more importantly, updated triggers - updateMonitor(randomPPLMonitor().copy(id = pplMonitor.id, version = pplMonitor.version)) - - // sleep so postIndex can expire the generated alert - OpenSearchTestCase.waitUntil({ - return@waitUntil false - }, 5, TimeUnit.SECONDS) - - val getAlertsResponsePostExpire = getAlerts() - val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 - assert(!alertsGeneratedPostExpire) - - assertEquals(1, getAlertV2HistoryDocCount()) - } - - fun `test generated alert gets expired because monitor was deleted with alert history disabled`() { - client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_ENABLED.key, "false") - - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) - - val pplMonitor = createRandomPPLMonitor( - randomPPLMonitor( - enabled = true, - schedule = IntervalSchedule(interval = 20, unit = MINUTES), - triggers = listOf( - randomPPLTrigger( - throttleDuration = null, - // for this test, configured expire can't be the reason for alert expiration - expireDuration = 1000L, - mode = TriggerMode.RESULT_SET, - conditionType = ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = NumResultsCondition.GREATER_THAN, - numResultsValue = 0L, - customCondition = null - ) - ), - query = "source = $TEST_INDEX_NAME | head 10" - ) - ) - - val executeResponse = executeMonitor(pplMonitor.id) - val triggered = isTriggered(pplMonitor, executeResponse) - - val getAlertsResponsePreExpire = getAlerts() - val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 - - assert(triggered) { "Monitor should have triggered but it didn't" } - assert(alertsGeneratedPreExpire) { "Alerts should have been generated but they weren't" } - - // delete the monitor - deleteMonitor(pplMonitor) - - // sleep so postDelete can expire the generated alert - OpenSearchTestCase.waitUntil({ - return@waitUntil false - }, 5, TimeUnit.SECONDS) - - val getAlertsResponsePostExpire = getAlerts() - val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 - assert(!alertsGeneratedPostExpire) - - assertEquals(0, getAlertV2HistoryDocCount()) - } - - fun `test generated alert gets expired because monitor was edited with alert history disabled`() { - client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_ENABLED.key, "false") - - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) - - // first create a ppl monitor that's guaranteed to generate an alert - val initialPplTrigger = randomPPLTrigger( - id = "initialID", - throttleDuration = null, - // for this test, configured expire can't be the reason for alert expiration - expireDuration = 1000L, - mode = TriggerMode.RESULT_SET, - conditionType = ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = NumResultsCondition.GREATER_THAN, - numResultsValue = 0L, - customCondition = null - ) - - val initialPplMonitorConfig = randomPPLMonitor( - enabled = true, - schedule = IntervalSchedule(interval = 20, unit = MINUTES), - triggers = listOf(initialPplTrigger), - query = "source = $TEST_INDEX_NAME | head 10" - ) - - val pplMonitor = createRandomPPLMonitor(initialPplMonitorConfig) - - val executeResponse = executeMonitor(pplMonitor.id) - val triggered = isTriggered(pplMonitor, executeResponse) - - val getAlertsResponsePreExpire = getAlerts() - val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 - - assert(triggered) { "Monitor should have triggered but it didn't" } - assert(alertsGeneratedPreExpire) { "Alerts should have been generated but they weren't" } - - // update the monitor to any new config - updateMonitor(randomPPLMonitor().copy(id = pplMonitor.id, version = pplMonitor.version)) - - // sleep so postIndex can expire the generated alert - OpenSearchTestCase.waitUntil({ - return@waitUntil false - }, 5, TimeUnit.SECONDS) - - val getAlertsResponsePostExpire = getAlerts() - val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 - assert(!alertsGeneratedPostExpire) - - assertEquals(0, getAlertV2HistoryDocCount()) - } - - private fun assertIndexExists(index: String) { - val response = client().makeRequest("HEAD", index) - assertEquals("Index $index does not exist.", RestStatus.OK, response.restStatus()) - } - - private fun assertIndexDoesNotExist(index: String) { - val response = client().makeRequest("HEAD", index) - assertEquals("Index $index exists when it shouldn't.", RestStatus.NOT_FOUND, response.restStatus()) - } - - private fun resetHistorySettings() { - client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_ENABLED.key, "true") - client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_ROLLOVER_PERIOD.key, "60s") - client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_RETENTION_PERIOD.key, "60s") - } - - private fun getAlertV2Indices(): List { - val response = client().makeRequest("GET", "/_cat/indices/${AlertV2Indices.ALL_ALERT_V2_INDEX_PATTERN}?format=json") - val xcp = createParser(XContentType.JSON.xContent(), response.entity.content) - val responseList = xcp.list() - val indices = mutableListOf() - responseList.filterIsInstance>().forEach { indices.add(it["index"] as String) } - - return indices - } - - // generates alerts by creating then executing a monitor - private fun generateAlertV2s( - pplMonitorConfig: Monitor = randomPPLMonitor( - query = "source = $TEST_INDEX_NAME | head 3", - triggers = listOf( - randomPPLTrigger( - conditionType = ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = NumResultsCondition.GREATER_THAN, - numResultsValue = 0L - ) - ) - ) - ): Monitor { - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) - indexDocFromSomeTimeAgo(2, MINUTES, "def", 10) - indexDocFromSomeTimeAgo(3, MINUTES, "ghi", 7) - - val pplMonitor = createRandomPPLMonitor(pplMonitorConfig) - - val executeResponse = executeMonitor(pplMonitor.id) - - // ensure execute call succeeded - val xcp = createParser(XContentType.JSON.xContent(), executeResponse.entity.content) - val output = xcp.map() - assertNull("Error running monitor v2", output["error"]) - - return pplMonitor - } -} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt index deb1b7cd0..90af7fb64 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt @@ -11,10 +11,14 @@ import org.apache.hc.core5.http.message.BasicHeader import org.opensearch.alerting.ALERTING_BASE_URI import org.opensearch.alerting.ALWAYS_RUN import org.opensearch.alerting.ANOMALY_DETECTOR_INDEX +import org.opensearch.alerting.AlertingPlugin.Companion.MONITOR_BASE_URI import org.opensearch.alerting.AlertingRestTestCase import org.opensearch.alerting.LEGACY_OPENDISTRO_ALERTING_BASE_URI +import org.opensearch.alerting.TEST_INDEX_MAPPINGS +import org.opensearch.alerting.TEST_INDEX_NAME import org.opensearch.alerting.alerts.AlertIndices import org.opensearch.alerting.anomalyDetectorIndexMapping +import org.opensearch.alerting.assertPplMonitorsEqual import org.opensearch.alerting.core.settings.ScheduledJobSettings import org.opensearch.alerting.makeRequest import org.opensearch.alerting.model.destination.Chime @@ -30,15 +34,24 @@ import org.opensearch.alerting.randomDocLevelMonitorInput import org.opensearch.alerting.randomDocLevelQuery import org.opensearch.alerting.randomDocumentLevelMonitor import org.opensearch.alerting.randomDocumentLevelTrigger +import org.opensearch.alerting.randomPPLMonitor +import org.opensearch.alerting.randomPPLTrigger import org.opensearch.alerting.randomQueryLevelMonitor import org.opensearch.alerting.randomQueryLevelTrigger +import org.opensearch.alerting.randomTemplateScript import org.opensearch.alerting.randomThrottle import org.opensearch.alerting.randomUser import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_MAX_MONITORS +import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_QUERY_LENGTH +import org.opensearch.alerting.settings.AlertingSettings.Companion.NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH +import org.opensearch.alerting.settings.AlertingSettings.Companion.NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH import org.opensearch.alerting.toJsonString import org.opensearch.alerting.util.DestinationType import org.opensearch.client.ResponseException import org.opensearch.client.WarningFailureException +import org.opensearch.common.UUIDs +import org.opensearch.common.settings.Settings import org.opensearch.common.unit.TimeValue import org.opensearch.common.xcontent.XContentType import org.opensearch.commons.alerting.model.Alert @@ -47,6 +60,7 @@ import org.opensearch.commons.alerting.model.DocLevelMonitorInput import org.opensearch.commons.alerting.model.DocLevelQuery import org.opensearch.commons.alerting.model.DocumentLevelTrigger import org.opensearch.commons.alerting.model.Monitor +import org.opensearch.commons.alerting.model.PPLSQLTrigger.ConditionType import org.opensearch.commons.alerting.model.QueryLevelTrigger import org.opensearch.commons.alerting.model.ScheduledJob import org.opensearch.commons.alerting.model.SearchInput @@ -55,6 +69,8 @@ import org.opensearch.core.common.bytes.BytesReference import org.opensearch.core.rest.RestStatus import org.opensearch.core.xcontent.ToXContent import org.opensearch.core.xcontent.XContentBuilder +import org.opensearch.core.xcontent.XContentParser +import org.opensearch.core.xcontent.XContentParserUtils import org.opensearch.index.query.QueryBuilders import org.opensearch.script.Script import org.opensearch.search.aggregations.AggregationBuilders @@ -66,6 +82,7 @@ import org.opensearch.test.rest.OpenSearchRestTestCase import java.time.Instant import java.time.ZoneId import java.time.temporal.ChronoUnit +import java.time.temporal.ChronoUnit.MINUTES import java.util.concurrent.TimeUnit @TestLogging("level:DEBUG", reason = "Debug for tests.") @@ -1530,6 +1547,379 @@ class MonitorRestApiIT : AlertingRestTestCase() { } } + /* PPL Monitor Simple Case Tests */ + fun `test create ppl monitor`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + val pplMonitor = randomPPLMonitor() + + val response = client().makeRequest("POST", MONITOR_BASE_URI, emptyMap(), pplMonitor.toHttpEntity()) + assertEquals("Unable to create a new monitor v2", RestStatus.CREATED, response.restStatus()) + + val responseBody = response.asMap() + val createdId = responseBody["_id"] as String + val createdVersion = responseBody["_version"] as Int + assertNotEquals("response is missing Id", Monitor.NO_ID, createdId) + assertEquals("incorrect version", 1, createdVersion) + } + + fun `test update ppl monitor`() { + val originalMonitor = createRandomPPLMonitor() + + val newMonitorConfig = randomPPLMonitor() + + val updateResponse = client().makeRequest( + "PUT", + "$MONITOR_BASE_URI/${originalMonitor.id}", + emptyMap(), newMonitorConfig.toHttpEntity() + ) + + assertEquals("Update monitor failed", RestStatus.OK, updateResponse.restStatus()) + val responseBody = updateResponse.asMap() + assertEquals("Updated monitor id doesn't match", originalMonitor.id, responseBody["_id"] as String) + assertEquals("Version not incremented", (originalMonitor.version + 1).toInt(), responseBody["_version"] as Int) + + val updatedMonitor = getMonitor(originalMonitor.id) + assertPplMonitorsEqual(newMonitorConfig, updatedMonitor) + } + + fun `test get ppl monitor`() { + // first create the monitor + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + val pplMonitor = randomPPLMonitor() + + val createResponse = client().makeRequest("POST", MONITOR_BASE_URI, emptyMap(), pplMonitor.toHttpEntity()) + assertEquals("Unable to create a new monitor v2", RestStatus.CREATED, createResponse.restStatus()) + + val responseBody = createResponse.asMap() + val pplMonitorId = responseBody["_id"] as String + val pplMonitorVersion = (responseBody["_version"] as Int).toLong() + + // then attempt to get it + val response = client().makeRequest("GET", "$MONITOR_BASE_URI/$pplMonitorId") + assertEquals("Unable to get monitorV2 $pplMonitorId", RestStatus.OK, response.restStatus()) + + val parser = createParser(XContentType.JSON.xContent(), response.entity.content) + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser) + + lateinit var id: String + var version: Long = 0 + lateinit var storedPplMonitor: Monitor + + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + parser.nextToken() + + when (parser.currentName()) { + "_id" -> id = parser.text() + "_version" -> version = parser.longValue() + "monitor" -> storedPplMonitor = Monitor.parse(parser) + "associated_workflows" -> { + XContentParserUtils.ensureExpectedToken( + XContentParser.Token.START_ARRAY, + parser.currentToken(), + parser + ) + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + // do nothing + } + } + } + } + + assertEquals( + "Monitor V2 ID from Get Monitor doesn't match one from Create Monitor response", + pplMonitorId, id + ) + assertEquals( + "Monitor V2 version from Get Monitor doesn't match one from Create Monitor response", + pplMonitorVersion, version + ) + assertPplMonitorsEqual(pplMonitor, storedPplMonitor) + } + + fun `test head ppl monitor`() { + val submittedPplMonitor = createRandomPPLMonitor() + val response = client().makeRequest("HEAD", "$MONITOR_BASE_URI/${submittedPplMonitor.id}") + assertEquals("Unable to get monitorV2 ${submittedPplMonitor.id}", RestStatus.OK, response.restStatus()) + } + + fun `test search ppl monitor with GET and match_all`() { + createRandomPPLMonitor() + + val search = SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).toString() + val searchResponse = client().makeRequest( + "GET", "$MONITOR_BASE_URI/_search", + emptyMap(), StringEntity(search, ContentType.APPLICATION_JSON) + ) + + assertEquals("Search monitor failed", RestStatus.OK, searchResponse.restStatus()) + val xcp = createParser(XContentType.JSON.xContent(), searchResponse.entity.content) + val hits = xcp.map()["hits"]!! as Map> + val numberDocsFound = hits["total"]?.get("value") + assertEquals("PPL Monitor not found during search", 1, numberDocsFound) + } + + fun `test search ppl monitor with POST and term query on ID`() { + val pplMonitor = createRandomPPLMonitor() + + val search = SearchSourceBuilder().query(QueryBuilders.termQuery("_id", pplMonitor.id)).toString() + val searchResponse = client().makeRequest( + "POST", "$MONITOR_BASE_URI/_search", + emptyMap(), StringEntity(search, ContentType.APPLICATION_JSON) + ) + + assertEquals("Search monitor failed", RestStatus.OK, searchResponse.restStatus()) + val xcp = createParser(XContentType.JSON.xContent(), searchResponse.entity.content) + val hits = xcp.map()["hits"]!! as Map> + val numberDocsFound = hits["total"]?.get("value") + assertEquals("PPL Monitor not found during search", 1, numberDocsFound) + } + + fun `test delete ppl monitor`() { + val pplMonitor = createRandomPPLMonitor() + + val deleteResponse = client().makeRequest("DELETE", "$MONITOR_BASE_URI/${pplMonitor.id}") + assertEquals("Delete failed", RestStatus.OK, deleteResponse.restStatus()) + + val getResponse = client().makeRequest("HEAD", "$MONITOR_BASE_URI/${pplMonitor.id}") + assertEquals("Deleted monitor still exists", RestStatus.NOT_FOUND, getResponse.restStatus()) + } + + fun `test parsing ppl monitor as a scheduled job`() { + val monitorV2 = createRandomPPLMonitor() + + val builder = monitorV2.toXContentWithUser(XContentBuilder.builder(XContentType.JSON.xContent()), USE_TYPED_KEYS) + val string = BytesReference.bytes(builder).utf8ToString() + val xcp = createParser(XContentType.JSON.xContent(), string) + val scheduledJob = ScheduledJob.parse(xcp, monitorV2.id, monitorV2.version) + assertEquals(monitorV2, scheduledJob) + } + + /* PPL Monitor Validation Tests */ + fun `test create ppl monitor that queries nonexistent index fails`() { + val pplMonitorConfig = randomPPLMonitor( + query = "source = nonexistent_index | head 10" + ) + + // ensure the request fails + try { + createRandomPPLMonitor(pplMonitorConfig) + fail("Expected request to fail with BAD_REQUEST but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + } + + // ensure no monitor was created + ensureNumMonitors(0) + } + + fun `test create ppl monitor with more than max allowed monitors fails`() { + adminClient().updateSettings(ALERTING_MAX_MONITORS.key, 1) + + createRandomPPLMonitor() + + // ensure the request fails + try { + createRandomPPLMonitor() + fail("Expected request to fail with BAD_REQUEST but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + } + + // ensure no monitor was created + ensureNumMonitors(1) + } + + fun `test create ppl monitor with invalid query fails`() { + // ensure the request fails + try { + createRandomPPLMonitor( + randomPPLMonitor( + query = "source = $TEST_INDEX_NAME | not valid ppl" + ) + ) + fail("Expected request to fail with BAD_REQUEST but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + } + + // ensure no monitor was created + ensureNumMonitors(0) + } + + fun `test create ppl monitor with query that's too long fails`() { + adminClient().updateSettings(ALERTING_V2_MAX_QUERY_LENGTH.key, 1) + + // ensure the request fails + try { + createRandomPPLMonitor( + randomPPLMonitor( + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + fail("Expected request to fail with BAD_REQUEST but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + } + + // ensure no monitor was created + ensureNumMonitors(0) + } + + fun `test create ppl monitor with invalid custom condition fails`() { + // ensure the request fails + try { + createRandomPPLMonitor( + randomPPLMonitor( + triggers = listOf( + randomPPLTrigger( + conditionType = ConditionType.CUSTOM, + customCondition = "not a valid PPL custom condition", + numResultsCondition = null, + numResultsValue = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + fail("Expected request to fail with BAD_REQUEST but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + } + + // ensure no monitor was created + ensureNumMonitors(0) + } + + fun `test create ppl monitor with custom condition that evals to num not bool fails`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 1) + indexDocFromSomeTimeAgo(2, MINUTES, "abc", 2) + + // ensure the request fails + try { + createRandomPPLMonitor( + randomPPLMonitor( + triggers = listOf( + randomPPLTrigger( + conditionType = ConditionType.CUSTOM, + customCondition = "eval something = sum * 2", + numResultsCondition = null, + numResultsValue = null + ) + ), + query = "source = $TEST_INDEX_NAME | stats sum(number) as sum by abc" + ) + ) + fail("Expected request to fail with BAD_REQUEST but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + } + + // ensure no monitor was created + ensureNumMonitors(0) + } + + fun `test create ppl monitor with notification subject source too long fails`() { + adminClient().updateSettings(NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH.key, 100) + + var subjectTooLong = "" + for (i in 0 until 101) { + subjectTooLong += "a" + } + + // ensure the request fails + try { + createRandomPPLMonitor( + randomPPLMonitor( + triggers = listOf( + randomPPLTrigger( + actions = listOf( + randomAction( + template = randomTemplateScript( + source = "some message" + ), + subjectTemplate = randomTemplateScript( + source = subjectTooLong + ) + ) + ) + ) + ) + ) + ) + fail("Expected request to fail with BAD_REQUEST but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + } + + // ensure no monitor was created + ensureNumMonitors(0) + } + + fun `test create ppl monitor with notification message source too long fails`() { + adminClient().updateSettings(NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH.key, 1000) + + var messageTooLong = "" + for (i in 0 until 1001) { + messageTooLong += "a" + } + + // ensure the request fails + try { + createRandomPPLMonitor( + randomPPLMonitor( + triggers = listOf( + randomPPLTrigger( + actions = listOf( + randomAction( + template = randomTemplateScript( + source = messageTooLong + ), + subjectTemplate = randomTemplateScript( + source = "some subject" + ) + ) + ) + ) + ) + ) + ) + fail("Expected request to fail with BAD_REQUEST but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + } + + // ensure no monitor was created + ensureNumMonitors(0) + } + + fun `test update nonexistent ppl monitor fails`() { + // the random monitor query searches index TEST_INDEX_NAME, + // so we need to create that first to ensure at least the request body is valid + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + + val monitorV2 = randomPPLMonitor() + val randomId = UUIDs.base64UUID() + + try { + client().makeRequest("PUT", "$MONITOR_BASE_URI/$randomId", emptyMap(), monitorV2.toHttpEntity()) + fail("Expected request to fail with NOT_FOUND but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.NOT_FOUND, e.response.restStatus()) + } + } + + fun `test delete nonexistent ppl monitor fails`() { + val randomId = UUIDs.base64UUID() + + try { + client().makeRequest("DELETE", "$MONITOR_BASE_URI/$randomId") + fail("Expected request to fail with NOT_FOUND but it succeeded") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.NOT_FOUND, e.response.restStatus()) + } + } + private fun validateAlertingStatsNodeResponse(nodesResponse: Map) { assertEquals("Incorrect number of nodes", numberOfNodes, nodesResponse["total"]) assertEquals("Failed nodes found during monitor stats call", 0, nodesResponse["failed"]) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorV2RestApiIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorV2RestApiIT.kt deleted file mode 100644 index 3c5c5920e..000000000 --- a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorV2RestApiIT.kt +++ /dev/null @@ -1,470 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.resthandler - -import org.apache.hc.core5.http.ContentType -import org.apache.hc.core5.http.io.entity.StringEntity -import org.opensearch.alerting.AlertingPlugin.Companion.MONITOR_BASE_URI -import org.opensearch.alerting.AlertingRestTestCase -import org.opensearch.alerting.TEST_INDEX_MAPPINGS -import org.opensearch.alerting.TEST_INDEX_NAME -import org.opensearch.alerting.assertPplMonitorsEqual -import org.opensearch.alerting.makeRequest -import org.opensearch.alerting.randomAction -import org.opensearch.alerting.randomPPLMonitor -import org.opensearch.alerting.randomPPLTrigger -import org.opensearch.alerting.randomTemplateScript -import org.opensearch.alerting.resthandler.MonitorRestApiIT.Companion.USE_TYPED_KEYS -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_MAX_MONITORS -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_EXPIRE_DURATION -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_QUERY_LENGTH -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_THROTTLE_DURATION -import org.opensearch.alerting.settings.AlertingSettings.Companion.NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH -import org.opensearch.alerting.settings.AlertingSettings.Companion.NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH -import org.opensearch.client.ResponseException -import org.opensearch.common.UUIDs -import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.XContentType -import org.opensearch.commons.alerting.model.Monitor -import org.opensearch.commons.alerting.model.PPLSQLTrigger.ConditionType -import org.opensearch.commons.alerting.model.ScheduledJob -import org.opensearch.core.common.bytes.BytesReference -import org.opensearch.core.rest.RestStatus -import org.opensearch.core.xcontent.XContentBuilder -import org.opensearch.core.xcontent.XContentParser -import org.opensearch.core.xcontent.XContentParserUtils -import org.opensearch.index.query.QueryBuilders -import org.opensearch.search.builder.SearchSourceBuilder -import org.opensearch.test.junit.annotations.TestLogging -import java.time.temporal.ChronoUnit.MINUTES - -/*** - * Tests Alerting V2 CRUD and validations - * - * Gradle command to run this suite: - * ./gradlew :alerting:integTest -Dhttps=true -Dsecurity=true -Duser=admin -Dpassword=admin \ - * --tests "org.opensearch.alerting.resthandler.MonitorV2RestApiIT" - */ -@TestLogging("level:DEBUG", reason = "Debug for tests.") -@Suppress("UNCHECKED_CAST") -class MonitorV2RestApiIT : AlertingRestTestCase() { - /* Simple Case Tests */ - fun `test create ppl monitor`() { - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - val pplMonitor = randomPPLMonitor() - - val response = client().makeRequest("POST", MONITOR_BASE_URI, emptyMap(), pplMonitor.toHttpEntity()) - assertEquals("Unable to create a new monitor v2", RestStatus.CREATED, response.restStatus()) - - val responseBody = response.asMap() - val createdId = responseBody["_id"] as String - val createdVersion = responseBody["_version"] as Int - assertNotEquals("response is missing Id", Monitor.NO_ID, createdId) - assertEquals("incorrect version", 1, createdVersion) - } - - fun `test update ppl monitor`() { - val originalMonitor = createRandomPPLMonitor() - - val newMonitorConfig = randomPPLMonitor() - - val updateResponse = client().makeRequest( - "PUT", - "$MONITOR_BASE_URI/${originalMonitor.id}", - emptyMap(), newMonitorConfig.toHttpEntity() - ) - - assertEquals("Update monitor failed", RestStatus.OK, updateResponse.restStatus()) - val responseBody = updateResponse.asMap() - assertEquals("Updated monitor id doesn't match", originalMonitor.id, responseBody["_id"] as String) - assertEquals("Version not incremented", (originalMonitor.version + 1).toInt(), responseBody["_version"] as Int) - - val updatedMonitor = getMonitor(originalMonitor.id) - assertPplMonitorsEqual(newMonitorConfig, updatedMonitor) - } - - fun `test get ppl monitor`() { - // first create the monitor - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - val pplMonitor = randomPPLMonitor() - - val createResponse = client().makeRequest("POST", MONITOR_BASE_URI, emptyMap(), pplMonitor.toHttpEntity()) - assertEquals("Unable to create a new monitor v2", RestStatus.CREATED, createResponse.restStatus()) - - val responseBody = createResponse.asMap() - val pplMonitorId = responseBody["_id"] as String - val pplMonitorVersion = (responseBody["_version"] as Int).toLong() - - // then attempt to get it - val response = client().makeRequest("GET", "$MONITOR_BASE_URI/$pplMonitorId") - assertEquals("Unable to get monitorV2 $pplMonitorId", RestStatus.OK, response.restStatus()) - - val parser = createParser(XContentType.JSON.xContent(), response.entity.content) - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser) - - lateinit var id: String - var version: Long = 0 - lateinit var storedPplMonitor: Monitor - - while (parser.nextToken() != XContentParser.Token.END_OBJECT) { - parser.nextToken() - - when (parser.currentName()) { - "_id" -> id = parser.text() - "_version" -> version = parser.longValue() - "monitor" -> storedPplMonitor = Monitor.parse(parser) - "associated_workflows" -> { - XContentParserUtils.ensureExpectedToken( - XContentParser.Token.START_ARRAY, - parser.currentToken(), - parser - ) - while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - // do nothing - } - } - } - } - - assertEquals( - "Monitor V2 ID from Get Monitor doesn't match one from Create Monitor response", - pplMonitorId, id - ) - assertEquals( - "Monitor V2 version from Get Monitor doesn't match one from Create Monitor response", - pplMonitorVersion, version - ) - assertPplMonitorsEqual(pplMonitor, storedPplMonitor) - } - - fun `test head ppl monitor`() { - val submittedPplMonitor = createRandomPPLMonitor() - val response = client().makeRequest("HEAD", "$MONITOR_BASE_URI/${submittedPplMonitor.id}") - assertEquals("Unable to get monitorV2 ${submittedPplMonitor.id}", RestStatus.OK, response.restStatus()) - } - - fun `test search ppl monitor with GET and match_all`() { - createRandomPPLMonitor() - - val search = SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).toString() - val searchResponse = client().makeRequest( - "GET", "$MONITOR_BASE_URI/_search", - emptyMap(), StringEntity(search, ContentType.APPLICATION_JSON) - ) - - assertEquals("Search monitor failed", RestStatus.OK, searchResponse.restStatus()) - val xcp = createParser(XContentType.JSON.xContent(), searchResponse.entity.content) - val hits = xcp.map()["hits"]!! as Map> - val numberDocsFound = hits["total"]?.get("value") - assertEquals("PPL Monitor not found during search", 1, numberDocsFound) - } - - fun `test search ppl monitor with POST and term query on ID`() { - val pplMonitor = createRandomPPLMonitor() - - val search = SearchSourceBuilder().query(QueryBuilders.termQuery("_id", pplMonitor.id)).toString() - val searchResponse = client().makeRequest( - "POST", "$MONITOR_BASE_URI/_search", - emptyMap(), StringEntity(search, ContentType.APPLICATION_JSON) - ) - - assertEquals("Search monitor failed", RestStatus.OK, searchResponse.restStatus()) - val xcp = createParser(XContentType.JSON.xContent(), searchResponse.entity.content) - val hits = xcp.map()["hits"]!! as Map> - val numberDocsFound = hits["total"]?.get("value") - assertEquals("PPL Monitor not found during search", 1, numberDocsFound) - } - - fun `test delete ppl monitor`() { - val pplMonitor = createRandomPPLMonitor() - - val deleteResponse = client().makeRequest("DELETE", "$MONITOR_BASE_URI/${pplMonitor.id}") - assertEquals("Delete failed", RestStatus.OK, deleteResponse.restStatus()) - - val getResponse = client().makeRequest("HEAD", "$MONITOR_BASE_URI/${pplMonitor.id}") - assertEquals("Deleted monitor still exists", RestStatus.NOT_FOUND, getResponse.restStatus()) - } - - fun `test parsing ppl monitor as a scheduled job`() { - val monitorV2 = createRandomPPLMonitor() - - val builder = monitorV2.toXContentWithUser(XContentBuilder.builder(XContentType.JSON.xContent()), USE_TYPED_KEYS) - val string = BytesReference.bytes(builder).utf8ToString() - val xcp = createParser(XContentType.JSON.xContent(), string) - val scheduledJob = ScheduledJob.parse(xcp, monitorV2.id, monitorV2.version) - assertEquals(monitorV2, scheduledJob) - } - - /* Validation Tests */ - fun `test create ppl monitor that queries nonexistent index fails`() { - val pplMonitorConfig = randomPPLMonitor( - query = "source = nonexistent_index | head 10" - ) - - // ensure the request fails - try { - createRandomPPLMonitor(pplMonitorConfig) - fail("Expected request to fail with BAD_REQUEST but it succeeded") - } catch (e: ResponseException) { - assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) - } - - // ensure no monitor was created - ensureNumMonitors(0) - } - - fun `test create ppl monitor with more than max allowed monitors fails`() { - adminClient().updateSettings(ALERTING_MAX_MONITORS.key, 1) - - createRandomPPLMonitor() - - // ensure the request fails - try { - createRandomPPLMonitor() - fail("Expected request to fail with BAD_REQUEST but it succeeded") - } catch (e: ResponseException) { - assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) - } - - // ensure no monitor was created - ensureNumMonitors(1) - } - - fun `test create ppl monitor with throttle greater than max fails`() { - val maxThrottleDuration = 60L - client().updateSettings(ALERTING_V2_MAX_THROTTLE_DURATION.key, maxThrottleDuration) - - // ensure the request fails - try { - createRandomPPLMonitor( - randomPPLMonitor( - triggers = listOf( - randomPPLTrigger(throttleDuration = maxThrottleDuration + 10) - ) - ) - ) - fail("Expected request to fail with BAD_REQUEST but it succeeded") - } catch (e: ResponseException) { - assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) - } - - // ensure no monitor was created - ensureNumMonitors(0) - } - - fun `test create ppl monitor with expire greater than max fails`() { - val maxExpireDuration = 60L - client().updateSettings(ALERTING_V2_MAX_EXPIRE_DURATION.key, maxExpireDuration) - - // ensure the request fails - try { - createRandomPPLMonitor( - randomPPLMonitor( - triggers = listOf( - randomPPLTrigger(expireDuration = maxExpireDuration + 10) - ) - ) - ) - fail("Expected request to fail with BAD_REQUEST but it succeeded") - } catch (e: ResponseException) { - assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) - } - - // ensure no monitor was created - ensureNumMonitors(0) - } - - fun `test create ppl monitor with invalid query fails`() { - // ensure the request fails - try { - createRandomPPLMonitor( - randomPPLMonitor( - query = "source = $TEST_INDEX_NAME | not valid ppl" - ) - ) - fail("Expected request to fail with BAD_REQUEST but it succeeded") - } catch (e: ResponseException) { - assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) - } - - // ensure no monitor was created - ensureNumMonitors(0) - } - - fun `test create ppl monitor with query that's too long fails`() { - adminClient().updateSettings(ALERTING_V2_MAX_QUERY_LENGTH.key, 1) - - // ensure the request fails - try { - createRandomPPLMonitor( - randomPPLMonitor( - query = "source = $TEST_INDEX_NAME | head 10" - ) - ) - fail("Expected request to fail with BAD_REQUEST but it succeeded") - } catch (e: ResponseException) { - assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) - } - - // ensure no monitor was created - ensureNumMonitors(0) - } - - fun `test create ppl monitor with invalid custom condition fails`() { - // ensure the request fails - try { - createRandomPPLMonitor( - randomPPLMonitor( - triggers = listOf( - randomPPLTrigger( - conditionType = ConditionType.CUSTOM, - customCondition = "not a valid PPL custom condition", - numResultsCondition = null, - numResultsValue = null - ) - ), - query = "source = $TEST_INDEX_NAME | head 10" - ) - ) - fail("Expected request to fail with BAD_REQUEST but it succeeded") - } catch (e: ResponseException) { - assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) - } - - // ensure no monitor was created - ensureNumMonitors(0) - } - - fun `test create ppl monitor with custom condition that evals to num not bool fails`() { - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - indexDocFromSomeTimeAgo(1, MINUTES, "abc", 1) - indexDocFromSomeTimeAgo(2, MINUTES, "abc", 2) - - // ensure the request fails - try { - createRandomPPLMonitor( - randomPPLMonitor( - triggers = listOf( - randomPPLTrigger( - conditionType = ConditionType.CUSTOM, - customCondition = "eval something = sum * 2", - numResultsCondition = null, - numResultsValue = null - ) - ), - query = "source = $TEST_INDEX_NAME | stats sum(number) as sum by abc" - ) - ) - fail("Expected request to fail with BAD_REQUEST but it succeeded") - } catch (e: ResponseException) { - assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) - } - - // ensure no monitor was created - ensureNumMonitors(0) - } - - fun `test create ppl monitor with notification subject source too long fails`() { - adminClient().updateSettings(NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH.key, 100) - - var subjectTooLong = "" - for (i in 0 until 101) { - subjectTooLong += "a" - } - - // ensure the request fails - try { - createRandomPPLMonitor( - randomPPLMonitor( - triggers = listOf( - randomPPLTrigger( - actions = listOf( - randomAction( - template = randomTemplateScript( - source = "some message" - ), - subjectTemplate = randomTemplateScript( - source = subjectTooLong - ) - ) - ) - ) - ) - ) - ) - fail("Expected request to fail with BAD_REQUEST but it succeeded") - } catch (e: ResponseException) { - assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) - } - - // ensure no monitor was created - ensureNumMonitors(0) - } - - fun `test create ppl monitor with notification message source too long fails`() { - adminClient().updateSettings(NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH.key, 1000) - - var messageTooLong = "" - for (i in 0 until 1001) { - messageTooLong += "a" - } - - // ensure the request fails - try { - createRandomPPLMonitor( - randomPPLMonitor( - triggers = listOf( - randomPPLTrigger( - actions = listOf( - randomAction( - template = randomTemplateScript( - source = messageTooLong - ), - subjectTemplate = randomTemplateScript( - source = "some subject" - ) - ) - ) - ) - ) - ) - ) - fail("Expected request to fail with BAD_REQUEST but it succeeded") - } catch (e: ResponseException) { - assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) - } - - // ensure no monitor was created - ensureNumMonitors(0) - } - - fun `test update nonexistent ppl monitor fails`() { - // the random monitor query searches index TEST_INDEX_NAME, - // so we need to create that first to ensure at least the request body is valid - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - - val monitorV2 = randomPPLMonitor() - val randomId = UUIDs.base64UUID() - - try { - client().makeRequest("PUT", "$MONITOR_BASE_URI/$randomId", emptyMap(), monitorV2.toHttpEntity()) - fail("Expected request to fail with NOT_FOUND but it succeeded") - } catch (e: ResponseException) { - assertEquals("Unexpected status", RestStatus.NOT_FOUND, e.response.restStatus()) - } - } - - fun `test delete nonexistent ppl monitor fails`() { - val randomId = UUIDs.base64UUID() - - try { - client().makeRequest("DELETE", "$MONITOR_BASE_URI/$randomId") - fail("Expected request to fail with NOT_FOUND but it succeeded") - } catch (e: ResponseException) { - assertEquals("Unexpected status", RestStatus.NOT_FOUND, e.response.restStatus()) - } - } -} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureMonitorV2RestApiIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureMonitorV2RestApiIT.kt deleted file mode 100644 index d937eeae8..000000000 --- a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureMonitorV2RestApiIT.kt +++ /dev/null @@ -1,870 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.resthandler - -import org.apache.hc.core5.http.ContentType -import org.apache.hc.core5.http.HttpHeaders -import org.apache.hc.core5.http.io.entity.StringEntity -import org.apache.hc.core5.http.message.BasicHeader -import org.junit.After -import org.junit.Before -import org.junit.BeforeClass -import org.opensearch.alerting.ALERTING_FULL_ACCESS_ROLE -import org.opensearch.alerting.ALL_ACCESS_ROLE -import org.opensearch.alerting.AlertingPlugin.Companion.MONITOR_BASE_URI -import org.opensearch.alerting.AlertingRestTestCase -import org.opensearch.alerting.PPL_FULL_ACCESS_ROLE -import org.opensearch.alerting.ROLE_TO_PERMISSION_MAPPING -import org.opensearch.alerting.TEST_INDEX_MAPPINGS -import org.opensearch.alerting.TEST_INDEX_NAME -import org.opensearch.alerting.makeRequest -import org.opensearch.alerting.randomPPLMonitor -import org.opensearch.alerting.randomPPLTrigger -import org.opensearch.client.ResponseException -import org.opensearch.client.RestClient -import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.XContentType -import org.opensearch.commons.alerting.model.IntervalSchedule -import org.opensearch.commons.alerting.model.PPLSQLTrigger.ConditionType -import org.opensearch.commons.alerting.model.PPLSQLTrigger.NumResultsCondition -import org.opensearch.commons.alerting.model.PPLSQLTrigger.TriggerMode -import org.opensearch.commons.rest.SecureRestClientBuilder -import org.opensearch.core.rest.RestStatus -import org.opensearch.index.query.QueryBuilders -import org.opensearch.search.builder.SearchSourceBuilder -import java.time.temporal.ChronoUnit.MINUTES - -/*** - * Tests Alerting V2 CRUD with role-based access control - * - * Gradle command to run this suite: - * ./gradlew :alerting:integTest -Dhttps=true -Dsecurity=true -Duser=admin -Dpassword=admin \ - * --tests "org.opensearch.alerting.resthandler.SecureMonitorV2RestApiIT" - */ -class SecureMonitorV2RestApiIT : AlertingRestTestCase() { - - companion object { - @BeforeClass - @JvmStatic fun setup() { - // things to execute once and keep around for the class - org.junit.Assume.assumeTrue(System.getProperty("security", "false")!!.toBoolean()) - } - } - - val user = "userD" - var userClient: RestClient? = null - - @Before - fun create() { - if (userClient == null) { - createUser(user, arrayOf()) - userClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), user, password) - .setSocketTimeout(60000) - .setConnectionRequestTimeout(180000) - .build() - } - } - - @After - fun cleanup() { - userClient?.close() - deleteUser(user) - } - - fun `test create monitor as user without alerting access fails`() { - if (!isHttps()) { - return - } - - val pplMonitorConfig = randomPPLMonitor() - - createUserWithTestDataAndCustomRole( - user, - TEST_INDEX_NAME, - "custom_role", - listOf(), - null - ) - - try { - createPPLIndexThenMonitorWithClient( - userClient!!, - monitor = pplMonitorConfig - ) - fail("Expected create monitor to fail as user does not have permissions to call alerting APIs") - } catch (e: ResponseException) { - assertEquals("Unexpected error status", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) - } - - ensureNumMonitors(0) - } - - fun `test create monitor that queries index user doesn't have access to fails`() { - if (!isHttps()) { - return - } - - createIndex("some_index", Settings.EMPTY) - - val pplMonitorConfig = randomPPLMonitor( - query = "source = some_index | head 10" - ) - - createUserWithTestDataAndCustomRole( - user, - "other_index", - "custom_role", - listOf(), - getClusterPermissionsFromCustomRole(ALL_ACCESS_ROLE) - ) - - try { - createPPLIndexThenMonitorWithClient( - userClient!!, - monitor = pplMonitorConfig - ) - fail("Expected create monitor to fail as user does not have permissions to index that monitor queries") - } catch (e: ResponseException) { - assertEquals("Unexpected error status", RestStatus.BAD_REQUEST.status, e.response.statusLine.statusCode) - } - - ensureNumMonitors(0) - } - - fun `test update monitor that queries index user doesn't have access to fails`() { - if (!isHttps()) { - return - } - - // RBAC is out of scope for this test, so give all users and requests the same one - val backendRole = "backend_role_a" - - val pplMonitorConfig = randomPPLMonitor() - - // first create the monitor with a user that has access to all indices - // (the FULL_ACCESS_ROLEs include full index permissions) - createUserWithRoles( - user, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf(backendRole), - false - ) - - // this function automatically creates index TEST_INDEX_NAME, then a monitor that queries it - val pplMonitor = createPPLIndexThenMonitorWithClient(userClient!!, pplMonitorConfig, listOf(backendRole)) - - /* - user: String, - index: String, - role: String, - backendRoles: List, - clusterPermissions: String?, - */ - - // prepare a user that has full access to all cluster actions, - // but only access to a specific unrelated index - val noIndicesUser = "noIndicesUser" - createUserWithTestDataAndCustomRole( - noIndicesUser, - "unrelated_index", - "unrelated_role", - listOf(backendRole), - listOf(ROLE_TO_PERMISSION_MAPPING[ALL_ACCESS_ROLE]) - ) - - val noIndicesUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), noIndicesUser, password) - .setSocketTimeout(60000) - .setConnectionRequestTimeout(180000) - .build() - - // update some field that isn't the PPL query and the index it's querying - val newMonitor = pplMonitorConfig.copy(name = "some_random_name") - - try { - // noIndicesUser, who only has access to index unrelated_index, should be blocked - // from updating a monitor that queries index TEST_INDEX_NAME because noIndicesUser - // has no access to TEST_INDEX_NAME - noIndicesUserClient!!.makeRequest( - "PUT", - "$MONITOR_BASE_URI/${pplMonitor.id}", - newMonitor.toHttpEntity(), - BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") - ) - fail("Expected update monitor to fail as user does not have permissions to index that monitor queries") - } catch (e: ResponseException) { - assertEquals("Unexpected error status", RestStatus.BAD_REQUEST.status, e.response.statusLine.statusCode) - } - - // cleanup - noIndicesUserClient.close() - } - - fun `test RBAC create monitor with backend roles user has access to succeeds`() { - enableFilterBy() - if (!isHttps()) { - return - } - - val pplMonitorConfig = randomPPLMonitor(enabled = true) - - createUserWithRoles( - user, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_a", "backend_role_b"), - false - ) - - createPPLIndexThenMonitorWithClient(userClient!!, monitor = pplMonitorConfig, listOf("backend_role_a")) - - ensureNumMonitors(1) - } - - fun `test RBAC create monitor with backend roles user has no access to fails`() { - enableFilterBy() - if (!isHttps()) { - return - } - - val pplMonitorConfig = randomPPLMonitor(enabled = true) - - createUserWithRoles( - user, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_a", "backend_role_b"), - false - ) - - try { - createPPLIndexThenMonitorWithClient( - userClient!!, - monitor = pplMonitorConfig, - listOf("backend_role_a", "backend_role_b", "backend_role_c") - ) - fail("Expected create monitor to fail as user does not have backend_role_c backend role") - } catch (e: ResponseException) { - assertEquals("Unexpected error status", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) - } - - ensureNumMonitors(0) - } - - fun `test RBAC update monitorV2 as user with correct backend roles succeeds`() { - enableFilterBy() - if (!isHttps()) { - return - } - val pplMonitorConfig = randomPPLMonitor(enabled = true) - - createUserWithRoles( - user, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_a", "backend_role_b"), - false - ) - - val pplMonitor = createPPLIndexThenMonitorWithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) - - // getUser should have access to the monitor above created by user - val updateUser = "updateUser" - - createUserWithRoles( - updateUser, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_a"), - true - ) - - val getUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), updateUser, password) - .setSocketTimeout(60000) - .setConnectionRequestTimeout(180000) - .build() - - val newMonitor = randomPPLMonitor() - val updateMonitorResponse = getUserClient!!.makeRequest( - "PUT", - "$MONITOR_BASE_URI/${pplMonitor.id}", - newMonitor.toHttpEntity(), - BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") - ) - assertEquals("Update monitorV2 failed", RestStatus.OK, updateMonitorResponse.restStatus()) - - // cleanup - getUserClient.close() - } - - fun `test RBAC update monitorV2 as user without correct backend roles fails`() { - enableFilterBy() - if (!isHttps()) { - return - } - val pplMonitorConfig = randomPPLMonitor(enabled = true) - - createUserWithRoles( - user, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_a", "backend_role_b"), - false - ) - - val pplMonitor = createPPLIndexThenMonitorWithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) - - // updateUser should have access to the monitor above created by user - val updateUser = "updateUser" - - createUserWithRoles( - updateUser, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_c"), - true - ) - - val getUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), updateUser, password) - .setSocketTimeout(60000) - .setConnectionRequestTimeout(180000) - .build() - - val newMonitor = randomPPLMonitor() - - try { - getUserClient!!.makeRequest( - "PUT", - "$MONITOR_BASE_URI/${pplMonitor.id}", - newMonitor.toHttpEntity(), - BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") - ) - fail("Expected update monitor to fail as user does not have the correct backend roles") - } catch (e: ResponseException) { - assertEquals("Unexpected error status", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) - } - - // cleanup - getUserClient.close() - } - - fun `test RBAC get monitorV2 as user with correct backend roles succeeds`() { - enableFilterBy() - if (!isHttps()) { - return - } - val pplMonitorConfig = randomPPLMonitor(enabled = true) - - createUserWithRoles( - user, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_a", "backend_role_b"), - false - ) - - val pplMonitor = createPPLIndexThenMonitorWithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) - - // getUser should have access to the monitor above created by user - val getUser = "getUser" - - createUserWithRoles( - getUser, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_a"), - true - ) - - val getUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), getUser, password) - .setSocketTimeout(60000) - .setConnectionRequestTimeout(180000) - .build() - - val getMonitorResponse = getUserClient!!.makeRequest( - "GET", - "$MONITOR_BASE_URI/${pplMonitor.id}", - null, - BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") - ) - assertEquals("Get monitorV2 failed", RestStatus.OK, getMonitorResponse.restStatus()) - - // cleanup - getUserClient.close() - } - - fun `test RBAC get monitorV2 as user without correct backend roles fails`() { - enableFilterBy() - if (!isHttps()) { - return - } - val pplMonitorConfig = randomPPLMonitor(enabled = true) - - createUserWithRoles( - user, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_a", "backend_role_b"), - false - ) - - val pplMonitor = createPPLIndexThenMonitorWithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) - - // getUser should not have access to the monitor above created by user - val getUser = "getUser" - - createUserWithRoles( - getUser, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_c"), - true - ) - - val getUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), getUser, password) - .setSocketTimeout(60000) - .setConnectionRequestTimeout(180000) - .build() - - try { - getUserClient!!.makeRequest( - "GET", - "$MONITOR_BASE_URI/${pplMonitor.id}", - null, - BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") - ) - fail("Expected Forbidden exception") - } catch (e: ResponseException) { - assertEquals("Unexpected get monitor status", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) - } finally { - getUserClient?.close() - } - } - - fun `test RBAC search monitorV2 as user with correct backend roles returns results`() { - enableFilterBy() - if (!isHttps()) { - return - } - val pplMonitorConfig = randomPPLMonitor(enabled = true) - - createUserWithRoles( - user, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_a", "backend_role_b"), - false - ) - - createPPLIndexThenMonitorWithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) - - // getUser should have access to the monitor above created by user - val searchUser = "searchUser" - - createUserWithRoles( - searchUser, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_a"), - true - ) - - val searchUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), searchUser, password) - .setSocketTimeout(60000) - .setConnectionRequestTimeout(180000) - .build() - - val search = SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).toString() - val searchMonitorResponse = searchUserClient!!.makeRequest( - "POST", - "$MONITOR_BASE_URI/_search", - StringEntity(search, ContentType.APPLICATION_JSON), - BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") - ) - assertEquals("Search monitorV2 failed", RestStatus.OK, searchMonitorResponse.restStatus()) - - createParser(XContentType.JSON.xContent(), searchMonitorResponse.entity.content).use { xcp -> - val hits = xcp.map()["hits"]!! as Map> - logger.info("hits: $hits") - val numberDocsFound = hits["total"]?.get("value") - assertEquals("Created PPL Monitor should be visible but was not", 1, numberDocsFound) - } - - // cleanup - searchUserClient.close() - } - - fun `test RBAC search monitorV2 as user without correct backend roles returns no results`() { - enableFilterBy() - if (!isHttps()) { - return - } - val pplMonitorConfig = randomPPLMonitor(enabled = true) - - createUserWithRoles( - user, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_a", "backend_role_b"), - false - ) - - createPPLIndexThenMonitorWithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) - - // getUser should have access to the monitor above created by user - val searchUser = "searchUser" - - createUserWithRoles( - searchUser, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_c"), - true - ) - - val searchUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), searchUser, password) - .setSocketTimeout(60000) - .setConnectionRequestTimeout(180000) - .build() - - val search = SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).toString() - val searchMonitorResponse = searchUserClient!!.makeRequest( - "POST", - "$MONITOR_BASE_URI/_search", - StringEntity(search, ContentType.APPLICATION_JSON), - BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") - ) - assertEquals("Search monitorV2 failed", RestStatus.OK, searchMonitorResponse.restStatus()) - - createParser(XContentType.JSON.xContent(), searchMonitorResponse.entity.content).use { xcp -> - val hits = xcp.map()["hits"]!! as Map> - val numberDocsFound = hits["total"]?.get("value") - assertEquals("Created PPL Monitor should be visible but was not", 0, numberDocsFound) - } - - // cleanup - searchUserClient.close() - } - - fun `test RBAC execute monitorV2 as user with correct backend roles succeeds`() { - enableFilterBy() - if (!isHttps()) { - return - } - val pplMonitorConfig = randomPPLMonitor(enabled = true) - - createUserWithRoles( - user, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_a", "backend_role_b"), - false - ) - - val pplMonitor = createPPLIndexThenMonitorWithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) - - // getUser should have access to the monitor above created by user - val executeUser = "executeUser" - - createUserWithRoles( - executeUser, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_a"), - true - ) - - val getUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), executeUser, password) - .setSocketTimeout(60000) - .setConnectionRequestTimeout(180000) - .build() - - val getMonitorResponse = getUserClient!!.makeRequest( - "POST", - "$MONITOR_BASE_URI/${pplMonitor.id}/_execute", - null, - BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") - ) - assertEquals("Get monitorV2 failed", RestStatus.OK, getMonitorResponse.restStatus()) - - // cleanup - getUserClient.close() - } - - fun `test RBAC execute monitorV2 as user without correct backend roles fails`() { - enableFilterBy() - if (!isHttps()) { - return - } - val pplMonitorConfig = randomPPLMonitor(enabled = true) - - createUserWithRoles( - user, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_a", "backend_role_b"), - false - ) - - val pplMonitor = createPPLIndexThenMonitorWithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) - - // getUser should not have access to the monitor above created by user - val executeUser = "executeUser" - - createUserWithRoles( - executeUser, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_c"), - true - ) - - val getUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), executeUser, password) - .setSocketTimeout(60000) - .setConnectionRequestTimeout(180000) - .build() - - try { - getUserClient!!.makeRequest( - "POST", - "$MONITOR_BASE_URI/${pplMonitor.id}/_execute", - null, - BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") - ) - fail("Expected Forbidden exception") - } catch (e: ResponseException) { - assertEquals("Unexpected delete monitor status", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) - } finally { - getUserClient?.close() - } - } - - fun `test RBAC get alerts v2 as user with correct backend roles succeeds`() { - enableFilterBy() - if (!isHttps()) { - return - } - - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - indexDocFromSomeTimeAgo(2, MINUTES, "abc", 5) - - val pplMonitorConfig = createRandomPPLMonitor( - randomPPLMonitor( - enabled = true, - schedule = IntervalSchedule(interval = 1, unit = MINUTES), - triggers = listOf( - randomPPLTrigger( - throttleDuration = null, - expireDuration = 5, - mode = TriggerMode.RESULT_SET, - conditionType = ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = NumResultsCondition.GREATER_THAN, - numResultsValue = 0L, - customCondition = null - ) - ), - query = "source = $TEST_INDEX_NAME | head 10" - ) - ) - - createUserWithRoles( - user, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_a", "backend_role_b"), - false - ) - - val pplMonitor = createPPLIndexThenMonitorWithClient( - userClient!!, - pplMonitorConfig, - null - ) - - val executeResponse = executeMonitor(pplMonitor.id) - val triggered = isTriggered(pplMonitor, executeResponse) - assertTrue(triggered) - - // TODO: creating this user overrides the ALERTING_FULL_ACCESS mapping and displaces "user" - // TODO: above, even though passing in isExistingRole = true should trigger an update - // TODO: role mappings call. doesn't block the test because "user" isn't used for the - // TODO: rest of the test, but this could lead to unexpected behavior for future test writers - // the get alerts user should be able to see the alerts - val getAlertsUser = "getAlertsUser" - createUserWithRoles( - getAlertsUser, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_a"), - true - ) - - val getAlertsUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), getAlertsUser, password) - .setSocketTimeout(60000) - .setConnectionRequestTimeout(180000) - .build() - - val getAlertsResponse = getAlertsUserClient!!.makeRequest( - "GET", - "$MONITOR_BASE_URI/alerts", - null, - BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") - ) - assertEquals("Get alerts v2 failed", RestStatus.OK, getAlertsResponse.restStatus()) - - val alertsGenerated = numAlerts(getAlertsResponse) > 0 - assert(alertsGenerated) - - // cleanup - getAlertsUserClient.close() - } - - fun `test RBAC get alerts v2 as user without correct backend roles fails`() { - enableFilterBy() - if (!isHttps()) { - return - } - - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - indexDocFromSomeTimeAgo(2, MINUTES, "abc", 5) - - val pplMonitorConfig = createRandomPPLMonitor( - randomPPLMonitor( - enabled = true, - schedule = IntervalSchedule(interval = 1, unit = MINUTES), - triggers = listOf( - randomPPLTrigger( - throttleDuration = null, - expireDuration = 5, - mode = TriggerMode.RESULT_SET, - conditionType = ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = NumResultsCondition.GREATER_THAN, - numResultsValue = 0L, - customCondition = null - ) - ), - query = "source = $TEST_INDEX_NAME | head 10" - ) - ) - - createUserWithRoles( - user, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_a", "backend_role_b"), - false - ) - - val pplMonitor = createPPLIndexThenMonitorWithClient( - userClient!!, - pplMonitorConfig, - null - ) - - val executeResponse = executeMonitor(pplMonitor.id) - val triggered = isTriggered(pplMonitor, executeResponse) - assertTrue(triggered) - - // the get alerts user should be able to see the alerts - val getAlertsUser = "getAlertsUser" - createUserWithRoles( - getAlertsUser, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_c"), - true - ) - - val getAlertsUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), getAlertsUser, password) - .setSocketTimeout(60000) - .setConnectionRequestTimeout(180000) - .build() - - val getAlertsResponse = getAlertsUserClient!!.makeRequest( - "GET", - "$MONITOR_BASE_URI/alerts", - null, - BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") - ) - assertEquals("Get alerts v2 failed", RestStatus.OK, getAlertsResponse.restStatus()) - - val alertsGenerated = numAlerts(getAlertsResponse) > 0 - assert(!alertsGenerated) - - // cleanup - getAlertsUserClient.close() - } - - fun `test RBAC delete monitorV2 as user with correct backend roles succeeds`() { - enableFilterBy() - if (!isHttps()) { - return - } - val pplMonitorConfig = randomPPLMonitor(enabled = true) - - createUserWithRoles( - user, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_a", "backend_role_b"), - false - ) - - val pplMonitor = createPPLIndexThenMonitorWithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) - - // getUser should have access to the monitor above created by user - val deleteUser = "deleteUser" - - createUserWithRoles( - deleteUser, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_a"), - true - ) - - val deleteUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), deleteUser, password) - .setSocketTimeout(60000) - .setConnectionRequestTimeout(180000) - .build() - - val getMonitorResponse = deleteUserClient!!.makeRequest( - "DELETE", - "$MONITOR_BASE_URI/${pplMonitor.id}", - null, - BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") - ) - assertEquals("Get monitorV2 failed", RestStatus.OK, getMonitorResponse.restStatus()) - - ensureNumMonitors(0) - - // cleanup - deleteUserClient.close() - } - - fun `test RBAC delete monitorV2 as user without correct backend roles fails`() { - enableFilterBy() - if (!isHttps()) { - return - } - val pplMonitorConfig = randomPPLMonitor(enabled = true) - - createUserWithRoles( - user, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_a", "backend_role_b"), - false - ) - - val pplMonitor = createPPLIndexThenMonitorWithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) - - // getUser should not have access to the monitor above created by user - val deleteUser = "deleteUser" - - createUserWithRoles( - deleteUser, - listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), - listOf("backend_role_c"), - true - ) - - val deleteUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), deleteUser, password) - .setSocketTimeout(60000) - .setConnectionRequestTimeout(180000) - .build() - - try { - deleteUserClient!!.makeRequest( - "DELETE", - "$MONITOR_BASE_URI/${pplMonitor.id}", - null, - BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") - ) - fail("Expected Forbidden exception") - } catch (e: ResponseException) { - assertEquals("Unexpected delete monitor status", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) - } finally { - deleteUserClient?.close() - } - - ensureNumMonitors(1) - } -} From 926dbed14cd5ee03dd587708058b5618f017dcbc Mon Sep 17 00:00:00 2001 From: Dennis Toepker Date: Thu, 9 Apr 2026 23:27:23 -0700 Subject: [PATCH 4/9] PPL Alerting: post load test optimization --- .../org/opensearch/alerting/AlertService.kt | 6 +- .../org/opensearch/alerting/InputService.kt | 89 ++++++++++++ .../org/opensearch/alerting/PPLUtils.kt | 7 +- .../alerting/QueryLevelMonitorRunner.kt | 119 ++++++++------- .../org/opensearch/alerting/TriggerService.kt | 137 +++++++++++------- .../script/PPLTriggerExecutionContext.kt | 34 ----- .../QueryLevelTriggerExecutionContext.kt | 9 +- .../alerting/MonitorRunnerServiceIT.kt | 44 +++++- 8 files changed, 298 insertions(+), 147 deletions(-) delete mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/script/PPLTriggerExecutionContext.kt diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertService.kt index 990b88ef9..49f495ad7 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertService.kt @@ -153,9 +153,7 @@ class AlertService( result: QueryLevelTriggerRunResult, alertError: AlertError?, executionId: String, - workflorwRunContext: WorkflowRunContext?, - monitorCtx: MonitorRunnerExecutionContext, - pplSqlQueryResult: List>? + workflorwRunContext: WorkflowRunContext? ): Alert? { val currentTime = Instant.now() val currentAlert = ctx.alert?.alert @@ -207,7 +205,7 @@ class AlertService( // populate PPL Monitor specific fields val query = if (ctx.monitor.isPplSqlMonitor()) (ctx.monitor.inputs[0] as PPLSQLInput).query else null - val queryResults = if (ctx.monitor.isPplSqlMonitor()) pplSqlQueryResult.orEmpty() else emptyList() + val queryResults = if (ctx.monitor.isPplSqlMonitor()) ctx.pplQueryResults else emptyList() // Merge the alert's error message to the current alert's history val updatedHistory = currentAlert?.errorHistory.update(alertError) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt index 3ee6d644d..e7c3e89ad 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt @@ -7,9 +7,14 @@ package org.opensearch.alerting import kotlinx.coroutines.newSingleThreadContext import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout import org.apache.logging.log4j.LogManager +import org.json.JSONObject import org.opensearch.action.search.SearchRequest import org.opensearch.action.search.SearchResponse +import org.opensearch.alerting.PPLUtils.appendDataRowsLimit +import org.opensearch.alerting.PPLUtils.capAndReformatPPLQueryResults +import org.opensearch.alerting.PPLUtils.executePplQuery import org.opensearch.alerting.opensearchapi.convertToMap import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.settings.AlertingSettings @@ -33,6 +38,7 @@ import org.opensearch.common.xcontent.XContentType import org.opensearch.commons.alerting.model.ClusterMetricsInput import org.opensearch.commons.alerting.model.InputRunResults import org.opensearch.commons.alerting.model.Monitor +import org.opensearch.commons.alerting.model.PPLSQLInput import org.opensearch.commons.alerting.model.SearchInput import org.opensearch.commons.alerting.model.TriggerAfterKey import org.opensearch.commons.alerting.model.WorkflowRunContext @@ -50,9 +56,11 @@ import org.opensearch.script.ScriptService import org.opensearch.script.ScriptType import org.opensearch.script.TemplateScript import org.opensearch.search.builder.SearchSourceBuilder +import org.opensearch.transport.TransportService import org.opensearch.transport.client.Client import java.time.Duration import java.time.Instant +import kotlin.time.measureTimedValue /** Service that handles the collection of input results for Monitor executions */ class InputService( @@ -216,6 +224,87 @@ class InputService( } } + suspend fun collectInputResultsForPPLMonitor( + monitor: Monitor, + monitorCtx: MonitorRunnerExecutionContext, + transportService: TransportService + ): InputRunResults { + return try { + // PPL Alerting: + // these query results are for number_of_results PPL triggers, + // only the number of results returned by base query matters + // for those triggers, not the contents themselves + val basePplQueryResults = runPPLBaseQuery( + monitor, + (monitor.inputs[0] as PPLSQLInput).query, + monitorCtx, + transportService + ) + val numPplResults = basePplQueryResults.getLong("total") + + // PPL Alerting: + // PPL Trigger evaluations won't read this input result in. + // for num results triggers, this is because the contents of the query results + // are unimportant, only the number of results matters. + // for custom trigger, this is because it'll be running its own query + // (base query + custom condition) and evaluating on those query results. + // thus, the size capped and reformatted base query results are included + // here to populate the final customer facing response of monitor execution + val cappedPPLBaseQueryResults = capAndReformatPPLQueryResults( + basePplQueryResults, + monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.ALERT_V2_QUERY_RESULTS_MAX_SIZE) + ) + + InputRunResults(emptyList(), null, null, cappedPPLBaseQueryResults, numPplResults) + } catch (e: Exception) { + logger.error( + "failed to run PPL Monitor base query " + + "from PPL Monitor ${monitor.name} (id: ${monitor.id}", + e + ) + InputRunResults(emptyList(), e, null, listOf(), null) + } + } + + // for PPL Monitors, the base PPL query is run once + // for number_of_results PPL triggers + private suspend fun runPPLBaseQuery( + pplSqlMonitor: Monitor, + baseQuery: String, + monitorCtx: MonitorRunnerExecutionContext, + transportService: TransportService + ): JSONObject { + + // TODO: change name to trigger max duration + val monitorExecutionDuration = monitorCtx + .clusterService!! + .clusterSettings + .get(AlertingSettings.ALERT_V2_MONITOR_EXECUTION_MAX_DURATION) + + var queryResponseJson: JSONObject? = null + + withTimeout(monitorExecutionDuration.millis) { + // limit the number of PPL query result data rows returned + val dataRowsLimit = monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS) + val limitedQueryToExecute = appendDataRowsLimit(baseQuery, dataRowsLimit) + + logger.debug("executing the base PPL query of monitor: ${pplSqlMonitor.id}") + val (queryResponseJsonReceived, timeTaken) = measureTimedValue { + executePplQuery( + limitedQueryToExecute, + monitorCtx.clusterService!!.state().nodes.localNode, + transportService + ) + } + logger.debug("base query results: $queryResponseJsonReceived") + logger.debug("time taken to execute base query against sql/ppl plugin: $timeTaken") + + queryResponseJson = queryResponseJsonReceived + } + + return queryResponseJson!! + } + fun getSearchRequest( monitor: Monitor, searchInput: SearchInput, diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt b/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt index f1021e2e3..ae10f4684 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt @@ -191,6 +191,12 @@ object PPLUtils { return evalResultVarIdx } + fun capAndReformatPPLQueryResults(rawQueryResults: JSONObject, maxSize: Long): List> { + val cappedQueryResults = capPPLQueryResultsSize(rawQueryResults, maxSize).toMap() + val reformattedQueryResults = constructPPLQueryResultsMap(cappedQueryResults) + return reformattedQueryResults + } + /** * Caps the size of PPL query results to prevent memory issues and oversized alert payloads. * @@ -236,7 +242,6 @@ object PPLUtils { val limitExceedMessageQueryResults = JSONObject() val schema = JSONArray().put(JSONObject(mapOf("name" to "message", "type" to "string"))) -// val schema = JSONArray(pplQueryResults.getJSONArray("schema").toList()) val datarows = JSONArray().put(JSONArray(listOf(PPL_RESULTS_SIZE_EXCEEDED_MESSAGE))) limitExceedMessageQueryResults.put("schema", schema) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/QueryLevelMonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/QueryLevelMonitorRunner.kt index a45eb82a7..a1625a40f 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/QueryLevelMonitorRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/QueryLevelMonitorRunner.kt @@ -6,9 +6,6 @@ package org.opensearch.alerting import org.apache.logging.log4j.LogManager -import org.json.JSONObject -import org.opensearch.alerting.PPLUtils.capPPLQueryResultsSize -import org.opensearch.alerting.PPLUtils.constructPPLQueryResultsMap import org.opensearch.alerting.model.AlertContext import org.opensearch.alerting.opensearchapi.InjectorContextElement import org.opensearch.alerting.opensearchapi.withClosableContext @@ -18,11 +15,11 @@ import org.opensearch.alerting.trigger.RemoteQueryLevelTriggerEvaluator import org.opensearch.alerting.util.CommentsUtils import org.opensearch.alerting.util.isADMonitor import org.opensearch.commons.alerting.model.Alert -import org.opensearch.commons.alerting.model.InputRunResults import org.opensearch.commons.alerting.model.Monitor import org.opensearch.commons.alerting.model.MonitorRunResult import org.opensearch.commons.alerting.model.PPLSQLInput import org.opensearch.commons.alerting.model.PPLSQLTrigger +import org.opensearch.commons.alerting.model.PPLSQLTrigger.ConditionType import org.opensearch.commons.alerting.model.QueryLevelTrigger import org.opensearch.commons.alerting.model.QueryLevelTriggerRunResult import org.opensearch.commons.alerting.model.SearchInput @@ -65,7 +62,26 @@ object QueryLevelMonitorRunner : MonitorRunner() { logger.error("Error loading alerts for monitor: $id", e) return monitorResult.copy(error = e) } - if (!isADMonitor(monitor)) { + + if (isADMonitor(monitor)) { + monitorResult = monitorResult.copy( + inputResults = monitorCtx.inputService!!.collectInputResultsForADMonitor(monitor, periodStart, periodEnd) + ) + } else if (monitor.isPplSqlMonitor()) { + withClosableContext( + InjectorContextElement( + monitor.id, + monitorCtx.settings!!, + monitorCtx.threadPool!!.threadContext, + monitor.user?.roles, + monitor.user + ) + ) { + monitorResult = monitorResult.copy( + inputResults = monitorCtx.inputService!!.collectInputResultsForPPLMonitor(monitor, monitorCtx, transportService) + ) + } + } else { withClosableContext( InjectorContextElement( monitor.id, @@ -75,26 +91,16 @@ object QueryLevelMonitorRunner : MonitorRunner() { monitor.user ) ) { - // If monitor is Query-level, proceed with collecting input results as usual. - // If monitor is PPLSQL Monitor, input results have to be collected during - // trigger execution because the PPL query run can be different per trigger. - // In the PPLSQL case, delay storing the input results until after trigger execution. - if (!monitor.isPplSqlMonitor()) { - monitorResult = monitorResult.copy( - inputResults = monitorCtx.inputService!!.collectInputResults( - monitor, - periodStart, - periodEnd, - null, - workflowRunContext - ) + monitorResult = monitorResult.copy( + inputResults = monitorCtx.inputService!!.collectInputResults( + monitor, + periodStart, + periodEnd, + null, + workflowRunContext ) - } + ) } - } else { - monitorResult = monitorResult.copy( - inputResults = monitorCtx.inputService!!.collectInputResultsForADMonitor(monitor, periodStart, periodEnd) - ) } val updatedAlerts = mutableListOf() @@ -127,10 +133,6 @@ object QueryLevelMonitorRunner : MonitorRunner() { maxComments ) - // if this is a PPL/SQL Monitor run, this will be populated with - // the query results for each trigger - val pplSqlQueryResults = mutableMapOf>() - for (trigger in monitor.triggers) { val currentAlert = currentAlerts[trigger] val currentAlertContext = currentAlert?.let { @@ -163,14 +165,24 @@ object QueryLevelMonitorRunner : MonitorRunner() { else monitorCtx.triggerService!!.runQueryLevelTrigger(monitor, trigger as QueryLevelTrigger, triggerCtx) } Monitor.MonitorType.PPL_MONITOR -> { - monitorCtx.triggerService!!.runPplSqlTrigger( - monitor, - trigger as PPLSQLTrigger, - (monitor.inputs[0] as PPLSQLInput).query, - pplSqlQueryResults, - monitorCtx, - transportService - ) + val pplSqlTrigger = trigger as PPLSQLTrigger + + if (pplSqlTrigger.conditionType == ConditionType.NUMBER_OF_RESULTS) { + // number of results trigger case + monitorCtx.triggerService!!.runPplSqlNumResultsTrigger( + pplSqlTrigger, + monitorResult.inputResults.pplBaseQueryNumResults + ) + } else { + // custom condition trigger case + monitorCtx.triggerService!!.runPplSqlCustomTrigger( + monitor, + pplSqlTrigger, + (monitor.inputs[0] as PPLSQLInput).query, + monitorCtx, + transportService + ) + } } else -> throw IllegalArgumentException("Unsupported monitor type: ${monitor.monitorType}.") @@ -179,17 +191,26 @@ object QueryLevelMonitorRunner : MonitorRunner() { triggerResults[trigger.id] = triggerResult - // the query results passed into notifications and alerts must be size-capped and reformatted - val sizeCappedFormattedPPLQueryResults = pplSqlQueryResults[trigger.id]?.let { - val sizeCappedRows = capPPLQueryResultsSize( - JSONObject(it), - monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.ALERT_V2_QUERY_RESULTS_MAX_SIZE) - ) - constructPPLQueryResultsMap(sizeCappedRows.toMap()) + // PPL Alerting: + // what query results get stored in the Alert and Notification depends on the trigger type. + // if number of results: evaluated on base query, so base query results are included. + // if custom: the custom trigger ran its own query (base query + custom condition), so that query's results are included + // trigger is not PPLSQLTrigger: this is a query-level monitor run, simply include an empty list + val pplQueryResultsToInclude = if (trigger is PPLSQLTrigger && trigger.conditionType == ConditionType.NUMBER_OF_RESULTS) { + monitorResult.inputResults.pplBaseQueryResults + } else if (trigger is PPLSQLTrigger && trigger.conditionType == ConditionType.CUSTOM) { + triggerResult.pplCustomQueryResults + } else { + listOf() } + // PPL Alerting: + // triggerCtx hasn't been populated yet with the correct PPL query results + // to include in the notification, so populate that here. + // if this is a query-level monitor run, triggerCtx.pplQueryResults simply + // stays an empty list val postRunTriggerCtx = triggerCtx.copy( - pplSqlQueryResult = sizeCappedFormattedPPLQueryResults + pplQueryResults = pplQueryResultsToInclude ) if (monitorCtx.triggerService!!.isQueryLevelTriggerActionable(triggerCtx, triggerResult, workflowRunContext)) { @@ -204,21 +225,11 @@ object QueryLevelMonitorRunner : MonitorRunner() { triggerResult, monitorResult.alertError() ?: triggerResult.alertError(), executionId, - workflowRunContext, - monitorCtx, - sizeCappedFormattedPPLQueryResults + workflowRunContext ) if (updatedAlert != null) updatedAlerts += updatedAlert } - // store input results after trigger runs, as each trigger could have - // run a different query - if (monitor.isPplSqlMonitor()) { - monitorResult = monitorResult.copy( - inputResults = InputRunResults(results = pplSqlQueryResults.values.toList()) - ) - } - // Don't save alerts if this is a test monitor if (!dryrun && monitor.id != Monitor.NO_ID) { monitorCtx.retryPolicy?.let { diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt index 1d8e27c52..8620ce999 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager import org.json.JSONObject import org.opensearch.alerting.PPLUtils.appendCustomCondition import org.opensearch.alerting.PPLUtils.appendDataRowsLimit +import org.opensearch.alerting.PPLUtils.capAndReformatPPLQueryResults import org.opensearch.alerting.PPLUtils.executePplQuery import org.opensearch.alerting.PPLUtils.findEvalResultVar import org.opensearch.alerting.PPLUtils.findEvalResultVarIdxInSchema @@ -40,7 +41,6 @@ import org.opensearch.commons.alerting.model.DocumentLevelTrigger import org.opensearch.commons.alerting.model.DocumentLevelTriggerRunResult import org.opensearch.commons.alerting.model.Monitor import org.opensearch.commons.alerting.model.PPLSQLTrigger -import org.opensearch.commons.alerting.model.PPLSQLTrigger.ConditionType import org.opensearch.commons.alerting.model.PPLSQLTrigger.NumResultsCondition import org.opensearch.commons.alerting.model.QueryLevelTrigger import org.opensearch.commons.alerting.model.QueryLevelTriggerRunResult @@ -251,31 +251,95 @@ class TriggerService(val scriptService: ScriptService) { return keyValuesList } - suspend fun runPplSqlTrigger( + fun runPplSqlNumResultsTrigger( + pplSqlTrigger: PPLSQLTrigger, + numResults: Long? + ): QueryLevelTriggerRunResult { + + if (numResults == null) { + return QueryLevelTriggerRunResult( + pplSqlTrigger.name, + true, + IllegalStateException("Did not receive a number of results from PPL query execution: ${pplSqlTrigger.id}") + ) + } + + if (pplSqlTrigger.numResultsCondition == null) { + return QueryLevelTriggerRunResult( + pplSqlTrigger.name, + true, + IllegalStateException("No number of results condition found for trigger: ${pplSqlTrigger.id}") + ) + } + + if (pplSqlTrigger.numResultsValue == null) { + return QueryLevelTriggerRunResult( + pplSqlTrigger.name, + true, + IllegalStateException("No number of results value found for trigger: ${pplSqlTrigger.id}") + ) + } + + val numResultsCondition = pplSqlTrigger.numResultsCondition!! + val numResultsValue = pplSqlTrigger.numResultsValue!! + + val triggered = when (numResultsCondition) { + NumResultsCondition.GREATER_THAN -> numResults > numResultsValue + NumResultsCondition.GREATER_THAN_EQUAL -> numResults >= numResultsValue + NumResultsCondition.LESS_THAN -> numResults < numResultsValue + NumResultsCondition.LESS_THAN_EQUAL -> numResults <= numResultsValue + NumResultsCondition.EQUAL -> numResults == numResultsValue + NumResultsCondition.NOT_EQUAL -> numResults != numResultsValue + } + + logger.debug("Number of Results PPLTrigger ${pplSqlTrigger.name} with ID ${pplSqlTrigger.id} triggered: $triggered") + + // unlike evaluating custom conditions, where we must include the query results because + // a custom condition executes its own PPL query, number of results trigger evaluation + // doesn't need to include query results because they're evaluated purely on the size + // of the results. the results themselves are already held in QueryLevelMonitorRunner.kt + // (the caller of this function), so passing the results here only to return them again + // inside the QueryLevelTriggerRunResult would be redundant + return QueryLevelTriggerRunResult(pplSqlTrigger.name, triggered, null) + } + + suspend fun runPplSqlCustomTrigger( pplSqlMonitor: Monitor, pplSqlTrigger: PPLSQLTrigger, query: String, - pplSqlQueryResults: MutableMap>, monitorCtx: MonitorRunnerExecutionContext, transportService: TransportService ): QueryLevelTriggerRunResult { + + if (pplSqlTrigger.customCondition == null) { + return QueryLevelTriggerRunResult( + pplSqlTrigger.name, + true, + IllegalStateException("No custom condition found for trigger: ${pplSqlTrigger.id}") + ) + } + // TODO: change name to trigger max duration val monitorExecutionDuration = monitorCtx .clusterService!! .clusterSettings .get(AlertingSettings.ALERT_V2_MONITOR_EXECUTION_MAX_DURATION) + val queryResultsSizeLimit = monitorCtx + .clusterService!! + .clusterSettings + .get(AlertingSettings.ALERT_V2_QUERY_RESULTS_MAX_SIZE) var triggered: Boolean? = null + var customConditionQueryResults: List>? = null try { withTimeout(monitorExecutionDuration.millis) { logger.debug("checking if custom condition is used and appending to base query") - // if trigger uses custom condition, append the custom condition to query, otherwise simply proceed - val queryToExecute = if (pplSqlTrigger.conditionType == ConditionType.NUMBER_OF_RESULTS) { // number of results trigger - query - } else { // custom condition trigger - appendCustomCondition(query, pplSqlTrigger.customCondition!!) - } + + val customCondition = pplSqlTrigger.customCondition!! + + // append the custom condition to query + val queryToExecute = appendCustomCondition(query, customCondition) // limit the number of PPL query result data rows returned val dataRowsLimit = monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS) @@ -287,7 +351,7 @@ class TriggerService(val scriptService: ScriptService) { // in the alert and notification must be added that results were excluded // and an alert that should have been generated might not have been - logger.debug("executing the PPL query of monitor: ${pplSqlMonitor.id}") + logger.debug("executing the PPL query of monitor: ${pplSqlMonitor.id} with custom condition: $customCondition") // execute the PPL query val (queryResponseJson, timeTaken) = measureTimedValue { withClosableContext( @@ -309,34 +373,25 @@ class TriggerService(val scriptService: ScriptService) { logger.debug("query results for trigger ${pplSqlTrigger.id}: $queryResponseJson") logger.debug("time taken to execute query against sql/ppl plugin: $timeTaken") - // store the query results for Execute Monitor API response - // unlike the query results stored in alerts and notifications, which must be size capped - // (because they will be stored in the OpenSearch cluster or sent as notification) and must be based - // on only the query results that met the trigger condition (because alerts should generate - // on query results that met trigger condition, not those that didn't), the pplQueryResults - // here will be returned as part of the Execute Monitor API response. This will return the original, - // untouched set of query results, and whether this causes size exceed errors is deferred - // to HTTP's response size limits - pplSqlQueryResults[pplSqlTrigger.id] = queryResponseJson.toMap() - - // determine if the trigger condition has been met - triggered = if (pplSqlTrigger.conditionType == ConditionType.NUMBER_OF_RESULTS) { // number of results trigger - evaluateNumResultsTrigger( - queryResponseJson, - pplSqlTrigger.numResultsCondition!!, - pplSqlTrigger.numResultsValue!! - ) - } else { // custom condition trigger - evaluateCustomTrigger(queryResponseJson, pplSqlTrigger.customCondition!!) - } + // determine if the custom trigger condition was met + triggered = evaluateCustomTrigger(queryResponseJson, pplSqlTrigger.customCondition!!) + + // cap and reformat the results to be included in trigger run result + customConditionQueryResults = capAndReformatPPLQueryResults(queryResponseJson, queryResultsSizeLimit) - logger.debug("PPLTrigger ${pplSqlTrigger.name} with ID ${pplSqlTrigger.id} triggered: $triggered") + logger.debug("Custom PPLTrigger ${pplSqlTrigger.name} with ID ${pplSqlTrigger.id} triggered: $triggered") } - return QueryLevelTriggerRunResult(pplSqlTrigger.name, triggered!!, null) + return QueryLevelTriggerRunResult( + pplSqlTrigger.name, + triggered!!, + null, + mutableMapOf(), + customConditionQueryResults!! + ) } catch (e: Exception) { logger.error( - "failed to run PPL Trigger ${pplSqlTrigger.name} (id: ${pplSqlTrigger.id} " + + "failed to run PPL Custom Trigger ${pplSqlTrigger.name} (id: ${pplSqlTrigger.id} " + "from PPL Monitor ${pplSqlMonitor.name} (id: ${pplSqlMonitor.id}", e ) @@ -345,22 +400,6 @@ class TriggerService(val scriptService: ScriptService) { } } - private fun evaluateNumResultsTrigger( - pplQueryResponse: JSONObject, - numResultsCondition: NumResultsCondition, - numResultsValue: Long - ): Boolean { - val numResults = pplQueryResponse.getLong("total") - return when (numResultsCondition) { - NumResultsCondition.GREATER_THAN -> numResults > numResultsValue - NumResultsCondition.GREATER_THAN_EQUAL -> numResults >= numResultsValue - NumResultsCondition.LESS_THAN -> numResults < numResultsValue - NumResultsCondition.LESS_THAN_EQUAL -> numResults <= numResultsValue - NumResultsCondition.EQUAL -> numResults == numResultsValue - NumResultsCondition.NOT_EQUAL -> numResults != numResultsValue - } - } - private fun evaluateCustomTrigger(pplQueryResponse: JSONObject, customCondition: String): Boolean { // find the name of the eval result variable defined in custom condition val evalResultVarName = findEvalResultVar(customCondition) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/script/PPLTriggerExecutionContext.kt b/alerting/src/main/kotlin/org/opensearch/alerting/script/PPLTriggerExecutionContext.kt deleted file mode 100644 index ced650c73..000000000 --- a/alerting/src/main/kotlin/org/opensearch/alerting/script/PPLTriggerExecutionContext.kt +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.alerting.script - -import org.opensearch.common.settings.ClusterSettings -import org.opensearch.commons.alerting.model.Monitor -import org.opensearch.commons.alerting.model.PPLSQLTrigger -import org.opensearch.commons.alerting.model.PPLSQLTrigger.Companion.PPL_SQL_TRIGGER_FIELD -import java.time.Instant - -data class PPLTriggerExecutionContext( - override val monitor: Monitor, - override val error: Exception? = null, - override val results: List>, - override val periodStart: Instant, - override val periodEnd: Instant, - override val clusterSettings: ClusterSettings, - val pplTrigger: PPLSQLTrigger, -) : TriggerExecutionContext(monitor, results, periodStart, periodEnd, error, clusterSettings) { - - override fun asTemplateArg(): Map { - val templateArg = super.asTemplateArg().toMutableMap() - templateArg[PPL_SQL_TRIGGER_FIELD] = pplTrigger.asTemplateArg() - templateArg[PPL_QUERY_RESULTS_FIELD] = results[0] // PPL/SQL Monitors only ever return one set of results - return templateArg.toMap() - } - - companion object { - const val PPL_QUERY_RESULTS_FIELD = "ppl_query_results" - } -} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/script/QueryLevelTriggerExecutionContext.kt b/alerting/src/main/kotlin/org/opensearch/alerting/script/QueryLevelTriggerExecutionContext.kt index 097564abf..23bdf4704 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/script/QueryLevelTriggerExecutionContext.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/script/QueryLevelTriggerExecutionContext.kt @@ -19,12 +19,12 @@ data class QueryLevelTriggerExecutionContext( override val monitor: Monitor, val trigger: Trigger, override val results: List>, + val pplQueryResults: List>, // each list element is a result row override val periodStart: Instant, override val periodEnd: Instant, val alert: AlertContext? = null, override val error: Exception? = null, override val clusterSettings: ClusterSettings, - val pplSqlQueryResult: List>? = null, // each list element is a result row ) : TriggerExecutionContext(monitor, results, periodStart, periodEnd, error, clusterSettings) { init { @@ -44,12 +44,13 @@ data class QueryLevelTriggerExecutionContext( monitor, trigger, monitorRunResult.inputResults.results, + // PPL Alerting: this empty list is overridden post PPL Trigger execution + listOf(), monitorRunResult.periodStart, monitorRunResult.periodEnd, alertContext, monitorRunResult.scriptContextError(trigger), - clusterSettings, - null + clusterSettings ) /** @@ -60,7 +61,7 @@ data class QueryLevelTriggerExecutionContext( val tempArg = super.asTemplateArg().toMutableMap() tempArg["trigger"] = trigger.asTemplateArg() tempArg["alert"] = alert?.asTemplateArg() // map "alert" templateArg field to AlertContext wrapper instead of Alert object - tempArg["ppl_query_results"] = pplSqlQueryResult + tempArg["ppl_query_results"] = pplQueryResults return tempArg } } diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt index a9b7b97d6..b16ac3a45 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt @@ -2232,7 +2232,7 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() { } } - fun `test execute PPL monitor execution timeout generates error alert`() { + fun `test execute num results PPL monitor execution timeout generates error alert`() { // Create test index with PPL-compatible mappings createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) indexDocFromSomeTimeAgo(2, MINUTES, "abc", 5) @@ -2274,6 +2274,48 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() { verifyAlert(alerts.single(), monitor, ERROR) } + fun `test execute custom condition PPL monitor execution timeout generates error alert`() { + // Create test index with PPL-compatible mappings + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(2, MINUTES, "abc", 5) + + // Create PPL monitor with NUMBER_OF_RESULTS trigger condition + val monitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + triggers = listOf( + randomPPLTrigger( + conditionType = PPLSQLTrigger.ConditionType.CUSTOM, + customCondition = "eval result = max_num > 5", + numResultsCondition = null, + numResultsValue = null + ) + ), + query = "source = $TEST_INDEX_NAME | stats max(number) as max_num by abc" + ) + ) + + // Set monitor execution timeout to 1 nanosecond to force a timeout + adminClient().updateSettings(AlertingSettings.ALERT_V2_MONITOR_EXECUTION_MAX_DURATION.key, TimeValue.timeValueNanos(1)) + + val response = executeMonitor(monitor.id) + + val output = entityAsMap(response) + assertEquals(monitor.name, output["monitor_name"]) + + // Verify execute response contains error + @Suppress("UNCHECKED_CAST") + for (triggerResult in output.objectMap("trigger_results").values) { + assertTrue("Missing trigger error message", (triggerResult?.get("error") as? String)?.isNotEmpty() == true) + } + + // Verify ERROR alert was created + val alerts = searchAlerts(monitor) + assertEquals("Alert not saved", 1, alerts.size) + verifyAlert(alerts.single(), monitor, ERROR) + } + fun `test execute PPL monitor with size exceeded query results`() { // Create test index with PPL-compatible mappings createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) From 2fd81b1a6be4f38ec18b7a7759457a9b117651fd Mon Sep 17 00:00:00 2001 From: Dennis Toepker Date: Mon, 13 Apr 2026 16:57:40 -0700 Subject: [PATCH 5/9] PPL Alerting: optimizing PPL Monitor create validation queries with explain instead of executing the queries --- .../org/opensearch/alerting/InputService.kt | 3 +- .../org/opensearch/alerting/PPLUtils.kt | 108 +++--------- .../org/opensearch/alerting/TriggerService.kt | 31 +--- .../transport/TransportIndexMonitorAction.kt | 46 ++---- .../alerting/MonitorRunnerServiceIT.kt | 28 +--- .../org/opensearch/alerting/PPLUtilsTests.kt | 155 +----------------- .../alerting/resthandler/MonitorRestApiIT.kt | 31 +--- .../alerting/core/ppl/PPLPluginInterface.kt | 5 + 8 files changed, 68 insertions(+), 339 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt index e7c3e89ad..686842db4 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt @@ -266,7 +266,7 @@ class InputService( } } - // for PPL Monitors, the base PPL query is run once + // for PPL Monitor execution, the base PPL query is run once // for number_of_results PPL triggers private suspend fun runPPLBaseQuery( pplSqlMonitor: Monitor, @@ -292,6 +292,7 @@ class InputService( val (queryResponseJsonReceived, timeTaken) = measureTimedValue { executePplQuery( limitedQueryToExecute, + false, monitorCtx.clusterService!!.state().nodes.localNode, transportService ) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt b/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt index ae10f4684..7cb9f85bd 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt @@ -15,12 +15,14 @@ import org.opensearch.transport.TransportService object PPLUtils { - // TODO: these are in-house PPL query parsers, find a PPL plugin dependency that does this for us - /* Regular Expressions */ - // captures the name of the result variable in a PPL monitor's custom condition - // e.g. custom condition: `eval apple = avg_latency > 100` - // captures: "apple" - private val evalResultVarRegex = """^(?!.*\|)\s*(?i:eval)\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*=""".toRegex() +// // TODO: these are in-house PPL query parsers, find a PPL plugin dependency that does this for us +// /* Regular Expressions */ +// // captures the name of the result variable in a PPL monitor's custom condition +// // e.g. custom condition: `eval apple = avg_latency > 100` +// // captures: "apple" +// private val evalResultVarRegex = """^(?!.*\|)\s*(?i:eval)\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*=""".toRegex() + + private val customConditionValidationRegex = """^\s*where\s+.+""".toRegex() const val PPL_RESULTS_SIZE_EXCEEDED_MESSAGE = "The PPL Query results were too large and thus excluded." @@ -73,6 +75,10 @@ object PPLUtils { return "$query | head $maxDataRows" } + fun customConditionIsValid(customCondition: String): Boolean { + return customCondition.matches(customConditionValidationRegex) + } + /** * Executes a PPL query and returns the response as a parsable JSONObject. * @@ -80,7 +86,9 @@ object PPLUtils { * and parses the response into a structured JSON format suitable for trigger evaluation * * @param query The PPL query string to execute - * @param client The NodeClient used to communicate with the PPL plugin + * @param explain true if the query should just be explained, false if the query should be executed + * @param localNode The node within which the request will be serviced + * @param transportService The transport service used to run the request * @return A JSONObject containing the query execution results * * @throws Exception if the query execution fails or the response cannot be parsed as JSON @@ -90,14 +98,21 @@ object PPLUtils { */ suspend fun executePplQuery( query: String, + explain: Boolean, localNode: DiscoveryNode, transportService: TransportService ): JSONObject { + val path = if (explain) { + "/_plugins/_ppl/_explain" + } else { + "/_plugins/_ppl" + } + // call PPL plugin to execute query val transportPplQueryRequest = TransportPPLQueryRequest( query, JSONObject(mapOf("query" to query)), - null // null path falls back to a default path internal to SQL/PPL Plugin + path ) val transportPplQueryResponse = PPLPluginInterface.suspendUntil { @@ -114,83 +129,6 @@ object PPLUtils { return queryResponseJson } - /** - * Searches a custom condition eval statement for the name of the eval result variable. - * - * Parses a PPL eval expression to extract the variable name being assigned. The eval - * statement must follow the format: `eval = `. This variable - * name is needed to reference the evaluation result in subsequent trigger condition checks. - * - * @param customCondition The PPL custom condition string containing an eval statement (e.g. eval result = avg > 3) - * @return The name of the eval result variable - * @throws IllegalArgumentException if no valid eval statement is found or the syntax is invalid - * - * @example - * ``` - * val condition = "eval error_rate = errors / total" - * val varName = findEvalResultVar(condition) - * // Returns: "error_rate" - * ``` - * - * @note A precheck of the base query + custom condition is assumed to have been done already. - * The function thus expects the PPL keyword "eval" followed by whitespace. Without the - * whitespace (e.g., "evalresult"), the PPL plugin would have thrown a syntax error - * during upstream validations - * @note Variable names must follow standard identifier rules: start with a letter or underscore, - * followed by letters, digits, or underscores (matching `[a-zA-Z_][a-zA-Z0-9_]*`). - * - * TODO: Replace this in-house parser with a PPL plugin dependency that provides proper - * query parsing functionality. - */ - fun findEvalResultVar(customCondition: String): String { - // TODO: these are in-house PPL query parsers, find a PPL plugin dependency that does this for us - val evalResultVar = evalResultVarRegex.find(customCondition)?.groupValues?.get(1) - ?: throw IllegalArgumentException("Given custom condition is invalid, could not find eval result variable") - return evalResultVar - } - - /** - * Finds the index of the eval result variable in the PPL query response schema. - * - * Searches through the schema array in the PPL query response to locate the column - * corresponding to the eval result variable. This index is used to extract the - * eval result values from the datarows in the query response. - * - * @param customConditionQueryResponse The JSONObject containing the PPL query response - * with "schema" and "datarows" fields - * @param evalResultVarName The name of the eval result variable to locate in the schema - * @return The zero-based index of the eval result variable in the schema array - * @throws IllegalStateException if the eval result variable is not found in the schema - * - * @note The eval result variable should always be present in the schema if the query - * executed successfully. If not found, this indicates an unexpected state. - * @note The query response schema is assumed to follow PPL plugin Execute API response schema - */ - fun findEvalResultVarIdxInSchema(customConditionQueryResponse: JSONObject, evalResultVarName: String): Int { - // find the index eval statement result variable in the PPL query response schema - val schemaList = customConditionQueryResponse.getJSONArray("schema") - var evalResultVarIdx = -1 - for (i in 0 until schemaList.length()) { - val schemaObj = schemaList.getJSONObject(i) - val columnName = schemaObj.getString("name") - - if (columnName == evalResultVarName) { - evalResultVarIdx = i - break - } - } - - // eval statement result variable should always be found - if (evalResultVarIdx == -1) { - throw IllegalStateException( - "Expected to find eval statement results variable \"$evalResultVarName\" in results " + - "of PPL query with custom condition, but did not." - ) - } - - return evalResultVarIdx - } - fun capAndReformatPPLQueryResults(rawQueryResults: JSONObject, maxSize: Long): List> { val cappedQueryResults = capPPLQueryResultsSize(rawQueryResults, maxSize).toMap() val reformattedQueryResults = constructPPLQueryResultsMap(cappedQueryResults) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt index 8620ce999..4dbfd7ef1 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt @@ -7,13 +7,10 @@ package org.opensearch.alerting import kotlinx.coroutines.withTimeout import org.apache.logging.log4j.LogManager -import org.json.JSONObject import org.opensearch.alerting.PPLUtils.appendCustomCondition import org.opensearch.alerting.PPLUtils.appendDataRowsLimit import org.opensearch.alerting.PPLUtils.capAndReformatPPLQueryResults import org.opensearch.alerting.PPLUtils.executePplQuery -import org.opensearch.alerting.PPLUtils.findEvalResultVar -import org.opensearch.alerting.PPLUtils.findEvalResultVarIdxInSchema import org.opensearch.alerting.chainedAlertCondition.parsers.ChainedAlertExpressionParser import org.opensearch.alerting.opensearchapi.InjectorContextElement import org.opensearch.alerting.opensearchapi.withClosableContext @@ -365,6 +362,7 @@ class TriggerService(val scriptService: ScriptService) { ) { executePplQuery( limitedQueryToExecute, + false, monitorCtx.clusterService!!.state().nodes.localNode, transportService ) @@ -373,8 +371,12 @@ class TriggerService(val scriptService: ScriptService) { logger.debug("query results for trigger ${pplSqlTrigger.id}: $queryResponseJson") logger.debug("time taken to execute query against sql/ppl plugin: $timeTaken") - // determine if the custom trigger condition was met - triggered = evaluateCustomTrigger(queryResponseJson, pplSqlTrigger.customCondition!!) + // val numPplResults = basePplQueryResults.getLong("total") + + // the custom condition query returns all buckets that met the custom condition, + // so if there are any results at all, the custom condition was met for at least one bukcet, + // this trigger has triggered. + triggered = queryResponseJson.getLong("total") > 0 // cap and reformat the results to be included in trigger run result customConditionQueryResults = capAndReformatPPLQueryResults(queryResponseJson, queryResultsSizeLimit) @@ -399,23 +401,4 @@ class TriggerService(val scriptService: ScriptService) { return QueryLevelTriggerRunResult(pplSqlTrigger.name, true, e) } } - - private fun evaluateCustomTrigger(pplQueryResponse: JSONObject, customCondition: String): Boolean { - // find the name of the eval result variable defined in custom condition - val evalResultVarName = findEvalResultVar(customCondition) - - // find the index eval statement result variable in the PPL query response schema - val evalResultVarIdx = findEvalResultVarIdxInSchema(pplQueryResponse, evalResultVarName) - - val dataRowList = pplQueryResponse.getJSONArray("datarows") - for (i in 0 until dataRowList.length()) { - val dataRow = dataRowList.getJSONArray(i) - val evalResult = dataRow.getBoolean(evalResultVarIdx) - if (evalResult) { - return true - } - } - - return false - } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt index cfb7e6051..9ef391e0d 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt @@ -32,9 +32,8 @@ import org.opensearch.alerting.AlertingPlugin import org.opensearch.alerting.MonitorMetadataService import org.opensearch.alerting.PPLUtils.appendCustomCondition import org.opensearch.alerting.PPLUtils.appendDataRowsLimit +import org.opensearch.alerting.PPLUtils.customConditionIsValid import org.opensearch.alerting.PPLUtils.executePplQuery -import org.opensearch.alerting.PPLUtils.findEvalResultVar -import org.opensearch.alerting.PPLUtils.findEvalResultVarIdxInSchema import org.opensearch.alerting.core.ScheduledJobIndices import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.service.DeleteMonitorService @@ -284,7 +283,7 @@ class TransportIndexMonitorAction @Inject constructor( user: User? ) { // declare upfront the validation listener that will move on to the next phase - // of monitor indexing after validations are complete + // of monitor indexing after PPL validations are complete val validationListener = object : ActionListener { // validationListener override fun onResponse(response: Unit) { // user permissions to indices have already been checked if we made it to the onResponse(), @@ -350,10 +349,10 @@ class TransportIndexMonitorAction @Inject constructor( val limitedQueryToExecute = appendDataRowsLimit(query, maxQueryResults) - // now run the base query as is. + // now PPL explain the base query as is. // if there are any PPL syntax, index not found, insufficient index permissions, or other errors, // this will throw an exception from the SQL/PPL plugin - executePplQuery(limitedQueryToExecute, clusterService.state().nodes.localNode, transportService) + executePplQuery(limitedQueryToExecute, true, clusterService.state().nodes.localNode, transportService) // scan all the triggers with custom conditions, and ensure each query constructed // from the base query + custom condition is valid @@ -364,38 +363,28 @@ class TransportIndexMonitorAction @Inject constructor( continue } - val evalResultVar = findEvalResultVar(pplTrigger.customCondition!!) + val customCondition = pplTrigger.customCondition!! - val queryWithCustomCondition = appendCustomCondition(query, pplTrigger.customCondition!!) - val limitedQueryWithCustomCondition = appendDataRowsLimit(queryWithCustomCondition, maxQueryResults) - - // if the custom condition is invalid, this will throw an exception - // from the SQL/PPL plugin - val executePplQueryResponse = executePplQuery( - limitedQueryWithCustomCondition, - clusterService.state().nodes.localNode, - transportService - ) - - val evalResultVarIdx = findEvalResultVarIdxInSchema(executePplQueryResponse, evalResultVar) - - val resultVarType = executePplQueryResponse - .getJSONArray("schema") - .getJSONObject(evalResultVarIdx) - .getString("type") - - // custom conditions must evaluate to a boolean result, otherwise it's invalid - if (resultVarType != "boolean") { + // validate the custom condition is a where statement and + // not some other valid PPL statement + if (!customConditionIsValid(customCondition)) { validationListener.onFailure( AlertingException.wrap( IllegalArgumentException( - "Custom condition in trigger ${pplTrigger.name} is invalid because it does not " + - "evaluate to a boolean, but instead to type: $resultVarType" + "Custom condition for trigger ${trigger.id} is invalid, " + + "custom condition must be a valid PPL where statement." ) ) ) return false } + + val queryWithCustomCondition = appendCustomCondition(query, customCondition) + val limitedQueryWithCustomCondition = appendDataRowsLimit(queryWithCustomCondition, maxQueryResults) + + // if the custom condition is invalid, this will throw an exception + // from the SQL/PPL plugin + executePplQuery(limitedQueryWithCustomCondition, true, clusterService.state().nodes.localNode, transportService) } } catch (e: Exception) { validationListener.onFailure( @@ -410,7 +399,6 @@ class TransportIndexMonitorAction @Inject constructor( } private fun validatePplSqlMonitor(pplSqlMonitor: Monitor, validationListener: ActionListener): Boolean { - // ensure the trigger throttle and expire durations are valid pplSqlMonitor.triggers.forEach { trigger -> val pplTrigger = trigger as PPLSQLTrigger diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt index b16ac3a45..e83463005 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt @@ -2287,7 +2287,7 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() { triggers = listOf( randomPPLTrigger( conditionType = PPLSQLTrigger.ConditionType.CUSTOM, - customCondition = "eval result = max_num > 5", + customCondition = "where max_num > 5", numResultsCondition = null, numResultsValue = null ) @@ -2489,7 +2489,7 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() { triggers = listOf( randomPPLTrigger( conditionType = PPLSQLTrigger.ConditionType.CUSTOM, - customCondition = "eval result = max_num > 5", + customCondition = "where max_num > 5", numResultsCondition = null, numResultsValue = null ) @@ -2520,43 +2520,31 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() { assertEquals("source = $TEST_INDEX_NAME | stats max(number) as max_num by abc", activeAlert.pplQuery) assertTrue("PPL query results should not be empty", activeAlert.pplQueryResults.isNotEmpty()) - // Verify the query results are in the new transformed format (list of maps) - assertEquals("Should have 3 aggregation result rows", 3, activeAlert.pplQueryResults.size) + // Verify the query results are in the new transformed format (list of maps), + // and only have the 2 buckets that met the custom condition + assertEquals("Should have 2 aggregation result rows", 2, activeAlert.pplQueryResults.size) // Convert results to a map of group name -> result row for easier validation val groupResults = activeAlert.pplQueryResults.associateBy { it["abc"] as String } // Verify all expected groups are present - assertTrue("Should contain group 'abc'", groupResults.containsKey("abc")) assertTrue("Should contain group 'def'", groupResults.containsKey("def")) assertTrue("Should contain group 'ghi'", groupResults.containsKey("ghi")) // Verify each result row has the expected structure and values - val abcResult = groupResults["abc"]!! - assertTrue("abc result should contain 'max_num' field", abcResult.containsKey("max_num")) - assertTrue("abc result should contain 'abc' field", abcResult.containsKey("abc")) - assertTrue("abc result should contain 'result' field", abcResult.containsKey("result")) - assertEquals("Group 'abc' max should be 3", 3, (abcResult["max_num"] as Number).toInt()) - assertEquals("Group 'abc' field should be 'abc'", "abc", abcResult["abc"]) - assertEquals("Group 'abc' eval result should be false (3 > 5 is false)", false, abcResult["result"]) - val defResult = groupResults["def"]!! assertTrue("def result should contain 'max_num' field", defResult.containsKey("max_num")) assertTrue("def result should contain 'abc' field", defResult.containsKey("abc")) - assertTrue("def result should contain 'result' field", defResult.containsKey("result")) assertEquals("Group 'def' max should be 6", 6, (defResult["max_num"] as Number).toInt()) - assertEquals("Group 'def' field should be 'def'", "def", defResult["abc"]) - assertEquals("Group 'def' eval result should be true (6 > 5 is true)", true, defResult["result"]) + assertEquals("Group 'def' abc field should be 'def'", "def", defResult["abc"]) val ghiResult = groupResults["ghi"]!! assertTrue("ghi result should contain 'max_num' field", ghiResult.containsKey("max_num")) assertTrue("ghi result should contain 'abc' field", ghiResult.containsKey("abc")) - assertTrue("ghi result should contain 'result' field", ghiResult.containsKey("result")) assertEquals("Group 'ghi' max should be 9", 9, (ghiResult["max_num"] as Number).toInt()) - assertEquals("Group 'ghi' field should be 'ghi'", "ghi", ghiResult["abc"]) - assertEquals("Group 'ghi' eval result should be true (9 > 5 is true)", true, ghiResult["result"]) + assertEquals("Group 'ghi' abc field should be 'ghi'", "ghi", ghiResult["abc"]) - // Custom condition "eval result = max_num > 5" should evaluate to true for def (6 > 5) and ghi (9 > 5) + // Custom condition "where max_num > 5" should evaluate to true for def (6 > 5) and ghi (9 > 5) // This caused the trigger to fire and create an ACTIVE alert // Delete documents from "def" and "ghi" groups to make condition no longer true diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/PPLUtilsTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/PPLUtilsTests.kt index b66fd1494..1ef5757ff 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/PPLUtilsTests.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/PPLUtilsTests.kt @@ -1,159 +1,14 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.alerting -import org.json.JSONArray -import org.json.JSONObject import org.opensearch.alerting.PPLUtils.PPL_RESULTS_SIZE_EXCEEDED_MESSAGE import org.opensearch.test.OpenSearchTestCase class PPLUtilsTests : OpenSearchTestCase() { - - fun `test findEvalResultVar with simple variable name`() { - val condition = "eval _some_result1 = avg_latency > 100" - assertEquals("_some_result1", PPLUtils.findEvalResultVar(condition)) - } - - fun `test findEvalResultVar with extra whitespace`() { - val condition = " eval myVar = value > 100" - assertEquals("myVar", PPLUtils.findEvalResultVar(condition)) - } - - fun `test findEvalResultVar with no whitespace after equals`() { - val condition = "eval apple=avg_latency > 100" - assertEquals("apple", PPLUtils.findEvalResultVar(condition)) - } - - fun `test findEvalResultVar with complex expression`() { - val condition = "eval is_high_latency = (avg_latency > 100 AND error_count > 5) OR max_latency > 500" - assertEquals("is_high_latency", PPLUtils.findEvalResultVar(condition)) - } - - fun `test findEvalResultVar is case insensitive for eval keyword`() { - val condition = "EVAL result = value > 100" - assertEquals("result", PPLUtils.findEvalResultVar(condition)) - } - - fun `test findEvalResultVar with nested parentheses`() { - val condition = "eval complex = ((a + b) * (c - d)) > threshold" - assertEquals("complex", PPLUtils.findEvalResultVar(condition)) - } - - fun `test findEvalResultVar with eval in middle of string`() { - val condition = "stats count() | eval threshold_met = count > 10" - assertThrows(IllegalArgumentException::class.java) { - PPLUtils.findEvalResultVar(condition) - } - } - - fun `test findEvalResultVar with multiple eval statements`() { - val condition = "eval first = a > 1 | eval second = b > 2" - assertThrows(IllegalArgumentException::class.java) { - PPLUtils.findEvalResultVar(condition) - } - } - - fun `test findEvalResultVar throws exception when eval keyword missing`() { - val condition = "result = avg_latency > 100" - assertThrows(IllegalArgumentException::class.java) { - PPLUtils.findEvalResultVar(condition) - } - } - - fun `test findEvalResultVar throws exception when variable name invalid`() { - val condition = "eval 123invalid = value > 100" - assertThrows(IllegalArgumentException::class.java) { - PPLUtils.findEvalResultVar(condition) - } - } - - fun `test findEvalResultVar throws exception when equals sign missing`() { - val condition = "eval result value > 100" - assertThrows(IllegalArgumentException::class.java) { - PPLUtils.findEvalResultVar(condition) - } - } - - fun `test findEvalResultVar throws exception with empty string`() { - assertThrows(IllegalArgumentException::class.java) { - PPLUtils.findEvalResultVar("") - } - } - - fun `test findEvalResultVar throws exception when eval not followed by whitespace`() { - val condition = "evalresult = value > 100" - assertThrows(IllegalArgumentException::class.java) { - PPLUtils.findEvalResultVar(condition) - } - } - - fun `test findEvalResultVarIdxInSchema finds variable at first position`() { - val queryResponse = JSONObject( - """ - { - "schema": [ - {"name": "count", "type": "integer"}, - {"name": "result", "type": "boolean"}, - {"name": "avg", "type": "double"} - ] - } - """.trimIndent() - ) - - val index = PPLUtils.findEvalResultVarIdxInSchema(queryResponse, "result") - assertEquals(1, index) - } - - fun `test findEvalResultVarIdxInSchema throws exception when variable not found`() { - val queryResponse = JSONObject( - """ - { - "schema": [ - {"name": "field1", "type": "string"}, - {"name": "field2", "type": "integer"} - ] - } - """.trimIndent() - ) - - val exception = assertThrows(IllegalStateException::class.java) { - PPLUtils.findEvalResultVarIdxInSchema(queryResponse, "nonexistent") - } - assertTrue(exception.message!!.contains("Expected to find eval statement results variable")) - assertTrue(exception.message!!.contains("nonexistent")) - } - - fun `test findEvalResultVarIdxInSchema with variable names containing special characters`() { - val queryResponse = JSONObject( - """ - { - "schema": [ - {"name": "field_with_underscores", "type": "boolean"}, - {"name": "field-with-dashes", "type": "boolean"}, - {"name": "field.with.dots", "type": "boolean"} - ] - } - """.trimIndent() - ) - - assertEquals(0, PPLUtils.findEvalResultVarIdxInSchema(queryResponse, "field_with_underscores")) - assertEquals(1, PPLUtils.findEvalResultVarIdxInSchema(queryResponse, "field-with-dashes")) - assertEquals(2, PPLUtils.findEvalResultVarIdxInSchema(queryResponse, "field.with.dots")) - } - - fun `test findEvalResultVarIdxInSchema with large schema`() { - val schemaBuilder = StringBuilder("[") - for (i in 0 until 50) { - if (i > 0) schemaBuilder.append(",") - schemaBuilder.append("""{"name": "field$i", "type": "string"}""") - } - schemaBuilder.append("]") - - val queryResponse = JSONObject() - queryResponse.put("schema", JSONArray(schemaBuilder.toString())) - - val index = PPLUtils.findEvalResultVarIdxInSchema(queryResponse, "field25") - assertEquals(25, index) - } - fun `test constructPPLQueryResultsMap with simple types`() { // Arrange: Simple query result with basic types val rawResults = mapOf( diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt index 90af7fb64..fc5edb199 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt @@ -1773,7 +1773,7 @@ class MonitorRestApiIT : AlertingRestTestCase() { triggers = listOf( randomPPLTrigger( conditionType = ConditionType.CUSTOM, - customCondition = "not a valid PPL custom condition", + customCondition = "eval result = 3 > 1", numResultsCondition = null, numResultsValue = null ) @@ -1790,35 +1790,6 @@ class MonitorRestApiIT : AlertingRestTestCase() { ensureNumMonitors(0) } - fun `test create ppl monitor with custom condition that evals to num not bool fails`() { - createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) - indexDocFromSomeTimeAgo(1, MINUTES, "abc", 1) - indexDocFromSomeTimeAgo(2, MINUTES, "abc", 2) - - // ensure the request fails - try { - createRandomPPLMonitor( - randomPPLMonitor( - triggers = listOf( - randomPPLTrigger( - conditionType = ConditionType.CUSTOM, - customCondition = "eval something = sum * 2", - numResultsCondition = null, - numResultsValue = null - ) - ), - query = "source = $TEST_INDEX_NAME | stats sum(number) as sum by abc" - ) - ) - fail("Expected request to fail with BAD_REQUEST but it succeeded") - } catch (e: ResponseException) { - assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) - } - - // ensure no monitor was created - ensureNumMonitors(0) - } - fun `test create ppl monitor with notification subject source too long fails`() { adminClient().updateSettings(NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH.key, 100) diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/ppl/PPLPluginInterface.kt b/core/src/main/kotlin/org/opensearch/alerting/core/ppl/PPLPluginInterface.kt index a121f6b93..e009b8845 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/core/ppl/PPLPluginInterface.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/core/ppl/PPLPluginInterface.kt @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.alerting.core.ppl import org.opensearch.action.ActionListenerResponseHandler From 7540f323e7d6e32b4e7379ccda826741aacc2693 Mon Sep 17 00:00:00 2001 From: Dennis Toepker Date: Tue, 14 Apr 2026 01:21:20 -0700 Subject: [PATCH 6/9] changing custom condition validation to use trigger name instead of trigger id --- alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt | 2 +- .../alerting/transport/TransportIndexMonitorAction.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt b/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt index 7cb9f85bd..9080361be 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt @@ -82,7 +82,7 @@ object PPLUtils { /** * Executes a PPL query and returns the response as a parsable JSONObject. * - * This method calls the PPL Plugin's Execute API via the transport layer to execute the provided query + * This method calls the PPL Plugin's Execute or Explain API via the transport layer to execute the provided query * and parses the response into a structured JSON format suitable for trigger evaluation * * @param query The PPL query string to execute diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt index 9ef391e0d..a2228090d 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt @@ -371,7 +371,7 @@ class TransportIndexMonitorAction @Inject constructor( validationListener.onFailure( AlertingException.wrap( IllegalArgumentException( - "Custom condition for trigger ${trigger.id} is invalid, " + + "Custom condition for trigger ${trigger.name} is invalid, " + "custom condition must be a valid PPL where statement." ) ) From 1e2a78dc1dd3eafcee9a268f0b0ff7ae3046feae Mon Sep 17 00:00:00 2001 From: Dennis Toepker Date: Fri, 17 Apr 2026 13:11:32 -0700 Subject: [PATCH 7/9] changing setting names and removing mentions of V2 --- .../org/opensearch/alerting/AlertingPlugin.kt | 14 +- .../org/opensearch/alerting/InputService.kt | 6 +- .../org/opensearch/alerting/TriggerService.kt | 6 +- .../alerting/settings/AlertingSettings.kt | 20 +- .../transport/TransportIndexMonitorAction.kt | 18 +- .../alerting/MonitorRunnerServiceIT.kt | 6 +- .../alerting/resthandler/MonitorRestApiIT.kt | 4 +- .../resources/mappings/scheduled-jobs.json | 195 ------------------ 8 files changed, 34 insertions(+), 235 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index 5e32d967c..0150ec286 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -304,7 +304,6 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R val settings = environment.settings() val lockService = LockService(client, clusterService) alertIndices = AlertIndices(settings, client, threadPool, clusterService) - alertV2Indices = AlertV2Indices(settings, client, threadPool, clusterService) val sdkClient: SdkClient = SdkClientFactory.createSdkClient( client, @@ -476,15 +475,10 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R AlertingSettings.REMOTE_METADATA_REGION, AlertingSettings.REMOTE_METADATA_SERVICE_NAME, AlertingSettings.MULTI_TENANT_TRIGGER_EVAL_ENABLED, - AlertingSettings.ALERT_V2_HISTORY_ENABLED, - AlertingSettings.ALERT_V2_HISTORY_ROLLOVER_PERIOD, - AlertingSettings.ALERT_V2_HISTORY_INDEX_MAX_AGE, - AlertingSettings.ALERT_V2_HISTORY_MAX_DOCS, - AlertingSettings.ALERT_V2_HISTORY_RETENTION_PERIOD, - AlertingSettings.ALERT_V2_MONITOR_EXECUTION_MAX_DURATION, - AlertingSettings.ALERTING_V2_MAX_QUERY_LENGTH, - AlertingSettings.ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS, - AlertingSettings.ALERT_V2_QUERY_RESULTS_MAX_SIZE, + AlertingSettings.PPL_MONITOR_EXECUTION_MAX_DURATION, + AlertingSettings.PPL_MAX_QUERY_LENGTH, + AlertingSettings.PPL_QUERY_RESULTS_MAX_DATAROWS, + AlertingSettings.PPL_QUERY_RESULTS_MAX_SIZE, AlertingSettings.NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH, AlertingSettings.NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH, ) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt index 686842db4..4e5592b8c 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt @@ -252,7 +252,7 @@ class InputService( // here to populate the final customer facing response of monitor execution val cappedPPLBaseQueryResults = capAndReformatPPLQueryResults( basePplQueryResults, - monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.ALERT_V2_QUERY_RESULTS_MAX_SIZE) + monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.PPL_QUERY_RESULTS_MAX_SIZE) ) InputRunResults(emptyList(), null, null, cappedPPLBaseQueryResults, numPplResults) @@ -279,13 +279,13 @@ class InputService( val monitorExecutionDuration = monitorCtx .clusterService!! .clusterSettings - .get(AlertingSettings.ALERT_V2_MONITOR_EXECUTION_MAX_DURATION) + .get(AlertingSettings.PPL_MONITOR_EXECUTION_MAX_DURATION) var queryResponseJson: JSONObject? = null withTimeout(monitorExecutionDuration.millis) { // limit the number of PPL query result data rows returned - val dataRowsLimit = monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS) + val dataRowsLimit = monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.PPL_QUERY_RESULTS_MAX_DATAROWS) val limitedQueryToExecute = appendDataRowsLimit(baseQuery, dataRowsLimit) logger.debug("executing the base PPL query of monitor: ${pplSqlMonitor.id}") diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt index 4dbfd7ef1..01acaa08d 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt @@ -320,11 +320,11 @@ class TriggerService(val scriptService: ScriptService) { val monitorExecutionDuration = monitorCtx .clusterService!! .clusterSettings - .get(AlertingSettings.ALERT_V2_MONITOR_EXECUTION_MAX_DURATION) + .get(AlertingSettings.PPL_MONITOR_EXECUTION_MAX_DURATION) val queryResultsSizeLimit = monitorCtx .clusterService!! .clusterSettings - .get(AlertingSettings.ALERT_V2_QUERY_RESULTS_MAX_SIZE) + .get(AlertingSettings.PPL_QUERY_RESULTS_MAX_SIZE) var triggered: Boolean? = null var customConditionQueryResults: List>? = null @@ -339,7 +339,7 @@ class TriggerService(val scriptService: ScriptService) { val queryToExecute = appendCustomCondition(query, customCondition) // limit the number of PPL query result data rows returned - val dataRowsLimit = monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS) + val dataRowsLimit = monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.PPL_QUERY_RESULTS_MAX_DATAROWS) val limitedQueryToExecute = appendDataRowsLimit(queryToExecute, dataRowsLimit) // TODO: after getting ppl query results, see if the number of results diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt index ad888c11e..e0bef8bfa 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt @@ -309,14 +309,14 @@ class AlertingSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ) - val ALERT_V2_MONITOR_EXECUTION_MAX_DURATION = Setting.positiveTimeSetting( - "plugins.alerting.v2.alert_monitor_execution_max_duration", + val PPL_MONITOR_EXECUTION_MAX_DURATION = Setting.positiveTimeSetting( + "plugins.alerting.ppl_monitor_max_execution_duration", TimeValue(1, TimeUnit.MINUTES), Setting.Property.NodeScope, Setting.Property.Dynamic ) - val ALERTING_V2_MAX_QUERY_LENGTH = Setting.longSetting( - "plugins.alerting.v2.monitor.max_query_length", + val PPL_MAX_QUERY_LENGTH = Setting.longSetting( + "plugins.alerting.ppl_monitor_max_query_length", 2000L, 0L, Setting.Property.NodeScope, Setting.Property.Dynamic @@ -324,30 +324,30 @@ class AlertingSettings { // max data rows to retrieve when executing PPL query against // SQL/PPL plugin during monitor execution - val ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS = Setting.longSetting( - "plugins.alerting.v2.query_results_max_datarows", + val PPL_QUERY_RESULTS_MAX_DATAROWS = Setting.longSetting( + "plugins.alerting.ppl_query_results_max_datarows", 10000L, 1L, Setting.Property.NodeScope, Setting.Property.Dynamic ) // max size of query results to store in alerts and notifications - val ALERT_V2_QUERY_RESULTS_MAX_SIZE = Setting.longSetting( - "plugins.alerting.v2.query_results_max_size", + val PPL_QUERY_RESULTS_MAX_SIZE = Setting.longSetting( + "plugins.alerting.ppl_query_results_max_size", 3000L, 0L, Setting.Property.NodeScope, Setting.Property.Dynamic ) val NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH = Setting.intSetting( - "plugins.alerting.v2.notification_subject_source_max_length", + "plugins.alerting.notification_subject_source_max_length", 1000, 100, Setting.Property.NodeScope, Setting.Property.Dynamic ) val NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH = Setting.intSetting( - "plugins.alerting.v2.notification_message_source_max_length", + "plugins.alerting.notification_message_source_max_length", 3000, 1000, Setting.Property.NodeScope, Setting.Property.Dynamic diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt index a2228090d..19a9de085 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt @@ -39,13 +39,13 @@ import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.service.DeleteMonitorService import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_MAX_MONITORS -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_QUERY_LENGTH -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS import org.opensearch.alerting.settings.AlertingSettings.Companion.INDEX_TIMEOUT import org.opensearch.alerting.settings.AlertingSettings.Companion.MAX_ACTION_THROTTLE_VALUE import org.opensearch.alerting.settings.AlertingSettings.Companion.MAX_TRIGGERS_PER_MONITOR import org.opensearch.alerting.settings.AlertingSettings.Companion.NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH import org.opensearch.alerting.settings.AlertingSettings.Companion.NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH +import org.opensearch.alerting.settings.AlertingSettings.Companion.PPL_MAX_QUERY_LENGTH +import org.opensearch.alerting.settings.AlertingSettings.Companion.PPL_QUERY_RESULTS_MAX_DATAROWS import org.opensearch.alerting.settings.AlertingSettings.Companion.REQUEST_TIMEOUT import org.opensearch.alerting.settings.DestinationSettings.Companion.ALLOW_LIST import org.opensearch.alerting.util.DocLevelMonitorQueries @@ -131,8 +131,8 @@ class TransportIndexMonitorAction @Inject constructor( @Volatile private var allowList = ALLOW_LIST.get(settings) // PPL Alerting related settings - @Volatile private var maxQueryLength = ALERTING_V2_MAX_QUERY_LENGTH.get(settings) - @Volatile private var maxQueryResults = ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS.get(settings) + @Volatile private var maxQueryLength = PPL_MAX_QUERY_LENGTH.get(settings) + @Volatile private var maxQueryResults = PPL_QUERY_RESULTS_MAX_DATAROWS.get(settings) @Volatile private var notificationSubjectMaxLength = NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH.get(settings) @Volatile private var notificationMessageMaxLength = NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH.get(settings) @@ -146,8 +146,8 @@ class TransportIndexMonitorAction @Inject constructor( clusterService.clusterSettings.addSettingsUpdateConsumer(MAX_ACTION_THROTTLE_VALUE) { maxActionThrottle = it } clusterService.clusterSettings.addSettingsUpdateConsumer(ALLOW_LIST) { allowList = it } - clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_MAX_QUERY_LENGTH) { maxQueryLength = it } - clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS) { maxQueryResults = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(PPL_MAX_QUERY_LENGTH) { maxQueryLength = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(PPL_QUERY_RESULTS_MAX_DATAROWS) { maxQueryResults = it } clusterService.clusterSettings.addSettingsUpdateConsumer(NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH) { notificationSubjectMaxLength = it } @@ -313,7 +313,7 @@ class TransportIndexMonitorAction @Inject constructor( // initiate the PPL monitor and PPL query validations client.threadPool().threadContext.stashContext().use { scope.launch { - val singleThreadContext = newSingleThreadContext("IndexMonitorV2ActionThread") + val singleThreadContext = newSingleThreadContext("IndexPPLMonitorActionThread") withContext(singleThreadContext) { it.restore() @@ -409,9 +409,9 @@ class TransportIndexMonitorAction @Inject constructor( AlertingException.wrap( IllegalArgumentException( "Trigger ${trigger.id} checks for number of results threshold of ${trigger.numResultsValue}, " + - "but Alerting V2 is configured only to retrieve $maxQueryResults query results maximum. " + + "but PPL Alerting is configured only to retrieve $maxQueryResults query results maximum. " + "Please lower the number of results value to one below this maximum value, or adjust the cluster " + - "setting: $ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS.key}" + "setting: $PPL_QUERY_RESULTS_MAX_DATAROWS.key}" ) ) ) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt index e83463005..8a51b261c 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt @@ -2255,7 +2255,7 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() { ) // Set monitor execution timeout to 1 nanosecond to force a timeout - adminClient().updateSettings(AlertingSettings.ALERT_V2_MONITOR_EXECUTION_MAX_DURATION.key, TimeValue.timeValueNanos(1)) + adminClient().updateSettings(AlertingSettings.PPL_MONITOR_EXECUTION_MAX_DURATION.key, TimeValue.timeValueNanos(1)) val response = executeMonitor(monitor.id) @@ -2297,7 +2297,7 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() { ) // Set monitor execution timeout to 1 nanosecond to force a timeout - adminClient().updateSettings(AlertingSettings.ALERT_V2_MONITOR_EXECUTION_MAX_DURATION.key, TimeValue.timeValueNanos(1)) + adminClient().updateSettings(AlertingSettings.PPL_MONITOR_EXECUTION_MAX_DURATION.key, TimeValue.timeValueNanos(1)) val response = executeMonitor(monitor.id) @@ -2339,7 +2339,7 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() { ) // Set max PPL query results size to guarantee it will be exceeded - adminClient().updateSettings(AlertingSettings.ALERT_V2_QUERY_RESULTS_MAX_SIZE.key, 5L) + adminClient().updateSettings(AlertingSettings.PPL_QUERY_RESULTS_MAX_SIZE.key, 5L) executeMonitor(monitor.id) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt index fc5edb199..6db45f514 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt @@ -43,9 +43,9 @@ import org.opensearch.alerting.randomThrottle import org.opensearch.alerting.randomUser import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_MAX_MONITORS -import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_QUERY_LENGTH import org.opensearch.alerting.settings.AlertingSettings.Companion.NOTIFICATION_MESSAGE_SOURCE_MAX_LENGTH import org.opensearch.alerting.settings.AlertingSettings.Companion.NOTIFICATION_SUBJECT_SOURCE_MAX_LENGTH +import org.opensearch.alerting.settings.AlertingSettings.Companion.PPL_MAX_QUERY_LENGTH import org.opensearch.alerting.toJsonString import org.opensearch.alerting.util.DestinationType import org.opensearch.client.ResponseException @@ -1747,7 +1747,7 @@ class MonitorRestApiIT : AlertingRestTestCase() { } fun `test create ppl monitor with query that's too long fails`() { - adminClient().updateSettings(ALERTING_V2_MAX_QUERY_LENGTH.key, 1) + adminClient().updateSettings(PPL_MAX_QUERY_LENGTH.key, 1) // ensure the request fails try { diff --git a/core/src/main/resources/mappings/scheduled-jobs.json b/core/src/main/resources/mappings/scheduled-jobs.json index ba5c0010d..154829d64 100644 --- a/core/src/main/resources/mappings/scheduled-jobs.json +++ b/core/src/main/resources/mappings/scheduled-jobs.json @@ -450,201 +450,6 @@ } } }, - "monitor_v2": { - "dynamic": "false", - "properties": { - "ppl_monitor": { - "dynamic": "false", - "properties": { - "schema_version": { - "type": "integer" - }, - "name": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "enabled": { - "type": "boolean" - }, - "schedule": { - "properties": { - "period": { - "properties": { - "interval": { - "type": "integer" - }, - "unit": { - "type": "keyword" - } - } - }, - "cron": { - "properties": { - "expression": { - "type": "text" - }, - "timezone": { - "type": "keyword" - } - } - } - } - }, - "look_back_window_minutes": { - "type": "long" - }, - "timestamp_field": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "last_update_time": { - "type": "date", - "format": "strict_date_time||epoch_millis" - }, - "enabled_time": { - "type": "date", - "format": "strict_date_time||epoch_millis" - }, - "description": { - "type": "text" - }, - "user": { - "properties": { - "name": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "backend_roles": { - "type" : "text", - "fields" : { - "keyword" : { - "type" : "keyword" - } - } - }, - "roles": { - "type" : "text", - "fields" : { - "keyword" : { - "type" : "keyword" - } - } - }, - "custom_attribute_names": { - "type" : "text", - "fields" : { - "keyword" : { - "type" : "keyword" - } - } - } - } - }, - "query_language": { - "type": "keyword" - }, - "query": { - "type": "text" - }, - "triggers": { - "type": "nested", - "properties": { - "name": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "severity": { - "type": "keyword" - }, - "throttle_minutes": { - "type": "long" - }, - "expires_minutes": { - "type": "long" - }, - "last_triggered_time": { - "type": "date", - "format": "strict_date_time||epoch_millis" - }, - "mode": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "num_results_condition": { - "type": "keyword" - }, - "num_results_value": { - "type": "long" - }, - "custom_condition": { - "type": "text" - }, - "actions": { - "type": "nested", - "properties": { - "name": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "destination_id": { - "type": "keyword" - }, - "subject_template": { - "type": "object", - "enabled": false - }, - "message_template": { - "type": "object", - "enabled": false - }, - "throttle_enabled": { - "type": "boolean" - }, - "throttle": { - "properties": { - "value": { - "type": "integer" - }, - "unit": { - "type": "keyword" - } - } - } - } - } - } - } - } - } - } - }, "destination": { "dynamic": "false", "properties": { From e09b57de92bb6bf41e860feb4f2211e936a65eb1 Mon Sep 17 00:00:00 2001 From: Dennis Toepker Date: Mon, 20 Apr 2026 13:59:26 -0700 Subject: [PATCH 8/9] post common utils refactor changes Signed-off-by: Dennis Toepker --- .../org/opensearch/alerting/AlertService.kt | 19 +++-- .../org/opensearch/alerting/AlertingPlugin.kt | 8 +- .../org/opensearch/alerting/InputService.kt | 8 +- .../alerting/MonitorRunnerService.kt | 8 +- .../alerting/QueryLevelMonitorRunner.kt | 30 ++++---- .../org/opensearch/alerting/TriggerService.kt | 66 ++++++++--------- .../resthandler/RestIndexMonitorAction.kt | 4 +- .../QueryLevelTriggerExecutionContext.kt | 6 +- .../transport/TransportIndexMonitorAction.kt | 46 ++++++------ .../alerting/alerts/alert_mapping.json | 4 +- .../alerting/AlertingRestTestCase.kt | 40 +--------- .../alerting/MonitorRunnerServiceIT.kt | 74 +++++++++---------- .../org/opensearch/alerting/TestHelpers.kt | 30 ++++---- .../alerting/resthandler/MonitorRestApiIT.kt | 3 +- 14 files changed, 156 insertions(+), 190 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertService.kt index 49f495ad7..9d3f5536a 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertService.kt @@ -35,13 +35,13 @@ import org.opensearch.commons.alerting.model.ClusterMetricsTriggerRunResult import org.opensearch.commons.alerting.model.DataSources import org.opensearch.commons.alerting.model.Monitor import org.opensearch.commons.alerting.model.NoOpTrigger -import org.opensearch.commons.alerting.model.PPLSQLInput +import org.opensearch.commons.alerting.model.PPLInput import org.opensearch.commons.alerting.model.QueryLevelTriggerRunResult import org.opensearch.commons.alerting.model.Trigger import org.opensearch.commons.alerting.model.Workflow import org.opensearch.commons.alerting.model.WorkflowRunContext import org.opensearch.commons.alerting.model.action.AlertCategory -import org.opensearch.commons.alerting.util.isPplSqlMonitor +import org.opensearch.commons.alerting.util.isPPLMonitor import org.opensearch.core.action.ActionListener import org.opensearch.core.common.bytes.BytesReference import org.opensearch.core.rest.RestStatus @@ -204,8 +204,8 @@ class AlertService( } // populate PPL Monitor specific fields - val query = if (ctx.monitor.isPplSqlMonitor()) (ctx.monitor.inputs[0] as PPLSQLInput).query else null - val queryResults = if (ctx.monitor.isPplSqlMonitor()) ctx.pplQueryResults else emptyList() + val query = if (ctx.monitor.isPPLMonitor()) (ctx.monitor.inputs[0] as PPLInput).query else null + val queryResults = if (ctx.monitor.isPPLMonitor()) ctx.pplQueryResults else emptyList() // Merge the alert's error message to the current alert's history val updatedHistory = currentAlert?.errorHistory.update(alertError) @@ -218,8 +218,8 @@ class AlertService( actionExecutionResults = updatedActionExecutionResults, schemaVersion = IndexUtils.alertIndexSchemaVersion, clusters = triggeredClusters, - pplQuery = query, - pplQueryResults = queryResults + query = query, + queryResults = queryResults ) } else if (alertError == null && currentAlert?.isAcknowledged() == true) { null @@ -233,22 +233,21 @@ class AlertService( actionExecutionResults = updatedActionExecutionResults, schemaVersion = IndexUtils.alertIndexSchemaVersion, clusters = triggeredClusters, - pplQuery = query, - pplQueryResults = queryResults + query = query, + queryResults = queryResults ) } else { val alertState = if (workflorwRunContext?.auditDelegateMonitorAlerts == true) { Alert.State.AUDIT } else if (alertError == null) Alert.State.ACTIVE else Alert.State.ERROR - // TODO: does the trigger field get populated as QueryLevel or PPLSQL Trigger correctly? Alert( monitor = ctx.monitor, trigger = ctx.trigger, startTime = currentTime, lastNotificationTime = currentTime, state = alertState, errorMessage = alertError?.message, errorHistory = updatedHistory, actionExecutionResults = updatedActionExecutionResults, schemaVersion = IndexUtils.alertIndexSchemaVersion, executionId = executionId, workflowId = workflorwRunContext?.workflowId ?: "", - clusters = triggeredClusters, pplQuery = query, pplQueryResults = queryResults + clusters = triggeredClusters, query = query, queryResults = queryResults ) } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index 0150ec286..1df921076 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -108,8 +108,8 @@ import org.opensearch.commons.alerting.model.ClusterMetricsInput import org.opensearch.commons.alerting.model.DocLevelMonitorInput import org.opensearch.commons.alerting.model.DocumentLevelTrigger import org.opensearch.commons.alerting.model.Monitor -import org.opensearch.commons.alerting.model.PPLSQLInput -import org.opensearch.commons.alerting.model.PPLSQLTrigger +import org.opensearch.commons.alerting.model.PPLInput +import org.opensearch.commons.alerting.model.PPLTrigger import org.opensearch.commons.alerting.model.QueryLevelTrigger import org.opensearch.commons.alerting.model.ScheduledJob import org.opensearch.commons.alerting.model.ScheduledJob.Companion.SCHEDULED_JOBS_INDEX @@ -275,14 +275,14 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R Monitor.XCONTENT_REGISTRY, SearchInput.XCONTENT_REGISTRY, DocLevelMonitorInput.XCONTENT_REGISTRY, - PPLSQLInput.XCONTENT_REGISTRY, + PPLInput.XCONTENT_REGISTRY, QueryLevelTrigger.XCONTENT_REGISTRY, BucketLevelTrigger.XCONTENT_REGISTRY, ClusterMetricsInput.XCONTENT_REGISTRY, DocumentLevelTrigger.XCONTENT_REGISTRY, ChainedAlertTrigger.XCONTENT_REGISTRY, RemoteMonitorTrigger.XCONTENT_REGISTRY, - PPLSQLTrigger.XCONTENT_REGISTRY, + PPLTrigger.XCONTENT_REGISTRY, Workflow.XCONTENT_REGISTRY ) } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt index 4e5592b8c..6b65f476d 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt @@ -38,7 +38,7 @@ import org.opensearch.common.xcontent.XContentType import org.opensearch.commons.alerting.model.ClusterMetricsInput import org.opensearch.commons.alerting.model.InputRunResults import org.opensearch.commons.alerting.model.Monitor -import org.opensearch.commons.alerting.model.PPLSQLInput +import org.opensearch.commons.alerting.model.PPLInput import org.opensearch.commons.alerting.model.SearchInput import org.opensearch.commons.alerting.model.TriggerAfterKey import org.opensearch.commons.alerting.model.WorkflowRunContext @@ -236,7 +236,7 @@ class InputService( // for those triggers, not the contents themselves val basePplQueryResults = runPPLBaseQuery( monitor, - (monitor.inputs[0] as PPLSQLInput).query, + (monitor.inputs[0] as PPLInput).query, monitorCtx, transportService ) @@ -269,7 +269,7 @@ class InputService( // for PPL Monitor execution, the base PPL query is run once // for number_of_results PPL triggers private suspend fun runPPLBaseQuery( - pplSqlMonitor: Monitor, + pplMonitor: Monitor, baseQuery: String, monitorCtx: MonitorRunnerExecutionContext, transportService: TransportService @@ -288,7 +288,7 @@ class InputService( val dataRowsLimit = monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.PPL_QUERY_RESULTS_MAX_DATAROWS) val limitedQueryToExecute = appendDataRowsLimit(baseQuery, dataRowsLimit) - logger.debug("executing the base PPL query of monitor: ${pplSqlMonitor.id}") + logger.debug("executing the base PPL query of monitor: ${pplMonitor.id}") val (queryResponseJsonReceived, timeTaken) = measureTimedValue { executePplQuery( limitedQueryToExecute, diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt index 9a7582d9b..71406d790 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt @@ -72,7 +72,7 @@ import org.opensearch.commons.alerting.util.AlertingException import org.opensearch.commons.alerting.util.IndexPatternUtils import org.opensearch.commons.alerting.util.isBucketLevelMonitor import org.opensearch.commons.alerting.util.isMonitorOfStandardType -import org.opensearch.commons.alerting.util.isPplSqlMonitor +import org.opensearch.commons.alerting.util.isPPLMonitor import org.opensearch.core.action.ActionListener import org.opensearch.core.rest.RestStatus import org.opensearch.core.xcontent.NamedXContentRegistry @@ -469,9 +469,9 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon "Executing scheduled monitor - id: ${monitor.id}, type: ${monitor.monitorType}, periodStart: $periodStart, " + "periodEnd: $periodEnd, dryrun: $dryrun, executionId: $executionId" ) - val runResult = if (monitor.isPplSqlMonitor()) { - // PPL/SQL Monitor runs with QueryLevelMonitorRunner - // as PPL/SQL Monitors are ultimately query-based + val runResult = if (monitor.isPPLMonitor()) { + // PPL Monitor runs with QueryLevelMonitorRunner + // as PPL Monitors are ultimately query-based QueryLevelMonitorRunner.runMonitor( monitor, monitorCtx, diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/QueryLevelMonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/QueryLevelMonitorRunner.kt index a1625a40f..3938714a1 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/QueryLevelMonitorRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/QueryLevelMonitorRunner.kt @@ -17,14 +17,14 @@ import org.opensearch.alerting.util.isADMonitor import org.opensearch.commons.alerting.model.Alert import org.opensearch.commons.alerting.model.Monitor import org.opensearch.commons.alerting.model.MonitorRunResult -import org.opensearch.commons.alerting.model.PPLSQLInput -import org.opensearch.commons.alerting.model.PPLSQLTrigger -import org.opensearch.commons.alerting.model.PPLSQLTrigger.ConditionType +import org.opensearch.commons.alerting.model.PPLInput +import org.opensearch.commons.alerting.model.PPLTrigger +import org.opensearch.commons.alerting.model.PPLTrigger.ConditionType import org.opensearch.commons.alerting.model.QueryLevelTrigger import org.opensearch.commons.alerting.model.QueryLevelTriggerRunResult import org.opensearch.commons.alerting.model.SearchInput import org.opensearch.commons.alerting.model.WorkflowRunContext -import org.opensearch.commons.alerting.util.isPplSqlMonitor +import org.opensearch.commons.alerting.util.isPPLMonitor import org.opensearch.transport.TransportService import java.time.Instant import java.util.Locale @@ -67,7 +67,7 @@ object QueryLevelMonitorRunner : MonitorRunner() { monitorResult = monitorResult.copy( inputResults = monitorCtx.inputService!!.collectInputResultsForADMonitor(monitor, periodStart, periodEnd) ) - } else if (monitor.isPplSqlMonitor()) { + } else if (monitor.isPPLMonitor()) { withClosableContext( InjectorContextElement( monitor.id, @@ -165,20 +165,20 @@ object QueryLevelMonitorRunner : MonitorRunner() { else monitorCtx.triggerService!!.runQueryLevelTrigger(monitor, trigger as QueryLevelTrigger, triggerCtx) } Monitor.MonitorType.PPL_MONITOR -> { - val pplSqlTrigger = trigger as PPLSQLTrigger + val pplTrigger = trigger as PPLTrigger - if (pplSqlTrigger.conditionType == ConditionType.NUMBER_OF_RESULTS) { + if (pplTrigger.conditionType == ConditionType.NUMBER_OF_RESULTS) { // number of results trigger case - monitorCtx.triggerService!!.runPplSqlNumResultsTrigger( - pplSqlTrigger, + monitorCtx.triggerService!!.runPplNumResultsTrigger( + pplTrigger, monitorResult.inputResults.pplBaseQueryNumResults ) } else { // custom condition trigger case - monitorCtx.triggerService!!.runPplSqlCustomTrigger( + monitorCtx.triggerService!!.runPplCustomTrigger( monitor, - pplSqlTrigger, - (monitor.inputs[0] as PPLSQLInput).query, + pplTrigger, + (monitor.inputs[0] as PPLInput).query, monitorCtx, transportService ) @@ -195,10 +195,10 @@ object QueryLevelMonitorRunner : MonitorRunner() { // what query results get stored in the Alert and Notification depends on the trigger type. // if number of results: evaluated on base query, so base query results are included. // if custom: the custom trigger ran its own query (base query + custom condition), so that query's results are included - // trigger is not PPLSQLTrigger: this is a query-level monitor run, simply include an empty list - val pplQueryResultsToInclude = if (trigger is PPLSQLTrigger && trigger.conditionType == ConditionType.NUMBER_OF_RESULTS) { + // trigger is not PPLTrigger: this is a query-level monitor run, simply include an empty list + val pplQueryResultsToInclude = if (trigger is PPLTrigger && trigger.conditionType == ConditionType.NUMBER_OF_RESULTS) { monitorResult.inputResults.pplBaseQueryResults - } else if (trigger is PPLSQLTrigger && trigger.conditionType == ConditionType.CUSTOM) { + } else if (trigger is PPLTrigger && trigger.conditionType == ConditionType.CUSTOM) { triggerResult.pplCustomQueryResults } else { listOf() diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt index 01acaa08d..88c003450 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt @@ -37,8 +37,8 @@ import org.opensearch.commons.alerting.model.DocLevelQuery import org.opensearch.commons.alerting.model.DocumentLevelTrigger import org.opensearch.commons.alerting.model.DocumentLevelTriggerRunResult import org.opensearch.commons.alerting.model.Monitor -import org.opensearch.commons.alerting.model.PPLSQLTrigger -import org.opensearch.commons.alerting.model.PPLSQLTrigger.NumResultsCondition +import org.opensearch.commons.alerting.model.PPLTrigger +import org.opensearch.commons.alerting.model.PPLTrigger.NumResultsCondition import org.opensearch.commons.alerting.model.QueryLevelTrigger import org.opensearch.commons.alerting.model.QueryLevelTriggerRunResult import org.opensearch.commons.alerting.model.Workflow @@ -248,37 +248,37 @@ class TriggerService(val scriptService: ScriptService) { return keyValuesList } - fun runPplSqlNumResultsTrigger( - pplSqlTrigger: PPLSQLTrigger, + fun runPplNumResultsTrigger( + pplTrigger: PPLTrigger, numResults: Long? ): QueryLevelTriggerRunResult { if (numResults == null) { return QueryLevelTriggerRunResult( - pplSqlTrigger.name, + pplTrigger.name, true, - IllegalStateException("Did not receive a number of results from PPL query execution: ${pplSqlTrigger.id}") + IllegalStateException("Did not receive a number of results from PPL query execution: ${pplTrigger.id}") ) } - if (pplSqlTrigger.numResultsCondition == null) { + if (pplTrigger.numResultsCondition == null) { return QueryLevelTriggerRunResult( - pplSqlTrigger.name, + pplTrigger.name, true, - IllegalStateException("No number of results condition found for trigger: ${pplSqlTrigger.id}") + IllegalStateException("No number of results condition found for trigger: ${pplTrigger.id}") ) } - if (pplSqlTrigger.numResultsValue == null) { + if (pplTrigger.numResultsValue == null) { return QueryLevelTriggerRunResult( - pplSqlTrigger.name, + pplTrigger.name, true, - IllegalStateException("No number of results value found for trigger: ${pplSqlTrigger.id}") + IllegalStateException("No number of results value found for trigger: ${pplTrigger.id}") ) } - val numResultsCondition = pplSqlTrigger.numResultsCondition!! - val numResultsValue = pplSqlTrigger.numResultsValue!! + val numResultsCondition = pplTrigger.numResultsCondition!! + val numResultsValue = pplTrigger.numResultsValue!! val triggered = when (numResultsCondition) { NumResultsCondition.GREATER_THAN -> numResults > numResultsValue @@ -289,7 +289,7 @@ class TriggerService(val scriptService: ScriptService) { NumResultsCondition.NOT_EQUAL -> numResults != numResultsValue } - logger.debug("Number of Results PPLTrigger ${pplSqlTrigger.name} with ID ${pplSqlTrigger.id} triggered: $triggered") + logger.debug("Number of Results PPLTrigger ${pplTrigger.name} with ID ${pplTrigger.id} triggered: $triggered") // unlike evaluating custom conditions, where we must include the query results because // a custom condition executes its own PPL query, number of results trigger evaluation @@ -297,22 +297,22 @@ class TriggerService(val scriptService: ScriptService) { // of the results. the results themselves are already held in QueryLevelMonitorRunner.kt // (the caller of this function), so passing the results here only to return them again // inside the QueryLevelTriggerRunResult would be redundant - return QueryLevelTriggerRunResult(pplSqlTrigger.name, triggered, null) + return QueryLevelTriggerRunResult(pplTrigger.name, triggered, null) } - suspend fun runPplSqlCustomTrigger( - pplSqlMonitor: Monitor, - pplSqlTrigger: PPLSQLTrigger, + suspend fun runPplCustomTrigger( + pplMonitor: Monitor, + pplTrigger: PPLTrigger, query: String, monitorCtx: MonitorRunnerExecutionContext, transportService: TransportService ): QueryLevelTriggerRunResult { - if (pplSqlTrigger.customCondition == null) { + if (pplTrigger.customCondition == null) { return QueryLevelTriggerRunResult( - pplSqlTrigger.name, + pplTrigger.name, true, - IllegalStateException("No custom condition found for trigger: ${pplSqlTrigger.id}") + IllegalStateException("No custom condition found for trigger: ${pplTrigger.id}") ) } @@ -333,7 +333,7 @@ class TriggerService(val scriptService: ScriptService) { withTimeout(monitorExecutionDuration.millis) { logger.debug("checking if custom condition is used and appending to base query") - val customCondition = pplSqlTrigger.customCondition!! + val customCondition = pplTrigger.customCondition!! // append the custom condition to query val queryToExecute = appendCustomCondition(query, customCondition) @@ -348,16 +348,16 @@ class TriggerService(val scriptService: ScriptService) { // in the alert and notification must be added that results were excluded // and an alert that should have been generated might not have been - logger.debug("executing the PPL query of monitor: ${pplSqlMonitor.id} with custom condition: $customCondition") + logger.debug("executing the PPL query of monitor: ${pplMonitor.id} with custom condition: $customCondition") // execute the PPL query val (queryResponseJson, timeTaken) = measureTimedValue { withClosableContext( InjectorContextElement( - pplSqlMonitor.id, + pplMonitor.id, monitorCtx.settings!!, monitorCtx.threadPool!!.threadContext, - pplSqlMonitor.user?.roles, - pplSqlMonitor.user + pplMonitor.user?.roles, + pplMonitor.user ) ) { executePplQuery( @@ -368,7 +368,7 @@ class TriggerService(val scriptService: ScriptService) { ) } } - logger.debug("query results for trigger ${pplSqlTrigger.id}: $queryResponseJson") + logger.debug("query results for trigger ${pplTrigger.id}: $queryResponseJson") logger.debug("time taken to execute query against sql/ppl plugin: $timeTaken") // val numPplResults = basePplQueryResults.getLong("total") @@ -381,11 +381,11 @@ class TriggerService(val scriptService: ScriptService) { // cap and reformat the results to be included in trigger run result customConditionQueryResults = capAndReformatPPLQueryResults(queryResponseJson, queryResultsSizeLimit) - logger.debug("Custom PPLTrigger ${pplSqlTrigger.name} with ID ${pplSqlTrigger.id} triggered: $triggered") + logger.debug("Custom PPLTrigger ${pplTrigger.name} with ID ${pplTrigger.id} triggered: $triggered") } return QueryLevelTriggerRunResult( - pplSqlTrigger.name, + pplTrigger.name, triggered!!, null, mutableMapOf(), @@ -393,12 +393,12 @@ class TriggerService(val scriptService: ScriptService) { ) } catch (e: Exception) { logger.error( - "failed to run PPL Custom Trigger ${pplSqlTrigger.name} (id: ${pplSqlTrigger.id} " + - "from PPL Monitor ${pplSqlMonitor.name} (id: ${pplSqlMonitor.id}", + "failed to run PPL Custom Trigger ${pplTrigger.name} (id: ${pplTrigger.id} " + + "from PPL Monitor ${pplMonitor.name} (id: ${pplMonitor.id}", e ) - return QueryLevelTriggerRunResult(pplSqlTrigger.name, true, e) + return QueryLevelTriggerRunResult(pplTrigger.name, true, e) } } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestIndexMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestIndexMonitorAction.kt index 815fbd85f..7a9c76e10 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestIndexMonitorAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestIndexMonitorAction.kt @@ -18,7 +18,7 @@ import org.opensearch.commons.alerting.model.BucketLevelTrigger import org.opensearch.commons.alerting.model.DocLevelMonitorInput import org.opensearch.commons.alerting.model.DocumentLevelTrigger import org.opensearch.commons.alerting.model.Monitor -import org.opensearch.commons.alerting.model.PPLSQLTrigger +import org.opensearch.commons.alerting.model.PPLTrigger import org.opensearch.commons.alerting.model.QueryLevelTrigger import org.opensearch.commons.alerting.model.ScheduledJob import org.opensearch.commons.alerting.util.AlertingException @@ -139,7 +139,7 @@ class RestIndexMonitorAction : BaseRestHandler() { Monitor.MonitorType.PPL_MONITOR -> { triggers.forEach { - if (it !is PPLSQLTrigger) { + if (it !is PPLTrigger) { throw IllegalArgumentException("Illegal trigger type, ${it.javaClass.name}, for PPL monitor") } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/script/QueryLevelTriggerExecutionContext.kt b/alerting/src/main/kotlin/org/opensearch/alerting/script/QueryLevelTriggerExecutionContext.kt index 23bdf4704..d2963a8c2 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/script/QueryLevelTriggerExecutionContext.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/script/QueryLevelTriggerExecutionContext.kt @@ -9,7 +9,7 @@ import org.opensearch.alerting.model.AlertContext import org.opensearch.common.settings.ClusterSettings import org.opensearch.commons.alerting.model.Monitor import org.opensearch.commons.alerting.model.MonitorRunResult -import org.opensearch.commons.alerting.model.PPLSQLTrigger +import org.opensearch.commons.alerting.model.PPLTrigger import org.opensearch.commons.alerting.model.QueryLevelTrigger import org.opensearch.commons.alerting.model.QueryLevelTriggerRunResult import org.opensearch.commons.alerting.model.Trigger @@ -28,9 +28,9 @@ data class QueryLevelTriggerExecutionContext( ) : TriggerExecutionContext(monitor, results, periodStart, periodEnd, error, clusterSettings) { init { - require(trigger is QueryLevelTrigger || trigger is PPLSQLTrigger) { + require(trigger is QueryLevelTrigger || trigger is PPLTrigger) { "QueryLevelTriggerExecutionContext must only store Triggers for per-query style monitoring, " + - "like QueryLevelTrigger or PPLSQLTrigger" + "like QueryLevelTrigger or PPLTrigger" } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt index 19a9de085..99e014f25 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt @@ -70,8 +70,8 @@ import org.opensearch.commons.alerting.model.DocLevelMonitorInput.Companion.DOC_ import org.opensearch.commons.alerting.model.Monitor import org.opensearch.commons.alerting.model.Monitor.MonitorType import org.opensearch.commons.alerting.model.MonitorMetadata -import org.opensearch.commons.alerting.model.PPLSQLInput -import org.opensearch.commons.alerting.model.PPLSQLTrigger +import org.opensearch.commons.alerting.model.PPLInput +import org.opensearch.commons.alerting.model.PPLTrigger import org.opensearch.commons.alerting.model.ScheduledJob import org.opensearch.commons.alerting.model.ScheduledJob.Companion.SCHEDULED_JOBS_INDEX import org.opensearch.commons.alerting.model.SearchInput @@ -208,7 +208,7 @@ class TransportIndexMonitorAction @Inject constructor( // check if user has access to any anomaly detector for AD monitor checkAnomalyDetectorAndExecute(client, actionListener, transformedRequest, user) } else if (transformedRequest.monitor.monitorType == MonitorType.PPL_MONITOR.value) { - checkPplSqlQueryAndExecute(actionListener, transformedRequest, user) + checkPPLQueryAndExecute(actionListener, transformedRequest, user) } else { checkIndicesAndExecute(client, actionListener, transformedRequest, user) } @@ -277,7 +277,7 @@ class TransportIndexMonitorAction @Inject constructor( ) } - private fun checkPplSqlQueryAndExecute( + private fun checkPPLQueryAndExecute( actionListener: ActionListener, indexMonitorRequest: IndexMonitorRequest, user: User? @@ -293,12 +293,12 @@ class TransportIndexMonitorAction @Inject constructor( // roles can be matched and checked downstream // roles can be matched and checked downstream client.threadPool().threadContext.stashContext().use { - val pplSqlMonitor = indexMonitorRequest.monitor + val pplMonitor = indexMonitorRequest.monitor if (user == null) { - indexMonitorRequest.monitor = pplSqlMonitor + indexMonitorRequest.monitor = pplMonitor .copy(user = User("", listOf(), listOf(), mapOf())) } else { - indexMonitorRequest.monitor = pplSqlMonitor + indexMonitorRequest.monitor = pplMonitor .copy(user = User(user.name, user.backendRoles, user.roles, user.customAttributes)) } IndexMonitorHandler(client, actionListener, indexMonitorRequest, user).resolveUserAndStart() @@ -317,18 +317,18 @@ class TransportIndexMonitorAction @Inject constructor( withContext(singleThreadContext) { it.restore() - val pplSqlMonitor = indexMonitorRequest.monitor + val pplMonitor = indexMonitorRequest.monitor // validate the PPL query syntax and that user has permissions to // the indices being queried - val pplQueryValid = validatePplSqlQuery(pplSqlMonitor, validationListener) + val pplQueryValid = validatePPLQuery(pplMonitor, validationListener) if (!pplQueryValid) { return@withContext } - // run basic validations against the PPL/SQL Monitor - val pplSqlMonitorValid = validatePplSqlMonitor(pplSqlMonitor, validationListener) - if (!pplSqlMonitorValid) { + // run basic validations against the PPL Monitor + val pplMonitorValid = validatePPLMonitor(pplMonitor, validationListener) + if (!pplMonitorValid) { return@withContext } @@ -338,14 +338,14 @@ class TransportIndexMonitorAction @Inject constructor( } } - private suspend fun validatePplSqlQuery( - pplSqlMonitor: Monitor, + private suspend fun validatePPLQuery( + pplMonitor: Monitor, validationListener: ActionListener ): Boolean { // first attempt to run the monitor query and all possible // extensions of it (from custom conditions) try { - val query = (pplSqlMonitor.inputs[0] as PPLSQLInput).query + val query = (pplMonitor.inputs[0] as PPLInput).query val limitedQueryToExecute = appendDataRowsLimit(query, maxQueryResults) @@ -356,10 +356,10 @@ class TransportIndexMonitorAction @Inject constructor( // scan all the triggers with custom conditions, and ensure each query constructed // from the base query + custom condition is valid - for (trigger in pplSqlMonitor.triggers) { - val pplTrigger = trigger as PPLSQLTrigger + for (trigger in pplMonitor.triggers) { + val pplTrigger = trigger as PPLTrigger - if (pplTrigger.conditionType != PPLSQLTrigger.ConditionType.CUSTOM) { + if (pplTrigger.conditionType != PPLTrigger.ConditionType.CUSTOM) { continue } @@ -398,11 +398,11 @@ class TransportIndexMonitorAction @Inject constructor( return true } - private fun validatePplSqlMonitor(pplSqlMonitor: Monitor, validationListener: ActionListener): Boolean { - pplSqlMonitor.triggers.forEach { trigger -> - val pplTrigger = trigger as PPLSQLTrigger + private fun validatePPLMonitor(pplMonitor: Monitor, validationListener: ActionListener): Boolean { + pplMonitor.triggers.forEach { trigger -> + val pplTrigger = trigger as PPLTrigger - if (pplTrigger.conditionType == PPLSQLTrigger.ConditionType.NUMBER_OF_RESULTS && + if (pplTrigger.conditionType == PPLTrigger.ConditionType.NUMBER_OF_RESULTS && pplTrigger.numResultsValue!! > maxQueryResults ) { validationListener.onFailure( @@ -443,7 +443,7 @@ class TransportIndexMonitorAction @Inject constructor( } } - val query = (pplSqlMonitor.inputs[0] as PPLSQLInput).query + val query = (pplMonitor.inputs[0] as PPLInput).query // ensure the query length doesn't exceed the limit if (query.length > maxQueryLength) { diff --git a/alerting/src/main/resources/org/opensearch/alerting/alerts/alert_mapping.json b/alerting/src/main/resources/org/opensearch/alerting/alerts/alert_mapping.json index 57dfc0fa4..12a3d86d2 100644 --- a/alerting/src/main/resources/org/opensearch/alerting/alerts/alert_mapping.json +++ b/alerting/src/main/resources/org/opensearch/alerting/alerts/alert_mapping.json @@ -178,10 +178,10 @@ } } }, - "ppl_query": { + "query": { "type": "text" }, - "ppl_query_results": { + "query_results": { "type": "nested", "dynamic": true } diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt b/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt index 9305ac25e..6cf36f982 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt @@ -42,12 +42,8 @@ import org.opensearch.common.xcontent.XContentFactory import org.opensearch.common.xcontent.XContentFactory.jsonBuilder import org.opensearch.common.xcontent.XContentType import org.opensearch.common.xcontent.json.JsonXContent.jsonXContent -import org.opensearch.commons.alerting.action.GetAlertsResponse.Companion.ALERTS_FIELD -import org.opensearch.commons.alerting.action.GetAlertsResponse.Companion.TOTAL_ALERTS_FIELD import org.opensearch.commons.alerting.action.GetFindingsResponse import org.opensearch.commons.alerting.model.Alert -import org.opensearch.commons.alerting.model.Alert.Companion.ERROR_MESSAGE_FIELD -import org.opensearch.commons.alerting.model.Alert.Companion.STATE_FIELD import org.opensearch.commons.alerting.model.BucketLevelTrigger import org.opensearch.commons.alerting.model.ChainedAlertTrigger import org.opensearch.commons.alerting.model.Comment @@ -58,8 +54,8 @@ import org.opensearch.commons.alerting.model.DocumentLevelTrigger import org.opensearch.commons.alerting.model.Finding import org.opensearch.commons.alerting.model.FindingWithDocs import org.opensearch.commons.alerting.model.Monitor -import org.opensearch.commons.alerting.model.PPLSQLInput -import org.opensearch.commons.alerting.model.PPLSQLTrigger +import org.opensearch.commons.alerting.model.PPLInput +import org.opensearch.commons.alerting.model.PPLTrigger import org.opensearch.commons.alerting.model.QueryLevelTrigger import org.opensearch.commons.alerting.model.ScheduledJob import org.opensearch.commons.alerting.model.SearchInput @@ -113,11 +109,11 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { Monitor.XCONTENT_REGISTRY, SearchInput.XCONTENT_REGISTRY, DocLevelMonitorInput.XCONTENT_REGISTRY, - PPLSQLInput.XCONTENT_REGISTRY, + PPLInput.XCONTENT_REGISTRY, QueryLevelTrigger.XCONTENT_REGISTRY, BucketLevelTrigger.XCONTENT_REGISTRY, DocumentLevelTrigger.XCONTENT_REGISTRY, - PPLSQLTrigger.XCONTENT_REGISTRY, + PPLTrigger.XCONTENT_REGISTRY, Workflow.XCONTENT_REGISTRY, ChainedAlertTrigger.XCONTENT_REGISTRY ) + SearchModule(Settings.EMPTY, emptyList()).namedXContents @@ -2082,32 +2078,4 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { val numberDocsFound = hits["total"]?.get("value") assertEquals("Unexpected number of PPL Monitors found in Search Monitors", expectedNum, numberDocsFound) } - - // takes in an execute monitor API response and returns true if the - // trigger condition was met. assumes the monitor executed only had 1 trigger - protected fun isTriggered(pplMonitor: Monitor, executeResponse: Response): Boolean { - val executeResponseMap = entityAsMap(executeResponse) - val triggerResultsObj = (executeResponseMap["trigger_results"] as Map)[pplMonitor.triggers[0].id] as Map - return triggerResultsObj["triggered"] as Boolean - } - - // takes in a get alerts API response and returns the current number of active alerts - protected fun numAlerts(getAlertsResponse: Response): Int { - logger.info("get alerts response: ${entityAsMap(getAlertsResponse)}") - return entityAsMap(getAlertsResponse)[TOTAL_ALERTS_FIELD] as Int - } - - protected fun containsErrorAlert(getAlertsResponse: Response): Boolean { - val getAlertsMap = entityAsMap(getAlertsResponse) - val alertsList = getAlertsMap[ALERTS_FIELD] as List> - alertsList.forEach { alert -> - val errorMessage = alert[ERROR_MESSAGE_FIELD] as String? - val state = Alert.State.valueOf((alert[STATE_FIELD] as String?)!!.uppercase(Locale.ROOT)) - - if (state == Alert.State.ERROR && errorMessage != null) { - return true - } - } - return false - } } diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt index 8a51b261c..d933d1029 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt @@ -34,7 +34,7 @@ import org.opensearch.commons.alerting.model.DocLevelMonitorInput import org.opensearch.commons.alerting.model.DocLevelQuery import org.opensearch.commons.alerting.model.IntervalSchedule import org.opensearch.commons.alerting.model.Monitor -import org.opensearch.commons.alerting.model.PPLSQLTrigger +import org.opensearch.commons.alerting.model.PPLTrigger import org.opensearch.commons.alerting.model.SearchInput import org.opensearch.commons.alerting.model.action.ActionExecutionPolicy import org.opensearch.commons.alerting.model.action.AlertCategory @@ -2244,8 +2244,8 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() { schedule = IntervalSchedule(interval = 1, unit = MINUTES), triggers = listOf( randomPPLTrigger( - conditionType = PPLSQLTrigger.ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = PPLSQLTrigger.NumResultsCondition.GREATER_THAN, + conditionType = PPLTrigger.ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = PPLTrigger.NumResultsCondition.GREATER_THAN, numResultsValue = 0L, customCondition = null ) @@ -2286,7 +2286,7 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() { schedule = IntervalSchedule(interval = 1, unit = MINUTES), triggers = listOf( randomPPLTrigger( - conditionType = PPLSQLTrigger.ConditionType.CUSTOM, + conditionType = PPLTrigger.ConditionType.CUSTOM, customCondition = "where max_num > 5", numResultsCondition = null, numResultsValue = null @@ -2328,8 +2328,8 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() { schedule = IntervalSchedule(interval = 1, unit = MINUTES), triggers = listOf( randomPPLTrigger( - conditionType = PPLSQLTrigger.ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = PPLSQLTrigger.NumResultsCondition.GREATER_THAN, + conditionType = PPLTrigger.ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = PPLTrigger.NumResultsCondition.GREATER_THAN, numResultsValue = 0L, customCondition = null ) @@ -2352,10 +2352,10 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() { assertEquals(monitor.triggers[0].id, alert.triggerId) // Verify ppl_query_results contains the size exceeded message - assertNotNull(alert.pplQueryResults) - assertEquals(1, alert.pplQueryResults.size) + assertNotNull(alert.queryResults) + assertEquals(1, alert.queryResults.size) - val firstResultRow = alert.pplQueryResults[0] + val firstResultRow = alert.queryResults[0] assertTrue(firstResultRow.containsKey("message")) assertEquals(PPL_RESULTS_SIZE_EXCEEDED_MESSAGE, firstResultRow["message"]) } @@ -2373,8 +2373,8 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() { schedule = IntervalSchedule(interval = 1, unit = MINUTES), triggers = listOf( randomPPLTrigger( - conditionType = PPLSQLTrigger.ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = PPLSQLTrigger.NumResultsCondition.GREATER_THAN, + conditionType = PPLTrigger.ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = PPLTrigger.NumResultsCondition.GREATER_THAN, numResultsValue = 0L, customCondition = null ) @@ -2403,15 +2403,15 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() { verifyAlert(activeAlert, monitor, ACTIVE) // Verify alert contains PPL query and results - assertNotNull("PPL query should be stored in alert", activeAlert.pplQuery) - assertEquals("source = $TEST_INDEX_NAME | head 10", activeAlert.pplQuery) - assertTrue("PPL query results should not be empty", activeAlert.pplQueryResults.isNotEmpty()) + assertNotNull("PPL query should be stored in alert", activeAlert.query) + assertEquals("source = $TEST_INDEX_NAME | head 10", activeAlert.query) + assertTrue("PPL query results should not be empty", activeAlert.queryResults.isNotEmpty()) // Verify the query results are in the new transformed format (list of maps) - assertEquals("Should have exactly 1 result row from the indexed document", 1, activeAlert.pplQueryResults.size) + assertEquals("Should have exactly 1 result row from the indexed document", 1, activeAlert.queryResults.size) // Get the first (and only) result row - val resultRow = activeAlert.pplQueryResults[0] + val resultRow = activeAlert.queryResults[0] // Verify the result row contains the expected fields from the indexed document assertTrue("Result should contain 'timestamp' field", resultRow.containsKey("timestamp")) @@ -2488,7 +2488,7 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() { schedule = IntervalSchedule(interval = 1, unit = MINUTES), triggers = listOf( randomPPLTrigger( - conditionType = PPLSQLTrigger.ConditionType.CUSTOM, + conditionType = PPLTrigger.ConditionType.CUSTOM, customCondition = "where max_num > 5", numResultsCondition = null, numResultsValue = null @@ -2516,16 +2516,16 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() { verifyAlert(activeAlert, monitor, ACTIVE) // Verify alert contains PPL query and aggregated results - assertNotNull("PPL query should be stored in alert", activeAlert.pplQuery) - assertEquals("source = $TEST_INDEX_NAME | stats max(number) as max_num by abc", activeAlert.pplQuery) - assertTrue("PPL query results should not be empty", activeAlert.pplQueryResults.isNotEmpty()) + assertNotNull("PPL query should be stored in alert", activeAlert.query) + assertEquals("source = $TEST_INDEX_NAME | stats max(number) as max_num by abc", activeAlert.query) + assertTrue("PPL query results should not be empty", activeAlert.queryResults.isNotEmpty()) // Verify the query results are in the new transformed format (list of maps), // and only have the 2 buckets that met the custom condition - assertEquals("Should have 2 aggregation result rows", 2, activeAlert.pplQueryResults.size) + assertEquals("Should have 2 aggregation result rows", 2, activeAlert.queryResults.size) // Convert results to a map of group name -> result row for easier validation - val groupResults = activeAlert.pplQueryResults.associateBy { it["abc"] as String } + val groupResults = activeAlert.queryResults.associateBy { it["abc"] as String } // Verify all expected groups are present assertTrue("Should contain group 'def'", groupResults.containsKey("def")) @@ -2613,8 +2613,8 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() { schedule = IntervalSchedule(interval = 1, unit = MINUTES), triggers = listOf( randomPPLTrigger( - conditionType = PPLSQLTrigger.ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = PPLSQLTrigger.NumResultsCondition.GREATER_THAN, + conditionType = PPLTrigger.ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = PPLTrigger.NumResultsCondition.GREATER_THAN, numResultsValue = 0L, customCondition = null ) @@ -2638,15 +2638,15 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() { verifyAlert(firstAlert, monitor, ACTIVE) // Verify alert contains PPL query and results with 1 document - assertNotNull("PPL query should be stored in alert", firstAlert.pplQuery) - assertEquals("source = $TEST_INDEX_NAME | head 10", firstAlert.pplQuery) - assertTrue("PPL query results should not be empty", firstAlert.pplQueryResults.isNotEmpty()) + assertNotNull("PPL query should be stored in alert", firstAlert.query) + assertEquals("source = $TEST_INDEX_NAME | head 10", firstAlert.query) + assertTrue("PPL query results should not be empty", firstAlert.queryResults.isNotEmpty()) // Verify the query results are in the new transformed format (list of maps) - assertEquals("Should have 1 result row from the first indexed document", 1, firstAlert.pplQueryResults.size) + assertEquals("Should have 1 result row from the first indexed document", 1, firstAlert.queryResults.size) // Get the first result row and verify it has the expected structure - val firstResultRow = firstAlert.pplQueryResults[0] + val firstResultRow = firstAlert.queryResults[0] assertTrue("Result should contain 'timestamp' field", firstResultRow.containsKey("timestamp")) assertTrue("Result should contain 'abc' field", firstResultRow.containsKey("abc")) assertTrue("Result should contain 'number' field", firstResultRow.containsKey("number")) @@ -2675,26 +2675,26 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() { assertEquals("Alert ID should be the same", firstAlert.id, updatedAlert.id) // Verify the alert's query results have been updated to include both documents - assertNotNull("PPL query should still be stored in alert", updatedAlert.pplQuery) - assertTrue("PPL query results should not be empty", updatedAlert.pplQueryResults.isNotEmpty()) + assertNotNull("PPL query should still be stored in alert", updatedAlert.query) + assertTrue("PPL query results should not be empty", updatedAlert.queryResults.isNotEmpty()) // Verify the query results now contain 2 rows (updated from previous 1 row) - assertEquals("Should now have 2 result rows from both indexed documents", 2, updatedAlert.pplQueryResults.size) + assertEquals("Should now have 2 result rows from both indexed documents", 2, updatedAlert.queryResults.size) // Verify both result rows have the expected structure - updatedAlert.pplQueryResults.forEach { row -> + updatedAlert.queryResults.forEach { row -> assertTrue("Each result row should contain 'timestamp' field", row.containsKey("timestamp")) assertTrue("Each result row should contain 'abc' field", row.containsKey("abc")) assertTrue("Each result row should contain 'number' field", row.containsKey("number")) } // Verify we can find both documents in the results by their unique 'abc' values - val abcValues = updatedAlert.pplQueryResults.map { it["abc"] }.toSet() + val abcValues = updatedAlert.queryResults.map { it["abc"] }.toSet() assertTrue("Results should contain first document with abc='test-value'", abcValues.contains("test-value")) assertTrue("Results should contain second document with abc='test-value-2'", abcValues.contains("test-value-2")) // Verify we can find both documents by their number values - val numberValues = updatedAlert.pplQueryResults.map { (it["number"] as Number).toInt() }.toSet() + val numberValues = updatedAlert.queryResults.map { (it["number"] as Number).toInt() }.toSet() assertTrue("Results should contain first document with number=5", numberValues.contains(5)) assertTrue("Results should contain second document with number=10", numberValues.contains(10)) @@ -2754,8 +2754,8 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() { schedule = IntervalSchedule(interval = 1, unit = MINUTES), triggers = listOf( randomPPLTrigger( - conditionType = PPLSQLTrigger.ConditionType.NUMBER_OF_RESULTS, - numResultsCondition = PPLSQLTrigger.NumResultsCondition.GREATER_THAN, + conditionType = PPLTrigger.ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = PPLTrigger.NumResultsCondition.GREATER_THAN, numResultsValue = 0L, customCondition = null, actions = listOf(action) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt b/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt index 331eb96c3..fdf43579a 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt @@ -47,11 +47,11 @@ import org.opensearch.commons.alerting.model.InputRunResults import org.opensearch.commons.alerting.model.IntervalSchedule import org.opensearch.commons.alerting.model.Monitor import org.opensearch.commons.alerting.model.MonitorRunResult -import org.opensearch.commons.alerting.model.PPLSQLInput -import org.opensearch.commons.alerting.model.PPLSQLInput.QueryLanguage -import org.opensearch.commons.alerting.model.PPLSQLTrigger -import org.opensearch.commons.alerting.model.PPLSQLTrigger.ConditionType -import org.opensearch.commons.alerting.model.PPLSQLTrigger.NumResultsCondition +import org.opensearch.commons.alerting.model.PPLInput +import org.opensearch.commons.alerting.model.PPLInput.QueryLanguage +import org.opensearch.commons.alerting.model.PPLTrigger +import org.opensearch.commons.alerting.model.PPLTrigger.ConditionType +import org.opensearch.commons.alerting.model.PPLTrigger.NumResultsCondition import org.opensearch.commons.alerting.model.QueryLevelTrigger import org.opensearch.commons.alerting.model.QueryLevelTriggerRunResult import org.opensearch.commons.alerting.model.Schedule @@ -246,7 +246,7 @@ fun randomPPLMonitor( schedule: Schedule = IntervalSchedule(interval = 5, unit = ChronoUnit.MINUTES), lastUpdateTime: Instant = Instant.now().truncatedTo(ChronoUnit.MILLIS), enabledTime: Instant? = if (enabled) Instant.now().truncatedTo(ChronoUnit.MILLIS) else null, - triggers: List = List(randomIntBetween(1, 5)) { randomPPLTrigger() }, + triggers: List = List(randomIntBetween(1, 5)) { randomPPLTrigger() }, user: User? = randomUser(), queryLanguage: QueryLanguage = QueryLanguage.PPL, query: String = "source = $TEST_INDEX_NAME | head 10" @@ -259,7 +259,7 @@ fun randomPPLMonitor( enabledTime = enabledTime, monitorType = Monitor.MonitorType.PPL_MONITOR.value, inputs = listOf( - PPLSQLInput( + PPLInput( query = query, queryLanguage = queryLanguage ) @@ -407,8 +407,8 @@ fun randomPPLTrigger( numResultsCondition: NumResultsCondition? = NumResultsCondition.entries.random(), numResultsValue: Long? = randomLongBetween(1L, 50L), customCondition: String? = null -): PPLSQLTrigger { - return PPLSQLTrigger( +): PPLTrigger { + return PPLTrigger( id = id, name = name, severity = severity, @@ -896,24 +896,24 @@ fun assertPplMonitorsEqual(pplMonitor1: Monitor, pplMonitor2: Monitor) { assertEquals("Monitor schedules not equal", pplMonitor1.schedule, pplMonitor2.schedule) assertEquals( "Monitor query languages not equal", - (pplMonitor1.inputs[0] as PPLSQLInput).queryLanguage, - (pplMonitor2.inputs[0] as PPLSQLInput).queryLanguage + (pplMonitor1.inputs[0] as PPLInput).queryLanguage, + (pplMonitor2.inputs[0] as PPLInput).queryLanguage ) assertEquals( "Monitor queries not equal", - (pplMonitor1.inputs[0] as PPLSQLInput).query, - (pplMonitor2.inputs[0] as PPLSQLInput).query + (pplMonitor1.inputs[0] as PPLInput).query, + (pplMonitor2.inputs[0] as PPLInput).query ) assertEquals("Number of triggers in monitor not equal", pplMonitor1.triggers.size, pplMonitor2.triggers.size) val sortedTriggers1 = pplMonitor1.triggers.sortedBy { it.id } val sortedTriggers2 = pplMonitor2.triggers.sortedBy { it.id } for (i in sortedTriggers1.indices) { - assertPplTriggersEqual(sortedTriggers1[i] as PPLSQLTrigger, sortedTriggers2[i] as PPLSQLTrigger) + assertPplTriggersEqual(sortedTriggers1[i] as PPLTrigger, sortedTriggers2[i] as PPLTrigger) } } -fun assertPplTriggersEqual(pplTrigger1: PPLSQLTrigger, pplTrigger2: PPLSQLTrigger) { +fun assertPplTriggersEqual(pplTrigger1: PPLTrigger, pplTrigger2: PPLTrigger) { assertEquals( "Monitor trigger IDs not equal", pplTrigger1.id, diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt index 6db45f514..183579a79 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt @@ -60,7 +60,7 @@ import org.opensearch.commons.alerting.model.DocLevelMonitorInput import org.opensearch.commons.alerting.model.DocLevelQuery import org.opensearch.commons.alerting.model.DocumentLevelTrigger import org.opensearch.commons.alerting.model.Monitor -import org.opensearch.commons.alerting.model.PPLSQLTrigger.ConditionType +import org.opensearch.commons.alerting.model.PPLTrigger.ConditionType import org.opensearch.commons.alerting.model.QueryLevelTrigger import org.opensearch.commons.alerting.model.ScheduledJob import org.opensearch.commons.alerting.model.SearchInput @@ -82,7 +82,6 @@ import org.opensearch.test.rest.OpenSearchRestTestCase import java.time.Instant import java.time.ZoneId import java.time.temporal.ChronoUnit -import java.time.temporal.ChronoUnit.MINUTES import java.util.concurrent.TimeUnit @TestLogging("level:DEBUG", reason = "Debug for tests.") From e2fc017a8cf2767c9df1365b3d503d7f3e9a5e49 Mon Sep 17 00:00:00 2001 From: Dennis Toepker Date: Mon, 20 Apr 2026 19:12:46 -0700 Subject: [PATCH 9/9] refactoring PPL call to use client.execute instead of transport service Signed-off-by: Dennis Toepker --- .../org/opensearch/alerting/InputService.kt | 4 +- .../org/opensearch/alerting/PPLUtils.kt | 9 ++-- .../org/opensearch/alerting/TriggerService.kt | 4 +- .../transport/TransportIndexMonitorAction.kt | 5 ++- .../alerting/core/ppl/PPLPluginInterface.kt | 41 ++++--------------- 5 files changed, 17 insertions(+), 46 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt index 6b65f476d..c09f4e1a4 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt @@ -58,6 +58,7 @@ import org.opensearch.script.TemplateScript import org.opensearch.search.builder.SearchSourceBuilder import org.opensearch.transport.TransportService import org.opensearch.transport.client.Client +import org.opensearch.transport.client.node.NodeClient import java.time.Duration import java.time.Instant import kotlin.time.measureTimedValue @@ -293,8 +294,7 @@ class InputService( executePplQuery( limitedQueryToExecute, false, - monitorCtx.clusterService!!.state().nodes.localNode, - transportService + monitorCtx.client!! as NodeClient ) } logger.debug("base query results: $queryResponseJsonReceived") diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt b/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt index 9080361be..2d7558c21 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt @@ -9,9 +9,8 @@ import org.json.JSONArray import org.json.JSONObject import org.opensearch.alerting.core.ppl.PPLPluginInterface import org.opensearch.alerting.opensearchapi.suspendUntil -import org.opensearch.cluster.node.DiscoveryNode import org.opensearch.sql.plugin.transport.TransportPPLQueryRequest -import org.opensearch.transport.TransportService +import org.opensearch.transport.client.node.NodeClient object PPLUtils { @@ -99,8 +98,7 @@ object PPLUtils { suspend fun executePplQuery( query: String, explain: Boolean, - localNode: DiscoveryNode, - transportService: TransportService + client: NodeClient ): JSONObject { val path = if (explain) { "/_plugins/_ppl/_explain" @@ -117,8 +115,7 @@ object PPLUtils { val transportPplQueryResponse = PPLPluginInterface.suspendUntil { this.executeQuery( - transportService, - localNode, + client, transportPplQueryRequest, it ) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt index 88c003450..321708a27 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt @@ -49,6 +49,7 @@ import org.opensearch.search.aggregations.Aggregation import org.opensearch.search.aggregations.Aggregations import org.opensearch.search.aggregations.support.AggregationPath import org.opensearch.transport.TransportService +import org.opensearch.transport.client.node.NodeClient import kotlin.time.measureTimedValue /** Service that handles executing Triggers */ @@ -363,8 +364,7 @@ class TriggerService(val scriptService: ScriptService) { executePplQuery( limitedQueryToExecute, false, - monitorCtx.clusterService!!.state().nodes.localNode, - transportService + monitorCtx.client!! as NodeClient ) } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt index 99e014f25..8c87371dd 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt @@ -100,6 +100,7 @@ import org.opensearch.search.builder.SearchSourceBuilder import org.opensearch.tasks.Task import org.opensearch.transport.TransportService import org.opensearch.transport.client.Client +import org.opensearch.transport.client.node.NodeClient import java.io.IOException import java.time.Duration import java.util.Locale @@ -352,7 +353,7 @@ class TransportIndexMonitorAction @Inject constructor( // now PPL explain the base query as is. // if there are any PPL syntax, index not found, insufficient index permissions, or other errors, // this will throw an exception from the SQL/PPL plugin - executePplQuery(limitedQueryToExecute, true, clusterService.state().nodes.localNode, transportService) + executePplQuery(limitedQueryToExecute, true, client as NodeClient) // scan all the triggers with custom conditions, and ensure each query constructed // from the base query + custom condition is valid @@ -384,7 +385,7 @@ class TransportIndexMonitorAction @Inject constructor( // if the custom condition is invalid, this will throw an exception // from the SQL/PPL plugin - executePplQuery(limitedQueryWithCustomCondition, true, clusterService.state().nodes.localNode, transportService) + executePplQuery(limitedQueryWithCustomCondition, true, client as NodeClient) } } catch (e: Exception) { validationListener.onFailure( diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/ppl/PPLPluginInterface.kt b/core/src/main/kotlin/org/opensearch/alerting/core/ppl/PPLPluginInterface.kt index e009b8845..8ff15165c 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/core/ppl/PPLPluginInterface.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/core/ppl/PPLPluginInterface.kt @@ -5,35 +5,24 @@ package org.opensearch.alerting.core.ppl -import org.opensearch.action.ActionListenerResponseHandler -import org.opensearch.cluster.node.DiscoveryNode -import org.opensearch.common.unit.TimeValue import org.opensearch.commons.utils.recreateObject import org.opensearch.core.action.ActionListener import org.opensearch.core.action.ActionResponse -import org.opensearch.core.common.io.stream.Writeable import org.opensearch.sql.plugin.transport.PPLQueryAction import org.opensearch.sql.plugin.transport.TransportPPLQueryRequest import org.opensearch.sql.plugin.transport.TransportPPLQueryResponse -import org.opensearch.transport.TransportException -import org.opensearch.transport.TransportRequestOptions -import org.opensearch.transport.TransportService +import org.opensearch.transport.client.node.NodeClient /** * Transport action plugin interfaces for the SQL/PPL plugin */ +@Suppress("UNCHECKED_CAST") object PPLPluginInterface { fun executeQuery( - transportService: TransportService, - localNode: DiscoveryNode, + client: NodeClient, request: TransportPPLQueryRequest, listener: ActionListener, ) { - - val responseReader = Writeable.Reader { - TransportPPLQueryResponse(it) - } - val wrappedListener = object : ActionListener { override fun onResponse(response: ActionResponse) { val recreated = recreateObject(response) { TransportPPLQueryResponse(it) } @@ -43,28 +32,12 @@ object PPLPluginInterface { override fun onFailure(exception: Exception) { listener.onFailure(exception) } - } + } as ActionListener - transportService.sendRequest( - localNode, - PPLQueryAction.NAME, + client.execute( + PPLQueryAction.INSTANCE, request, - TransportRequestOptions - .builder() - .withTimeout(TimeValue.timeValueMinutes(1)) - .build(), - object : ActionListenerResponseHandler( - wrappedListener, - responseReader - ) { - override fun handleResponse(response: ActionResponse) { - wrappedListener.onResponse(response) - } - - override fun handleException(e: TransportException) { - wrappedListener.onFailure(e) - } - } + wrappedListener ) } }