Skip to content

Ssoteam 2296 #84

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
thegentlemanphysicist merged 3 commits into from
Apr 2, 2025
Merged

Ssoteam 2296 #84

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c346160
chore: update alerts
thegentlemanphysicist Apr 2, 2025
a0d9fc4
chore: delete obsolete metrics
thegentlemanphysicist Apr 2, 2025
9ce06be
chore: resolve merge conflict
thegentlemanphysicist Apr 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 0 additions & 110 deletions terraform/gold-eb75ad-team/alerts.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -504,116 +504,6 @@ resource "sysdig_monitor_alert_promql" "prod_kc_disk_log_pv_usage_sixty" {
}
}

# NO LONGER WORKING IN SYSDIG
# # The Keycloak deployment has a max surge of 3, and the keycloak pods
# # require 2 cores. Setting limit at (3*2+1) cores
# ## Max surge as of Aug 2024 1, Pod count 3, CPU limit 4 (1*4+1) = 5
# resource "sysdig_monitor_alert_promql" "prod_gold_cpu_limit" {
# name = "[GOLD PROD] CPU limit of pods nearing namespace limit"
# description = "There is no longer CPU room to accomodate the keycloak max surge of 1 pod"
# severity = 2
# enabled = true

# promql = "(kube_resourcequota_sysdig_limits_cpu_hard -kube_resourcequota_sysdig_limits_cpu_used{namespace=\"eb75ad-prod\",kube_cluster_name=\"gold\",kube_resourcequota_label_scope_notterminating='true'}) < 5"
# trigger_after_minutes = 2

# notification_channels = [132277, 57336]
# custom_notification {
# title = "{{__alert_name__}} is {{__alert_status__}}"
# }
# }

# # The Keycloak deployment has a max surge of 3, and the keycloak pods
# # require 2 cores. Setting limit at (3*2+1) cores
# ## Max surge as of Aug 2024 1, Pod count 3, CPU limit 4 (1*4+1) = 5
# resource "sysdig_monitor_alert_promql" "test_gold_cpu_limit" {
# name = "[GOLD TEST] CPU limit of pods nearing namespace limit"
# description = "There is no longer CPU room to accomodate the keycloak max surge of 1 pod"
# severity = 4
# enabled = true

# promql = "(kube_resourcequota_sysdig_limits_cpu_hard -kube_resourcequota_sysdig_limits_cpu_used{namespace=\"eb75ad-test\",kube_cluster_name=\"gold\",kube_resourcequota_label_scope_notterminating='true'}) < 5"
# trigger_after_minutes = 2

# notification_channels = [132277, 57336]
# custom_notification {
# title = "{{__alert_name__}} is {{__alert_status__}}"
# }
# }

# # The Keycloak deployment has a max surge of 3, and the keycloak pods
# # require 2 cores. Setting limit at (3*2+1) cores
# ## Max surge as of Aug 2024 1, Pod count 3, CPU limit 4 (1*4+1) = 5
# resource "sysdig_monitor_alert_promql" "dev_gold_cpu_limit" {
# name = "[GOLD DEV] CPU limit of pods nearing namespace limit"
# description = "There is no longer CPU room to accomodate the keycloak max surge of 1 pod"
# severity = 4
# enabled = true

# promql = "(kube_resourcequota_sysdig_limits_cpu_hard -kube_resourcequota_sysdig_limits_cpu_used{namespace=\"eb75ad-dev\",kube_cluster_name=\"gold\",kube_resourcequota_label_scope_notterminating='true'}) < 5"
# trigger_after_minutes = 2

# notification_channels = [132277, 57336]
# custom_notification {
# title = "{{__alert_name__}} is {{__alert_status__}}"
# }
# }

# # The Keycloak deployment has a max surge of 3, and the keycloak pods
# # require 4Gi of memory. Setting limit at (3*4+4)Gi
# ## Max surge as of Aug 2024 1, Setting limit at (1*4+4)Gi
# resource "sysdig_monitor_alert_promql" "prod_gold_memory_limit" {
# name = "[GOLD PROD] Memory limit of pods nearing namespace limit"
# description = "There is almost no Memory room to accomodate the keycloak max surge of 1 pod"
# severity = 2
# enabled = true

# promql = "(kube_resourcequota_sysdig_limits_memory_hard -kube_resourcequota_sysdig_limits_memory_used{namespace=\"eb75ad-prod\",kube_cluster_name=\"gold\",kube_resourcequota_label_scope_notterminating='true'}) < 8000000000"
# trigger_after_minutes = 2

# notification_channels = [132277, 57336]
# custom_notification {
# title = "{{__alert_name__}} is {{__alert_status__}}"
# }
# }

# # The Keycloak deployment has a max surge of 3, and the keycloak pods
# # require 4Gi of memory. Setting limit at (3*4+4)Gi
# ## Max surge as of Aug 2024 1, Setting limit at (1*4+4)Gi
# resource "sysdig_monitor_alert_promql" "dev_gold_memory_limit" {
# name = "[GOLD DEV] Memory limit of pods nearing namespace limit"
# description = "There is almost no Memory room to accomodate the keycloak max surge of 1 pod"
# severity = 2
# enabled = true

# promql = "(kube_resourcequota_sysdig_limits_memory_hard -kube_resourcequota_sysdig_limits_memory_used{namespace=\"eb75ad-dev\",kube_cluster_name=\"gold\",kube_resourcequota_label_scope_notterminating='true'}) < 8000000000"
# trigger_after_minutes = 2

# notification_channels = [132277, 57336]
# custom_notification {
# title = "{{__alert_name__}} is {{__alert_status__}}"
# }
# }

# # The Keycloak deployment has a max surge of 3, and the keycloak pods
# # require 4Gi of memory. Setting limit at (3*4+4)Gi
# ## Max surge as of Aug 2024 1, Setting limit at (1*4+4)Gi

# resource "sysdig_monitor_alert_promql" "test_gold_memory_limit" {
# name = "[GOLD TEST] Memory limit of pods nearing namespace limit"
# description = "There is almost no Memory room to accomodate the keycloak max surge of 1 pod"
# severity = 2
# enabled = true

# promql = "(kube_resourcequota_sysdig_limits_memory_hard -kube_resourcequota_sysdig_limits_memory_used{namespace=\"eb75ad-test\",kube_cluster_name=\"gold\",kube_resourcequota_label_scope_notterminating='true'}) < 8000000000"
# trigger_after_minutes = 2

# notification_channels = [132277, 57336]
# custom_notification {
# title = "{{__alert_name__}} is {{__alert_status__}}"
# }
# }

resource "sysdig_monitor_alert_v2_change" "prod_pvcuse_spike" {

name = "[PROD] the prod PVC usage surged"
Expand Down