[Feature] Member Failure Discovery (#1882)

Author: ajanikow

CHANGELOG.md

@@ -19,6 +19,7 @@
 - (Feature) DebugPackage Improvements
 - (Feature) Improve Bootstrap time
 - (Feature) Move Condition Actions to Internal
+- (Bugfix) Improve Member Failure discovery
 
 ## [1.2.47](https://github.com/arangodb/kube-arangodb/tree/1.2.47) (2025-03-28)
 - (Bugfix) Use Profile Annotations

pkg/apis/deployment/v1/member_status.go

@@ -138,7 +138,7 @@ func (s MemberStatus) Age() time.Duration {
 
 // AppendLastTermination appends termination to the list if required
 func (s *MemberStatus) AppendLastTermination(termination time.Time) bool {
-	if s == nil {
+	if s == nil || termination.IsZero() {
 		return false
 	}
 

pkg/apis/deployment/v2alpha1/member_status.go

@@ -138,7 +138,7 @@ func (s MemberStatus) Age() time.Duration {
 
 // AppendLastTermination appends termination to the list if required
 func (s *MemberStatus) AppendLastTermination(termination time.Time) bool {
-	if s == nil {
+	if s == nil || termination.IsZero() {
 		return false
 	}
 

pkg/deployment/resources/pod_inspector_restarts.go

@@ -38,6 +38,10 @@ func (r *Resources) failedContainerHandler(log logging.Logger, memberStatus api.
 
 	for _, c := range pod.Status.InitContainerStatuses {
 		if t := c.State.Terminated; t != nil {
+			if t.ExitCode == 0 {
+				continue
+			}
+
 			if q := t.FinishedAt.Time; q.After(last) {
 				if !q.Before(current) {
 					current = q
@@ -46,10 +50,6 @@ func (r *Resources) failedContainerHandler(log logging.Logger, memberStatus api.
 				continue
 			}
 
-			if t.ExitCode == 0 {
-				continue
-			}
-
 			log.Str("member", memberStatus.ID).
 				Str("pod", pod.GetName()).
 				Str("container", c.Name).
@@ -64,6 +64,10 @@ func (r *Resources) failedContainerHandler(log logging.Logger, memberStatus api.
 
 			r.metrics.IncMemberInitContainerRestarts(memberStatus.ID, c.Name, t.Reason, t.ExitCode)
 		} else if t := c.LastTerminationState.Terminated; t != nil {
+			if t.ExitCode == 0 {
+				continue
+			}
+
 			if q := t.FinishedAt.Time; q.After(last) {
 				if !q.Before(current) {
 					current = q
@@ -72,10 +76,6 @@ func (r *Resources) failedContainerHandler(log logging.Logger, memberStatus api.
 				continue
 			}
 
-			if t.ExitCode == 0 {
-				continue
-			}
-
 			log.Str("member", memberStatus.ID).
 				Str("pod", pod.GetName()).
 				Str("container", c.Name).
@@ -94,6 +94,10 @@ func (r *Resources) failedContainerHandler(log logging.Logger, memberStatus api.
 
 	for _, c := range pod.Status.ContainerStatuses {
 		if t := c.State.Terminated; t != nil {
+			if t.ExitCode == 0 {
+				continue
+			}
+
 			if q := t.FinishedAt.Time; q.After(last) {
 				if !q.Before(current) {
 					current = q
@@ -102,10 +106,6 @@ func (r *Resources) failedContainerHandler(log logging.Logger, memberStatus api.
 				continue
 			}
 
-			if t.ExitCode == 0 {
-				continue
-			}
-
 			log.Str("member", memberStatus.ID).
 				Str("pod", pod.GetName()).
 				Str("container", c.Name).
@@ -120,6 +120,10 @@ func (r *Resources) failedContainerHandler(log logging.Logger, memberStatus api.
 
 			r.metrics.IncMemberContainerRestarts(memberStatus.ID, c.Name, t.Reason, t.ExitCode)
 		} else if t := c.LastTerminationState.Terminated; t != nil {
+			if t.ExitCode == 0 {
+				continue
+			}
+
 			if q := t.FinishedAt.Time; q.After(last) {
 				if !q.Before(current) {
 					current = q
@@ -128,10 +132,6 @@ func (r *Resources) failedContainerHandler(log logging.Logger, memberStatus api.
 				continue
 			}
 
-			if t.ExitCode == 0 {
-				continue
-			}
-
 			log.Str("member", memberStatus.ID).
 				Str("pod", pod.GetName()).
 				Str("container", c.Name).
@@ -150,6 +150,10 @@ func (r *Resources) failedContainerHandler(log logging.Logger, memberStatus api.
 
 	for _, c := range pod.Status.EphemeralContainerStatuses {
 		if t := c.State.Terminated; t != nil {
+			if t.ExitCode == 0 {
+				continue
+			}
+
 			if q := t.FinishedAt.Time; q.After(last) {
 				if !q.Before(current) {
 					current = q
@@ -158,10 +162,6 @@ func (r *Resources) failedContainerHandler(log logging.Logger, memberStatus api.
 				continue
 			}
 
-			if t.ExitCode == 0 {
-				continue
-			}
-
 			log.Str("member", memberStatus.ID).
 				Str("pod", pod.GetName()).
 				Str("container", c.Name).
@@ -176,6 +176,10 @@ func (r *Resources) failedContainerHandler(log logging.Logger, memberStatus api.
 
 			r.metrics.IncMemberEphemeralContainerRestarts(memberStatus.ID, c.Name, t.Reason, t.ExitCode)
 		} else if t := c.LastTerminationState.Terminated; t != nil {
+			if t.ExitCode == 0 {
+				continue
+			}
+
 			if q := t.FinishedAt.Time; q.After(last) {
 				if !q.Before(current) {
 					current = q
@@ -184,10 +188,6 @@ func (r *Resources) failedContainerHandler(log logging.Logger, memberStatus api.
 				continue
 			}
 
-			if t.ExitCode == 0 {
-				continue
-			}
-
 			log.Str("member", memberStatus.ID).
 				Str("pod", pod.GetName()).
 				Str("container", c.Name).