diff --git a/internal/controller/grpcroute_controller.go b/internal/controller/grpcroute_controller.go
index 5d80ec6..b19fc67 100644
--- a/internal/controller/grpcroute_controller.go
+++ b/internal/controller/grpcroute_controller.go
@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/util/retry"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -94,10 +95,28 @@ func (r *GRPCRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 				return ctrl.Result{}, err
 			}
 		}
-		// remove our finalizer from the list and update it.
-		controllerutil.RemoveFinalizer(&grpcroute, FinalizerSecurityPolicy)
-		if err := r.Update(ctx, &grpcroute); err != nil {
-			return ctrl.Result{}, err
+		err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
+			var latest gatewayv1.GRPCRoute
+			// Get the latest version of the HTTPRoute object.
+			if err := r.Get(ctx, req.NamespacedName, &latest); err != nil {
+				// If the object is already gone, nothing to do
+				if client.IgnoreNotFound(err) == nil {
+					return nil
+				}
+				return err
+			}
+
+			controllerutil.RemoveFinalizer(&latest, FinalizerSecurityPolicy)
+			// Try to update. Return error to RetryOnConflict which wil trigger a new attempt if object is stale.
+			return r.Update(ctx, &latest)
+		})
+		if err != nil {
+			// Ignore not found errors - the object might have been deleted by another reconciliation
+			if client.IgnoreNotFound(err) != nil {
+				log.Error(err, "Failed to remove finalizer for", "GRPCRoute", req.NamespacedName)
+				return ctrl.Result{}, err
+			}
+			log.Info("GRPCRoute already deleted", "name", req.NamespacedName)
 		}
 		// Stop reconciliation as the item is being deleted
 		return ctrl.Result{}, nil
diff --git a/internal/controller/httproute_controller.go b/internal/controller/httproute_controller.go
index d65801f..46263c7 100644
--- a/internal/controller/httproute_controller.go
+++ b/internal/controller/httproute_controller.go
@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/util/retry"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -94,9 +95,30 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 			}
 		}
 		// remove our finalizer from the list and update it.
-		controllerutil.RemoveFinalizer(&httproute, FinalizerSecurityPolicy)
-		if err := r.Update(ctx, &httproute); err != nil {
-			return ctrl.Result{}, err
+		err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
+			var latest gatewayv1.HTTPRoute
+			// Get the latest version of the HTTPRoute object.
+			if err := r.Get(ctx, req.NamespacedName, &latest); err != nil {
+				// If the object is already gone, nothing to do
+				if client.IgnoreNotFound(err) == nil {
+					return nil
+				}
+				return err
+			}
+
+			controllerutil.RemoveFinalizer(&latest, FinalizerSecurityPolicy)
+			// Try to update. Return error to RetryOnConflict which wil trigger a new attempt if object is stale.
+			return r.Update(ctx, &latest)
+		})
+		if err != nil {
+			// Ignore not found errors - the object might have been deleted by another reconciliation
+			if client.IgnoreNotFound(err) != nil {
+				log.Error(err, "Failed to remove finalizer for", "HTTPRoute", req.NamespacedName)
+				return ctrl.Result{}, err
+			}
+			log.Info("HTTPRoute already deleted", "name", req.NamespacedName)
+		} else {
+			log.Info("Removed finalizer from HTTPRoute", "name", req.NamespacedName)
 		}
 		// Stop reconciliation as the item is being deleted
 		return ctrl.Result{}, nil
diff --git a/internal/controller/update_security_policy.go b/internal/controller/update_security_policy.go
index 4eb77ed..80dbfb4 100644
--- a/internal/controller/update_security_policy.go
+++ b/internal/controller/update_security_policy.go
@@ -61,7 +61,7 @@ func updateSecurityPolicy(ctx context.Context, r Client, securitypolicy envoyv1.
 		if err := r.Update(ctx, &securitypolicy); err != nil {
 			return fmt.Errorf("failed to update SecurityPolicy: %w", err)
 		}
-		return fmt.Errorf("removed all security policy rules")
+		return nil
 	}
 
 	// Convert string slice to CIDR slice