From 78f97b5a1d1358cedffa68f145d327f36ecc4a7b Mon Sep 17 00:00:00 2001
From: echken <echken@localhost.localdomain>
Date: Mon, 14 Mar 2022 22:29:21 -0400
Subject: [PATCH 01/32] add monitor mode support

1. add monitor mode openflow table
2. add monirot mode tower obj

Signed-off-by: echken <echken@localhost.localdomain>
---
 .../fake/fake_policyenforcementmode.go        | 122 -------------
 .../v1alpha1/fake/fake_security_client.go     |   4 -
 .../security/v1alpha1/generated_expansion.go  |   2 -
 .../v1alpha1/policyenforcementmode.go         | 168 ------------------
 .../security/v1alpha1/security_client.go      |   5 -
 .../externalversions/generic.go               |   2 -
 .../security/v1alpha1/interface.go            |   7 -
 .../v1alpha1/policyenforcementmode.go         |  89 ----------
 .../security/v1alpha1/expansion_generated.go  |   4 -
 .../v1alpha1/policyenforcementmode.go         |  65 -------
 10 files changed, 468 deletions(-)
 delete mode 100644 pkg/client/clientset_generated/clientset/typed/security/v1alpha1/fake/fake_policyenforcementmode.go
 delete mode 100644 pkg/client/clientset_generated/clientset/typed/security/v1alpha1/policyenforcementmode.go
 delete mode 100644 pkg/client/informers_generated/externalversions/security/v1alpha1/policyenforcementmode.go
 delete mode 100644 pkg/client/listers_generated/security/v1alpha1/policyenforcementmode.go

diff --git a/pkg/client/clientset_generated/clientset/typed/security/v1alpha1/fake/fake_policyenforcementmode.go b/pkg/client/clientset_generated/clientset/typed/security/v1alpha1/fake/fake_policyenforcementmode.go
deleted file mode 100644
index f6b6a2253..000000000
--- a/pkg/client/clientset_generated/clientset/typed/security/v1alpha1/fake/fake_policyenforcementmode.go
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
-Copyright 2021 The Everoute Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by client-gen. DO NOT EDIT.
-
-package fake
-
-import (
-	"context"
-
-	v1alpha1 "github.com/everoute/everoute/pkg/apis/security/v1alpha1"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	labels "k8s.io/apimachinery/pkg/labels"
-	schema "k8s.io/apimachinery/pkg/runtime/schema"
-	types "k8s.io/apimachinery/pkg/types"
-	watch "k8s.io/apimachinery/pkg/watch"
-	testing "k8s.io/client-go/testing"
-)
-
-// FakePolicyEnforcementModes implements PolicyEnforcementModeInterface
-type FakePolicyEnforcementModes struct {
-	Fake *FakeSecurityV1alpha1
-}
-
-var policyenforcementmodesResource = schema.GroupVersionResource{Group: "security.everoute.io", Version: "v1alpha1", Resource: "policyenforcementmodes"}
-
-var policyenforcementmodesKind = schema.GroupVersionKind{Group: "security.everoute.io", Version: "v1alpha1", Kind: "PolicyEnforcementMode"}
-
-// Get takes name of the policyEnforcementMode, and returns the corresponding policyEnforcementMode object, and an error if there is any.
-func (c *FakePolicyEnforcementModes) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.PolicyEnforcementMode, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewRootGetAction(policyenforcementmodesResource, name), &v1alpha1.PolicyEnforcementMode{})
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*v1alpha1.PolicyEnforcementMode), err
-}
-
-// List takes label and field selectors, and returns the list of PolicyEnforcementModes that match those selectors.
-func (c *FakePolicyEnforcementModes) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.PolicyEnforcementModeList, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewRootListAction(policyenforcementmodesResource, policyenforcementmodesKind, opts), &v1alpha1.PolicyEnforcementModeList{})
-	if obj == nil {
-		return nil, err
-	}
-
-	label, _, _ := testing.ExtractFromListOptions(opts)
-	if label == nil {
-		label = labels.Everything()
-	}
-	list := &v1alpha1.PolicyEnforcementModeList{ListMeta: obj.(*v1alpha1.PolicyEnforcementModeList).ListMeta}
-	for _, item := range obj.(*v1alpha1.PolicyEnforcementModeList).Items {
-		if label.Matches(labels.Set(item.Labels)) {
-			list.Items = append(list.Items, item)
-		}
-	}
-	return list, err
-}
-
-// Watch returns a watch.Interface that watches the requested policyEnforcementModes.
-func (c *FakePolicyEnforcementModes) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) {
-	return c.Fake.
-		InvokesWatch(testing.NewRootWatchAction(policyenforcementmodesResource, opts))
-}
-
-// Create takes the representation of a policyEnforcementMode and creates it.  Returns the server's representation of the policyEnforcementMode, and an error, if there is any.
-func (c *FakePolicyEnforcementModes) Create(ctx context.Context, policyEnforcementMode *v1alpha1.PolicyEnforcementMode, opts v1.CreateOptions) (result *v1alpha1.PolicyEnforcementMode, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewRootCreateAction(policyenforcementmodesResource, policyEnforcementMode), &v1alpha1.PolicyEnforcementMode{})
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*v1alpha1.PolicyEnforcementMode), err
-}
-
-// Update takes the representation of a policyEnforcementMode and updates it. Returns the server's representation of the policyEnforcementMode, and an error, if there is any.
-func (c *FakePolicyEnforcementModes) Update(ctx context.Context, policyEnforcementMode *v1alpha1.PolicyEnforcementMode, opts v1.UpdateOptions) (result *v1alpha1.PolicyEnforcementMode, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewRootUpdateAction(policyenforcementmodesResource, policyEnforcementMode), &v1alpha1.PolicyEnforcementMode{})
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*v1alpha1.PolicyEnforcementMode), err
-}
-
-// Delete takes name of the policyEnforcementMode and deletes it. Returns an error if one occurs.
-func (c *FakePolicyEnforcementModes) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error {
-	_, err := c.Fake.
-		Invokes(testing.NewRootDeleteAction(policyenforcementmodesResource, name), &v1alpha1.PolicyEnforcementMode{})
-	return err
-}
-
-// DeleteCollection deletes a collection of objects.
-func (c *FakePolicyEnforcementModes) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error {
-	action := testing.NewRootDeleteCollectionAction(policyenforcementmodesResource, listOpts)
-
-	_, err := c.Fake.Invokes(action, &v1alpha1.PolicyEnforcementModeList{})
-	return err
-}
-
-// Patch applies the patch and returns the patched policyEnforcementMode.
-func (c *FakePolicyEnforcementModes) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.PolicyEnforcementMode, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewRootPatchSubresourceAction(policyenforcementmodesResource, name, pt, data, subresources...), &v1alpha1.PolicyEnforcementMode{})
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*v1alpha1.PolicyEnforcementMode), err
-}
diff --git a/pkg/client/clientset_generated/clientset/typed/security/v1alpha1/fake/fake_security_client.go b/pkg/client/clientset_generated/clientset/typed/security/v1alpha1/fake/fake_security_client.go
index 279bab321..09019e39f 100644
--- a/pkg/client/clientset_generated/clientset/typed/security/v1alpha1/fake/fake_security_client.go
+++ b/pkg/client/clientset_generated/clientset/typed/security/v1alpha1/fake/fake_security_client.go
@@ -37,10 +37,6 @@ func (c *FakeSecurityV1alpha1) GlobalPolicies() v1alpha1.GlobalPolicyInterface {
 	return &FakeGlobalPolicies{c}
 }
 
-func (c *FakeSecurityV1alpha1) PolicyEnforcementModes() v1alpha1.PolicyEnforcementModeInterface {
-	return &FakePolicyEnforcementModes{c}
-}
-
 func (c *FakeSecurityV1alpha1) SecurityPolicies(namespace string) v1alpha1.SecurityPolicyInterface {
 	return &FakeSecurityPolicies{c, namespace}
 }
diff --git a/pkg/client/clientset_generated/clientset/typed/security/v1alpha1/generated_expansion.go b/pkg/client/clientset_generated/clientset/typed/security/v1alpha1/generated_expansion.go
index 9e00a39c2..f7c650e87 100644
--- a/pkg/client/clientset_generated/clientset/typed/security/v1alpha1/generated_expansion.go
+++ b/pkg/client/clientset_generated/clientset/typed/security/v1alpha1/generated_expansion.go
@@ -22,6 +22,4 @@ type EndpointExpansion interface{}
 
 type GlobalPolicyExpansion interface{}
 
-type PolicyEnforcementModeExpansion interface{}
-
 type SecurityPolicyExpansion interface{}
diff --git a/pkg/client/clientset_generated/clientset/typed/security/v1alpha1/policyenforcementmode.go b/pkg/client/clientset_generated/clientset/typed/security/v1alpha1/policyenforcementmode.go
deleted file mode 100644
index 308e11684..000000000
--- a/pkg/client/clientset_generated/clientset/typed/security/v1alpha1/policyenforcementmode.go
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
-Copyright 2021 The Everoute Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by client-gen. DO NOT EDIT.
-
-package v1alpha1
-
-import (
-	"context"
-	"time"
-
-	v1alpha1 "github.com/everoute/everoute/pkg/apis/security/v1alpha1"
-	scheme "github.com/everoute/everoute/pkg/client/clientset_generated/clientset/scheme"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	types "k8s.io/apimachinery/pkg/types"
-	watch "k8s.io/apimachinery/pkg/watch"
-	rest "k8s.io/client-go/rest"
-)
-
-// PolicyEnforcementModesGetter has a method to return a PolicyEnforcementModeInterface.
-// A group's client should implement this interface.
-type PolicyEnforcementModesGetter interface {
-	PolicyEnforcementModes() PolicyEnforcementModeInterface
-}
-
-// PolicyEnforcementModeInterface has methods to work with PolicyEnforcementMode resources.
-type PolicyEnforcementModeInterface interface {
-	Create(ctx context.Context, policyEnforcementMode *v1alpha1.PolicyEnforcementMode, opts v1.CreateOptions) (*v1alpha1.PolicyEnforcementMode, error)
-	Update(ctx context.Context, policyEnforcementMode *v1alpha1.PolicyEnforcementMode, opts v1.UpdateOptions) (*v1alpha1.PolicyEnforcementMode, error)
-	Delete(ctx context.Context, name string, opts v1.DeleteOptions) error
-	DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error
-	Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.PolicyEnforcementMode, error)
-	List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.PolicyEnforcementModeList, error)
-	Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error)
-	Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.PolicyEnforcementMode, err error)
-	PolicyEnforcementModeExpansion
-}
-
-// policyEnforcementModes implements PolicyEnforcementModeInterface
-type policyEnforcementModes struct {
-	client rest.Interface
-}
-
-// newPolicyEnforcementModes returns a PolicyEnforcementModes
-func newPolicyEnforcementModes(c *SecurityV1alpha1Client) *policyEnforcementModes {
-	return &policyEnforcementModes{
-		client: c.RESTClient(),
-	}
-}
-
-// Get takes name of the policyEnforcementMode, and returns the corresponding policyEnforcementMode object, and an error if there is any.
-func (c *policyEnforcementModes) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.PolicyEnforcementMode, err error) {
-	result = &v1alpha1.PolicyEnforcementMode{}
-	err = c.client.Get().
-		Resource("policyenforcementmodes").
-		Name(name).
-		VersionedParams(&options, scheme.ParameterCodec).
-		Do(ctx).
-		Into(result)
-	return
-}
-
-// List takes label and field selectors, and returns the list of PolicyEnforcementModes that match those selectors.
-func (c *policyEnforcementModes) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.PolicyEnforcementModeList, err error) {
-	var timeout time.Duration
-	if opts.TimeoutSeconds != nil {
-		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
-	}
-	result = &v1alpha1.PolicyEnforcementModeList{}
-	err = c.client.Get().
-		Resource("policyenforcementmodes").
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Timeout(timeout).
-		Do(ctx).
-		Into(result)
-	return
-}
-
-// Watch returns a watch.Interface that watches the requested policyEnforcementModes.
-func (c *policyEnforcementModes) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) {
-	var timeout time.Duration
-	if opts.TimeoutSeconds != nil {
-		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
-	}
-	opts.Watch = true
-	return c.client.Get().
-		Resource("policyenforcementmodes").
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Timeout(timeout).
-		Watch(ctx)
-}
-
-// Create takes the representation of a policyEnforcementMode and creates it.  Returns the server's representation of the policyEnforcementMode, and an error, if there is any.
-func (c *policyEnforcementModes) Create(ctx context.Context, policyEnforcementMode *v1alpha1.PolicyEnforcementMode, opts v1.CreateOptions) (result *v1alpha1.PolicyEnforcementMode, err error) {
-	result = &v1alpha1.PolicyEnforcementMode{}
-	err = c.client.Post().
-		Resource("policyenforcementmodes").
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Body(policyEnforcementMode).
-		Do(ctx).
-		Into(result)
-	return
-}
-
-// Update takes the representation of a policyEnforcementMode and updates it. Returns the server's representation of the policyEnforcementMode, and an error, if there is any.
-func (c *policyEnforcementModes) Update(ctx context.Context, policyEnforcementMode *v1alpha1.PolicyEnforcementMode, opts v1.UpdateOptions) (result *v1alpha1.PolicyEnforcementMode, err error) {
-	result = &v1alpha1.PolicyEnforcementMode{}
-	err = c.client.Put().
-		Resource("policyenforcementmodes").
-		Name(policyEnforcementMode.Name).
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Body(policyEnforcementMode).
-		Do(ctx).
-		Into(result)
-	return
-}
-
-// Delete takes name of the policyEnforcementMode and deletes it. Returns an error if one occurs.
-func (c *policyEnforcementModes) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error {
-	return c.client.Delete().
-		Resource("policyenforcementmodes").
-		Name(name).
-		Body(&opts).
-		Do(ctx).
-		Error()
-}
-
-// DeleteCollection deletes a collection of objects.
-func (c *policyEnforcementModes) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error {
-	var timeout time.Duration
-	if listOpts.TimeoutSeconds != nil {
-		timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second
-	}
-	return c.client.Delete().
-		Resource("policyenforcementmodes").
-		VersionedParams(&listOpts, scheme.ParameterCodec).
-		Timeout(timeout).
-		Body(&opts).
-		Do(ctx).
-		Error()
-}
-
-// Patch applies the patch and returns the patched policyEnforcementMode.
-func (c *policyEnforcementModes) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.PolicyEnforcementMode, err error) {
-	result = &v1alpha1.PolicyEnforcementMode{}
-	err = c.client.Patch(pt).
-		Resource("policyenforcementmodes").
-		Name(name).
-		SubResource(subresources...).
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Body(data).
-		Do(ctx).
-		Into(result)
-	return
-}
diff --git a/pkg/client/clientset_generated/clientset/typed/security/v1alpha1/security_client.go b/pkg/client/clientset_generated/clientset/typed/security/v1alpha1/security_client.go
index e1d02ce30..6f25b07bc 100644
--- a/pkg/client/clientset_generated/clientset/typed/security/v1alpha1/security_client.go
+++ b/pkg/client/clientset_generated/clientset/typed/security/v1alpha1/security_client.go
@@ -29,7 +29,6 @@ type SecurityV1alpha1Interface interface {
 	RESTClient() rest.Interface
 	EndpointsGetter
 	GlobalPoliciesGetter
-	PolicyEnforcementModesGetter
 	SecurityPoliciesGetter
 }
 
@@ -46,10 +45,6 @@ func (c *SecurityV1alpha1Client) GlobalPolicies() GlobalPolicyInterface {
 	return newGlobalPolicies(c)
 }
 
-func (c *SecurityV1alpha1Client) PolicyEnforcementModes() PolicyEnforcementModeInterface {
-	return newPolicyEnforcementModes(c)
-}
-
 func (c *SecurityV1alpha1Client) SecurityPolicies(namespace string) SecurityPolicyInterface {
 	return newSecurityPolicies(c, namespace)
 }
diff --git a/pkg/client/informers_generated/externalversions/generic.go b/pkg/client/informers_generated/externalversions/generic.go
index 5ffcfc5d4..8e39325e4 100644
--- a/pkg/client/informers_generated/externalversions/generic.go
+++ b/pkg/client/informers_generated/externalversions/generic.go
@@ -72,8 +72,6 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource
 		return &genericInformer{resource: resource.GroupResource(), informer: f.Security().V1alpha1().Endpoints().Informer()}, nil
 	case securityv1alpha1.SchemeGroupVersion.WithResource("globalpolicies"):
 		return &genericInformer{resource: resource.GroupResource(), informer: f.Security().V1alpha1().GlobalPolicies().Informer()}, nil
-	case securityv1alpha1.SchemeGroupVersion.WithResource("policyenforcementmodes"):
-		return &genericInformer{resource: resource.GroupResource(), informer: f.Security().V1alpha1().PolicyEnforcementModes().Informer()}, nil
 	case securityv1alpha1.SchemeGroupVersion.WithResource("securitypolicies"):
 		return &genericInformer{resource: resource.GroupResource(), informer: f.Security().V1alpha1().SecurityPolicies().Informer()}, nil
 
diff --git a/pkg/client/informers_generated/externalversions/security/v1alpha1/interface.go b/pkg/client/informers_generated/externalversions/security/v1alpha1/interface.go
index a807192e0..79a5a403e 100644
--- a/pkg/client/informers_generated/externalversions/security/v1alpha1/interface.go
+++ b/pkg/client/informers_generated/externalversions/security/v1alpha1/interface.go
@@ -28,8 +28,6 @@ type Interface interface {
 	Endpoints() EndpointInformer
 	// GlobalPolicies returns a GlobalPolicyInformer.
 	GlobalPolicies() GlobalPolicyInformer
-	// PolicyEnforcementModes returns a PolicyEnforcementModeInformer.
-	PolicyEnforcementModes() PolicyEnforcementModeInformer
 	// SecurityPolicies returns a SecurityPolicyInformer.
 	SecurityPolicies() SecurityPolicyInformer
 }
@@ -55,11 +53,6 @@ func (v *version) GlobalPolicies() GlobalPolicyInformer {
 	return &globalPolicyInformer{factory: v.factory, tweakListOptions: v.tweakListOptions}
 }
 
-// PolicyEnforcementModes returns a PolicyEnforcementModeInformer.
-func (v *version) PolicyEnforcementModes() PolicyEnforcementModeInformer {
-	return &policyEnforcementModeInformer{factory: v.factory, tweakListOptions: v.tweakListOptions}
-}
-
 // SecurityPolicies returns a SecurityPolicyInformer.
 func (v *version) SecurityPolicies() SecurityPolicyInformer {
 	return &securityPolicyInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
diff --git a/pkg/client/informers_generated/externalversions/security/v1alpha1/policyenforcementmode.go b/pkg/client/informers_generated/externalversions/security/v1alpha1/policyenforcementmode.go
deleted file mode 100644
index ff2aabe7e..000000000
--- a/pkg/client/informers_generated/externalversions/security/v1alpha1/policyenforcementmode.go
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
-Copyright 2021 The Everoute Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by informer-gen. DO NOT EDIT.
-
-package v1alpha1
-
-import (
-	"context"
-	time "time"
-
-	securityv1alpha1 "github.com/everoute/everoute/pkg/apis/security/v1alpha1"
-	clientset "github.com/everoute/everoute/pkg/client/clientset_generated/clientset"
-	internalinterfaces "github.com/everoute/everoute/pkg/client/informers_generated/externalversions/internalinterfaces"
-	v1alpha1 "github.com/everoute/everoute/pkg/client/listers_generated/security/v1alpha1"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	runtime "k8s.io/apimachinery/pkg/runtime"
-	watch "k8s.io/apimachinery/pkg/watch"
-	cache "k8s.io/client-go/tools/cache"
-)
-
-// PolicyEnforcementModeInformer provides access to a shared informer and lister for
-// PolicyEnforcementModes.
-type PolicyEnforcementModeInformer interface {
-	Informer() cache.SharedIndexInformer
-	Lister() v1alpha1.PolicyEnforcementModeLister
-}
-
-type policyEnforcementModeInformer struct {
-	factory          internalinterfaces.SharedInformerFactory
-	tweakListOptions internalinterfaces.TweakListOptionsFunc
-}
-
-// NewPolicyEnforcementModeInformer constructs a new informer for PolicyEnforcementMode type.
-// Always prefer using an informer factory to get a shared informer instead of getting an independent
-// one. This reduces memory footprint and number of connections to the server.
-func NewPolicyEnforcementModeInformer(client clientset.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer {
-	return NewFilteredPolicyEnforcementModeInformer(client, resyncPeriod, indexers, nil)
-}
-
-// NewFilteredPolicyEnforcementModeInformer constructs a new informer for PolicyEnforcementMode type.
-// Always prefer using an informer factory to get a shared informer instead of getting an independent
-// one. This reduces memory footprint and number of connections to the server.
-func NewFilteredPolicyEnforcementModeInformer(client clientset.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer {
-	return cache.NewSharedIndexInformer(
-		&cache.ListWatch{
-			ListFunc: func(options v1.ListOptions) (runtime.Object, error) {
-				if tweakListOptions != nil {
-					tweakListOptions(&options)
-				}
-				return client.SecurityV1alpha1().PolicyEnforcementModes().List(context.TODO(), options)
-			},
-			WatchFunc: func(options v1.ListOptions) (watch.Interface, error) {
-				if tweakListOptions != nil {
-					tweakListOptions(&options)
-				}
-				return client.SecurityV1alpha1().PolicyEnforcementModes().Watch(context.TODO(), options)
-			},
-		},
-		&securityv1alpha1.PolicyEnforcementMode{},
-		resyncPeriod,
-		indexers,
-	)
-}
-
-func (f *policyEnforcementModeInformer) defaultInformer(client clientset.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
-	return NewFilteredPolicyEnforcementModeInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions)
-}
-
-func (f *policyEnforcementModeInformer) Informer() cache.SharedIndexInformer {
-	return f.factory.InformerFor(&securityv1alpha1.PolicyEnforcementMode{}, f.defaultInformer)
-}
-
-func (f *policyEnforcementModeInformer) Lister() v1alpha1.PolicyEnforcementModeLister {
-	return v1alpha1.NewPolicyEnforcementModeLister(f.Informer().GetIndexer())
-}
diff --git a/pkg/client/listers_generated/security/v1alpha1/expansion_generated.go b/pkg/client/listers_generated/security/v1alpha1/expansion_generated.go
index c8775f3b1..395804f85 100644
--- a/pkg/client/listers_generated/security/v1alpha1/expansion_generated.go
+++ b/pkg/client/listers_generated/security/v1alpha1/expansion_generated.go
@@ -30,10 +30,6 @@ type EndpointNamespaceListerExpansion interface{}
 // GlobalPolicyLister.
 type GlobalPolicyListerExpansion interface{}
 
-// PolicyEnforcementModeListerExpansion allows custom methods to be added to
-// PolicyEnforcementModeLister.
-type PolicyEnforcementModeListerExpansion interface{}
-
 // SecurityPolicyListerExpansion allows custom methods to be added to
 // SecurityPolicyLister.
 type SecurityPolicyListerExpansion interface{}
diff --git a/pkg/client/listers_generated/security/v1alpha1/policyenforcementmode.go b/pkg/client/listers_generated/security/v1alpha1/policyenforcementmode.go
deleted file mode 100644
index 0f76e2f39..000000000
--- a/pkg/client/listers_generated/security/v1alpha1/policyenforcementmode.go
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
-Copyright 2021 The Everoute Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by lister-gen. DO NOT EDIT.
-
-package v1alpha1
-
-import (
-	v1alpha1 "github.com/everoute/everoute/pkg/apis/security/v1alpha1"
-	"k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/client-go/tools/cache"
-)
-
-// PolicyEnforcementModeLister helps list PolicyEnforcementModes.
-type PolicyEnforcementModeLister interface {
-	// List lists all PolicyEnforcementModes in the indexer.
-	List(selector labels.Selector) (ret []*v1alpha1.PolicyEnforcementMode, err error)
-	// Get retrieves the PolicyEnforcementMode from the index for a given name.
-	Get(name string) (*v1alpha1.PolicyEnforcementMode, error)
-	PolicyEnforcementModeListerExpansion
-}
-
-// policyEnforcementModeLister implements the PolicyEnforcementModeLister interface.
-type policyEnforcementModeLister struct {
-	indexer cache.Indexer
-}
-
-// NewPolicyEnforcementModeLister returns a new PolicyEnforcementModeLister.
-func NewPolicyEnforcementModeLister(indexer cache.Indexer) PolicyEnforcementModeLister {
-	return &policyEnforcementModeLister{indexer: indexer}
-}
-
-// List lists all PolicyEnforcementModes in the indexer.
-func (s *policyEnforcementModeLister) List(selector labels.Selector) (ret []*v1alpha1.PolicyEnforcementMode, err error) {
-	err = cache.ListAll(s.indexer, selector, func(m interface{}) {
-		ret = append(ret, m.(*v1alpha1.PolicyEnforcementMode))
-	})
-	return ret, err
-}
-
-// Get retrieves the PolicyEnforcementMode from the index for a given name.
-func (s *policyEnforcementModeLister) Get(name string) (*v1alpha1.PolicyEnforcementMode, error) {
-	obj, exists, err := s.indexer.GetByKey(name)
-	if err != nil {
-		return nil, err
-	}
-	if !exists {
-		return nil, errors.NewNotFound(v1alpha1.Resource("policyenforcementmode"), name)
-	}
-	return obj.(*v1alpha1.PolicyEnforcementMode), nil
-}

From 72b72afd0d87d78d4508a7c149021e28920fcd59 Mon Sep 17 00:00:00 2001
From: echken <echken@localhost.localdomain>
Date: Mon, 14 Mar 2022 22:29:21 -0400
Subject: [PATCH 02/32] add monitor mode support

1. add monitor mode openflow table
2. add monirot mode tower obj

Signed-off-by: echken <echken@localhost.localdomain>
---
 ...ty.everoute.io_policyenforcementmodes.yaml |  51 --
 deploy/everoute-agent/role.yaml               |   1 -
 deploy/everoute-controller/role.yaml          |   1 -
 deploy/everoute.yaml                          |  53 --
 docs/content/en/docs/reference/apidocs.html   | 107 ---
 pkg/agent/controller/policy/cache/rule.go     |  51 +-
 .../controller/policy/policy_controller.go    |  43 +-
 .../policy/policy_controller_helper.go        |   6 +-
 .../policy/policy_enforcement_controller.go   |  55 --
 pkg/agent/datapath/clsBridge.go               |   7 +-
 pkg/agent/datapath/localBridge.go             |   6 +-
 pkg/agent/datapath/multiBridgeDatapath.go     |  41 +-
 .../datapath/multiBridgeDatapath_test.go      |  10 +-
 pkg/agent/datapath/policyBridge.go            | 448 +++++-----
 pkg/agent/datapath/uplinkBridge.go            |   7 +-
 pkg/apis/security/v1alpha1/register.go        |   2 -
 pkg/apis/security/v1alpha1/types.go           |  41 +-
 .../v1alpha1/zz_generated.deepcopy.go         |  76 --
 pkg/openapi/openapi_generated.go              | 762 ++++++++----------
 19 files changed, 665 insertions(+), 1103 deletions(-)
 delete mode 100644 deploy/crds/security.everoute.io_policyenforcementmodes.yaml
 delete mode 100644 pkg/agent/controller/policy/policy_enforcement_controller.go

diff --git a/deploy/crds/security.everoute.io_policyenforcementmodes.yaml b/deploy/crds/security.everoute.io_policyenforcementmodes.yaml
deleted file mode 100644
index 210761ce0..000000000
--- a/deploy/crds/security.everoute.io_policyenforcementmodes.yaml
+++ /dev/null
@@ -1,51 +0,0 @@
-
----
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    controller-gen.kubebuilder.io/version: v0.6.2
-  creationTimestamp: null
-  name: policyenforcementmodes.security.everoute.io
-spec:
-  group: security.everoute.io
-  names:
-    kind: PolicyEnforcementMode
-    listKind: PolicyEnforcementModeList
-    plural: policyenforcementmodes
-    singular: policyenforcementmode
-  scope: Cluster
-  versions:
-  - name: v1alpha1
-    schema:
-      openAPIV3Schema:
-        properties:
-          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
-              of an object. Servers should convert recognized schemas to the latest
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-            type: string
-          kind:
-            description: 'Kind is a string value representing the REST resource this
-              object represents. Servers may infer this from the endpoint the client
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-            type: string
-          metadata:
-            type: object
-          spec:
-            description: Specification of the desired behavior for this SecurityPolicy.
-            properties:
-              defaultEnforcementMode:
-                type: string
-              enforcementMode:
-                type: string
-            type: object
-        type: object
-    served: true
-    storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []
diff --git a/deploy/everoute-agent/role.yaml b/deploy/everoute-agent/role.yaml
index 811c25cb5..0c9dc7cb6 100644
--- a/deploy/everoute-agent/role.yaml
+++ b/deploy/everoute-agent/role.yaml
@@ -44,7 +44,6 @@ rules:
   resources:
     - securitypolicies
     - globalpolicies
-    - policyenforcementmodes
   verbs:
     - get
     - list
diff --git a/deploy/everoute-controller/role.yaml b/deploy/everoute-controller/role.yaml
index 00ca61897..115a9d0a2 100644
--- a/deploy/everoute-controller/role.yaml
+++ b/deploy/everoute-controller/role.yaml
@@ -78,7 +78,6 @@ rules:
   - endpoints
   - endpoints/status
   - globalpolicies
-  - policyenforcementmodes
   verbs:
   - patch
   - create
diff --git a/deploy/everoute.yaml b/deploy/everoute.yaml
index 1d503e4a1..018ce2338 100644
--- a/deploy/everoute.yaml
+++ b/deploy/everoute.yaml
@@ -736,57 +736,6 @@ status:
   conditions: []
   storedVersions: []
 
----
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    controller-gen.kubebuilder.io/version: v0.6.2
-  creationTimestamp: null
-  name: policyenforcementmodes.security.everoute.io
-spec:
-  group: security.everoute.io
-  names:
-    kind: PolicyEnforcementMode
-    listKind: PolicyEnforcementModeList
-    plural: policyenforcementmodes
-    singular: policyenforcementmode
-  scope: Cluster
-  versions:
-  - name: v1alpha1
-    schema:
-      openAPIV3Schema:
-        properties:
-          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
-              of an object. Servers should convert recognized schemas to the latest
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-            type: string
-          kind:
-            description: 'Kind is a string value representing the REST resource this
-              object represents. Servers may infer this from the endpoint the client
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-            type: string
-          metadata:
-            type: object
-          spec:
-            description: Specification of the desired behavior for this SecurityPolicy.
-            properties:
-              defaultEnforcementMode:
-                type: string
-              enforcementMode:
-                type: string
-            type: object
-        type: object
-    served: true
-    storage: true
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
@@ -1958,7 +1907,6 @@ rules:
   resources:
     - securitypolicies
     - globalpolicies
-    - policyenforcementmodes
   verbs:
     - get
     - list
@@ -2118,7 +2066,6 @@ rules:
   - endpoints
   - endpoints/status
   - globalpolicies
-  - policyenforcementmodes
   verbs:
   - patch
   - create
diff --git a/docs/content/en/docs/reference/apidocs.html b/docs/content/en/docs/reference/apidocs.html
index 0eb70b3b1..c9e634e4e 100644
--- a/docs/content/en/docs/reference/apidocs.html
+++ b/docs/content/en/docs/reference/apidocs.html
@@ -762,118 +762,11 @@ <h3 id="security.everoute.io/v1alpha1.NamespacedName">NamespacedName
 </tr>
 </tbody>
 </table>
-<h3 id="security.everoute.io/v1alpha1.PolicyEnforcementMode">PolicyEnforcementMode
-</h3>
-<table class="table table-striped">
-<thead style="background-color: rgb(160,180,190)">
-<tr>
-<th>Field</th>
-<th>Description</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td>
-<code>metadata</code><br/>
-<em>
-<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#objectmeta-v1-meta">
-metav1.ObjectMeta
-</a>
-</em>
-</td>
-<td>
-Refer to the Kubernetes API documentation for the fields of the
-<code>metadata</code> field.
-</td>
-</tr>
-<tr>
-<td>
-<code>spec</code><br/>
-<em>
-<a href="#security.everoute.io/v1alpha1.PolicyEnforcementModeSpec">
-PolicyEnforcementModeSpec
-</a>
-</em>
-</td>
-<td>
-<p>Specification of the desired behavior for this SecurityPolicy.</p>
-<br/><br/>
-<table>
-<tr>
-<td>
-<code>defaultEnforcementMode</code><br/>
-<em>
-<a href="#security.everoute.io/v1alpha1.PolicyMode">
-PolicyMode
-</a>
-</em>
-</td>
-<td>
-</td>
-</tr>
-<tr>
-<td>
-<code>enforcementMode</code><br/>
-<em>
-<a href="#security.everoute.io/v1alpha1.PolicyMode">
-PolicyMode
-</a>
-</em>
-</td>
-<td>
-</td>
-</tr>
-</table>
-</td>
-</tr>
-</tbody>
-</table>
-<h3 id="security.everoute.io/v1alpha1.PolicyEnforcementModeSpec">PolicyEnforcementModeSpec
-</h3>
-<p>
-(<em>Appears in:</em>
-<a href="#security.everoute.io/v1alpha1.PolicyEnforcementMode">PolicyEnforcementMode</a>)
-</p>
-<table class="table table-striped">
-<thead style="background-color: rgb(160,180,190)">
-<tr>
-<th>Field</th>
-<th>Description</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td>
-<code>defaultEnforcementMode</code><br/>
-<em>
-<a href="#security.everoute.io/v1alpha1.PolicyMode">
-PolicyMode
-</a>
-</em>
-</td>
-<td>
-</td>
-</tr>
-<tr>
-<td>
-<code>enforcementMode</code><br/>
-<em>
-<a href="#security.everoute.io/v1alpha1.PolicyMode">
-PolicyMode
-</a>
-</em>
-</td>
-<td>
-</td>
-</tr>
-</tbody>
-</table>
 <h3 id="security.everoute.io/v1alpha1.PolicyMode">PolicyMode
 (<code>string</code> alias)</h3>
 <p>
 (<em>Appears in:</em>
 <a href="#security.everoute.io/v1alpha1.GlobalPolicySpec">GlobalPolicySpec</a>, 
-<a href="#security.everoute.io/v1alpha1.PolicyEnforcementModeSpec">PolicyEnforcementModeSpec</a>, 
 <a href="#security.everoute.io/v1alpha1.SecurityPolicySpec">SecurityPolicySpec</a>)
 </p>
 <table class="table table-striped">
diff --git a/pkg/agent/controller/policy/cache/rule.go b/pkg/agent/controller/policy/cache/rule.go
index f33336fad..fe65722cf 100644
--- a/pkg/agent/controller/policy/cache/rule.go
+++ b/pkg/agent/controller/policy/cache/rule.go
@@ -50,16 +50,17 @@ type PolicyRule struct {
 	Action RuleAction `json:"action"`
 
 	// match fields
-	Direction   RuleDirection `json:"direction"`
-	RuleType    RuleType      `json:"ruleType"`
-	Tier        string        `json:"tier,omitempty"`
-	SrcIPAddr   string        `json:"srcIPAddr,omitempty"`
-	DstIPAddr   string        `json:"dstIPAddr,omitempty"`
-	IPProtocol  string        `json:"ipProtocol"`
-	SrcPort     uint16        `json:"srcPort,omitempty"`
-	DstPort     uint16        `json:"dstPort,omitempty"`
-	SrcPortMask uint16        `json:"srcPortMask,omitempty"`
-	DstPortMask uint16        `json:"dstPortMask,omitempty"`
+	Direction       RuleDirection `json:"direction"`
+	RuleType        RuleType      `json:"ruleType"`
+	Tier            string        `json:"tier,omitempty"`
+	EnforcementMode string        `json:"enforcementMode,omitempty"`
+	SrcIPAddr       string        `json:"srcIPAddr,omitempty"`
+	DstIPAddr       string        `json:"dstIPAddr,omitempty"`
+	IPProtocol      string        `json:"ipProtocol"`
+	SrcPort         uint16        `json:"srcPort,omitempty"`
+	DstPort         uint16        `json:"dstPort,omitempty"`
+	SrcPortMask     uint16        `json:"srcPortMask,omitempty"`
+	DstPortMask     uint16        `json:"dstPortMask,omitempty"`
 }
 
 type DeepCopyBase interface {
@@ -97,9 +98,10 @@ type CompleteRule struct {
 	// RuleID is a unique identifier of rule, it's always set to policyNamespace/policyName/ruleName.
 	RuleID string
 
-	Tier      string
-	Action    RuleAction
-	Direction RuleDirection
+	Tier            string
+	EnforcementMode string
+	Action          RuleAction
+	Direction       RuleDirection
 
 	// SymmetricMode will ignore direction, generate both ingress and egress rule
 	SymmetricMode bool
@@ -202,17 +204,18 @@ func (rule *CompleteRule) generateRule(srcIPBlock, dstIPBlock string, direction
 	}
 
 	policyRule := PolicyRule{
-		Direction:   direction,
-		RuleType:    ruleType,
-		Tier:        rule.Tier,
-		SrcIPAddr:   srcIPBlock,
-		DstIPAddr:   dstIPBlock,
-		IPProtocol:  string(port.Protocol),
-		SrcPort:     port.SrcPort,
-		DstPort:     port.DstPort,
-		SrcPortMask: port.SrcPortMask,
-		DstPortMask: port.DstPortMask,
-		Action:      rule.Action,
+		Direction:       direction,
+		RuleType:        ruleType,
+		Tier:            rule.Tier,
+		EnforcementMode: rule.EnforcementMode,
+		SrcIPAddr:       srcIPBlock,
+		DstIPAddr:       dstIPBlock,
+		IPProtocol:      string(port.Protocol),
+		SrcPort:         port.SrcPort,
+		DstPort:         port.DstPort,
+		SrcPortMask:     port.SrcPortMask,
+		DstPortMask:     port.DstPortMask,
+		Action:          rule.Action,
 	}
 
 	// todo: it is not appropriate to calculate the flowkey here
diff --git a/pkg/agent/controller/policy/policy_controller.go b/pkg/agent/controller/policy/policy_controller.go
index 3247752f9..a1578f22c 100644
--- a/pkg/agent/controller/policy/policy_controller.go
+++ b/pkg/agent/controller/policy/policy_controller.go
@@ -144,7 +144,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
 	}
 
 	var err error
-	var policyController, patchController, globalPolicyController, policyEnforcementModeController controller.Controller
+	var policyController, patchController, globalPolicyController controller.Controller
 
 	// ignore not empty ruleCache for future cache inject
 	if r.ruleCache == nil {
@@ -211,17 +211,6 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
 		return err
 	}
 
-	if policyEnforcementModeController, err = controller.New("policy_enforcement_controller", mgr, controller.Options{
-		MaxConcurrentReconciles: 1,
-		Reconciler:              reconcile.Func(r.ReconcilePolicyEnforcementMode),
-	}); err != nil {
-		return err
-	}
-
-	if err = policyEnforcementModeController.Watch(&source.Kind{Type: &securityv1alpha1.PolicyEnforcementMode{}}, &handler.EnqueueRequestForObject{}); err != nil {
-		return err
-	}
-
 	return nil
 }
 
@@ -324,13 +313,14 @@ func (r *Reconciler) completePolicy(policy *securityv1alpha1.SecurityPolicy) ([]
 	if ingressEnabled {
 		for _, rule := range policy.Spec.IngressRules {
 			ingressRule := &policycache.CompleteRule{
-				RuleID:        fmt.Sprintf("%s/%s/%s.%s", policy.Namespace, policy.Name, "ingress", rule.Name),
-				Tier:          policy.Spec.Tier,
-				Action:        policycache.RuleActionAllow,
-				Direction:     policycache.RuleDirectionIn,
-				SymmetricMode: policy.Spec.SymmetricMode,
-				DstGroups:     policycache.DeepCopyMap(appliedGroups).(map[string]int32),
-				DstIPBlocks:   policycache.DeepCopyMap(appliedIPBlocks).(map[string]*policycache.IPBlockItem),
+				RuleID:          fmt.Sprintf("%s/%s/%s/%s.%s", policy.Namespace, policy.Name, policycache.NormalPolicy, "ingress", rule.Name),
+				Tier:            policy.Spec.Tier,
+				EnforcementMode: policy.Spec.SecurityPolicyEnforcementMode.String(),
+				Action:          policycache.RuleActionAllow,
+				Direction:       policycache.RuleDirectionIn,
+				SymmetricMode:   policy.Spec.SymmetricMode,
+				DstGroups:       policycache.DeepCopyMap(appliedGroups).(map[string]int32),
+				DstIPBlocks:     policycache.DeepCopyMap(appliedIPBlocks).(map[string]*policycache.IPBlockItem),
 			}
 
 			if len(rule.From) == 0 {
@@ -361,6 +351,7 @@ func (r *Reconciler) completePolicy(policy *securityv1alpha1.SecurityPolicy) ([]
 			defaultIngressRule := &policycache.CompleteRule{
 				RuleID:            fmt.Sprintf("%s/%s/%s.%s", policy.Namespace, policy.Name, "default", "ingress"),
 				Tier:              policy.Spec.Tier,
+				EnforcementMode:   policy.Spec.SecurityPolicyEnforcementMode.String(),
 				Action:            policycache.RuleActionDrop,
 				Direction:         policycache.RuleDirectionIn,
 				SymmetricMode:     false, // never generate symmetric rule for default rule
@@ -377,6 +368,7 @@ func (r *Reconciler) completePolicy(policy *securityv1alpha1.SecurityPolicy) ([]
 	if egressEnabled {
 		for _, rule := range policy.Spec.EgressRules {
 			egressRule := &policycache.CompleteRule{
+<<<<<<< HEAD
 				RuleID:        fmt.Sprintf("%s/%s/%s.%s", policy.Namespace, policy.Name, "egress", rule.Name),
 				Tier:          policy.Spec.Tier,
 				Action:        policycache.RuleActionAllow,
@@ -384,6 +376,16 @@ func (r *Reconciler) completePolicy(policy *securityv1alpha1.SecurityPolicy) ([]
 				SymmetricMode: policy.Spec.SymmetricMode,
 				SrcGroups:     policycache.DeepCopyMap(appliedGroups).(map[string]int32),
 				SrcIPBlocks:   policycache.DeepCopyMap(appliedIPBlocks).(map[string]*policycache.IPBlockItem),
+=======
+				RuleID:          fmt.Sprintf("%s/%s/%s/%s.%s", policy.Namespace, policy.Name, policycache.NormalPolicy, "egress", rule.Name),
+				Tier:            policy.Spec.Tier,
+				EnforcementMode: policy.Spec.SecurityPolicyEnforcementMode.String(),
+				Action:          policycache.RuleActionAllow,
+				Direction:       policycache.RuleDirectionOut,
+				SymmetricMode:   policy.Spec.SymmetricMode,
+				SrcGroups:       policycache.DeepCopyMap(appliedGroups).(map[string]int32),
+				SrcIPBlocks:     policycache.DeepCopyMap(appliedIPBlocks).(map[string]*policycache.IPBlockItem),
+>>>>>>> 1f9a0f0 (add monitor mode support)
 			}
 
 			if len(rule.To) == 0 {
@@ -414,6 +416,7 @@ func (r *Reconciler) completePolicy(policy *securityv1alpha1.SecurityPolicy) ([]
 			defaultEgressRule := &policycache.CompleteRule{
 				RuleID:            fmt.Sprintf("%s/%s/%s.%s", policy.Namespace, policy.Name, "default", "egress"),
 				Tier:              policy.Spec.Tier,
+				EnforcementMode:   policy.Spec.SecurityPolicyEnforcementMode.String(),
 				Action:            policycache.RuleActionDrop,
 				Direction:         policycache.RuleDirectionOut,
 				SymmetricMode:     false, // never generate symmetric rule for default rule
@@ -543,5 +546,5 @@ func (r *Reconciler) addPolicyRuleToDatapath(ruleID string, rule *policycache.Po
 	ruleDirection := getRuleDirection(rule.Direction)
 	ruleTier := getRuleTier(rule.Tier)
 
-	return r.DatapathManager.AddEveroutePolicyRule(everoutePolicyRule, rule.Name, ruleDirection, ruleTier)
+	return r.DatapathManager.AddEveroutePolicyRule(everoutePolicyRule, rule.Name, ruleDirection, ruleTier, rule.EnforcementMode)
 }
diff --git a/pkg/agent/controller/policy/policy_controller_helper.go b/pkg/agent/controller/policy/policy_controller_helper.go
index f11b3d62b..04506900c 100644
--- a/pkg/agent/controller/policy/policy_controller_helper.go
+++ b/pkg/agent/controller/policy/policy_controller_helper.go
@@ -108,11 +108,11 @@ func getRuleTier(ruleTier string) uint8 {
 	var tier uint8
 	switch ruleTier {
 	case "tier0":
-		tier = datapath.POLICY_TIER0
-	case "tier1":
 		tier = datapath.POLICY_TIER1
-	case "tier2":
+	case "tier1":
 		tier = datapath.POLICY_TIER2
+	case "tier2":
+		tier = datapath.POLICY_TIER3
 	default:
 		debug.PrintStack()
 		klog.Fatalf("unsupport ruleTier %s in policyRule.", ruleTier)
diff --git a/pkg/agent/controller/policy/policy_enforcement_controller.go b/pkg/agent/controller/policy/policy_enforcement_controller.go
deleted file mode 100644
index 161690e54..000000000
--- a/pkg/agent/controller/policy/policy_enforcement_controller.go
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
-Copyright 2021 The Everoute Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package policy
-
-import (
-	"context"
-
-	"k8s.io/klog"
-	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	securityv1alpha1 "github.com/everoute/everoute/pkg/apis/security/v1alpha1"
-)
-
-func (r *Reconciler) ReconcilePolicyEnforcementMode(request ctrl.Request) (ctrl.Result, error) {
-	var policyMode securityv1alpha1.PolicyEnforcementMode
-	var ctx = context.Background()
-
-	r.reconcilerLock.Lock()
-	defer r.reconcilerLock.Unlock()
-
-	err := r.Get(ctx, request.NamespacedName, &policyMode)
-	if client.IgnoreNotFound(err) != nil {
-		klog.Errorf("unable to fetch policy %s: %s", request.Name, err.Error())
-		return ctrl.Result{}, err
-	}
-
-	if err := r.UpdatePolicyEnforcementMode(policyMode); err != nil {
-		return ctrl.Result{}, err
-	}
-
-	return ctrl.Result{}, nil
-}
-
-func (r *Reconciler) UpdatePolicyEnforcementMode(newMode securityv1alpha1.PolicyEnforcementMode) error {
-	if err := r.DatapathManager.UpdateEveroutePolicyEnforcementMode(newMode.Spec.EnforcementMode.String()); err != nil {
-		return err
-	}
-
-	return nil
-}
diff --git a/pkg/agent/datapath/clsBridge.go b/pkg/agent/datapath/clsBridge.go
index f68e6c5b9..3136fcf2e 100644
--- a/pkg/agent/datapath/clsBridge.go
+++ b/pkg/agent/datapath/clsBridge.go
@@ -325,7 +325,7 @@ func (c *ClsBridge) RemoveLocalEndpoint(endpoint *Endpoint) error {
 	return nil
 }
 
-func (c *ClsBridge) AddMicroSegmentRule(rule *EveroutePolicyRule, direction uint8, tier uint8) (*FlowEntry, error) {
+func (c *ClsBridge) AddMicroSegmentRule(rule *EveroutePolicyRule, direction uint8, tier uint8, mode string) (*FlowEntry, error) {
 	return nil, nil
 }
 
@@ -333,11 +333,6 @@ func (c *ClsBridge) RemoveMicroSegmentRule(rule *EveroutePolicyRule) error {
 	return nil
 }
 
-func (c *ClsBridge) UpdatePolicyEnforcementMode(newMode string) error {
-
-	return nil
-}
-
 func (c *ClsBridge) AddVNFInstance() error {
 	return nil
 }
diff --git a/pkg/agent/datapath/localBridge.go b/pkg/agent/datapath/localBridge.go
index 7b9a646a7..f8be2bc84 100644
--- a/pkg/agent/datapath/localBridge.go
+++ b/pkg/agent/datapath/localBridge.go
@@ -764,7 +764,7 @@ func (l *LocalBridge) RemoveLocalEndpoint(endpoint *Endpoint) error {
 	return nil
 }
 
-func (l *LocalBridge) AddMicroSegmentRule(rule *EveroutePolicyRule, direction uint8, tier uint8) (*FlowEntry, error) {
+func (l *LocalBridge) AddMicroSegmentRule(rule *EveroutePolicyRule, direction uint8, tier uint8, mode string) (*FlowEntry, error) {
 	return nil, nil
 }
 
@@ -772,10 +772,6 @@ func (l *LocalBridge) RemoveMicroSegmentRule(rule *EveroutePolicyRule) error {
 	return nil
 }
 
-func (l *LocalBridge) UpdatePolicyEnforcementMode(mode string) error {
-	return nil
-}
-
 func (l *LocalBridge) AddVNFInstance() error {
 	return nil
 }
diff --git a/pkg/agent/datapath/multiBridgeDatapath.go b/pkg/agent/datapath/multiBridgeDatapath.go
index 15fe64fcf..ec0503584 100644
--- a/pkg/agent/datapath/multiBridgeDatapath.go
+++ b/pkg/agent/datapath/multiBridgeDatapath.go
@@ -63,9 +63,9 @@ const (
 
 //nolint
 const (
-	POLICY_TIER0 = 50
-	POLICY_TIER1 = 100
-	POLICY_TIER2 = 150
+	POLICY_TIER1 = 50
+	POLICY_TIER2 = 100
+	POLICY_TIER3 = 150
 )
 
 //nolint
@@ -80,6 +80,16 @@ const (
 	LOOP_BACK_ADDR    = "127.0.0.1"
 )
 
+//nolink
+const (
+	FLOW_ROUND_NUM_LENGTH = 10
+	FLOW_SEQ_NUM_LENGTH   = 22
+	FLOW_ROUND_NUM_MASK   = 0xffc
+	FLOW_SEQ_NUM_MASK     = 0x003fffff
+
+	DEFAULT_POLICY_ENFORCEMENT_MODE = "work"
+)
+
 //nolint
 const (
 	PROTOCOL_ARP = 0x0806
@@ -137,9 +147,8 @@ type Bridge interface {
 
 	AddSFCRule() error
 	RemoveSFCRule() error
-	AddMicroSegmentRule(rule *EveroutePolicyRule, direction uint8, tier uint8) (*FlowEntry, error)
+	AddMicroSegmentRule(rule *EveroutePolicyRule, direction uint8, tier uint8, mode string) (*FlowEntry, error)
 	RemoveMicroSegmentRule(rule *EveroutePolicyRule) error
-	UpdatePolicyEnforcementMode(mode string) error
 
 	IsSwitchConnected() bool
 
@@ -234,6 +243,7 @@ type EveroutePolicyRuleEntry struct {
 	EveroutePolicyRule  *EveroutePolicyRule
 	Direction           uint8
 	Tier                uint8
+	Mode                string
 	RuleFlowMap         map[string]*FlowEntry
 	PolicyRuleReference sets.String
 }
@@ -295,12 +305,12 @@ func (datapathManager *DpManager) InitializeDatapath(stopChan <-chan struct{}) {
 	// add rules for internalIP
 	for _, internalIP := range datapathManager.datapathConfig.InternalIPs {
 		// internal ingress rule
-		err := datapathManager.AddEveroutePolicyRule(newInternalIngressRule(internalIP), InternalIngressRulePrefix, POLICY_DIRECTION_IN, POLICY_TIER2)
+		err := datapathManager.AddEveroutePolicyRule(newInternalIngressRule(internalIP), InternalIngressRulePrefix, POLICY_DIRECTION_IN, POLICY_TIER3, "work")
 		if err != nil {
 			log.Fatalf("Failed to add internal whitelist: %v", err)
 		}
 		// internal egress rule
-		err = datapathManager.AddEveroutePolicyRule(newInternalEgressRule(internalIP), InternalEgressRulePrefix, POLICY_DIRECTION_OUT, POLICY_TIER2)
+		err = datapathManager.AddEveroutePolicyRule(newInternalEgressRule(internalIP), InternalEgressRulePrefix, POLICY_DIRECTION_OUT, POLICY_TIER3, "work")
 		if err != nil {
 			log.Fatalf("Failed to add internal whitelist: %v", err)
 		}
@@ -614,7 +624,7 @@ func (datapathManager *DpManager) ReplayVDSMicroSegmentFlow(vdsID string) error
 	for ruleID, erPolicyRuleEntry := range datapathManager.Rules {
 		// Add new policy rule flow to datapath
 		flowEntry, err := datapathManager.BridgeChainMap[vdsID][POLICY_BRIDGE_KEYWORD].AddMicroSegmentRule(erPolicyRuleEntry.EveroutePolicyRule,
-			erPolicyRuleEntry.Direction, erPolicyRuleEntry.Tier)
+			erPolicyRuleEntry.Direction, erPolicyRuleEntry.Tier, erPolicyRuleEntry.Mode)
 		if err != nil {
 			return fmt.Errorf("failed to add microsegment rule to vdsID %v, bridge %s, error: %v", vdsID, datapathManager.BridgeChainMap[vdsID][POLICY_BRIDGE_KEYWORD], err)
 		}
@@ -760,17 +770,7 @@ func (datapathManager *DpManager) RemoveLocalEndpoint(endpoint *Endpoint) error
 	return nil
 }
 
-func (datapathManager *DpManager) UpdateEveroutePolicyEnforcementMode(newMode string) error {
-	for vdsID, ovsbrname := range datapathManager.datapathConfig.ManagedVDSMap {
-		err := datapathManager.BridgeChainMap[vdsID][POLICY_BRIDGE_KEYWORD].UpdatePolicyEnforcementMode(newMode)
-		if err != nil {
-			return fmt.Errorf("failed to update policy enforcement mode to %v for vds %v : bridge %v, error: %v", newMode, vdsID, ovsbrname, err)
-		}
-	}
-	return nil
-}
-
-func (datapathManager *DpManager) AddEveroutePolicyRule(rule *EveroutePolicyRule, ruleName string, direction uint8, tier uint8) error {
+func (datapathManager *DpManager) AddEveroutePolicyRule(rule *EveroutePolicyRule, ruleName string, direction uint8, tier uint8, mode string) error {
 	datapathManager.flowReplayMutex.Lock()
 	defer datapathManager.flowReplayMutex.Unlock()
 	if !datapathManager.IsBridgesConnected() {
@@ -795,7 +795,7 @@ func (datapathManager *DpManager) AddEveroutePolicyRule(rule *EveroutePolicyRule
 	ruleFlowMap := make(map[string]*FlowEntry)
 	// Install policy rule flow to datapath
 	for vdsID, bridgeChain := range datapathManager.BridgeChainMap {
-		flowEntry, err := bridgeChain[POLICY_BRIDGE_KEYWORD].AddMicroSegmentRule(rule, direction, tier)
+		flowEntry, err := bridgeChain[POLICY_BRIDGE_KEYWORD].AddMicroSegmentRule(rule, direction, tier, mode)
 		if err != nil {
 			log.Errorf("Failed to add microsegment rule to vdsID %v, bridge %s, error: %v", vdsID, bridgeChain[POLICY_BRIDGE_KEYWORD], err)
 			return err
@@ -811,6 +811,7 @@ func (datapathManager *DpManager) AddEveroutePolicyRule(rule *EveroutePolicyRule
 	}
 	ruleEntry.Direction = direction
 	ruleEntry.Tier = tier
+	ruleEntry.Mode = mode
 	ruleEntry.EveroutePolicyRule = rule
 	ruleEntry.RuleFlowMap = ruleFlowMap
 
diff --git a/pkg/agent/datapath/multiBridgeDatapath_test.go b/pkg/agent/datapath/multiBridgeDatapath_test.go
index 8127ba50c..0f849a23b 100644
--- a/pkg/agent/datapath/multiBridgeDatapath_test.go
+++ b/pkg/agent/datapath/multiBridgeDatapath_test.go
@@ -142,7 +142,7 @@ func testLocalEndpoint(t *testing.T) {
 
 func testERPolicyRule(t *testing.T) {
 	t.Run("test ER policy rule", func(t *testing.T) {
-		if err := datapathManager.AddEveroutePolicyRule(rule1, "rule1", POLICY_DIRECTION_IN, POLICY_TIER1); err != nil {
+		if err := datapathManager.AddEveroutePolicyRule(rule1, "rule1", POLICY_DIRECTION_IN, POLICY_TIER2, "work"); err != nil {
 			t.Errorf("Failed to add ER policy rule: %v, error: %v", rule1, err)
 		}
 		if _, ok := datapathManager.Rules[rule1.RuleID]; !ok {
@@ -156,13 +156,13 @@ func testERPolicyRule(t *testing.T) {
 			t.Errorf("Failed to remove ER policy rule, rule %v in cache", rule1)
 		}
 
-		if err := datapathManager.AddEveroutePolicyRule(rule2, "rule2", POLICY_DIRECTION_OUT, POLICY_TIER0); err != nil {
+		if err := datapathManager.AddEveroutePolicyRule(rule2, "rule2", POLICY_DIRECTION_OUT, POLICY_TIER1, "work"); err != nil {
 			t.Errorf("Failed to add ER policy rule: %v, error: %v", rule2, err)
 		}
 		if _, ok := datapathManager.Rules[rule2.RuleID]; !ok {
 			t.Errorf("Failed to add ER policy rule, not found %v in cache", rule2)
 		}
-		if err := datapathManager.AddEveroutePolicyRule(rule2, "rule2", POLICY_DIRECTION_OUT, POLICY_TIER0); err != nil {
+		if err := datapathManager.AddEveroutePolicyRule(rule2, "rule2", POLICY_DIRECTION_OUT, POLICY_TIER1, "work"); err != nil {
 			t.Errorf("Failed to add ER policy rule: %v, error: %v", rule2, err)
 		}
 	})
@@ -176,8 +176,8 @@ func testFlowReplay(t *testing.T) {
 	}
 	t.Run("add ER policy rule", func(t *testing.T) {
 		Eventually(func() error {
-			log.Infof("add policy rule to datapath, tier: %d", POLICY_TIER2)
-			return datapathManager.AddEveroutePolicyRule(rule1, "rule1", POLICY_DIRECTION_IN, POLICY_TIER2)
+			log.Infof("add policy rule to datapath, tier: %d", POLICY_TIER3)
+			return datapathManager.AddEveroutePolicyRule(rule1, "rule1", POLICY_DIRECTION_IN, POLICY_TIER3, "work")
 		}, timeout, interval).Should(Succeed())
 	})
 
diff --git a/pkg/agent/datapath/policyBridge.go b/pkg/agent/datapath/policyBridge.go
index 501943990..24478fd53 100644
--- a/pkg/agent/datapath/policyBridge.go
+++ b/pkg/agent/datapath/policyBridge.go
@@ -15,20 +15,24 @@ import (
 
 //nolint
 const (
-	INPUT_TABLE               = 0
-	CT_STATE_TABLE            = 1
-	DIRECTION_SELECTION_TABLE = 10
-	EGRESS_TIER0_TABLE        = 20
-	EGRESS_TIER1_TABLE        = 25
-	EGRESS_TIER2_TABLE        = 30
-	EGRESS_TIER2_DROP_TABLR   = 31
-	INGRESS_TIER0_TABLE       = 50
-	INGRESS_TIER1_TABLE       = 55
-	INGRESS_TIER2_TABLE       = 60
-	INGRESS_TIER2_DROP_TABLE  = 61
-	CT_COMMIT_TABLE           = 70
-	SFC_POLICY_TABLE          = 80
-	POLICY_FORWARDING_TABLE   = 90
+	INPUT_TABLE                 = 0
+	CT_STATE_TABLE              = 1
+	DIRECTION_SELECTION_TABLE   = 10
+	EGRESS_TIER1_TABLE          = 20
+	EGRESS_TIER2_MONITOR_TABLE  = 24
+	EGRESS_TIER2_TABLE          = 25
+	EGRESS_TIER3_MONITOR_TABLE  = 29
+	EGRESS_TIER3_TABLE          = 30
+	EGRESS_DROP_TABLR           = 31
+	INGRESS_TIER1_TABLE         = 50
+	INGRESS_TIER2_MONITOR_TABLE = 54
+	INGRESS_TIER2_TABLE         = 55
+	INGRESS_TIER3_MONITOR_TABLE = 59
+	INGRESS_TIER3_TABLE         = 60
+	INGRESS_DROP_TABLE          = 61
+	CT_COMMIT_TABLE             = 70
+	SFC_POLICY_TABLE            = 80
+	POLICY_FORWARDING_TABLE     = 90
 )
 
 type PolicyBridge struct {
@@ -36,20 +40,24 @@ type PolicyBridge struct {
 	OfSwitch        *ofctrl.OFSwitch
 	datapathManager *DpManager
 
-	inputTable              *ofctrl.Table
-	ctStateTable            *ofctrl.Table
-	directionSelectionTable *ofctrl.Table
-	egressTier0PolicyTable  *ofctrl.Table
-	egressTier1PolicyTable  *ofctrl.Table
-	egressTier2PolicyTable  *ofctrl.Table
-	egressTier2DropTable    *ofctrl.Table
-	ingressTier0PolicyTable *ofctrl.Table
-	ingressTier1PolicyTable *ofctrl.Table
-	ingressTier2PolicyTable *ofctrl.Table
-	ingressTier2DropTable   *ofctrl.Table
-	ctCommitTable           *ofctrl.Table
-	sfcPolicyTable          *ofctrl.Table
-	policyForwardingTable   *ofctrl.Table
+	inputTable                     *ofctrl.Table
+	ctStateTable                   *ofctrl.Table
+	directionSelectionTable        *ofctrl.Table
+	egressTier1PolicyTable         *ofctrl.Table
+	egressTier2PolicyMonitorTable  *ofctrl.Table
+	egressTier2PolicyTable         *ofctrl.Table
+	egressTier3PolicyMonitorTable  *ofctrl.Table
+	egressTier3PolicyTable         *ofctrl.Table
+	egressDropTable                *ofctrl.Table
+	ingressTier1PolicyTable        *ofctrl.Table
+	ingressTier2PolicyMonitorTable *ofctrl.Table
+	ingressTier2PolicyTable        *ofctrl.Table
+	ingressTier3PolicyMonitorTable *ofctrl.Table
+	ingressTier3PolicyTable        *ofctrl.Table
+	ingressDropTable               *ofctrl.Table
+	ctCommitTable                  *ofctrl.Table
+	sfcPolicyTable                 *ofctrl.Table
+	policyForwardingTable          *ofctrl.Table
 
 	policySwitchStatusMutex sync.RWMutex
 	isPolicySwitchConnected bool
@@ -115,14 +123,18 @@ func (p *PolicyBridge) BridgeInit() {
 	p.inputTable = sw.DefaultTable()
 	p.ctStateTable, _ = sw.NewTable(CT_STATE_TABLE)
 	p.directionSelectionTable, _ = sw.NewTable(DIRECTION_SELECTION_TABLE)
-	p.ingressTier0PolicyTable, _ = sw.NewTable(INGRESS_TIER0_TABLE)
 	p.ingressTier1PolicyTable, _ = sw.NewTable(INGRESS_TIER1_TABLE)
+	p.ingressTier2PolicyMonitorTable, _ = sw.NewTable(INGRESS_TIER2_MONITOR_TABLE)
 	p.ingressTier2PolicyTable, _ = sw.NewTable(INGRESS_TIER2_TABLE)
-	p.ingressTier2DropTable, _ = sw.NewTable(INGRESS_TIER2_DROP_TABLE)
-	p.egressTier0PolicyTable, _ = sw.NewTable(EGRESS_TIER0_TABLE)
+	p.ingressTier3PolicyMonitorTable, _ = sw.NewTable(INGRESS_TIER3_MONITOR_TABLE)
+	p.ingressTier3PolicyTable, _ = sw.NewTable(INGRESS_TIER3_TABLE)
+	p.ingressDropTable, _ = sw.NewTable(INGRESS_DROP_TABLE)
 	p.egressTier1PolicyTable, _ = sw.NewTable(EGRESS_TIER1_TABLE)
+	p.egressTier2PolicyMonitorTable, _ = sw.NewTable(EGRESS_TIER2_MONITOR_TABLE)
 	p.egressTier2PolicyTable, _ = sw.NewTable(EGRESS_TIER2_TABLE)
-	p.egressTier2DropTable, _ = sw.NewTable(EGRESS_TIER2_DROP_TABLR)
+	p.egressTier3PolicyMonitorTable, _ = sw.NewTable(EGRESS_TIER3_MONITOR_TABLE)
+	p.egressTier3PolicyTable, _ = sw.NewTable(EGRESS_TIER3_TABLE)
+	p.egressDropTable, _ = sw.NewTable(EGRESS_DROP_TABLR)
 	p.ctCommitTable, _ = sw.NewTable(CT_COMMIT_TABLE)
 	p.sfcPolicyTable, _ = sw.NewTable(SFC_POLICY_TABLE)
 	p.policyForwardingTable, _ = sw.NewTable(POLICY_FORWARDING_TABLE)
@@ -150,14 +162,14 @@ func (p *PolicyBridge) initDirectionSelectionTable() error {
 		Priority:  MID_MATCH_FLOW_PRIORITY,
 		InputPort: uint32(p.datapathManager.BridgeChainPortMap[localBrName][PolicyToLocalSuffix]),
 	})
-	if err := fromLocalToEgressFlow.Next(p.egressTier0PolicyTable); err != nil {
+	if err := fromLocalToEgressFlow.Next(p.egressTier1PolicyTable); err != nil {
 		return fmt.Errorf("failed to install from local to egress flow, error: %v", err)
 	}
 	fromUpstreamToIngressFlow, _ := p.directionSelectionTable.NewFlow(ofctrl.FlowMatch{
 		Priority:  MID_MATCH_FLOW_PRIORITY,
 		InputPort: uint32(p.datapathManager.BridgeChainPortMap[localBrName][PolicyToClsSuffix]),
 	})
-	if err := fromUpstreamToIngressFlow.Next(p.ingressTier0PolicyTable); err != nil {
+	if err := fromUpstreamToIngressFlow.Next(p.ingressTier1PolicyTable); err != nil {
 		return fmt.Errorf("failed to install from upstream to ingress flow, error: %v", err)
 	}
 
@@ -253,11 +265,8 @@ func (p *PolicyBridge) initCTFlow(sw *ofctrl.OFSwitch) error {
 			},
 		},
 	})
-	if err := ctByPassFlow1.Resubmit(nil, &p.sfcPolicyTable.TableId); err != nil {
-		return fmt.Errorf("failed to install ct bypass flow 1, error: %v", err)
-	}
-	if err := ctByPassFlow1.Next(ofctrl.NewEmptyElem()); err != nil {
-		return fmt.Errorf("failed to install ct bypass flow 1, error: %v", err)
+	if err := ctByPassFlow1.Next(p.OfSwitch.DropAction()); err != nil {
+		return fmt.Errorf("failed to install ct drop flow, error: %v", err)
 	}
 	ctByPassFlow2, _ := p.ctCommitTable.NewFlow(ofctrl.FlowMatch{
 		Priority: MID_MATCH_FLOW_PRIORITY + FLOW_MATCH_OFFSET,
@@ -303,36 +312,48 @@ func (p *PolicyBridge) initCTFlow(sw *ofctrl.OFSwitch) error {
 
 func (p *PolicyBridge) initPolicyTable() error {
 	// egress policy table
-	egressTier1DefaultFlow, _ := p.egressTier0PolicyTable.NewFlow(ofctrl.FlowMatch{
+	egressTier1DefaultFlow, _ := p.egressTier1PolicyTable.NewFlow(ofctrl.FlowMatch{
 		Priority: DEFAULT_FLOW_MISS_PRIORITY,
 	})
-	if err := egressTier1DefaultFlow.Next(p.egressTier1PolicyTable); err != nil {
+	if err := egressTier1DefaultFlow.Next(p.egressTier2PolicyMonitorTable); err != nil {
 		return fmt.Errorf("failed to install egress tier1 default flow, error: %v", err)
 	}
-	egressTier2DefaultFlow, _ := p.egressTier1PolicyTable.NewFlow(ofctrl.FlowMatch{
+	egressTier2MonitorDefaultFlow, _ := p.egressTier2PolicyMonitorTable.NewFlow(ofctrl.FlowMatch{
+		Priority: DEFAULT_FLOW_MISS_PRIORITY,
+	})
+	if err := egressTier2MonitorDefaultFlow.Next(p.egressTier2PolicyTable); err != nil {
+		return fmt.Errorf("failed to install egress tier2 monitor table default flow, error: %v", err)
+	}
+	egressTier2DefaultFlow, _ := p.egressTier2PolicyTable.NewFlow(ofctrl.FlowMatch{
 		Priority: DEFAULT_FLOW_MISS_PRIORITY,
 	})
-	if err := egressTier2DefaultFlow.Next(p.egressTier2PolicyTable); err != nil {
+	if err := egressTier2DefaultFlow.Next(p.egressTier3PolicyMonitorTable); err != nil {
 		return fmt.Errorf("failed to install egress tier2 default flow, error: %v", err)
 	}
-	egressTier3DefaultFlow, _ := p.egressTier2PolicyTable.NewFlow(ofctrl.FlowMatch{
+	egressTier3MonitorDefaultFlow, _ := p.egressTier3PolicyMonitorTable.NewFlow(ofctrl.FlowMatch{
+		Priority: DEFAULT_FLOW_MISS_PRIORITY,
+	})
+	if err := egressTier3MonitorDefaultFlow.Next(p.egressTier3PolicyTable); err != nil {
+		return fmt.Errorf("failed to install egress tier2 monitor table default flow, error: %v", err)
+	}
+	egressTier3DefaultFlow, _ := p.egressTier3PolicyTable.NewFlow(ofctrl.FlowMatch{
 		Priority: DEFAULT_FLOW_MISS_PRIORITY,
 	})
 	if err := egressTier3DefaultFlow.Next(p.ctCommitTable); err != nil {
 		return fmt.Errorf("failed to install egress tier3 default flow, error: %v", err)
 	}
 
-	// egress tier3 drop table
-	egressTier2DropFlow, _ := p.egressTier2DropTable.NewFlow(ofctrl.FlowMatch{
+	// egress drop table
+	egressDropFlow, _ := p.egressDropTable.NewFlow(ofctrl.FlowMatch{
 		Priority: DEFAULT_FLOW_MISS_PRIORITY,
 	})
-	if err := egressTier2DropFlow.Next(p.ctCommitTable); err != nil {
+	if err := egressDropFlow.Next(p.ctCommitTable); err != nil {
 		return fmt.Errorf("failed to install egress tier3 drop table flow, error: %v", err)
 	}
 	ctTrkState := openflow13.NewCTStates()
 	ctTrkState.SetNew()
 	ctTrkState.SetTrk()
-	egressTier2DropTableCtCommitFlow, _ := p.egressTier2DropTable.NewFlow(ofctrl.FlowMatch{
+	egressDropTableCtCommitFlow, _ := p.egressDropTable.NewFlow(ofctrl.FlowMatch{
 		Priority:  NORMAL_MATCH_FLOW_PRIORITY,
 		Ethertype: PROTOCOL_IP,
 		CtStates:  ctTrkState,
@@ -342,22 +363,34 @@ func (p *PolicyBridge) initPolicyTable() error {
 	dstField, _ := openflow13.FindFieldHeaderByName("nxm_nx_ct_label", false)
 	moveAct := openflow13.NewNXActionRegMove(64, 0, 0, srcField, dstField)
 	ctCommitAction := ofctrl.NewConntrackAction(true, false, &p.ctCommitTable.TableId, &policyConntrackZone, moveAct)
-	_ = egressTier2DropTableCtCommitFlow.SetConntrack(ctCommitAction)
+	_ = egressDropTableCtCommitFlow.SetConntrack(ctCommitAction)
 
 	// ingress policy table
-	ingressTier1DefaultFlow, _ := p.ingressTier0PolicyTable.NewFlow(ofctrl.FlowMatch{
+	ingressTier1DefaultFlow, _ := p.ingressTier1PolicyTable.NewFlow(ofctrl.FlowMatch{
 		Priority: DEFAULT_FLOW_MISS_PRIORITY,
 	})
-	if err := ingressTier1DefaultFlow.Next(p.ingressTier1PolicyTable); err != nil {
+	if err := ingressTier1DefaultFlow.Next(p.ingressTier2PolicyMonitorTable); err != nil {
 		return fmt.Errorf("failed to install ingress tier1 default flow, error: %v", err)
 	}
-	ingressTier2DefaultFlow, _ := p.ingressTier1PolicyTable.NewFlow(ofctrl.FlowMatch{
+	ingressTier2MonitorDefaultFlow, _ := p.ingressTier2PolicyMonitorTable.NewFlow(ofctrl.FlowMatch{
 		Priority: DEFAULT_FLOW_MISS_PRIORITY,
 	})
-	if err := ingressTier2DefaultFlow.Next(p.ingressTier2PolicyTable); err != nil {
+	if err := ingressTier2MonitorDefaultFlow.Next(p.ingressTier2PolicyTable); err != nil {
+		return fmt.Errorf("failed to install ingress tier2 monitor table default flow, error: %v", err)
+	}
+	ingressTier2DefaultFlow, _ := p.ingressTier2PolicyTable.NewFlow(ofctrl.FlowMatch{
+		Priority: DEFAULT_FLOW_MISS_PRIORITY,
+	})
+	if err := ingressTier2DefaultFlow.Next(p.ingressTier3PolicyMonitorTable); err != nil {
 		return fmt.Errorf("failed to install ingress tier2 default flow, error: %v", err)
 	}
-	ingressTier3DefaultFlow, _ := p.ingressTier2PolicyTable.NewFlow(ofctrl.FlowMatch{
+	ingressTier3MonitorDefaultFlow, _ := p.ingressTier3PolicyMonitorTable.NewFlow(ofctrl.FlowMatch{
+		Priority: DEFAULT_FLOW_MISS_PRIORITY,
+	})
+	if err := ingressTier3MonitorDefaultFlow.Next(p.ingressTier3PolicyTable); err != nil {
+		return fmt.Errorf("failed to install ingress tier3 monitor table default flow, error: %v", err)
+	}
+	ingressTier3DefaultFlow, _ := p.ingressTier3PolicyTable.NewFlow(ofctrl.FlowMatch{
 		Priority: DEFAULT_FLOW_MISS_PRIORITY,
 	})
 	if err := ingressTier3DefaultFlow.Next(p.ctCommitTable); err != nil {
@@ -365,20 +398,20 @@ func (p *PolicyBridge) initPolicyTable() error {
 	}
 
 	// ingress tier3 drop table
-	ingressTier2DropFlow, _ := p.ingressTier2DropTable.NewFlow(ofctrl.FlowMatch{
+	ingressDropFlow, _ := p.ingressDropTable.NewFlow(ofctrl.FlowMatch{
 		Priority: DEFAULT_FLOW_MISS_PRIORITY,
 	})
-	if err := ingressTier2DropFlow.Next(p.ctCommitTable); err != nil {
+	if err := ingressDropFlow.Next(p.ctCommitTable); err != nil {
 		return fmt.Errorf("failed to install ingress tier3 drop table flow, error: %v", err)
 	}
-	ingressTier2DropTableCtCommitFlow, _ := p.ingressTier2DropTable.NewFlow(ofctrl.FlowMatch{
+	ingressDropTableCtCommitFlow, _ := p.ingressDropTable.NewFlow(ofctrl.FlowMatch{
 		Priority:  NORMAL_MATCH_FLOW_PRIORITY,
 		Ethertype: PROTOCOL_IP,
 		CtStates:  ctTrkState,
 	})
 	moveAct = openflow13.NewNXActionRegMove(64, 0, 64, srcField, dstField)
 	ctCommitAction = ofctrl.NewConntrackAction(true, false, &p.ctCommitTable.TableId, &policyConntrackZone, moveAct)
-	_ = ingressTier2DropTableCtCommitFlow.SetConntrack(ctCommitAction)
+	_ = ingressDropTableCtCommitFlow.SetConntrack(ctCommitAction)
 
 	// sfc policy table
 	sfcPolicyTableDefaultFlow, _ := p.sfcPolicyTable.NewFlow(ofctrl.FlowMatch{
@@ -440,52 +473,152 @@ func (p *PolicyBridge) RemoveLocalEndpoint(endpoint *Endpoint) error {
 	return nil
 }
 
-func (p *PolicyBridge) GetTierTable(direction uint8, tier uint8) (*ofctrl.Table, *ofctrl.Table, error) {
+func (p *PolicyBridge) GetTierTable(direction uint8, tier uint8, mode string) (*ofctrl.Table, *ofctrl.Table, error) {
 	var policyTable, nextTable *ofctrl.Table
 	// POLICY_TIER0 for endpoint isolation policy:
 	// 1) high priority rule is whitelist for support forensic policyrule, thus packet that match
 	//    that rules should passthrough other policy tier ---- send to ctCommitTable;
 	// 2) low priority rule is blacklist for support general isolation policyrule.
-	switch direction {
-	case POLICY_DIRECTION_OUT:
-		switch tier {
-		case POLICY_TIER0:
-			policyTable = p.egressTier0PolicyTable
-			nextTable = p.egressTier1PolicyTable
-		case POLICY_TIER1:
-			policyTable = p.egressTier1PolicyTable
-			nextTable = p.ctCommitTable
-		case POLICY_TIER2:
-			policyTable = p.egressTier2PolicyTable
-			nextTable = p.ctCommitTable
-		default:
-			return nil, nil, errors.New("unknow policy tier")
+	switch mode {
+	case "work":
+		switch direction {
+		case POLICY_DIRECTION_OUT:
+			switch tier {
+			case POLICY_TIER1:
+				policyTable = p.egressTier1PolicyTable
+				nextTable = p.egressTier2PolicyTable
+			case POLICY_TIER2:
+				policyTable = p.egressTier2PolicyTable
+				nextTable = p.ctCommitTable
+			case POLICY_TIER3:
+				policyTable = p.egressTier3PolicyTable
+				nextTable = p.ctCommitTable
+			default:
+				return nil, nil, errors.New("unknow policy tier")
+			}
+		case POLICY_DIRECTION_IN:
+			switch tier {
+			case POLICY_TIER1:
+				policyTable = p.ingressTier1PolicyTable
+				nextTable = p.ingressTier2PolicyTable
+			case POLICY_TIER2:
+				policyTable = p.ingressTier2PolicyTable
+				nextTable = p.ctCommitTable
+			case POLICY_TIER3:
+				policyTable = p.ingressTier3PolicyTable
+				nextTable = p.ctCommitTable
+			default:
+				return nil, nil, errors.New("unknow policy tier")
+			}
 		}
-	case POLICY_DIRECTION_IN:
-		switch tier {
-		case POLICY_TIER0:
-			policyTable = p.ingressTier0PolicyTable
-			nextTable = p.ingressTier1PolicyTable
-		case POLICY_TIER1:
-			policyTable = p.ingressTier1PolicyTable
-			nextTable = p.ctCommitTable
-		case POLICY_TIER2:
-			policyTable = p.ingressTier2PolicyTable
-			nextTable = p.ctCommitTable
-		default:
-			return nil, nil, errors.New("unknow policy tier")
+	case "monitor":
+		switch direction {
+		case POLICY_DIRECTION_OUT:
+			switch tier {
+			case POLICY_TIER1:
+			case POLICY_TIER2:
+				policyTable = p.egressTier2PolicyMonitorTable
+				nextTable = p.egressTier2PolicyTable
+			case POLICY_TIER3:
+				policyTable = p.egressTier3PolicyMonitorTable
+				nextTable = p.egressTier3PolicyTable
+			default:
+				return nil, nil, errors.New("unknow policy tier")
+			}
+		case POLICY_DIRECTION_IN:
+			switch tier {
+			case POLICY_TIER1:
+			case POLICY_TIER2:
+				policyTable = p.ingressTier2PolicyMonitorTable
+				nextTable = p.ingressTier2PolicyTable
+			case POLICY_TIER3:
+				policyTable = p.ingressTier3PolicyMonitorTable
+				nextTable = p.ingressTier3PolicyTable
+			default:
+				return nil, nil, errors.New("unknow policy tier")
+			}
 		}
 	}
 
 	return policyTable, nextTable, nil
 }
 
-func (p *PolicyBridge) AddMicroSegmentRule(rule *EveroutePolicyRule, direction uint8, tier uint8) (*FlowEntry, error) {
+func (p *PolicyBridge) AddMonitorModeMicroSegmentRule(rule *EveroutePolicyRule, direction uint8, tier uint8) (*FlowEntry, error) {
+	var ipDa *net.IP = nil
+	var ipDaMask *net.IP = nil
+	var ipSa *net.IP = nil
+	var ipSaMask *net.IP = nil
+	var err error
+
+	// make sure switch is connected
+	if !p.IsSwitchConnected() {
+		p.WaitForSwitchConnection()
+	}
+
+	// Different tier have different nextTable select strategy:
+	policyTable, nextTable, e := p.GetTierTable(direction, tier, "monitor")
+	if e != nil {
+		log.Errorf("Failed to get policy table tier %v", tier)
+		return nil, errors.New("failed get policy table")
+	}
+
+	// Parse dst ip
+	if rule.DstIPAddr != "" {
+		ipDa, ipDaMask, err = ParseIPAddrMaskString(rule.DstIPAddr)
+		if err != nil {
+			log.Errorf("Failed to parse dst ip %s. Err: %v", rule.DstIPAddr, err)
+			return nil, err
+		}
+	}
+
+	// parse src ip
+	if rule.SrcIPAddr != "" {
+		ipSa, ipSaMask, err = ParseIPAddrMaskString(rule.SrcIPAddr)
+		if err != nil {
+			log.Errorf("Failed to parse src ip %s. Err: %v", rule.SrcIPAddr, err)
+			return nil, err
+		}
+	}
+
+	// Install the rule in policy table
+	ruleFlow, err := policyTable.NewFlow(ofctrl.FlowMatch{
+		Priority:       uint16(rule.Priority),
+		Ethertype:      PROTOCOL_IP,
+		IpDa:           ipDa,
+		IpDaMask:       ipDaMask,
+		IpSa:           ipSa,
+		IpSaMask:       ipSaMask,
+		IpProto:        rule.IPProtocol,
+		TcpSrcPort:     rule.SrcPort,
+		TcpSrcPortMask: rule.SrcPortMask,
+		TcpDstPort:     rule.DstPort,
+		TcpDstPortMask: rule.DstPortMask,
+		UdpSrcPort:     rule.SrcPort,
+		UdpSrcPortMask: rule.SrcPortMask,
+		UdpDstPort:     rule.DstPort,
+		UdpDstPortMask: rule.DstPortMask,
+	})
+	if err != nil {
+		log.Errorf("Failed to add flow for rule {%v}. Err: %v", rule, err)
+		return nil, err
+	}
+
+	if rule.Action == "deny" {
+		ruleFlow.Next(nextTable)
+	}
+
+	return nil, nil
+}
+
+func (p *PolicyBridge) AddMicroSegmentRule(rule *EveroutePolicyRule, direction uint8, tier uint8, mode string) (*FlowEntry, error) {
 	var ipDa *net.IP = nil
 	var ipDaMask *net.IP = nil
 	var ipSa *net.IP = nil
 	var ipSaMask *net.IP = nil
 	var err error
+	if mode == "" {
+		mode = DEFAULT_POLICY_ENFORCEMENT_MODE
+	}
 
 	// make sure switch is connected
 	if !p.IsSwitchConnected() {
@@ -493,7 +626,7 @@ func (p *PolicyBridge) AddMicroSegmentRule(rule *EveroutePolicyRule, direction u
 	}
 
 	// Different tier have different nextTable select strategy:
-	policyTable, nextTable, e := p.GetTierTable(direction, tier)
+	policyTable, nextTable, e := p.GetTierTable(direction, tier, mode)
 	if e != nil {
 		log.Errorf("Failed to get policy table tier %v", tier)
 		return nil, errors.New("failed get policy table")
@@ -540,90 +673,50 @@ func (p *PolicyBridge) AddMicroSegmentRule(rule *EveroutePolicyRule, direction u
 		return nil, err
 	}
 
-	if tier == POLICY_TIER2 {
+	switch mode {
+	case "monitor":
+		if tier == POLICY_TIER1 {
+			return nil, fmt.Errorf("policy tier1 without monitor mode support")
+		}
+
+		// load flowID[0..9] -> xxreg0[0..9]
+		if err := ruleFlow.LoadField("nxm_nx_xxreg0", ruleFlow.FlowID>>FLOW_ROUND_NUM_LENGTH, openflow13.NewNXRange(0, 9)); err != nil {
+			return nil, err
+		}
+		if err := ruleFlow.LoadField("nxm_nx_xxreg0", ruleFlow.FlowID&FLOW_SEQ_NUM_MASK, openflow13.NewNXRange(32, 53)); err != nil {
+			return nil, err
+		}
+
+		if err := ruleFlow.Next(nextTable); err != nil {
+			return nil, err
+		}
+	case "work":
 		switch rule.Action {
 		case "allow":
-			if rule.Priority == GLOBAL_DEFAULT_POLICY_FLOW_PRIORITY {
-				if err := ruleFlow.LoadField("nxm_nx_reg0", 0x30, openflow13.NewNXRange(0, 15)); err != nil {
-					return nil, err
-				}
-				if err := ruleFlow.LoadField("nxm_nx_xxreg0", ruleFlow.FlowID, openflow13.NewNXRange(0, 63)); err != nil {
-					return nil, err
-				}
-				if direction == POLICY_DIRECTION_IN {
-					if err := ruleFlow.Resubmit(nil, &p.ingressTier2DropTable.TableId); err != nil {
-						return nil, fmt.Errorf("failed to install egress global drop flow, error: %v", err)
-					}
-					if err := ruleFlow.Next(ofctrl.NewEmptyElem()); err != nil {
-						return nil, fmt.Errorf("failed to install egress global drop flow, error: %v", err)
-					}
-				} else {
-					if err := ruleFlow.Resubmit(nil, &p.egressTier2DropTable.TableId); err != nil {
-						return nil, fmt.Errorf("failed to install egress global drop flow, error: %v", err)
-					}
-					if err := ruleFlow.Next(ofctrl.NewEmptyElem()); err != nil {
-						return nil, fmt.Errorf("failed to install ingress global drop flow, error: %v", err)
-					}
-				}
-			} else {
-				if err := ruleFlow.LoadField("nxm_nx_xxreg0", ruleFlow.FlowID, openflow13.NewNXRange(0, 63)); err != nil {
-					return nil, err
-				}
-				if err := ruleFlow.Resubmit(nil, &p.ctCommitTable.TableId); err != nil {
-					return nil, fmt.Errorf("failed to install microsegment policy rule flow %v, error: %v", ruleFlow, err)
-				}
-				if err := ruleFlow.Next(ofctrl.NewEmptyElem()); err != nil {
-					return nil, fmt.Errorf("failed to install microsegment policy rule flow %v, error: %v", ruleFlow, err)
-				}
+			if err := ruleFlow.LoadField("nxm_nx_reg0", 0x30, openflow13.NewNXRange(0, 15)); err != nil {
+				return nil, err
 			}
 		case "deny":
 			if err := ruleFlow.LoadField("nxm_nx_reg0", 0x20, openflow13.NewNXRange(0, 15)); err != nil {
 				return nil, err
 			}
-			if err := ruleFlow.LoadField("nxm_nx_xxreg0", ruleFlow.FlowID, openflow13.NewNXRange(0, 63)); err != nil {
-				return nil, err
-			}
-			if direction == POLICY_DIRECTION_IN {
-				if err := ruleFlow.Resubmit(nil, &p.ingressTier2DropTable.TableId); err != nil {
-					return nil, fmt.Errorf("failed to install ingress drop flow, error: %v", err)
-				}
-				if err := ruleFlow.Next(ofctrl.NewEmptyElem()); err != nil {
-					return nil, fmt.Errorf("failed to install ingress drop flow, error: %v", err)
-				}
-			} else {
-				if err := ruleFlow.Resubmit(nil, &p.egressTier2DropTable.TableId); err != nil {
-					return nil, fmt.Errorf("failed to install egress drop flow, error: %v", err)
-				}
-				if err := ruleFlow.Next(ofctrl.NewEmptyElem()); err != nil {
-					return nil, fmt.Errorf("failed to install ingress drop flow, error: %v", err)
-				}
-			}
+		default:
+			return nil, fmt.Errorf("unknown action")
 		}
 
-		return &FlowEntry{
-			Table:    policyTable,
-			Priority: ruleFlow.Match.Priority,
-			FlowID:   ruleFlow.FlowID,
-		}, nil
-	}
-
-	switch rule.Action {
-	case "allow":
-		err = ruleFlow.Next(nextTable)
-		if err != nil {
-			log.Errorf("Failed to install flow {%+v}. Err: %v", ruleFlow, err)
+		// load flowID[0..9] -> xxreg0[0..9]
+		if err := ruleFlow.LoadField("nxm_nx_xxreg0", ruleFlow.FlowID>>FLOW_ROUND_NUM_LENGTH, openflow13.NewNXRange(0, 9)); err != nil {
 			return nil, err
 		}
-	case "deny":
-		err = ruleFlow.Next(p.OfSwitch.DropAction())
-		if err != nil {
-			log.Errorf("Failed to install flow {%+v}. Err: %v", ruleFlow, err)
+		if err := ruleFlow.LoadField("nxm_nx_xxreg0", ruleFlow.FlowID&FLOW_SEQ_NUM_MASK, openflow13.NewNXRange(54, 75)); err != nil {
+			return nil, err
+		}
+
+		if err := ruleFlow.Next(nextTable); err != nil {
 			return nil, err
 		}
-	default:
-		log.Errorf("Unknown action in rule {%+v}", rule)
-		return nil, errors.New("unknown action in rule")
 	}
+
 	return &FlowEntry{
 		Table:    policyTable,
 		Priority: ruleFlow.Match.Priority,
@@ -635,45 +728,6 @@ func (p *PolicyBridge) RemoveMicroSegmentRule(rule *EveroutePolicyRule) error {
 	return nil
 }
 
-func (p *PolicyBridge) UpdatePolicyEnforcementMode(newMode string) error {
-	if newMode == "monitor" {
-		ctByPassFlow1, _ := p.ctCommitTable.NewFlow(ofctrl.FlowMatch{
-			Priority: MID_MATCH_FLOW_PRIORITY + FLOW_MATCH_OFFSET,
-			Regs: []*ofctrl.NXRegister{
-				{
-					RegID: 0,
-					Data:  0x20,
-					Range: openflow13.NewNXRange(0, 15),
-				},
-			},
-		})
-		if err := ctByPassFlow1.Resubmit(nil, &p.sfcPolicyTable.TableId); err != nil {
-			return fmt.Errorf("failed to install ct bypass flow 1, error: %v", err)
-		}
-		if err := ctByPassFlow1.Next(ofctrl.NewEmptyElem()); err != nil {
-			return fmt.Errorf("failed to install ct bypass flow 1, error: %v", err)
-		}
-	}
-
-	if newMode == "work" {
-		ctDropFlow1, _ := p.ctCommitTable.NewFlow(ofctrl.FlowMatch{
-			Priority: MID_MATCH_FLOW_PRIORITY + FLOW_MATCH_OFFSET,
-			Regs: []*ofctrl.NXRegister{
-				{
-					RegID: 0,
-					Data:  0x20,
-					Range: openflow13.NewNXRange(0, 15),
-				},
-			},
-		})
-		if err := ctDropFlow1.Next(p.OfSwitch.DropAction()); err != nil {
-			return fmt.Errorf("failed to install ct drop flow 1, error: %v", err)
-		}
-	}
-
-	return nil
-}
-
 func (p *PolicyBridge) AddVNFInstance() error {
 	return nil
 }
diff --git a/pkg/agent/datapath/uplinkBridge.go b/pkg/agent/datapath/uplinkBridge.go
index f33d28eb3..d21b765f4 100644
--- a/pkg/agent/datapath/uplinkBridge.go
+++ b/pkg/agent/datapath/uplinkBridge.go
@@ -113,7 +113,7 @@ func (u *UplinkBridge) RemoveLocalEndpoint(endpoint *Endpoint) error {
 	return nil
 }
 
-func (u *UplinkBridge) AddMicroSegmentRule(rule *EveroutePolicyRule, direction uint8, tier uint8) (*FlowEntry, error) {
+func (u *UplinkBridge) AddMicroSegmentRule(rule *EveroutePolicyRule, direction uint8, tier uint8, mode string) (*FlowEntry, error) {
 	return nil, nil
 }
 
@@ -121,11 +121,6 @@ func (u *UplinkBridge) RemoveMicroSegmentRule(rule *EveroutePolicyRule) error {
 	return nil
 }
 
-func (u *UplinkBridge) UpdatePolicyEnforcementMode(newMode string) error {
-
-	return nil
-}
-
 func (u *UplinkBridge) AddVNFInstance() error {
 	return nil
 }
diff --git a/pkg/apis/security/v1alpha1/register.go b/pkg/apis/security/v1alpha1/register.go
index a0372c79e..dbaa17b0b 100644
--- a/pkg/apis/security/v1alpha1/register.go
+++ b/pkg/apis/security/v1alpha1/register.go
@@ -32,8 +32,6 @@ func init() {
 		&SecurityPolicyList{},
 		&GlobalPolicy{},
 		&GlobalPolicyList{},
-		&PolicyEnforcementMode{},
-		&PolicyEnforcementModeList{},
 	)
 }
 
diff --git a/pkg/apis/security/v1alpha1/types.go b/pkg/apis/security/v1alpha1/types.go
index 468210fd9..fb30c05ec 100644
--- a/pkg/apis/security/v1alpha1/types.go
+++ b/pkg/apis/security/v1alpha1/types.go
@@ -25,6 +25,13 @@ import (
 	"github.com/everoute/everoute/pkg/types"
 )
 
+type PolicyMode string
+
+const (
+	WorkMode    PolicyMode = "work"
+	MonitorMode PolicyMode = "monitor"
+)
+
 // +genclient
 // +k8s:openapi-gen=true
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -367,37 +374,3 @@ type GlobalPolicyList struct {
 	metav1.ListMeta `json:"metadata,omitempty"`
 	Items           []GlobalPolicy `json:"items"`
 }
-
-type PolicyMode string
-
-const (
-	WorkMode    PolicyMode = "work"
-	MonitorMode PolicyMode = "monitor"
-)
-
-// +genclient
-// +genclient:nonNamespaced
-// +genclient:noStatus
-// +k8s:openapi-gen=true
-// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
-// +kubebuilder:resource:scope=Cluster
-// +kubebuilder:object:root=true
-type PolicyEnforcementMode struct {
-	metav1.TypeMeta   `json:",inline"`
-	metav1.ObjectMeta `json:"metadata,omitempty"`
-
-	// Specification of the desired behavior for this SecurityPolicy.
-	Spec PolicyEnforcementModeSpec `json:"spec,omitempty"`
-}
-
-type PolicyEnforcementModeSpec struct {
-	DefaultEnforcementMode PolicyMode `json:"defaultEnforcementMode,omitempty"`
-	EnforcementMode        PolicyMode `json:"enforcementMode,omitempty"`
-}
-
-// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
-type PolicyEnforcementModeList struct {
-	metav1.TypeMeta `json:",inline"`
-	metav1.ListMeta `json:"metadata,omitempty"`
-	Items           []PolicyEnforcementMode `json:"items"`
-}
diff --git a/pkg/apis/security/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/security/v1alpha1/zz_generated.deepcopy.go
index d213a7127..90553fede 100644
--- a/pkg/apis/security/v1alpha1/zz_generated.deepcopy.go
+++ b/pkg/apis/security/v1alpha1/zz_generated.deepcopy.go
@@ -282,82 +282,6 @@ func (in *NamespacedName) DeepCopy() *NamespacedName {
 	return out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *PolicyEnforcementMode) DeepCopyInto(out *PolicyEnforcementMode) {
-	*out = *in
-	out.TypeMeta = in.TypeMeta
-	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
-	out.Spec = in.Spec
-	return
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PolicyEnforcementMode.
-func (in *PolicyEnforcementMode) DeepCopy() *PolicyEnforcementMode {
-	if in == nil {
-		return nil
-	}
-	out := new(PolicyEnforcementMode)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
-func (in *PolicyEnforcementMode) DeepCopyObject() runtime.Object {
-	if c := in.DeepCopy(); c != nil {
-		return c
-	}
-	return nil
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *PolicyEnforcementModeList) DeepCopyInto(out *PolicyEnforcementModeList) {
-	*out = *in
-	out.TypeMeta = in.TypeMeta
-	in.ListMeta.DeepCopyInto(&out.ListMeta)
-	if in.Items != nil {
-		in, out := &in.Items, &out.Items
-		*out = make([]PolicyEnforcementMode, len(*in))
-		for i := range *in {
-			(*in)[i].DeepCopyInto(&(*out)[i])
-		}
-	}
-	return
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PolicyEnforcementModeList.
-func (in *PolicyEnforcementModeList) DeepCopy() *PolicyEnforcementModeList {
-	if in == nil {
-		return nil
-	}
-	out := new(PolicyEnforcementModeList)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
-func (in *PolicyEnforcementModeList) DeepCopyObject() runtime.Object {
-	if c := in.DeepCopy(); c != nil {
-		return c
-	}
-	return nil
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *PolicyEnforcementModeSpec) DeepCopyInto(out *PolicyEnforcementModeSpec) {
-	*out = *in
-	return
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PolicyEnforcementModeSpec.
-func (in *PolicyEnforcementModeSpec) DeepCopy() *PolicyEnforcementModeSpec {
-	if in == nil {
-		return nil
-	}
-	out := new(PolicyEnforcementModeSpec)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Rule) DeepCopyInto(out *Rule) {
 	*out = *in
diff --git a/pkg/openapi/openapi_generated.go b/pkg/openapi/openapi_generated.go
index ff4507d4c..069109900 100644
--- a/pkg/openapi/openapi_generated.go
+++ b/pkg/openapi/openapi_generated.go
@@ -32,334 +32,331 @@ import (
 
 func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition {
 	return map[string]common.OpenAPIDefinition{
-		"github.com/everoute/everoute/pkg/apis/agent/v1alpha1.AgentCondition":               schema_pkg_apis_agent_v1alpha1_AgentCondition(ref),
-		"github.com/everoute/everoute/pkg/apis/agent/v1alpha1.AgentInfo":                    schema_pkg_apis_agent_v1alpha1_AgentInfo(ref),
-		"github.com/everoute/everoute/pkg/apis/agent/v1alpha1.AgentInfoList":                schema_pkg_apis_agent_v1alpha1_AgentInfoList(ref),
-		"github.com/everoute/everoute/pkg/apis/agent/v1alpha1.BondConfig":                   schema_pkg_apis_agent_v1alpha1_BondConfig(ref),
-		"github.com/everoute/everoute/pkg/apis/agent/v1alpha1.OVSBridge":                    schema_pkg_apis_agent_v1alpha1_OVSBridge(ref),
-		"github.com/everoute/everoute/pkg/apis/agent/v1alpha1.OVSInfo":                      schema_pkg_apis_agent_v1alpha1_OVSInfo(ref),
-		"github.com/everoute/everoute/pkg/apis/agent/v1alpha1.OVSInterface":                 schema_pkg_apis_agent_v1alpha1_OVSInterface(ref),
-		"github.com/everoute/everoute/pkg/apis/agent/v1alpha1.OVSPort":                      schema_pkg_apis_agent_v1alpha1_OVSPort(ref),
-		"github.com/everoute/everoute/pkg/apis/agent/v1alpha1.VlanConfig":                   schema_pkg_apis_agent_v1alpha1_VlanConfig(ref),
-		"github.com/everoute/everoute/pkg/apis/group/v1alpha1.EndpointGroup":                schema_pkg_apis_group_v1alpha1_EndpointGroup(ref),
-		"github.com/everoute/everoute/pkg/apis/group/v1alpha1.EndpointGroupList":            schema_pkg_apis_group_v1alpha1_EndpointGroupList(ref),
-		"github.com/everoute/everoute/pkg/apis/group/v1alpha1.EndpointGroupSpec":            schema_pkg_apis_group_v1alpha1_EndpointGroupSpec(ref),
-		"github.com/everoute/everoute/pkg/apis/group/v1alpha1.EndpointReference":            schema_pkg_apis_group_v1alpha1_EndpointReference(ref),
-		"github.com/everoute/everoute/pkg/apis/group/v1alpha1.GroupMember":                  schema_pkg_apis_group_v1alpha1_GroupMember(ref),
-		"github.com/everoute/everoute/pkg/apis/group/v1alpha1.GroupMembers":                 schema_pkg_apis_group_v1alpha1_GroupMembers(ref),
-		"github.com/everoute/everoute/pkg/apis/group/v1alpha1.GroupMembersList":             schema_pkg_apis_group_v1alpha1_GroupMembersList(ref),
-		"github.com/everoute/everoute/pkg/apis/group/v1alpha1.GroupMembersPatch":            schema_pkg_apis_group_v1alpha1_GroupMembersPatch(ref),
-		"github.com/everoute/everoute/pkg/apis/group/v1alpha1.GroupMembersPatchList":        schema_pkg_apis_group_v1alpha1_GroupMembersPatchList(ref),
-		"github.com/everoute/everoute/pkg/apis/group/v1alpha1.GroupMembersReference":        schema_pkg_apis_group_v1alpha1_GroupMembersReference(ref),
-		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.ApplyToPeer":               schema_pkg_apis_security_v1alpha1_ApplyToPeer(ref),
-		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.Endpoint":                  schema_pkg_apis_security_v1alpha1_Endpoint(ref),
-		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.EndpointList":              schema_pkg_apis_security_v1alpha1_EndpointList(ref),
-		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.EndpointReference":         schema_pkg_apis_security_v1alpha1_EndpointReference(ref),
-		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.EndpointSpec":              schema_pkg_apis_security_v1alpha1_EndpointSpec(ref),
-		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.EndpointStatus":            schema_pkg_apis_security_v1alpha1_EndpointStatus(ref),
-		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.GlobalPolicy":              schema_pkg_apis_security_v1alpha1_GlobalPolicy(ref),
-		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.GlobalPolicyList":          schema_pkg_apis_security_v1alpha1_GlobalPolicyList(ref),
-		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.GlobalPolicySpec":          schema_pkg_apis_security_v1alpha1_GlobalPolicySpec(ref),
-		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.NamespacedName":            schema_pkg_apis_security_v1alpha1_NamespacedName(ref),
-		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.PolicyEnforcementMode":     schema_pkg_apis_security_v1alpha1_PolicyEnforcementMode(ref),
-		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.PolicyEnforcementModeList": schema_pkg_apis_security_v1alpha1_PolicyEnforcementModeList(ref),
-		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.PolicyEnforcementModeSpec": schema_pkg_apis_security_v1alpha1_PolicyEnforcementModeSpec(ref),
-		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.Rule":                      schema_pkg_apis_security_v1alpha1_Rule(ref),
-		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.SecurityPolicy":            schema_pkg_apis_security_v1alpha1_SecurityPolicy(ref),
-		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.SecurityPolicyList":        schema_pkg_apis_security_v1alpha1_SecurityPolicyList(ref),
-		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.SecurityPolicyPeer":        schema_pkg_apis_security_v1alpha1_SecurityPolicyPeer(ref),
-		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.SecurityPolicyPort":        schema_pkg_apis_security_v1alpha1_SecurityPolicyPort(ref),
-		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.SecurityPolicySpec":        schema_pkg_apis_security_v1alpha1_SecurityPolicySpec(ref),
-		"k8s.io/api/apps/v1.ControllerRevision":                                             schema_k8sio_api_apps_v1_ControllerRevision(ref),
-		"k8s.io/api/apps/v1.ControllerRevisionList":                                         schema_k8sio_api_apps_v1_ControllerRevisionList(ref),
-		"k8s.io/api/apps/v1.DaemonSet":                                                      schema_k8sio_api_apps_v1_DaemonSet(ref),
-		"k8s.io/api/apps/v1.DaemonSetCondition":                                             schema_k8sio_api_apps_v1_DaemonSetCondition(ref),
-		"k8s.io/api/apps/v1.DaemonSetList":                                                  schema_k8sio_api_apps_v1_DaemonSetList(ref),
-		"k8s.io/api/apps/v1.DaemonSetSpec":                                                  schema_k8sio_api_apps_v1_DaemonSetSpec(ref),
-		"k8s.io/api/apps/v1.DaemonSetStatus":                                                schema_k8sio_api_apps_v1_DaemonSetStatus(ref),
-		"k8s.io/api/apps/v1.DaemonSetUpdateStrategy":                                        schema_k8sio_api_apps_v1_DaemonSetUpdateStrategy(ref),
-		"k8s.io/api/apps/v1.Deployment":                                                     schema_k8sio_api_apps_v1_Deployment(ref),
-		"k8s.io/api/apps/v1.DeploymentCondition":                                            schema_k8sio_api_apps_v1_DeploymentCondition(ref),
-		"k8s.io/api/apps/v1.DeploymentList":                                                 schema_k8sio_api_apps_v1_DeploymentList(ref),
-		"k8s.io/api/apps/v1.DeploymentSpec":                                                 schema_k8sio_api_apps_v1_DeploymentSpec(ref),
-		"k8s.io/api/apps/v1.DeploymentStatus":                                               schema_k8sio_api_apps_v1_DeploymentStatus(ref),
-		"k8s.io/api/apps/v1.DeploymentStrategy":                                             schema_k8sio_api_apps_v1_DeploymentStrategy(ref),
-		"k8s.io/api/apps/v1.ReplicaSet":                                                     schema_k8sio_api_apps_v1_ReplicaSet(ref),
-		"k8s.io/api/apps/v1.ReplicaSetCondition":                                            schema_k8sio_api_apps_v1_ReplicaSetCondition(ref),
-		"k8s.io/api/apps/v1.ReplicaSetList":                                                 schema_k8sio_api_apps_v1_ReplicaSetList(ref),
-		"k8s.io/api/apps/v1.ReplicaSetSpec":                                                 schema_k8sio_api_apps_v1_ReplicaSetSpec(ref),
-		"k8s.io/api/apps/v1.ReplicaSetStatus":                                               schema_k8sio_api_apps_v1_ReplicaSetStatus(ref),
-		"k8s.io/api/apps/v1.RollingUpdateDaemonSet":                                         schema_k8sio_api_apps_v1_RollingUpdateDaemonSet(ref),
-		"k8s.io/api/apps/v1.RollingUpdateDeployment":                                        schema_k8sio_api_apps_v1_RollingUpdateDeployment(ref),
-		"k8s.io/api/apps/v1.RollingUpdateStatefulSetStrategy":                               schema_k8sio_api_apps_v1_RollingUpdateStatefulSetStrategy(ref),
-		"k8s.io/api/apps/v1.StatefulSet":                                                    schema_k8sio_api_apps_v1_StatefulSet(ref),
-		"k8s.io/api/apps/v1.StatefulSetCondition":                                           schema_k8sio_api_apps_v1_StatefulSetCondition(ref),
-		"k8s.io/api/apps/v1.StatefulSetList":                                                schema_k8sio_api_apps_v1_StatefulSetList(ref),
-		"k8s.io/api/apps/v1.StatefulSetSpec":                                                schema_k8sio_api_apps_v1_StatefulSetSpec(ref),
-		"k8s.io/api/apps/v1.StatefulSetStatus":                                              schema_k8sio_api_apps_v1_StatefulSetStatus(ref),
-		"k8s.io/api/apps/v1.StatefulSetUpdateStrategy":                                      schema_k8sio_api_apps_v1_StatefulSetUpdateStrategy(ref),
-		"k8s.io/api/core/v1.AWSElasticBlockStoreVolumeSource":                               schema_k8sio_api_core_v1_AWSElasticBlockStoreVolumeSource(ref),
-		"k8s.io/api/core/v1.Affinity":                                                       schema_k8sio_api_core_v1_Affinity(ref),
-		"k8s.io/api/core/v1.AttachedVolume":                                                 schema_k8sio_api_core_v1_AttachedVolume(ref),
-		"k8s.io/api/core/v1.AvoidPods":                                                      schema_k8sio_api_core_v1_AvoidPods(ref),
-		"k8s.io/api/core/v1.AzureDiskVolumeSource":                                          schema_k8sio_api_core_v1_AzureDiskVolumeSource(ref),
-		"k8s.io/api/core/v1.AzureFilePersistentVolumeSource":                                schema_k8sio_api_core_v1_AzureFilePersistentVolumeSource(ref),
-		"k8s.io/api/core/v1.AzureFileVolumeSource":                                          schema_k8sio_api_core_v1_AzureFileVolumeSource(ref),
-		"k8s.io/api/core/v1.Binding":                                                        schema_k8sio_api_core_v1_Binding(ref),
-		"k8s.io/api/core/v1.CSIPersistentVolumeSource":                                      schema_k8sio_api_core_v1_CSIPersistentVolumeSource(ref),
-		"k8s.io/api/core/v1.CSIVolumeSource":                                                schema_k8sio_api_core_v1_CSIVolumeSource(ref),
-		"k8s.io/api/core/v1.Capabilities":                                                   schema_k8sio_api_core_v1_Capabilities(ref),
-		"k8s.io/api/core/v1.CephFSPersistentVolumeSource":                                   schema_k8sio_api_core_v1_CephFSPersistentVolumeSource(ref),
-		"k8s.io/api/core/v1.CephFSVolumeSource":                                             schema_k8sio_api_core_v1_CephFSVolumeSource(ref),
-		"k8s.io/api/core/v1.CinderPersistentVolumeSource":                                   schema_k8sio_api_core_v1_CinderPersistentVolumeSource(ref),
-		"k8s.io/api/core/v1.CinderVolumeSource":                                             schema_k8sio_api_core_v1_CinderVolumeSource(ref),
-		"k8s.io/api/core/v1.ClientIPConfig":                                                 schema_k8sio_api_core_v1_ClientIPConfig(ref),
-		"k8s.io/api/core/v1.ComponentCondition":                                             schema_k8sio_api_core_v1_ComponentCondition(ref),
-		"k8s.io/api/core/v1.ComponentStatus":                                                schema_k8sio_api_core_v1_ComponentStatus(ref),
-		"k8s.io/api/core/v1.ComponentStatusList":                                            schema_k8sio_api_core_v1_ComponentStatusList(ref),
-		"k8s.io/api/core/v1.ConfigMap":                                                      schema_k8sio_api_core_v1_ConfigMap(ref),
-		"k8s.io/api/core/v1.ConfigMapEnvSource":                                             schema_k8sio_api_core_v1_ConfigMapEnvSource(ref),
-		"k8s.io/api/core/v1.ConfigMapKeySelector":                                           schema_k8sio_api_core_v1_ConfigMapKeySelector(ref),
-		"k8s.io/api/core/v1.ConfigMapList":                                                  schema_k8sio_api_core_v1_ConfigMapList(ref),
-		"k8s.io/api/core/v1.ConfigMapNodeConfigSource":                                      schema_k8sio_api_core_v1_ConfigMapNodeConfigSource(ref),
-		"k8s.io/api/core/v1.ConfigMapProjection":                                            schema_k8sio_api_core_v1_ConfigMapProjection(ref),
-		"k8s.io/api/core/v1.ConfigMapVolumeSource":                                          schema_k8sio_api_core_v1_ConfigMapVolumeSource(ref),
-		"k8s.io/api/core/v1.Container":                                                      schema_k8sio_api_core_v1_Container(ref),
-		"k8s.io/api/core/v1.ContainerImage":                                                 schema_k8sio_api_core_v1_ContainerImage(ref),
-		"k8s.io/api/core/v1.ContainerPort":                                                  schema_k8sio_api_core_v1_ContainerPort(ref),
-		"k8s.io/api/core/v1.ContainerState":                                                 schema_k8sio_api_core_v1_ContainerState(ref),
-		"k8s.io/api/core/v1.ContainerStateRunning":                                          schema_k8sio_api_core_v1_ContainerStateRunning(ref),
-		"k8s.io/api/core/v1.ContainerStateTerminated":                                       schema_k8sio_api_core_v1_ContainerStateTerminated(ref),
-		"k8s.io/api/core/v1.ContainerStateWaiting":                                          schema_k8sio_api_core_v1_ContainerStateWaiting(ref),
-		"k8s.io/api/core/v1.ContainerStatus":                                                schema_k8sio_api_core_v1_ContainerStatus(ref),
-		"k8s.io/api/core/v1.DaemonEndpoint":                                                 schema_k8sio_api_core_v1_DaemonEndpoint(ref),
-		"k8s.io/api/core/v1.DownwardAPIProjection":                                          schema_k8sio_api_core_v1_DownwardAPIProjection(ref),
-		"k8s.io/api/core/v1.DownwardAPIVolumeFile":                                          schema_k8sio_api_core_v1_DownwardAPIVolumeFile(ref),
-		"k8s.io/api/core/v1.DownwardAPIVolumeSource":                                        schema_k8sio_api_core_v1_DownwardAPIVolumeSource(ref),
-		"k8s.io/api/core/v1.EmptyDirVolumeSource":                                           schema_k8sio_api_core_v1_EmptyDirVolumeSource(ref),
-		"k8s.io/api/core/v1.EndpointAddress":                                                schema_k8sio_api_core_v1_EndpointAddress(ref),
-		"k8s.io/api/core/v1.EndpointPort":                                                   schema_k8sio_api_core_v1_EndpointPort(ref),
-		"k8s.io/api/core/v1.EndpointSubset":                                                 schema_k8sio_api_core_v1_EndpointSubset(ref),
-		"k8s.io/api/core/v1.Endpoints":                                                      schema_k8sio_api_core_v1_Endpoints(ref),
-		"k8s.io/api/core/v1.EndpointsList":                                                  schema_k8sio_api_core_v1_EndpointsList(ref),
-		"k8s.io/api/core/v1.EnvFromSource":                                                  schema_k8sio_api_core_v1_EnvFromSource(ref),
-		"k8s.io/api/core/v1.EnvVar":                                                         schema_k8sio_api_core_v1_EnvVar(ref),
-		"k8s.io/api/core/v1.EnvVarSource":                                                   schema_k8sio_api_core_v1_EnvVarSource(ref),
-		"k8s.io/api/core/v1.EphemeralContainer":                                             schema_k8sio_api_core_v1_EphemeralContainer(ref),
-		"k8s.io/api/core/v1.EphemeralContainerCommon":                                       schema_k8sio_api_core_v1_EphemeralContainerCommon(ref),
-		"k8s.io/api/core/v1.EphemeralContainers":                                            schema_k8sio_api_core_v1_EphemeralContainers(ref),
-		"k8s.io/api/core/v1.EphemeralVolumeSource":                                          schema_k8sio_api_core_v1_EphemeralVolumeSource(ref),
-		"k8s.io/api/core/v1.Event":                                                          schema_k8sio_api_core_v1_Event(ref),
-		"k8s.io/api/core/v1.EventList":                                                      schema_k8sio_api_core_v1_EventList(ref),
-		"k8s.io/api/core/v1.EventSeries":                                                    schema_k8sio_api_core_v1_EventSeries(ref),
-		"k8s.io/api/core/v1.EventSource":                                                    schema_k8sio_api_core_v1_EventSource(ref),
-		"k8s.io/api/core/v1.ExecAction":                                                     schema_k8sio_api_core_v1_ExecAction(ref),
-		"k8s.io/api/core/v1.FCVolumeSource":                                                 schema_k8sio_api_core_v1_FCVolumeSource(ref),
-		"k8s.io/api/core/v1.FlexPersistentVolumeSource":                                     schema_k8sio_api_core_v1_FlexPersistentVolumeSource(ref),
-		"k8s.io/api/core/v1.FlexVolumeSource":                                               schema_k8sio_api_core_v1_FlexVolumeSource(ref),
-		"k8s.io/api/core/v1.FlockerVolumeSource":                                            schema_k8sio_api_core_v1_FlockerVolumeSource(ref),
-		"k8s.io/api/core/v1.GCEPersistentDiskVolumeSource":                                  schema_k8sio_api_core_v1_GCEPersistentDiskVolumeSource(ref),
-		"k8s.io/api/core/v1.GitRepoVolumeSource":                                            schema_k8sio_api_core_v1_GitRepoVolumeSource(ref),
-		"k8s.io/api/core/v1.GlusterfsPersistentVolumeSource":                                schema_k8sio_api_core_v1_GlusterfsPersistentVolumeSource(ref),
-		"k8s.io/api/core/v1.GlusterfsVolumeSource":                                          schema_k8sio_api_core_v1_GlusterfsVolumeSource(ref),
-		"k8s.io/api/core/v1.HTTPGetAction":                                                  schema_k8sio_api_core_v1_HTTPGetAction(ref),
-		"k8s.io/api/core/v1.HTTPHeader":                                                     schema_k8sio_api_core_v1_HTTPHeader(ref),
-		"k8s.io/api/core/v1.Handler":                                                        schema_k8sio_api_core_v1_Handler(ref),
-		"k8s.io/api/core/v1.HostAlias":                                                      schema_k8sio_api_core_v1_HostAlias(ref),
-		"k8s.io/api/core/v1.HostPathVolumeSource":                                           schema_k8sio_api_core_v1_HostPathVolumeSource(ref),
-		"k8s.io/api/core/v1.ISCSIPersistentVolumeSource":                                    schema_k8sio_api_core_v1_ISCSIPersistentVolumeSource(ref),
-		"k8s.io/api/core/v1.ISCSIVolumeSource":                                              schema_k8sio_api_core_v1_ISCSIVolumeSource(ref),
-		"k8s.io/api/core/v1.KeyToPath":                                                      schema_k8sio_api_core_v1_KeyToPath(ref),
-		"k8s.io/api/core/v1.Lifecycle":                                                      schema_k8sio_api_core_v1_Lifecycle(ref),
-		"k8s.io/api/core/v1.LimitRange":                                                     schema_k8sio_api_core_v1_LimitRange(ref),
-		"k8s.io/api/core/v1.LimitRangeItem":                                                 schema_k8sio_api_core_v1_LimitRangeItem(ref),
-		"k8s.io/api/core/v1.LimitRangeList":                                                 schema_k8sio_api_core_v1_LimitRangeList(ref),
-		"k8s.io/api/core/v1.LimitRangeSpec":                                                 schema_k8sio_api_core_v1_LimitRangeSpec(ref),
-		"k8s.io/api/core/v1.List":                                                           schema_k8sio_api_core_v1_List(ref),
-		"k8s.io/api/core/v1.LoadBalancerIngress":                                            schema_k8sio_api_core_v1_LoadBalancerIngress(ref),
-		"k8s.io/api/core/v1.LoadBalancerStatus":                                             schema_k8sio_api_core_v1_LoadBalancerStatus(ref),
-		"k8s.io/api/core/v1.LocalObjectReference":                                           schema_k8sio_api_core_v1_LocalObjectReference(ref),
-		"k8s.io/api/core/v1.LocalVolumeSource":                                              schema_k8sio_api_core_v1_LocalVolumeSource(ref),
-		"k8s.io/api/core/v1.NFSVolumeSource":                                                schema_k8sio_api_core_v1_NFSVolumeSource(ref),
-		"k8s.io/api/core/v1.Namespace":                                                      schema_k8sio_api_core_v1_Namespace(ref),
-		"k8s.io/api/core/v1.NamespaceCondition":                                             schema_k8sio_api_core_v1_NamespaceCondition(ref),
-		"k8s.io/api/core/v1.NamespaceList":                                                  schema_k8sio_api_core_v1_NamespaceList(ref),
-		"k8s.io/api/core/v1.NamespaceSpec":                                                  schema_k8sio_api_core_v1_NamespaceSpec(ref),
-		"k8s.io/api/core/v1.NamespaceStatus":                                                schema_k8sio_api_core_v1_NamespaceStatus(ref),
-		"k8s.io/api/core/v1.Node":                                                           schema_k8sio_api_core_v1_Node(ref),
-		"k8s.io/api/core/v1.NodeAddress":                                                    schema_k8sio_api_core_v1_NodeAddress(ref),
-		"k8s.io/api/core/v1.NodeAffinity":                                                   schema_k8sio_api_core_v1_NodeAffinity(ref),
-		"k8s.io/api/core/v1.NodeCondition":                                                  schema_k8sio_api_core_v1_NodeCondition(ref),
-		"k8s.io/api/core/v1.NodeConfigSource":                                               schema_k8sio_api_core_v1_NodeConfigSource(ref),
-		"k8s.io/api/core/v1.NodeConfigStatus":                                               schema_k8sio_api_core_v1_NodeConfigStatus(ref),
-		"k8s.io/api/core/v1.NodeDaemonEndpoints":                                            schema_k8sio_api_core_v1_NodeDaemonEndpoints(ref),
-		"k8s.io/api/core/v1.NodeList":                                                       schema_k8sio_api_core_v1_NodeList(ref),
-		"k8s.io/api/core/v1.NodeProxyOptions":                                               schema_k8sio_api_core_v1_NodeProxyOptions(ref),
-		"k8s.io/api/core/v1.NodeResources":                                                  schema_k8sio_api_core_v1_NodeResources(ref),
-		"k8s.io/api/core/v1.NodeSelector":                                                   schema_k8sio_api_core_v1_NodeSelector(ref),
-		"k8s.io/api/core/v1.NodeSelectorRequirement":                                        schema_k8sio_api_core_v1_NodeSelectorRequirement(ref),
-		"k8s.io/api/core/v1.NodeSelectorTerm":                                               schema_k8sio_api_core_v1_NodeSelectorTerm(ref),
-		"k8s.io/api/core/v1.NodeSpec":                                                       schema_k8sio_api_core_v1_NodeSpec(ref),
-		"k8s.io/api/core/v1.NodeStatus":                                                     schema_k8sio_api_core_v1_NodeStatus(ref),
-		"k8s.io/api/core/v1.NodeSystemInfo":                                                 schema_k8sio_api_core_v1_NodeSystemInfo(ref),
-		"k8s.io/api/core/v1.ObjectFieldSelector":                                            schema_k8sio_api_core_v1_ObjectFieldSelector(ref),
-		"k8s.io/api/core/v1.ObjectReference":                                                schema_k8sio_api_core_v1_ObjectReference(ref),
-		"k8s.io/api/core/v1.PersistentVolume":                                               schema_k8sio_api_core_v1_PersistentVolume(ref),
-		"k8s.io/api/core/v1.PersistentVolumeClaim":                                          schema_k8sio_api_core_v1_PersistentVolumeClaim(ref),
-		"k8s.io/api/core/v1.PersistentVolumeClaimCondition":                                 schema_k8sio_api_core_v1_PersistentVolumeClaimCondition(ref),
-		"k8s.io/api/core/v1.PersistentVolumeClaimList":                                      schema_k8sio_api_core_v1_PersistentVolumeClaimList(ref),
-		"k8s.io/api/core/v1.PersistentVolumeClaimSpec":                                      schema_k8sio_api_core_v1_PersistentVolumeClaimSpec(ref),
-		"k8s.io/api/core/v1.PersistentVolumeClaimStatus":                                    schema_k8sio_api_core_v1_PersistentVolumeClaimStatus(ref),
-		"k8s.io/api/core/v1.PersistentVolumeClaimTemplate":                                  schema_k8sio_api_core_v1_PersistentVolumeClaimTemplate(ref),
-		"k8s.io/api/core/v1.PersistentVolumeClaimVolumeSource":                              schema_k8sio_api_core_v1_PersistentVolumeClaimVolumeSource(ref),
-		"k8s.io/api/core/v1.PersistentVolumeList":                                           schema_k8sio_api_core_v1_PersistentVolumeList(ref),
-		"k8s.io/api/core/v1.PersistentVolumeSource":                                         schema_k8sio_api_core_v1_PersistentVolumeSource(ref),
-		"k8s.io/api/core/v1.PersistentVolumeSpec":                                           schema_k8sio_api_core_v1_PersistentVolumeSpec(ref),
-		"k8s.io/api/core/v1.PersistentVolumeStatus":                                         schema_k8sio_api_core_v1_PersistentVolumeStatus(ref),
-		"k8s.io/api/core/v1.PhotonPersistentDiskVolumeSource":                               schema_k8sio_api_core_v1_PhotonPersistentDiskVolumeSource(ref),
-		"k8s.io/api/core/v1.Pod":                                                            schema_k8sio_api_core_v1_Pod(ref),
-		"k8s.io/api/core/v1.PodAffinity":                                                    schema_k8sio_api_core_v1_PodAffinity(ref),
-		"k8s.io/api/core/v1.PodAffinityTerm":                                                schema_k8sio_api_core_v1_PodAffinityTerm(ref),
-		"k8s.io/api/core/v1.PodAntiAffinity":                                                schema_k8sio_api_core_v1_PodAntiAffinity(ref),
-		"k8s.io/api/core/v1.PodAttachOptions":                                               schema_k8sio_api_core_v1_PodAttachOptions(ref),
-		"k8s.io/api/core/v1.PodCondition":                                                   schema_k8sio_api_core_v1_PodCondition(ref),
-		"k8s.io/api/core/v1.PodDNSConfig":                                                   schema_k8sio_api_core_v1_PodDNSConfig(ref),
-		"k8s.io/api/core/v1.PodDNSConfigOption":                                             schema_k8sio_api_core_v1_PodDNSConfigOption(ref),
-		"k8s.io/api/core/v1.PodExecOptions":                                                 schema_k8sio_api_core_v1_PodExecOptions(ref),
-		"k8s.io/api/core/v1.PodIP":                                                          schema_k8sio_api_core_v1_PodIP(ref),
-		"k8s.io/api/core/v1.PodList":                                                        schema_k8sio_api_core_v1_PodList(ref),
-		"k8s.io/api/core/v1.PodLogOptions":                                                  schema_k8sio_api_core_v1_PodLogOptions(ref),
-		"k8s.io/api/core/v1.PodPortForwardOptions":                                          schema_k8sio_api_core_v1_PodPortForwardOptions(ref),
-		"k8s.io/api/core/v1.PodProxyOptions":                                                schema_k8sio_api_core_v1_PodProxyOptions(ref),
-		"k8s.io/api/core/v1.PodReadinessGate":                                               schema_k8sio_api_core_v1_PodReadinessGate(ref),
-		"k8s.io/api/core/v1.PodSecurityContext":                                             schema_k8sio_api_core_v1_PodSecurityContext(ref),
-		"k8s.io/api/core/v1.PodSignature":                                                   schema_k8sio_api_core_v1_PodSignature(ref),
-		"k8s.io/api/core/v1.PodSpec":                                                        schema_k8sio_api_core_v1_PodSpec(ref),
-		"k8s.io/api/core/v1.PodStatus":                                                      schema_k8sio_api_core_v1_PodStatus(ref),
-		"k8s.io/api/core/v1.PodStatusResult":                                                schema_k8sio_api_core_v1_PodStatusResult(ref),
-		"k8s.io/api/core/v1.PodTemplate":                                                    schema_k8sio_api_core_v1_PodTemplate(ref),
-		"k8s.io/api/core/v1.PodTemplateList":                                                schema_k8sio_api_core_v1_PodTemplateList(ref),
-		"k8s.io/api/core/v1.PodTemplateSpec":                                                schema_k8sio_api_core_v1_PodTemplateSpec(ref),
-		"k8s.io/api/core/v1.PortStatus":                                                     schema_k8sio_api_core_v1_PortStatus(ref),
-		"k8s.io/api/core/v1.PortworxVolumeSource":                                           schema_k8sio_api_core_v1_PortworxVolumeSource(ref),
-		"k8s.io/api/core/v1.PreferAvoidPodsEntry":                                           schema_k8sio_api_core_v1_PreferAvoidPodsEntry(ref),
-		"k8s.io/api/core/v1.PreferredSchedulingTerm":                                        schema_k8sio_api_core_v1_PreferredSchedulingTerm(ref),
-		"k8s.io/api/core/v1.Probe":                                                          schema_k8sio_api_core_v1_Probe(ref),
-		"k8s.io/api/core/v1.ProjectedVolumeSource":                                          schema_k8sio_api_core_v1_ProjectedVolumeSource(ref),
-		"k8s.io/api/core/v1.QuobyteVolumeSource":                                            schema_k8sio_api_core_v1_QuobyteVolumeSource(ref),
-		"k8s.io/api/core/v1.RBDPersistentVolumeSource":                                      schema_k8sio_api_core_v1_RBDPersistentVolumeSource(ref),
-		"k8s.io/api/core/v1.RBDVolumeSource":                                                schema_k8sio_api_core_v1_RBDVolumeSource(ref),
-		"k8s.io/api/core/v1.RangeAllocation":                                                schema_k8sio_api_core_v1_RangeAllocation(ref),
-		"k8s.io/api/core/v1.ReplicationController":                                          schema_k8sio_api_core_v1_ReplicationController(ref),
-		"k8s.io/api/core/v1.ReplicationControllerCondition":                                 schema_k8sio_api_core_v1_ReplicationControllerCondition(ref),
-		"k8s.io/api/core/v1.ReplicationControllerList":                                      schema_k8sio_api_core_v1_ReplicationControllerList(ref),
-		"k8s.io/api/core/v1.ReplicationControllerSpec":                                      schema_k8sio_api_core_v1_ReplicationControllerSpec(ref),
-		"k8s.io/api/core/v1.ReplicationControllerStatus":                                    schema_k8sio_api_core_v1_ReplicationControllerStatus(ref),
-		"k8s.io/api/core/v1.ResourceFieldSelector":                                          schema_k8sio_api_core_v1_ResourceFieldSelector(ref),
-		"k8s.io/api/core/v1.ResourceQuota":                                                  schema_k8sio_api_core_v1_ResourceQuota(ref),
-		"k8s.io/api/core/v1.ResourceQuotaList":                                              schema_k8sio_api_core_v1_ResourceQuotaList(ref),
-		"k8s.io/api/core/v1.ResourceQuotaSpec":                                              schema_k8sio_api_core_v1_ResourceQuotaSpec(ref),
-		"k8s.io/api/core/v1.ResourceQuotaStatus":                                            schema_k8sio_api_core_v1_ResourceQuotaStatus(ref),
-		"k8s.io/api/core/v1.ResourceRequirements":                                           schema_k8sio_api_core_v1_ResourceRequirements(ref),
-		"k8s.io/api/core/v1.SELinuxOptions":                                                 schema_k8sio_api_core_v1_SELinuxOptions(ref),
-		"k8s.io/api/core/v1.ScaleIOPersistentVolumeSource":                                  schema_k8sio_api_core_v1_ScaleIOPersistentVolumeSource(ref),
-		"k8s.io/api/core/v1.ScaleIOVolumeSource":                                            schema_k8sio_api_core_v1_ScaleIOVolumeSource(ref),
-		"k8s.io/api/core/v1.ScopeSelector":                                                  schema_k8sio_api_core_v1_ScopeSelector(ref),
-		"k8s.io/api/core/v1.ScopedResourceSelectorRequirement":                              schema_k8sio_api_core_v1_ScopedResourceSelectorRequirement(ref),
-		"k8s.io/api/core/v1.SeccompProfile":                                                 schema_k8sio_api_core_v1_SeccompProfile(ref),
-		"k8s.io/api/core/v1.Secret":                                                         schema_k8sio_api_core_v1_Secret(ref),
-		"k8s.io/api/core/v1.SecretEnvSource":                                                schema_k8sio_api_core_v1_SecretEnvSource(ref),
-		"k8s.io/api/core/v1.SecretKeySelector":                                              schema_k8sio_api_core_v1_SecretKeySelector(ref),
-		"k8s.io/api/core/v1.SecretList":                                                     schema_k8sio_api_core_v1_SecretList(ref),
-		"k8s.io/api/core/v1.SecretProjection":                                               schema_k8sio_api_core_v1_SecretProjection(ref),
-		"k8s.io/api/core/v1.SecretReference":                                                schema_k8sio_api_core_v1_SecretReference(ref),
-		"k8s.io/api/core/v1.SecretVolumeSource":                                             schema_k8sio_api_core_v1_SecretVolumeSource(ref),
-		"k8s.io/api/core/v1.SecurityContext":                                                schema_k8sio_api_core_v1_SecurityContext(ref),
-		"k8s.io/api/core/v1.SerializedReference":                                            schema_k8sio_api_core_v1_SerializedReference(ref),
-		"k8s.io/api/core/v1.Service":                                                        schema_k8sio_api_core_v1_Service(ref),
-		"k8s.io/api/core/v1.ServiceAccount":                                                 schema_k8sio_api_core_v1_ServiceAccount(ref),
-		"k8s.io/api/core/v1.ServiceAccountList":                                             schema_k8sio_api_core_v1_ServiceAccountList(ref),
-		"k8s.io/api/core/v1.ServiceAccountTokenProjection":                                  schema_k8sio_api_core_v1_ServiceAccountTokenProjection(ref),
-		"k8s.io/api/core/v1.ServiceList":                                                    schema_k8sio_api_core_v1_ServiceList(ref),
-		"k8s.io/api/core/v1.ServicePort":                                                    schema_k8sio_api_core_v1_ServicePort(ref),
-		"k8s.io/api/core/v1.ServiceProxyOptions":                                            schema_k8sio_api_core_v1_ServiceProxyOptions(ref),
-		"k8s.io/api/core/v1.ServiceSpec":                                                    schema_k8sio_api_core_v1_ServiceSpec(ref),
-		"k8s.io/api/core/v1.ServiceStatus":                                                  schema_k8sio_api_core_v1_ServiceStatus(ref),
-		"k8s.io/api/core/v1.SessionAffinityConfig":                                          schema_k8sio_api_core_v1_SessionAffinityConfig(ref),
-		"k8s.io/api/core/v1.StorageOSPersistentVolumeSource":                                schema_k8sio_api_core_v1_StorageOSPersistentVolumeSource(ref),
-		"k8s.io/api/core/v1.StorageOSVolumeSource":                                          schema_k8sio_api_core_v1_StorageOSVolumeSource(ref),
-		"k8s.io/api/core/v1.Sysctl":                                                         schema_k8sio_api_core_v1_Sysctl(ref),
-		"k8s.io/api/core/v1.TCPSocketAction":                                                schema_k8sio_api_core_v1_TCPSocketAction(ref),
-		"k8s.io/api/core/v1.Taint":                                                          schema_k8sio_api_core_v1_Taint(ref),
-		"k8s.io/api/core/v1.Toleration":                                                     schema_k8sio_api_core_v1_Toleration(ref),
-		"k8s.io/api/core/v1.TopologySelectorLabelRequirement":                               schema_k8sio_api_core_v1_TopologySelectorLabelRequirement(ref),
-		"k8s.io/api/core/v1.TopologySelectorTerm":                                           schema_k8sio_api_core_v1_TopologySelectorTerm(ref),
-		"k8s.io/api/core/v1.TopologySpreadConstraint":                                       schema_k8sio_api_core_v1_TopologySpreadConstraint(ref),
-		"k8s.io/api/core/v1.TypedLocalObjectReference":                                      schema_k8sio_api_core_v1_TypedLocalObjectReference(ref),
-		"k8s.io/api/core/v1.Volume":                                                         schema_k8sio_api_core_v1_Volume(ref),
-		"k8s.io/api/core/v1.VolumeDevice":                                                   schema_k8sio_api_core_v1_VolumeDevice(ref),
-		"k8s.io/api/core/v1.VolumeMount":                                                    schema_k8sio_api_core_v1_VolumeMount(ref),
-		"k8s.io/api/core/v1.VolumeNodeAffinity":                                             schema_k8sio_api_core_v1_VolumeNodeAffinity(ref),
-		"k8s.io/api/core/v1.VolumeProjection":                                               schema_k8sio_api_core_v1_VolumeProjection(ref),
-		"k8s.io/api/core/v1.VolumeSource":                                                   schema_k8sio_api_core_v1_VolumeSource(ref),
-		"k8s.io/api/core/v1.VsphereVirtualDiskVolumeSource":                                 schema_k8sio_api_core_v1_VsphereVirtualDiskVolumeSource(ref),
-		"k8s.io/api/core/v1.WeightedPodAffinityTerm":                                        schema_k8sio_api_core_v1_WeightedPodAffinityTerm(ref),
-		"k8s.io/api/core/v1.WindowsSecurityContextOptions":                                  schema_k8sio_api_core_v1_WindowsSecurityContextOptions(ref),
-		"k8s.io/apimachinery/pkg/api/resource.Quantity":                                     schema_apimachinery_pkg_api_resource_Quantity(ref),
-		"k8s.io/apimachinery/pkg/api/resource.int64Amount":                                  schema_apimachinery_pkg_api_resource_int64Amount(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.APIGroup":                                     schema_pkg_apis_meta_v1_APIGroup(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.APIGroupList":                                 schema_pkg_apis_meta_v1_APIGroupList(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.APIResource":                                  schema_pkg_apis_meta_v1_APIResource(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.APIResourceList":                              schema_pkg_apis_meta_v1_APIResourceList(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.APIVersions":                                  schema_pkg_apis_meta_v1_APIVersions(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.Condition":                                    schema_pkg_apis_meta_v1_Condition(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.CreateOptions":                                schema_pkg_apis_meta_v1_CreateOptions(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.DeleteOptions":                                schema_pkg_apis_meta_v1_DeleteOptions(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.Duration":                                     schema_pkg_apis_meta_v1_Duration(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.ExportOptions":                                schema_pkg_apis_meta_v1_ExportOptions(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.FieldsV1":                                     schema_pkg_apis_meta_v1_FieldsV1(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.GetOptions":                                   schema_pkg_apis_meta_v1_GetOptions(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.GroupKind":                                    schema_pkg_apis_meta_v1_GroupKind(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.GroupResource":                                schema_pkg_apis_meta_v1_GroupResource(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.GroupVersion":                                 schema_pkg_apis_meta_v1_GroupVersion(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.GroupVersionForDiscovery":                     schema_pkg_apis_meta_v1_GroupVersionForDiscovery(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.GroupVersionKind":                             schema_pkg_apis_meta_v1_GroupVersionKind(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.GroupVersionResource":                         schema_pkg_apis_meta_v1_GroupVersionResource(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.InternalEvent":                                schema_pkg_apis_meta_v1_InternalEvent(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector":                                schema_pkg_apis_meta_v1_LabelSelector(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelectorRequirement":                     schema_pkg_apis_meta_v1_LabelSelectorRequirement(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.List":                                         schema_pkg_apis_meta_v1_List(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta":                                     schema_pkg_apis_meta_v1_ListMeta(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.ListOptions":                                  schema_pkg_apis_meta_v1_ListOptions(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.ManagedFieldsEntry":                           schema_pkg_apis_meta_v1_ManagedFieldsEntry(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.MicroTime":                                    schema_pkg_apis_meta_v1_MicroTime(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta":                                   schema_pkg_apis_meta_v1_ObjectMeta(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.OwnerReference":                               schema_pkg_apis_meta_v1_OwnerReference(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.PartialObjectMetadata":                        schema_pkg_apis_meta_v1_PartialObjectMetadata(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.PartialObjectMetadataList":                    schema_pkg_apis_meta_v1_PartialObjectMetadataList(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.Patch":                                        schema_pkg_apis_meta_v1_Patch(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.PatchOptions":                                 schema_pkg_apis_meta_v1_PatchOptions(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.Preconditions":                                schema_pkg_apis_meta_v1_Preconditions(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.RootPaths":                                    schema_pkg_apis_meta_v1_RootPaths(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.ServerAddressByClientCIDR":                    schema_pkg_apis_meta_v1_ServerAddressByClientCIDR(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.Status":                                       schema_pkg_apis_meta_v1_Status(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.StatusCause":                                  schema_pkg_apis_meta_v1_StatusCause(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.StatusDetails":                                schema_pkg_apis_meta_v1_StatusDetails(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.Table":                                        schema_pkg_apis_meta_v1_Table(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.TableColumnDefinition":                        schema_pkg_apis_meta_v1_TableColumnDefinition(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.TableOptions":                                 schema_pkg_apis_meta_v1_TableOptions(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.TableRow":                                     schema_pkg_apis_meta_v1_TableRow(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.TableRowCondition":                            schema_pkg_apis_meta_v1_TableRowCondition(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.Time":                                         schema_pkg_apis_meta_v1_Time(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.Timestamp":                                    schema_pkg_apis_meta_v1_Timestamp(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta":                                     schema_pkg_apis_meta_v1_TypeMeta(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.UpdateOptions":                                schema_pkg_apis_meta_v1_UpdateOptions(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.WatchEvent":                                   schema_pkg_apis_meta_v1_WatchEvent(ref),
-		"k8s.io/apimachinery/pkg/runtime.RawExtension":                                      schema_k8sio_apimachinery_pkg_runtime_RawExtension(ref),
-		"k8s.io/apimachinery/pkg/runtime.TypeMeta":                                          schema_k8sio_apimachinery_pkg_runtime_TypeMeta(ref),
-		"k8s.io/apimachinery/pkg/runtime.Unknown":                                           schema_k8sio_apimachinery_pkg_runtime_Unknown(ref),
-		"k8s.io/apimachinery/pkg/util/intstr.IntOrString":                                   schema_apimachinery_pkg_util_intstr_IntOrString(ref),
-		"k8s.io/apimachinery/pkg/version.Info":                                              schema_k8sio_apimachinery_pkg_version_Info(ref),
+		"github.com/everoute/everoute/pkg/apis/agent/v1alpha1.AgentCondition":        schema_pkg_apis_agent_v1alpha1_AgentCondition(ref),
+		"github.com/everoute/everoute/pkg/apis/agent/v1alpha1.AgentInfo":             schema_pkg_apis_agent_v1alpha1_AgentInfo(ref),
+		"github.com/everoute/everoute/pkg/apis/agent/v1alpha1.AgentInfoList":         schema_pkg_apis_agent_v1alpha1_AgentInfoList(ref),
+		"github.com/everoute/everoute/pkg/apis/agent/v1alpha1.BondConfig":            schema_pkg_apis_agent_v1alpha1_BondConfig(ref),
+		"github.com/everoute/everoute/pkg/apis/agent/v1alpha1.OVSBridge":             schema_pkg_apis_agent_v1alpha1_OVSBridge(ref),
+		"github.com/everoute/everoute/pkg/apis/agent/v1alpha1.OVSInfo":               schema_pkg_apis_agent_v1alpha1_OVSInfo(ref),
+		"github.com/everoute/everoute/pkg/apis/agent/v1alpha1.OVSInterface":          schema_pkg_apis_agent_v1alpha1_OVSInterface(ref),
+		"github.com/everoute/everoute/pkg/apis/agent/v1alpha1.OVSPort":               schema_pkg_apis_agent_v1alpha1_OVSPort(ref),
+		"github.com/everoute/everoute/pkg/apis/agent/v1alpha1.VlanConfig":            schema_pkg_apis_agent_v1alpha1_VlanConfig(ref),
+		"github.com/everoute/everoute/pkg/apis/group/v1alpha1.EndpointGroup":         schema_pkg_apis_group_v1alpha1_EndpointGroup(ref),
+		"github.com/everoute/everoute/pkg/apis/group/v1alpha1.EndpointGroupList":     schema_pkg_apis_group_v1alpha1_EndpointGroupList(ref),
+		"github.com/everoute/everoute/pkg/apis/group/v1alpha1.EndpointGroupSpec":     schema_pkg_apis_group_v1alpha1_EndpointGroupSpec(ref),
+		"github.com/everoute/everoute/pkg/apis/group/v1alpha1.EndpointReference":     schema_pkg_apis_group_v1alpha1_EndpointReference(ref),
+		"github.com/everoute/everoute/pkg/apis/group/v1alpha1.GroupMember":           schema_pkg_apis_group_v1alpha1_GroupMember(ref),
+		"github.com/everoute/everoute/pkg/apis/group/v1alpha1.GroupMembers":          schema_pkg_apis_group_v1alpha1_GroupMembers(ref),
+		"github.com/everoute/everoute/pkg/apis/group/v1alpha1.GroupMembersList":      schema_pkg_apis_group_v1alpha1_GroupMembersList(ref),
+		"github.com/everoute/everoute/pkg/apis/group/v1alpha1.GroupMembersPatch":     schema_pkg_apis_group_v1alpha1_GroupMembersPatch(ref),
+		"github.com/everoute/everoute/pkg/apis/group/v1alpha1.GroupMembersPatchList": schema_pkg_apis_group_v1alpha1_GroupMembersPatchList(ref),
+		"github.com/everoute/everoute/pkg/apis/group/v1alpha1.GroupMembersReference": schema_pkg_apis_group_v1alpha1_GroupMembersReference(ref),
+		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.ApplyToPeer":        schema_pkg_apis_security_v1alpha1_ApplyToPeer(ref),
+		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.Endpoint":           schema_pkg_apis_security_v1alpha1_Endpoint(ref),
+		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.EndpointList":       schema_pkg_apis_security_v1alpha1_EndpointList(ref),
+		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.EndpointReference":  schema_pkg_apis_security_v1alpha1_EndpointReference(ref),
+		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.EndpointSpec":       schema_pkg_apis_security_v1alpha1_EndpointSpec(ref),
+		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.EndpointStatus":     schema_pkg_apis_security_v1alpha1_EndpointStatus(ref),
+		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.GlobalPolicy":       schema_pkg_apis_security_v1alpha1_GlobalPolicy(ref),
+		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.GlobalPolicyList":   schema_pkg_apis_security_v1alpha1_GlobalPolicyList(ref),
+		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.GlobalPolicySpec":   schema_pkg_apis_security_v1alpha1_GlobalPolicySpec(ref),
+		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.NamespacedName":     schema_pkg_apis_security_v1alpha1_NamespacedName(ref),
+		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.Rule":               schema_pkg_apis_security_v1alpha1_Rule(ref),
+		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.SecurityPolicy":     schema_pkg_apis_security_v1alpha1_SecurityPolicy(ref),
+		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.SecurityPolicyList": schema_pkg_apis_security_v1alpha1_SecurityPolicyList(ref),
+		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.SecurityPolicyPeer": schema_pkg_apis_security_v1alpha1_SecurityPolicyPeer(ref),
+		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.SecurityPolicyPort": schema_pkg_apis_security_v1alpha1_SecurityPolicyPort(ref),
+		"github.com/everoute/everoute/pkg/apis/security/v1alpha1.SecurityPolicySpec": schema_pkg_apis_security_v1alpha1_SecurityPolicySpec(ref),
+		"k8s.io/api/apps/v1.ControllerRevision":                                      schema_k8sio_api_apps_v1_ControllerRevision(ref),
+		"k8s.io/api/apps/v1.ControllerRevisionList":                                  schema_k8sio_api_apps_v1_ControllerRevisionList(ref),
+		"k8s.io/api/apps/v1.DaemonSet":                                               schema_k8sio_api_apps_v1_DaemonSet(ref),
+		"k8s.io/api/apps/v1.DaemonSetCondition":                                      schema_k8sio_api_apps_v1_DaemonSetCondition(ref),
+		"k8s.io/api/apps/v1.DaemonSetList":                                           schema_k8sio_api_apps_v1_DaemonSetList(ref),
+		"k8s.io/api/apps/v1.DaemonSetSpec":                                           schema_k8sio_api_apps_v1_DaemonSetSpec(ref),
+		"k8s.io/api/apps/v1.DaemonSetStatus":                                         schema_k8sio_api_apps_v1_DaemonSetStatus(ref),
+		"k8s.io/api/apps/v1.DaemonSetUpdateStrategy":                                 schema_k8sio_api_apps_v1_DaemonSetUpdateStrategy(ref),
+		"k8s.io/api/apps/v1.Deployment":                                              schema_k8sio_api_apps_v1_Deployment(ref),
+		"k8s.io/api/apps/v1.DeploymentCondition":                                     schema_k8sio_api_apps_v1_DeploymentCondition(ref),
+		"k8s.io/api/apps/v1.DeploymentList":                                          schema_k8sio_api_apps_v1_DeploymentList(ref),
+		"k8s.io/api/apps/v1.DeploymentSpec":                                          schema_k8sio_api_apps_v1_DeploymentSpec(ref),
+		"k8s.io/api/apps/v1.DeploymentStatus":                                        schema_k8sio_api_apps_v1_DeploymentStatus(ref),
+		"k8s.io/api/apps/v1.DeploymentStrategy":                                      schema_k8sio_api_apps_v1_DeploymentStrategy(ref),
+		"k8s.io/api/apps/v1.ReplicaSet":                                              schema_k8sio_api_apps_v1_ReplicaSet(ref),
+		"k8s.io/api/apps/v1.ReplicaSetCondition":                                     schema_k8sio_api_apps_v1_ReplicaSetCondition(ref),
+		"k8s.io/api/apps/v1.ReplicaSetList":                                          schema_k8sio_api_apps_v1_ReplicaSetList(ref),
+		"k8s.io/api/apps/v1.ReplicaSetSpec":                                          schema_k8sio_api_apps_v1_ReplicaSetSpec(ref),
+		"k8s.io/api/apps/v1.ReplicaSetStatus":                                        schema_k8sio_api_apps_v1_ReplicaSetStatus(ref),
+		"k8s.io/api/apps/v1.RollingUpdateDaemonSet":                                  schema_k8sio_api_apps_v1_RollingUpdateDaemonSet(ref),
+		"k8s.io/api/apps/v1.RollingUpdateDeployment":                                 schema_k8sio_api_apps_v1_RollingUpdateDeployment(ref),
+		"k8s.io/api/apps/v1.RollingUpdateStatefulSetStrategy":                        schema_k8sio_api_apps_v1_RollingUpdateStatefulSetStrategy(ref),
+		"k8s.io/api/apps/v1.StatefulSet":                                             schema_k8sio_api_apps_v1_StatefulSet(ref),
+		"k8s.io/api/apps/v1.StatefulSetCondition":                                    schema_k8sio_api_apps_v1_StatefulSetCondition(ref),
+		"k8s.io/api/apps/v1.StatefulSetList":                                         schema_k8sio_api_apps_v1_StatefulSetList(ref),
+		"k8s.io/api/apps/v1.StatefulSetSpec":                                         schema_k8sio_api_apps_v1_StatefulSetSpec(ref),
+		"k8s.io/api/apps/v1.StatefulSetStatus":                                       schema_k8sio_api_apps_v1_StatefulSetStatus(ref),
+		"k8s.io/api/apps/v1.StatefulSetUpdateStrategy":                               schema_k8sio_api_apps_v1_StatefulSetUpdateStrategy(ref),
+		"k8s.io/api/core/v1.AWSElasticBlockStoreVolumeSource":                        schema_k8sio_api_core_v1_AWSElasticBlockStoreVolumeSource(ref),
+		"k8s.io/api/core/v1.Affinity":                                                schema_k8sio_api_core_v1_Affinity(ref),
+		"k8s.io/api/core/v1.AttachedVolume":                                          schema_k8sio_api_core_v1_AttachedVolume(ref),
+		"k8s.io/api/core/v1.AvoidPods":                                               schema_k8sio_api_core_v1_AvoidPods(ref),
+		"k8s.io/api/core/v1.AzureDiskVolumeSource":                                   schema_k8sio_api_core_v1_AzureDiskVolumeSource(ref),
+		"k8s.io/api/core/v1.AzureFilePersistentVolumeSource":                         schema_k8sio_api_core_v1_AzureFilePersistentVolumeSource(ref),
+		"k8s.io/api/core/v1.AzureFileVolumeSource":                                   schema_k8sio_api_core_v1_AzureFileVolumeSource(ref),
+		"k8s.io/api/core/v1.Binding":                                                 schema_k8sio_api_core_v1_Binding(ref),
+		"k8s.io/api/core/v1.CSIPersistentVolumeSource":                               schema_k8sio_api_core_v1_CSIPersistentVolumeSource(ref),
+		"k8s.io/api/core/v1.CSIVolumeSource":                                         schema_k8sio_api_core_v1_CSIVolumeSource(ref),
+		"k8s.io/api/core/v1.Capabilities":                                            schema_k8sio_api_core_v1_Capabilities(ref),
+		"k8s.io/api/core/v1.CephFSPersistentVolumeSource":                            schema_k8sio_api_core_v1_CephFSPersistentVolumeSource(ref),
+		"k8s.io/api/core/v1.CephFSVolumeSource":                                      schema_k8sio_api_core_v1_CephFSVolumeSource(ref),
+		"k8s.io/api/core/v1.CinderPersistentVolumeSource":                            schema_k8sio_api_core_v1_CinderPersistentVolumeSource(ref),
+		"k8s.io/api/core/v1.CinderVolumeSource":                                      schema_k8sio_api_core_v1_CinderVolumeSource(ref),
+		"k8s.io/api/core/v1.ClientIPConfig":                                          schema_k8sio_api_core_v1_ClientIPConfig(ref),
+		"k8s.io/api/core/v1.ComponentCondition":                                      schema_k8sio_api_core_v1_ComponentCondition(ref),
+		"k8s.io/api/core/v1.ComponentStatus":                                         schema_k8sio_api_core_v1_ComponentStatus(ref),
+		"k8s.io/api/core/v1.ComponentStatusList":                                     schema_k8sio_api_core_v1_ComponentStatusList(ref),
+		"k8s.io/api/core/v1.ConfigMap":                                               schema_k8sio_api_core_v1_ConfigMap(ref),
+		"k8s.io/api/core/v1.ConfigMapEnvSource":                                      schema_k8sio_api_core_v1_ConfigMapEnvSource(ref),
+		"k8s.io/api/core/v1.ConfigMapKeySelector":                                    schema_k8sio_api_core_v1_ConfigMapKeySelector(ref),
+		"k8s.io/api/core/v1.ConfigMapList":                                           schema_k8sio_api_core_v1_ConfigMapList(ref),
+		"k8s.io/api/core/v1.ConfigMapNodeConfigSource":                               schema_k8sio_api_core_v1_ConfigMapNodeConfigSource(ref),
+		"k8s.io/api/core/v1.ConfigMapProjection":                                     schema_k8sio_api_core_v1_ConfigMapProjection(ref),
+		"k8s.io/api/core/v1.ConfigMapVolumeSource":                                   schema_k8sio_api_core_v1_ConfigMapVolumeSource(ref),
+		"k8s.io/api/core/v1.Container":                                               schema_k8sio_api_core_v1_Container(ref),
+		"k8s.io/api/core/v1.ContainerImage":                                          schema_k8sio_api_core_v1_ContainerImage(ref),
+		"k8s.io/api/core/v1.ContainerPort":                                           schema_k8sio_api_core_v1_ContainerPort(ref),
+		"k8s.io/api/core/v1.ContainerState":                                          schema_k8sio_api_core_v1_ContainerState(ref),
+		"k8s.io/api/core/v1.ContainerStateRunning":                                   schema_k8sio_api_core_v1_ContainerStateRunning(ref),
+		"k8s.io/api/core/v1.ContainerStateTerminated":                                schema_k8sio_api_core_v1_ContainerStateTerminated(ref),
+		"k8s.io/api/core/v1.ContainerStateWaiting":                                   schema_k8sio_api_core_v1_ContainerStateWaiting(ref),
+		"k8s.io/api/core/v1.ContainerStatus":                                         schema_k8sio_api_core_v1_ContainerStatus(ref),
+		"k8s.io/api/core/v1.DaemonEndpoint":                                          schema_k8sio_api_core_v1_DaemonEndpoint(ref),
+		"k8s.io/api/core/v1.DownwardAPIProjection":                                   schema_k8sio_api_core_v1_DownwardAPIProjection(ref),
+		"k8s.io/api/core/v1.DownwardAPIVolumeFile":                                   schema_k8sio_api_core_v1_DownwardAPIVolumeFile(ref),
+		"k8s.io/api/core/v1.DownwardAPIVolumeSource":                                 schema_k8sio_api_core_v1_DownwardAPIVolumeSource(ref),
+		"k8s.io/api/core/v1.EmptyDirVolumeSource":                                    schema_k8sio_api_core_v1_EmptyDirVolumeSource(ref),
+		"k8s.io/api/core/v1.EndpointAddress":                                         schema_k8sio_api_core_v1_EndpointAddress(ref),
+		"k8s.io/api/core/v1.EndpointPort":                                            schema_k8sio_api_core_v1_EndpointPort(ref),
+		"k8s.io/api/core/v1.EndpointSubset":                                          schema_k8sio_api_core_v1_EndpointSubset(ref),
+		"k8s.io/api/core/v1.Endpoints":                                               schema_k8sio_api_core_v1_Endpoints(ref),
+		"k8s.io/api/core/v1.EndpointsList":                                           schema_k8sio_api_core_v1_EndpointsList(ref),
+		"k8s.io/api/core/v1.EnvFromSource":                                           schema_k8sio_api_core_v1_EnvFromSource(ref),
+		"k8s.io/api/core/v1.EnvVar":                                                  schema_k8sio_api_core_v1_EnvVar(ref),
+		"k8s.io/api/core/v1.EnvVarSource":                                            schema_k8sio_api_core_v1_EnvVarSource(ref),
+		"k8s.io/api/core/v1.EphemeralContainer":                                      schema_k8sio_api_core_v1_EphemeralContainer(ref),
+		"k8s.io/api/core/v1.EphemeralContainerCommon":                                schema_k8sio_api_core_v1_EphemeralContainerCommon(ref),
+		"k8s.io/api/core/v1.EphemeralContainers":                                     schema_k8sio_api_core_v1_EphemeralContainers(ref),
+		"k8s.io/api/core/v1.EphemeralVolumeSource":                                   schema_k8sio_api_core_v1_EphemeralVolumeSource(ref),
+		"k8s.io/api/core/v1.Event":                                                   schema_k8sio_api_core_v1_Event(ref),
+		"k8s.io/api/core/v1.EventList":                                               schema_k8sio_api_core_v1_EventList(ref),
+		"k8s.io/api/core/v1.EventSeries":                                             schema_k8sio_api_core_v1_EventSeries(ref),
+		"k8s.io/api/core/v1.EventSource":                                             schema_k8sio_api_core_v1_EventSource(ref),
+		"k8s.io/api/core/v1.ExecAction":                                              schema_k8sio_api_core_v1_ExecAction(ref),
+		"k8s.io/api/core/v1.FCVolumeSource":                                          schema_k8sio_api_core_v1_FCVolumeSource(ref),
+		"k8s.io/api/core/v1.FlexPersistentVolumeSource":                              schema_k8sio_api_core_v1_FlexPersistentVolumeSource(ref),
+		"k8s.io/api/core/v1.FlexVolumeSource":                                        schema_k8sio_api_core_v1_FlexVolumeSource(ref),
+		"k8s.io/api/core/v1.FlockerVolumeSource":                                     schema_k8sio_api_core_v1_FlockerVolumeSource(ref),
+		"k8s.io/api/core/v1.GCEPersistentDiskVolumeSource":                           schema_k8sio_api_core_v1_GCEPersistentDiskVolumeSource(ref),
+		"k8s.io/api/core/v1.GitRepoVolumeSource":                                     schema_k8sio_api_core_v1_GitRepoVolumeSource(ref),
+		"k8s.io/api/core/v1.GlusterfsPersistentVolumeSource":                         schema_k8sio_api_core_v1_GlusterfsPersistentVolumeSource(ref),
+		"k8s.io/api/core/v1.GlusterfsVolumeSource":                                   schema_k8sio_api_core_v1_GlusterfsVolumeSource(ref),
+		"k8s.io/api/core/v1.HTTPGetAction":                                           schema_k8sio_api_core_v1_HTTPGetAction(ref),
+		"k8s.io/api/core/v1.HTTPHeader":                                              schema_k8sio_api_core_v1_HTTPHeader(ref),
+		"k8s.io/api/core/v1.Handler":                                                 schema_k8sio_api_core_v1_Handler(ref),
+		"k8s.io/api/core/v1.HostAlias":                                               schema_k8sio_api_core_v1_HostAlias(ref),
+		"k8s.io/api/core/v1.HostPathVolumeSource":                                    schema_k8sio_api_core_v1_HostPathVolumeSource(ref),
+		"k8s.io/api/core/v1.ISCSIPersistentVolumeSource":                             schema_k8sio_api_core_v1_ISCSIPersistentVolumeSource(ref),
+		"k8s.io/api/core/v1.ISCSIVolumeSource":                                       schema_k8sio_api_core_v1_ISCSIVolumeSource(ref),
+		"k8s.io/api/core/v1.KeyToPath":                                               schema_k8sio_api_core_v1_KeyToPath(ref),
+		"k8s.io/api/core/v1.Lifecycle":                                               schema_k8sio_api_core_v1_Lifecycle(ref),
+		"k8s.io/api/core/v1.LimitRange":                                              schema_k8sio_api_core_v1_LimitRange(ref),
+		"k8s.io/api/core/v1.LimitRangeItem":                                          schema_k8sio_api_core_v1_LimitRangeItem(ref),
+		"k8s.io/api/core/v1.LimitRangeList":                                          schema_k8sio_api_core_v1_LimitRangeList(ref),
+		"k8s.io/api/core/v1.LimitRangeSpec":                                          schema_k8sio_api_core_v1_LimitRangeSpec(ref),
+		"k8s.io/api/core/v1.List":                                                    schema_k8sio_api_core_v1_List(ref),
+		"k8s.io/api/core/v1.LoadBalancerIngress":                                     schema_k8sio_api_core_v1_LoadBalancerIngress(ref),
+		"k8s.io/api/core/v1.LoadBalancerStatus":                                      schema_k8sio_api_core_v1_LoadBalancerStatus(ref),
+		"k8s.io/api/core/v1.LocalObjectReference":                                    schema_k8sio_api_core_v1_LocalObjectReference(ref),
+		"k8s.io/api/core/v1.LocalVolumeSource":                                       schema_k8sio_api_core_v1_LocalVolumeSource(ref),
+		"k8s.io/api/core/v1.NFSVolumeSource":                                         schema_k8sio_api_core_v1_NFSVolumeSource(ref),
+		"k8s.io/api/core/v1.Namespace":                                               schema_k8sio_api_core_v1_Namespace(ref),
+		"k8s.io/api/core/v1.NamespaceCondition":                                      schema_k8sio_api_core_v1_NamespaceCondition(ref),
+		"k8s.io/api/core/v1.NamespaceList":                                           schema_k8sio_api_core_v1_NamespaceList(ref),
+		"k8s.io/api/core/v1.NamespaceSpec":                                           schema_k8sio_api_core_v1_NamespaceSpec(ref),
+		"k8s.io/api/core/v1.NamespaceStatus":                                         schema_k8sio_api_core_v1_NamespaceStatus(ref),
+		"k8s.io/api/core/v1.Node":                                                    schema_k8sio_api_core_v1_Node(ref),
+		"k8s.io/api/core/v1.NodeAddress":                                             schema_k8sio_api_core_v1_NodeAddress(ref),
+		"k8s.io/api/core/v1.NodeAffinity":                                            schema_k8sio_api_core_v1_NodeAffinity(ref),
+		"k8s.io/api/core/v1.NodeCondition":                                           schema_k8sio_api_core_v1_NodeCondition(ref),
+		"k8s.io/api/core/v1.NodeConfigSource":                                        schema_k8sio_api_core_v1_NodeConfigSource(ref),
+		"k8s.io/api/core/v1.NodeConfigStatus":                                        schema_k8sio_api_core_v1_NodeConfigStatus(ref),
+		"k8s.io/api/core/v1.NodeDaemonEndpoints":                                     schema_k8sio_api_core_v1_NodeDaemonEndpoints(ref),
+		"k8s.io/api/core/v1.NodeList":                                                schema_k8sio_api_core_v1_NodeList(ref),
+		"k8s.io/api/core/v1.NodeProxyOptions":                                        schema_k8sio_api_core_v1_NodeProxyOptions(ref),
+		"k8s.io/api/core/v1.NodeResources":                                           schema_k8sio_api_core_v1_NodeResources(ref),
+		"k8s.io/api/core/v1.NodeSelector":                                            schema_k8sio_api_core_v1_NodeSelector(ref),
+		"k8s.io/api/core/v1.NodeSelectorRequirement":                                 schema_k8sio_api_core_v1_NodeSelectorRequirement(ref),
+		"k8s.io/api/core/v1.NodeSelectorTerm":                                        schema_k8sio_api_core_v1_NodeSelectorTerm(ref),
+		"k8s.io/api/core/v1.NodeSpec":                                                schema_k8sio_api_core_v1_NodeSpec(ref),
+		"k8s.io/api/core/v1.NodeStatus":                                              schema_k8sio_api_core_v1_NodeStatus(ref),
+		"k8s.io/api/core/v1.NodeSystemInfo":                                          schema_k8sio_api_core_v1_NodeSystemInfo(ref),
+		"k8s.io/api/core/v1.ObjectFieldSelector":                                     schema_k8sio_api_core_v1_ObjectFieldSelector(ref),
+		"k8s.io/api/core/v1.ObjectReference":                                         schema_k8sio_api_core_v1_ObjectReference(ref),
+		"k8s.io/api/core/v1.PersistentVolume":                                        schema_k8sio_api_core_v1_PersistentVolume(ref),
+		"k8s.io/api/core/v1.PersistentVolumeClaim":                                   schema_k8sio_api_core_v1_PersistentVolumeClaim(ref),
+		"k8s.io/api/core/v1.PersistentVolumeClaimCondition":                          schema_k8sio_api_core_v1_PersistentVolumeClaimCondition(ref),
+		"k8s.io/api/core/v1.PersistentVolumeClaimList":                               schema_k8sio_api_core_v1_PersistentVolumeClaimList(ref),
+		"k8s.io/api/core/v1.PersistentVolumeClaimSpec":                               schema_k8sio_api_core_v1_PersistentVolumeClaimSpec(ref),
+		"k8s.io/api/core/v1.PersistentVolumeClaimStatus":                             schema_k8sio_api_core_v1_PersistentVolumeClaimStatus(ref),
+		"k8s.io/api/core/v1.PersistentVolumeClaimTemplate":                           schema_k8sio_api_core_v1_PersistentVolumeClaimTemplate(ref),
+		"k8s.io/api/core/v1.PersistentVolumeClaimVolumeSource":                       schema_k8sio_api_core_v1_PersistentVolumeClaimVolumeSource(ref),
+		"k8s.io/api/core/v1.PersistentVolumeList":                                    schema_k8sio_api_core_v1_PersistentVolumeList(ref),
+		"k8s.io/api/core/v1.PersistentVolumeSource":                                  schema_k8sio_api_core_v1_PersistentVolumeSource(ref),
+		"k8s.io/api/core/v1.PersistentVolumeSpec":                                    schema_k8sio_api_core_v1_PersistentVolumeSpec(ref),
+		"k8s.io/api/core/v1.PersistentVolumeStatus":                                  schema_k8sio_api_core_v1_PersistentVolumeStatus(ref),
+		"k8s.io/api/core/v1.PhotonPersistentDiskVolumeSource":                        schema_k8sio_api_core_v1_PhotonPersistentDiskVolumeSource(ref),
+		"k8s.io/api/core/v1.Pod":                                                     schema_k8sio_api_core_v1_Pod(ref),
+		"k8s.io/api/core/v1.PodAffinity":                                             schema_k8sio_api_core_v1_PodAffinity(ref),
+		"k8s.io/api/core/v1.PodAffinityTerm":                                         schema_k8sio_api_core_v1_PodAffinityTerm(ref),
+		"k8s.io/api/core/v1.PodAntiAffinity":                                         schema_k8sio_api_core_v1_PodAntiAffinity(ref),
+		"k8s.io/api/core/v1.PodAttachOptions":                                        schema_k8sio_api_core_v1_PodAttachOptions(ref),
+		"k8s.io/api/core/v1.PodCondition":                                            schema_k8sio_api_core_v1_PodCondition(ref),
+		"k8s.io/api/core/v1.PodDNSConfig":                                            schema_k8sio_api_core_v1_PodDNSConfig(ref),
+		"k8s.io/api/core/v1.PodDNSConfigOption":                                      schema_k8sio_api_core_v1_PodDNSConfigOption(ref),
+		"k8s.io/api/core/v1.PodExecOptions":                                          schema_k8sio_api_core_v1_PodExecOptions(ref),
+		"k8s.io/api/core/v1.PodIP":                                                   schema_k8sio_api_core_v1_PodIP(ref),
+		"k8s.io/api/core/v1.PodList":                                                 schema_k8sio_api_core_v1_PodList(ref),
+		"k8s.io/api/core/v1.PodLogOptions":                                           schema_k8sio_api_core_v1_PodLogOptions(ref),
+		"k8s.io/api/core/v1.PodPortForwardOptions":                                   schema_k8sio_api_core_v1_PodPortForwardOptions(ref),
+		"k8s.io/api/core/v1.PodProxyOptions":                                         schema_k8sio_api_core_v1_PodProxyOptions(ref),
+		"k8s.io/api/core/v1.PodReadinessGate":                                        schema_k8sio_api_core_v1_PodReadinessGate(ref),
+		"k8s.io/api/core/v1.PodSecurityContext":                                      schema_k8sio_api_core_v1_PodSecurityContext(ref),
+		"k8s.io/api/core/v1.PodSignature":                                            schema_k8sio_api_core_v1_PodSignature(ref),
+		"k8s.io/api/core/v1.PodSpec":                                                 schema_k8sio_api_core_v1_PodSpec(ref),
+		"k8s.io/api/core/v1.PodStatus":                                               schema_k8sio_api_core_v1_PodStatus(ref),
+		"k8s.io/api/core/v1.PodStatusResult":                                         schema_k8sio_api_core_v1_PodStatusResult(ref),
+		"k8s.io/api/core/v1.PodTemplate":                                             schema_k8sio_api_core_v1_PodTemplate(ref),
+		"k8s.io/api/core/v1.PodTemplateList":                                         schema_k8sio_api_core_v1_PodTemplateList(ref),
+		"k8s.io/api/core/v1.PodTemplateSpec":                                         schema_k8sio_api_core_v1_PodTemplateSpec(ref),
+		"k8s.io/api/core/v1.PortStatus":                                              schema_k8sio_api_core_v1_PortStatus(ref),
+		"k8s.io/api/core/v1.PortworxVolumeSource":                                    schema_k8sio_api_core_v1_PortworxVolumeSource(ref),
+		"k8s.io/api/core/v1.PreferAvoidPodsEntry":                                    schema_k8sio_api_core_v1_PreferAvoidPodsEntry(ref),
+		"k8s.io/api/core/v1.PreferredSchedulingTerm":                                 schema_k8sio_api_core_v1_PreferredSchedulingTerm(ref),
+		"k8s.io/api/core/v1.Probe":                                                   schema_k8sio_api_core_v1_Probe(ref),
+		"k8s.io/api/core/v1.ProjectedVolumeSource":                                   schema_k8sio_api_core_v1_ProjectedVolumeSource(ref),
+		"k8s.io/api/core/v1.QuobyteVolumeSource":                                     schema_k8sio_api_core_v1_QuobyteVolumeSource(ref),
+		"k8s.io/api/core/v1.RBDPersistentVolumeSource":                               schema_k8sio_api_core_v1_RBDPersistentVolumeSource(ref),
+		"k8s.io/api/core/v1.RBDVolumeSource":                                         schema_k8sio_api_core_v1_RBDVolumeSource(ref),
+		"k8s.io/api/core/v1.RangeAllocation":                                         schema_k8sio_api_core_v1_RangeAllocation(ref),
+		"k8s.io/api/core/v1.ReplicationController":                                   schema_k8sio_api_core_v1_ReplicationController(ref),
+		"k8s.io/api/core/v1.ReplicationControllerCondition":                          schema_k8sio_api_core_v1_ReplicationControllerCondition(ref),
+		"k8s.io/api/core/v1.ReplicationControllerList":                               schema_k8sio_api_core_v1_ReplicationControllerList(ref),
+		"k8s.io/api/core/v1.ReplicationControllerSpec":                               schema_k8sio_api_core_v1_ReplicationControllerSpec(ref),
+		"k8s.io/api/core/v1.ReplicationControllerStatus":                             schema_k8sio_api_core_v1_ReplicationControllerStatus(ref),
+		"k8s.io/api/core/v1.ResourceFieldSelector":                                   schema_k8sio_api_core_v1_ResourceFieldSelector(ref),
+		"k8s.io/api/core/v1.ResourceQuota":                                           schema_k8sio_api_core_v1_ResourceQuota(ref),
+		"k8s.io/api/core/v1.ResourceQuotaList":                                       schema_k8sio_api_core_v1_ResourceQuotaList(ref),
+		"k8s.io/api/core/v1.ResourceQuotaSpec":                                       schema_k8sio_api_core_v1_ResourceQuotaSpec(ref),
+		"k8s.io/api/core/v1.ResourceQuotaStatus":                                     schema_k8sio_api_core_v1_ResourceQuotaStatus(ref),
+		"k8s.io/api/core/v1.ResourceRequirements":                                    schema_k8sio_api_core_v1_ResourceRequirements(ref),
+		"k8s.io/api/core/v1.SELinuxOptions":                                          schema_k8sio_api_core_v1_SELinuxOptions(ref),
+		"k8s.io/api/core/v1.ScaleIOPersistentVolumeSource":                           schema_k8sio_api_core_v1_ScaleIOPersistentVolumeSource(ref),
+		"k8s.io/api/core/v1.ScaleIOVolumeSource":                                     schema_k8sio_api_core_v1_ScaleIOVolumeSource(ref),
+		"k8s.io/api/core/v1.ScopeSelector":                                           schema_k8sio_api_core_v1_ScopeSelector(ref),
+		"k8s.io/api/core/v1.ScopedResourceSelectorRequirement":                       schema_k8sio_api_core_v1_ScopedResourceSelectorRequirement(ref),
+		"k8s.io/api/core/v1.SeccompProfile":                                          schema_k8sio_api_core_v1_SeccompProfile(ref),
+		"k8s.io/api/core/v1.Secret":                                                  schema_k8sio_api_core_v1_Secret(ref),
+		"k8s.io/api/core/v1.SecretEnvSource":                                         schema_k8sio_api_core_v1_SecretEnvSource(ref),
+		"k8s.io/api/core/v1.SecretKeySelector":                                       schema_k8sio_api_core_v1_SecretKeySelector(ref),
+		"k8s.io/api/core/v1.SecretList":                                              schema_k8sio_api_core_v1_SecretList(ref),
+		"k8s.io/api/core/v1.SecretProjection":                                        schema_k8sio_api_core_v1_SecretProjection(ref),
+		"k8s.io/api/core/v1.SecretReference":                                         schema_k8sio_api_core_v1_SecretReference(ref),
+		"k8s.io/api/core/v1.SecretVolumeSource":                                      schema_k8sio_api_core_v1_SecretVolumeSource(ref),
+		"k8s.io/api/core/v1.SecurityContext":                                         schema_k8sio_api_core_v1_SecurityContext(ref),
+		"k8s.io/api/core/v1.SerializedReference":                                     schema_k8sio_api_core_v1_SerializedReference(ref),
+		"k8s.io/api/core/v1.Service":                                                 schema_k8sio_api_core_v1_Service(ref),
+		"k8s.io/api/core/v1.ServiceAccount":                                          schema_k8sio_api_core_v1_ServiceAccount(ref),
+		"k8s.io/api/core/v1.ServiceAccountList":                                      schema_k8sio_api_core_v1_ServiceAccountList(ref),
+		"k8s.io/api/core/v1.ServiceAccountTokenProjection":                           schema_k8sio_api_core_v1_ServiceAccountTokenProjection(ref),
+		"k8s.io/api/core/v1.ServiceList":                                             schema_k8sio_api_core_v1_ServiceList(ref),
+		"k8s.io/api/core/v1.ServicePort":                                             schema_k8sio_api_core_v1_ServicePort(ref),
+		"k8s.io/api/core/v1.ServiceProxyOptions":                                     schema_k8sio_api_core_v1_ServiceProxyOptions(ref),
+		"k8s.io/api/core/v1.ServiceSpec":                                             schema_k8sio_api_core_v1_ServiceSpec(ref),
+		"k8s.io/api/core/v1.ServiceStatus":                                           schema_k8sio_api_core_v1_ServiceStatus(ref),
+		"k8s.io/api/core/v1.SessionAffinityConfig":                                   schema_k8sio_api_core_v1_SessionAffinityConfig(ref),
+		"k8s.io/api/core/v1.StorageOSPersistentVolumeSource":                         schema_k8sio_api_core_v1_StorageOSPersistentVolumeSource(ref),
+		"k8s.io/api/core/v1.StorageOSVolumeSource":                                   schema_k8sio_api_core_v1_StorageOSVolumeSource(ref),
+		"k8s.io/api/core/v1.Sysctl":                                                  schema_k8sio_api_core_v1_Sysctl(ref),
+		"k8s.io/api/core/v1.TCPSocketAction":                                         schema_k8sio_api_core_v1_TCPSocketAction(ref),
+		"k8s.io/api/core/v1.Taint":                                                   schema_k8sio_api_core_v1_Taint(ref),
+		"k8s.io/api/core/v1.Toleration":                                              schema_k8sio_api_core_v1_Toleration(ref),
+		"k8s.io/api/core/v1.TopologySelectorLabelRequirement":                        schema_k8sio_api_core_v1_TopologySelectorLabelRequirement(ref),
+		"k8s.io/api/core/v1.TopologySelectorTerm":                                    schema_k8sio_api_core_v1_TopologySelectorTerm(ref),
+		"k8s.io/api/core/v1.TopologySpreadConstraint":                                schema_k8sio_api_core_v1_TopologySpreadConstraint(ref),
+		"k8s.io/api/core/v1.TypedLocalObjectReference":                               schema_k8sio_api_core_v1_TypedLocalObjectReference(ref),
+		"k8s.io/api/core/v1.Volume":                                                  schema_k8sio_api_core_v1_Volume(ref),
+		"k8s.io/api/core/v1.VolumeDevice":                                            schema_k8sio_api_core_v1_VolumeDevice(ref),
+		"k8s.io/api/core/v1.VolumeMount":                                             schema_k8sio_api_core_v1_VolumeMount(ref),
+		"k8s.io/api/core/v1.VolumeNodeAffinity":                                      schema_k8sio_api_core_v1_VolumeNodeAffinity(ref),
+		"k8s.io/api/core/v1.VolumeProjection":                                        schema_k8sio_api_core_v1_VolumeProjection(ref),
+		"k8s.io/api/core/v1.VolumeSource":                                            schema_k8sio_api_core_v1_VolumeSource(ref),
+		"k8s.io/api/core/v1.VsphereVirtualDiskVolumeSource":                          schema_k8sio_api_core_v1_VsphereVirtualDiskVolumeSource(ref),
+		"k8s.io/api/core/v1.WeightedPodAffinityTerm":                                 schema_k8sio_api_core_v1_WeightedPodAffinityTerm(ref),
+		"k8s.io/api/core/v1.WindowsSecurityContextOptions":                           schema_k8sio_api_core_v1_WindowsSecurityContextOptions(ref),
+		"k8s.io/apimachinery/pkg/api/resource.Quantity":                              schema_apimachinery_pkg_api_resource_Quantity(ref),
+		"k8s.io/apimachinery/pkg/api/resource.int64Amount":                           schema_apimachinery_pkg_api_resource_int64Amount(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.APIGroup":                              schema_pkg_apis_meta_v1_APIGroup(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.APIGroupList":                          schema_pkg_apis_meta_v1_APIGroupList(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.APIResource":                           schema_pkg_apis_meta_v1_APIResource(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.APIResourceList":                       schema_pkg_apis_meta_v1_APIResourceList(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.APIVersions":                           schema_pkg_apis_meta_v1_APIVersions(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.Condition":                             schema_pkg_apis_meta_v1_Condition(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.CreateOptions":                         schema_pkg_apis_meta_v1_CreateOptions(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.DeleteOptions":                         schema_pkg_apis_meta_v1_DeleteOptions(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.Duration":                              schema_pkg_apis_meta_v1_Duration(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.ExportOptions":                         schema_pkg_apis_meta_v1_ExportOptions(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.FieldsV1":                              schema_pkg_apis_meta_v1_FieldsV1(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.GetOptions":                            schema_pkg_apis_meta_v1_GetOptions(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.GroupKind":                             schema_pkg_apis_meta_v1_GroupKind(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.GroupResource":                         schema_pkg_apis_meta_v1_GroupResource(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.GroupVersion":                          schema_pkg_apis_meta_v1_GroupVersion(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.GroupVersionForDiscovery":              schema_pkg_apis_meta_v1_GroupVersionForDiscovery(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.GroupVersionKind":                      schema_pkg_apis_meta_v1_GroupVersionKind(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.GroupVersionResource":                  schema_pkg_apis_meta_v1_GroupVersionResource(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.InternalEvent":                         schema_pkg_apis_meta_v1_InternalEvent(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector":                         schema_pkg_apis_meta_v1_LabelSelector(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelectorRequirement":              schema_pkg_apis_meta_v1_LabelSelectorRequirement(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.List":                                  schema_pkg_apis_meta_v1_List(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta":                              schema_pkg_apis_meta_v1_ListMeta(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.ListOptions":                           schema_pkg_apis_meta_v1_ListOptions(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.ManagedFieldsEntry":                    schema_pkg_apis_meta_v1_ManagedFieldsEntry(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.MicroTime":                             schema_pkg_apis_meta_v1_MicroTime(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta":                            schema_pkg_apis_meta_v1_ObjectMeta(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.OwnerReference":                        schema_pkg_apis_meta_v1_OwnerReference(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.PartialObjectMetadata":                 schema_pkg_apis_meta_v1_PartialObjectMetadata(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.PartialObjectMetadataList":             schema_pkg_apis_meta_v1_PartialObjectMetadataList(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.Patch":                                 schema_pkg_apis_meta_v1_Patch(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.PatchOptions":                          schema_pkg_apis_meta_v1_PatchOptions(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.Preconditions":                         schema_pkg_apis_meta_v1_Preconditions(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.RootPaths":                             schema_pkg_apis_meta_v1_RootPaths(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.ServerAddressByClientCIDR":             schema_pkg_apis_meta_v1_ServerAddressByClientCIDR(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.Status":                                schema_pkg_apis_meta_v1_Status(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.StatusCause":                           schema_pkg_apis_meta_v1_StatusCause(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.StatusDetails":                         schema_pkg_apis_meta_v1_StatusDetails(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.Table":                                 schema_pkg_apis_meta_v1_Table(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.TableColumnDefinition":                 schema_pkg_apis_meta_v1_TableColumnDefinition(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.TableOptions":                          schema_pkg_apis_meta_v1_TableOptions(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.TableRow":                              schema_pkg_apis_meta_v1_TableRow(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.TableRowCondition":                     schema_pkg_apis_meta_v1_TableRowCondition(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.Time":                                  schema_pkg_apis_meta_v1_Time(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.Timestamp":                             schema_pkg_apis_meta_v1_Timestamp(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta":                              schema_pkg_apis_meta_v1_TypeMeta(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.UpdateOptions":                         schema_pkg_apis_meta_v1_UpdateOptions(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.WatchEvent":                            schema_pkg_apis_meta_v1_WatchEvent(ref),
+		"k8s.io/apimachinery/pkg/runtime.RawExtension":                               schema_k8sio_apimachinery_pkg_runtime_RawExtension(ref),
+		"k8s.io/apimachinery/pkg/runtime.TypeMeta":                                   schema_k8sio_apimachinery_pkg_runtime_TypeMeta(ref),
+		"k8s.io/apimachinery/pkg/runtime.Unknown":                                    schema_k8sio_apimachinery_pkg_runtime_Unknown(ref),
+		"k8s.io/apimachinery/pkg/util/intstr.IntOrString":                            schema_apimachinery_pkg_util_intstr_IntOrString(ref),
+		"k8s.io/apimachinery/pkg/version.Info":                                       schema_k8sio_apimachinery_pkg_version_Info(ref),
 	}
 }
 
@@ -1588,115 +1585,6 @@ func schema_pkg_apis_security_v1alpha1_NamespacedName(ref common.ReferenceCallba
 	}
 }
 
-func schema_pkg_apis_security_v1alpha1_PolicyEnforcementMode(ref common.ReferenceCallback) common.OpenAPIDefinition {
-	return common.OpenAPIDefinition{
-		Schema: spec.Schema{
-			SchemaProps: spec.SchemaProps{
-				Type: []string{"object"},
-				Properties: map[string]spec.Schema{
-					"kind": {
-						SchemaProps: spec.SchemaProps{
-							Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds",
-							Type:        []string{"string"},
-							Format:      "",
-						},
-					},
-					"apiVersion": {
-						SchemaProps: spec.SchemaProps{
-							Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources",
-							Type:        []string{"string"},
-							Format:      "",
-						},
-					},
-					"metadata": {
-						SchemaProps: spec.SchemaProps{
-							Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"),
-						},
-					},
-					"spec": {
-						SchemaProps: spec.SchemaProps{
-							Description: "Specification of the desired behavior for this SecurityPolicy.",
-							Ref:         ref("github.com/everoute/everoute/pkg/apis/security/v1alpha1.PolicyEnforcementModeSpec"),
-						},
-					},
-				},
-			},
-		},
-		Dependencies: []string{
-			"github.com/everoute/everoute/pkg/apis/security/v1alpha1.PolicyEnforcementModeSpec", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"},
-	}
-}
-
-func schema_pkg_apis_security_v1alpha1_PolicyEnforcementModeList(ref common.ReferenceCallback) common.OpenAPIDefinition {
-	return common.OpenAPIDefinition{
-		Schema: spec.Schema{
-			SchemaProps: spec.SchemaProps{
-				Type: []string{"object"},
-				Properties: map[string]spec.Schema{
-					"kind": {
-						SchemaProps: spec.SchemaProps{
-							Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds",
-							Type:        []string{"string"},
-							Format:      "",
-						},
-					},
-					"apiVersion": {
-						SchemaProps: spec.SchemaProps{
-							Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources",
-							Type:        []string{"string"},
-							Format:      "",
-						},
-					},
-					"metadata": {
-						SchemaProps: spec.SchemaProps{
-							Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"),
-						},
-					},
-					"items": {
-						SchemaProps: spec.SchemaProps{
-							Type: []string{"array"},
-							Items: &spec.SchemaOrArray{
-								Schema: &spec.Schema{
-									SchemaProps: spec.SchemaProps{
-										Ref: ref("github.com/everoute/everoute/pkg/apis/security/v1alpha1.PolicyEnforcementMode"),
-									},
-								},
-							},
-						},
-					},
-				},
-				Required: []string{"items"},
-			},
-		},
-		Dependencies: []string{
-			"github.com/everoute/everoute/pkg/apis/security/v1alpha1.PolicyEnforcementMode", "k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"},
-	}
-}
-
-func schema_pkg_apis_security_v1alpha1_PolicyEnforcementModeSpec(ref common.ReferenceCallback) common.OpenAPIDefinition {
-	return common.OpenAPIDefinition{
-		Schema: spec.Schema{
-			SchemaProps: spec.SchemaProps{
-				Type: []string{"object"},
-				Properties: map[string]spec.Schema{
-					"defaultEnforcementMode": {
-						SchemaProps: spec.SchemaProps{
-							Type:   []string{"string"},
-							Format: "",
-						},
-					},
-					"enforcementMode": {
-						SchemaProps: spec.SchemaProps{
-							Type:   []string{"string"},
-							Format: "",
-						},
-					},
-				},
-			},
-		},
-	}
-}
-
 func schema_pkg_apis_security_v1alpha1_Rule(ref common.ReferenceCallback) common.OpenAPIDefinition {
 	return common.OpenAPIDefinition{
 		Schema: spec.Schema{

From 7d3897d573aab0eb72f985ca15c0e1a3f3feb261 Mon Sep 17 00:00:00 2001
From: echken <echken@localhost.localdomain>
Date: Wed, 16 Mar 2022 02:13:40 -0400
Subject: [PATCH 03/32] fix upgrade incompatable problem

Signed-off-by: echken <echken@localhost.localdomain>
---
 build/images/release/Dockerfile               |   2 +-
 ...security.everoute.io_securitypolicies.yaml |   2 +-
 deploy/everoute.yaml                          |   2 +-
 go.mod                                        |  30 +-
 go.sum                                        |  29 +-
 pkg/agent/datapath/multiBridgeDatapath.go     |   1 +
 pkg/agent/datapath/policyBridge.go            |  83 +----
 pkg/openapi/openapi_generated.go              | 325 +++++++++---------
 8 files changed, 212 insertions(+), 262 deletions(-)

diff --git a/build/images/release/Dockerfile b/build/images/release/Dockerfile
index 97d14e8c7..c13effc27 100644
--- a/build/images/release/Dockerfile
+++ b/build/images/release/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.15 as builder
+FROM golang:1.16 as builder
 
 # Download deps
 RUN mkdir -p $GOPATH/src/everoute
diff --git a/deploy/crds/security.everoute.io_securitypolicies.yaml b/deploy/crds/security.everoute.io_securitypolicies.yaml
index 7b3e0877f..e5a2fdf08 100644
--- a/deploy/crds/security.everoute.io_securitypolicies.yaml
+++ b/deploy/crds/security.everoute.io_securitypolicies.yaml
@@ -906,7 +906,7 @@ spec:
                   that include "Egress" (since such a policy would not include an
                   Egress section and would otherwise default to just [ "Ingress" ]).
                 items:
-                  description: Policy Type string describes the NetworkPolicy type
+                  description: PolicyType string describes the NetworkPolicy type
                     This type is beta-level in 1.8
                   type: string
                 type: array
diff --git a/deploy/everoute.yaml b/deploy/everoute.yaml
index 018ce2338..082eec5de 100644
--- a/deploy/everoute.yaml
+++ b/deploy/everoute.yaml
@@ -1643,7 +1643,7 @@ spec:
                   that include "Egress" (since such a policy would not include an
                   Egress section and would otherwise default to just [ "Ingress" ]).
                 items:
-                  description: Policy Type string describes the NetworkPolicy type
+                  description: PolicyType string describes the NetworkPolicy type
                     This type is beta-level in 1.8
                   type: string
                 type: array
diff --git a/go.mod b/go.mod
index 73341e029..d3a9e940c 100644
--- a/go.mod
+++ b/go.mod
@@ -16,7 +16,7 @@ require (
 	github.com/fatih/color v1.7.0
 	github.com/fsnotify/fsnotify v1.5.1
 	github.com/gertd/go-pluralize v0.1.7
-	github.com/go-logr/logr v0.4.0 // indirect
+	github.com/go-logr/logr v0.2.1 // indirect
 	github.com/go-openapi/spec v0.19.3
 	github.com/go-ping/ping v0.0.0-20210506233800-ff8be3320020
 	github.com/google/go-cmp v0.5.5
@@ -26,30 +26,36 @@ require (
 	github.com/onsi/ginkgo v1.16.4
 	github.com/onsi/gomega v1.15.0
 	github.com/pkg/errors v0.9.1
-	github.com/spf13/cobra v1.1.1
+	github.com/spf13/cobra v1.1.3
 	github.com/streamrail/concurrent-map v0.0.0-20160823150647-8bf1e9bacbf6
 	github.com/vektah/gqlparser/v2 v2.1.0
 	github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5
-	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d
-	golang.org/x/sys v0.0.0-20210917161153-d61c044b1678
-	google.golang.org/grpc v1.35.0
+	golang.org/x/crypto v0.0.0-20220128200615-198e4374d7ed
+	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e
+	google.golang.org/grpc v1.38.0
 	google.golang.org/protobuf v1.26.0
 	gopkg.in/yaml.v2 v2.4.0
-	k8s.io/api v0.20.6
-	k8s.io/apimachinery v0.20.6
+	k8s.io/api v0.22.2
+	k8s.io/apimachinery v0.22.2
 	k8s.io/apiserver v0.20.6
 	k8s.io/cli-runtime v0.20.1
-	k8s.io/client-go v0.20.6
+	k8s.io/client-go v0.22.2
 	k8s.io/klog v1.0.0
-	k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd
-	k8s.io/utils v0.0.0-20201110183641-67b214c5f920
-	sigs.k8s.io/controller-runtime v0.6.0
+	k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e
+	k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a
+	sigs.k8s.io/controller-runtime v0.10.2
 )
 
 replace (
 	github.com/contiv/libOpenflow => github.com/everoute/libOpenflow v0.0.0-20210716071814-4ef09249fae5
 	github.com/contiv/libovsdb => github.com/everoute/libovsdb v0.0.0-20210326110222-6c508538aa65
 	//github.com/contiv/ofnet => github.com/everoute/ofnet v0.0.0-20211229083707-5cb1e875a38b
-	github.com/contiv/ofnet => github.com/echkenluo/ofnet v0.0.0-20220221070635-77d8eaabab24
+	github.com/contiv/ofnet => github.com/echkenluo/ofnet v0.0.0-20220316073105-f24ad3ca4c99
 	github.com/osrg/gobgp => github.com/everoute/gobgp v0.0.0-20210127101833-12edfc1f4514
+	k8s.io/api v0.22.2 => k8s.io/api v0.20.6
+	k8s.io/apimachinery v0.22.2 => k8s.io/apimachinery v0.20.6
+	k8s.io/client-go v0.22.2 => k8s.io/client-go v0.20.6
+	k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e => k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd
+	k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a => k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a
+	sigs.k8s.io/controller-runtime v0.10.2 => sigs.k8s.io/controller-runtime v0.6.0
 )
diff --git a/go.sum b/go.sum
index cdd1b90ab..1868df722 100644
--- a/go.sum
+++ b/go.sum
@@ -300,8 +300,8 @@ github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:Htrtb
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
-github.com/echkenluo/ofnet v0.0.0-20220221070635-77d8eaabab24 h1:ufGfwD9HAgZ9CeQvK3mu2/mb4+vr93GZTf992h1+RGE=
-github.com/echkenluo/ofnet v0.0.0-20220221070635-77d8eaabab24/go.mod h1:VI923BFjPJipkV0SbiOmQ1qXefJnBBkjGxA3F56fenM=
+github.com/echkenluo/ofnet v0.0.0-20220316073105-f24ad3ca4c99 h1:WQWeCnqASk/iKFSPYjosYoonzaugeXqACRbINXfA7hc=
+github.com/echkenluo/ofnet v0.0.0-20220316073105-f24ad3ca4c99/go.mod h1:cJR/tu7mdjR0Rm4EXUqw7nJU9cMCoWg2lEO7jfwq7MM=
 github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -311,6 +311,7 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
@@ -321,8 +322,6 @@ github.com/everoute/libOpenflow v0.0.0-20210716071814-4ef09249fae5 h1:+z3ZBux28V
 github.com/everoute/libOpenflow v0.0.0-20210716071814-4ef09249fae5/go.mod h1:xsbTUkWYuDqVv4jMxwinRGdjlYwj0aZS8Rzf6HoNLdg=
 github.com/everoute/libovsdb v0.0.0-20210326110222-6c508538aa65 h1:a3dwYQt/Y9Yfz1XKpUelQKS5YDgmG4NtVpBrRguQY9k=
 github.com/everoute/libovsdb v0.0.0-20210326110222-6c508538aa65/go.mod h1:N8fR1bYMqi4pQ7jL17QlBlpJ0hHmqRAXVKkBwuIjClw=
-github.com/everoute/ofnet v0.0.0-20211229083707-5cb1e875a38b h1:rz+7uv9i5TuqXoy17XQ0yfxi27Y2E7s/wJs6M3MX4hM=
-github.com/everoute/ofnet v0.0.0-20211229083707-5cb1e875a38b/go.mod h1:VI923BFjPJipkV0SbiOmQ1qXefJnBBkjGxA3F56fenM=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
@@ -353,8 +352,8 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
-github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-logr/logr v0.2.1 h1:fV3MLmabKIZ383XifUjFSwcoGee0v9qgPp8wy5svibE=
+github.com/go-logr/logr v0.2.1/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-logr/zapr v0.1.0 h1:h+WVe9j6HAA01niTJPA/kKH0i7e0rLZBCwauQFcRE54=
 github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk=
 github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
@@ -803,8 +802,9 @@ github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKv
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
-github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=
 github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
+github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
+github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
@@ -918,8 +918,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
-golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220128200615-198e4374d7ed h1:YoWVYYAfvQ4ddHv3OKmIvX7NCAhFGTj62VP2l2kfBbA=
+golang.org/x/crypto v0.0.0-20220128200615-198e4374d7ed/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -996,6 +996,7 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -1090,8 +1091,8 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210917161153-d61c044b1678 h1:J27LZFQBFoihqXoegpscI10HpjZ7B5WQLLKL2FZXQKw=
-golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1227,8 +1228,9 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8=
 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0=
+google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -1334,8 +1336,9 @@ k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhD
 k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
 k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
 k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
-k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g=
+k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
diff --git a/pkg/agent/datapath/multiBridgeDatapath.go b/pkg/agent/datapath/multiBridgeDatapath.go
index ec0503584..095be2ef0 100644
--- a/pkg/agent/datapath/multiBridgeDatapath.go
+++ b/pkg/agent/datapath/multiBridgeDatapath.go
@@ -838,6 +838,7 @@ func (datapathManager *DpManager) RemoveEveroutePolicyRule(ruleID string, ruleNa
 		return nil
 	}
 
+	log.Infof("Received remove rule: %+v", ruleName)
 	for vdsID := range datapathManager.BridgeChainMap {
 		err := ofctrl.DeleteFlow(pRule.RuleFlowMap[vdsID].Table, pRule.RuleFlowMap[vdsID].Priority, pRule.RuleFlowMap[vdsID].FlowID)
 		if err != nil {
diff --git a/pkg/agent/datapath/policyBridge.go b/pkg/agent/datapath/policyBridge.go
index 24478fd53..2d631e6f5 100644
--- a/pkg/agent/datapath/policyBridge.go
+++ b/pkg/agent/datapath/policyBridge.go
@@ -361,7 +361,7 @@ func (p *PolicyBridge) initPolicyTable() error {
 	var policyConntrackZone uint16 = 65520
 	srcField, _ := openflow13.FindFieldHeaderByName("nxm_nx_xxreg0", false)
 	dstField, _ := openflow13.FindFieldHeaderByName("nxm_nx_ct_label", false)
-	moveAct := openflow13.NewNXActionRegMove(64, 0, 0, srcField, dstField)
+	moveAct := openflow13.NewNXActionRegMove(128, 0, 0, srcField, dstField)
 	ctCommitAction := ofctrl.NewConntrackAction(true, false, &p.ctCommitTable.TableId, &policyConntrackZone, moveAct)
 	_ = egressDropTableCtCommitFlow.SetConntrack(ctCommitAction)
 
@@ -409,7 +409,7 @@ func (p *PolicyBridge) initPolicyTable() error {
 		Ethertype: PROTOCOL_IP,
 		CtStates:  ctTrkState,
 	})
-	moveAct = openflow13.NewNXActionRegMove(64, 0, 64, srcField, dstField)
+	moveAct = openflow13.NewNXActionRegMove(128, 0, 0, srcField, dstField)
 	ctCommitAction = ofctrl.NewConntrackAction(true, false, &p.ctCommitTable.TableId, &policyConntrackZone, moveAct)
 	_ = ingressDropTableCtCommitFlow.SetConntrack(ctCommitAction)
 
@@ -543,73 +543,6 @@ func (p *PolicyBridge) GetTierTable(direction uint8, tier uint8, mode string) (*
 	return policyTable, nextTable, nil
 }
 
-func (p *PolicyBridge) AddMonitorModeMicroSegmentRule(rule *EveroutePolicyRule, direction uint8, tier uint8) (*FlowEntry, error) {
-	var ipDa *net.IP = nil
-	var ipDaMask *net.IP = nil
-	var ipSa *net.IP = nil
-	var ipSaMask *net.IP = nil
-	var err error
-
-	// make sure switch is connected
-	if !p.IsSwitchConnected() {
-		p.WaitForSwitchConnection()
-	}
-
-	// Different tier have different nextTable select strategy:
-	policyTable, nextTable, e := p.GetTierTable(direction, tier, "monitor")
-	if e != nil {
-		log.Errorf("Failed to get policy table tier %v", tier)
-		return nil, errors.New("failed get policy table")
-	}
-
-	// Parse dst ip
-	if rule.DstIPAddr != "" {
-		ipDa, ipDaMask, err = ParseIPAddrMaskString(rule.DstIPAddr)
-		if err != nil {
-			log.Errorf("Failed to parse dst ip %s. Err: %v", rule.DstIPAddr, err)
-			return nil, err
-		}
-	}
-
-	// parse src ip
-	if rule.SrcIPAddr != "" {
-		ipSa, ipSaMask, err = ParseIPAddrMaskString(rule.SrcIPAddr)
-		if err != nil {
-			log.Errorf("Failed to parse src ip %s. Err: %v", rule.SrcIPAddr, err)
-			return nil, err
-		}
-	}
-
-	// Install the rule in policy table
-	ruleFlow, err := policyTable.NewFlow(ofctrl.FlowMatch{
-		Priority:       uint16(rule.Priority),
-		Ethertype:      PROTOCOL_IP,
-		IpDa:           ipDa,
-		IpDaMask:       ipDaMask,
-		IpSa:           ipSa,
-		IpSaMask:       ipSaMask,
-		IpProto:        rule.IPProtocol,
-		TcpSrcPort:     rule.SrcPort,
-		TcpSrcPortMask: rule.SrcPortMask,
-		TcpDstPort:     rule.DstPort,
-		TcpDstPortMask: rule.DstPortMask,
-		UdpSrcPort:     rule.SrcPort,
-		UdpSrcPortMask: rule.SrcPortMask,
-		UdpDstPort:     rule.DstPort,
-		UdpDstPortMask: rule.DstPortMask,
-	})
-	if err != nil {
-		log.Errorf("Failed to add flow for rule {%v}. Err: %v", rule, err)
-		return nil, err
-	}
-
-	if rule.Action == "deny" {
-		ruleFlow.Next(nextTable)
-	}
-
-	return nil, nil
-}
-
 func (p *PolicyBridge) AddMicroSegmentRule(rule *EveroutePolicyRule, direction uint8, tier uint8, mode string) (*FlowEntry, error) {
 	var ipDa *net.IP = nil
 	var ipDaMask *net.IP = nil
@@ -679,8 +612,7 @@ func (p *PolicyBridge) AddMicroSegmentRule(rule *EveroutePolicyRule, direction u
 			return nil, fmt.Errorf("policy tier1 without monitor mode support")
 		}
 
-		// load flowID[0..9] -> xxreg0[0..9]
-		if err := ruleFlow.LoadField("nxm_nx_xxreg0", ruleFlow.FlowID>>FLOW_ROUND_NUM_LENGTH, openflow13.NewNXRange(0, 9)); err != nil {
+		if err := ruleFlow.LoadField("nxm_nx_xxreg0", ruleFlow.FlowID>>FLOW_SEQ_NUM_LENGTH, openflow13.NewNXRange(0, 9)); err != nil {
 			return nil, err
 		}
 		if err := ruleFlow.LoadField("nxm_nx_xxreg0", ruleFlow.FlowID&FLOW_SEQ_NUM_MASK, openflow13.NewNXRange(32, 53)); err != nil {
@@ -693,8 +625,10 @@ func (p *PolicyBridge) AddMicroSegmentRule(rule *EveroutePolicyRule, direction u
 	case "work":
 		switch rule.Action {
 		case "allow":
-			if err := ruleFlow.LoadField("nxm_nx_reg0", 0x30, openflow13.NewNXRange(0, 15)); err != nil {
-				return nil, err
+			if rule.Priority == GLOBAL_DEFAULT_POLICY_FLOW_PRIORITY {
+				if err := ruleFlow.LoadField("nxm_nx_reg0", 0x30, openflow13.NewNXRange(0, 15)); err != nil {
+					return nil, err
+				}
 			}
 		case "deny":
 			if err := ruleFlow.LoadField("nxm_nx_reg0", 0x20, openflow13.NewNXRange(0, 15)); err != nil {
@@ -704,8 +638,7 @@ func (p *PolicyBridge) AddMicroSegmentRule(rule *EveroutePolicyRule, direction u
 			return nil, fmt.Errorf("unknown action")
 		}
 
-		// load flowID[0..9] -> xxreg0[0..9]
-		if err := ruleFlow.LoadField("nxm_nx_xxreg0", ruleFlow.FlowID>>FLOW_ROUND_NUM_LENGTH, openflow13.NewNXRange(0, 9)); err != nil {
+		if err := ruleFlow.LoadField("nxm_nx_xxreg0", ruleFlow.FlowID>>FLOW_SEQ_NUM_LENGTH, openflow13.NewNXRange(0, 9)); err != nil {
 			return nil, err
 		}
 		if err := ruleFlow.LoadField("nxm_nx_xxreg0", ruleFlow.FlowID&FLOW_SEQ_NUM_MASK, openflow13.NewNXRange(54, 75)); err != nil {
diff --git a/pkg/openapi/openapi_generated.go b/pkg/openapi/openapi_generated.go
index 069109900..827f0ae17 100644
--- a/pkg/openapi/openapi_generated.go
+++ b/pkg/openapi/openapi_generated.go
@@ -144,7 +144,6 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA
 		"k8s.io/api/core/v1.EnvVarSource":                                            schema_k8sio_api_core_v1_EnvVarSource(ref),
 		"k8s.io/api/core/v1.EphemeralContainer":                                      schema_k8sio_api_core_v1_EphemeralContainer(ref),
 		"k8s.io/api/core/v1.EphemeralContainerCommon":                                schema_k8sio_api_core_v1_EphemeralContainerCommon(ref),
-		"k8s.io/api/core/v1.EphemeralContainers":                                     schema_k8sio_api_core_v1_EphemeralContainers(ref),
 		"k8s.io/api/core/v1.EphemeralVolumeSource":                                   schema_k8sio_api_core_v1_EphemeralVolumeSource(ref),
 		"k8s.io/api/core/v1.Event":                                                   schema_k8sio_api_core_v1_Event(ref),
 		"k8s.io/api/core/v1.EventList":                                               schema_k8sio_api_core_v1_EventList(ref),
@@ -309,11 +308,11 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA
 		"k8s.io/apimachinery/pkg/apis/meta/v1.APIResource":                           schema_pkg_apis_meta_v1_APIResource(ref),
 		"k8s.io/apimachinery/pkg/apis/meta/v1.APIResourceList":                       schema_pkg_apis_meta_v1_APIResourceList(ref),
 		"k8s.io/apimachinery/pkg/apis/meta/v1.APIVersions":                           schema_pkg_apis_meta_v1_APIVersions(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.ApplyOptions":                          schema_pkg_apis_meta_v1_ApplyOptions(ref),
 		"k8s.io/apimachinery/pkg/apis/meta/v1.Condition":                             schema_pkg_apis_meta_v1_Condition(ref),
 		"k8s.io/apimachinery/pkg/apis/meta/v1.CreateOptions":                         schema_pkg_apis_meta_v1_CreateOptions(ref),
 		"k8s.io/apimachinery/pkg/apis/meta/v1.DeleteOptions":                         schema_pkg_apis_meta_v1_DeleteOptions(ref),
 		"k8s.io/apimachinery/pkg/apis/meta/v1.Duration":                              schema_pkg_apis_meta_v1_Duration(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.ExportOptions":                         schema_pkg_apis_meta_v1_ExportOptions(ref),
 		"k8s.io/apimachinery/pkg/apis/meta/v1.FieldsV1":                              schema_pkg_apis_meta_v1_FieldsV1(ref),
 		"k8s.io/apimachinery/pkg/apis/meta/v1.GetOptions":                            schema_pkg_apis_meta_v1_GetOptions(ref),
 		"k8s.io/apimachinery/pkg/apis/meta/v1.GroupKind":                             schema_pkg_apis_meta_v1_GroupKind(ref),
@@ -2340,7 +2339,7 @@ func schema_k8sio_api_apps_v1_Deployment(ref common.ReferenceCallback) common.Op
 					},
 					"metadata": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Standard object metadata.",
+							Description: "Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata",
 							Ref:         ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"),
 						},
 					},
@@ -2921,7 +2920,13 @@ func schema_k8sio_api_apps_v1_RollingUpdateDaemonSet(ref common.ReferenceCallbac
 				Properties: map[string]spec.Schema{
 					"maxUnavailable": {
 						SchemaProps: spec.SchemaProps{
-							Description: "The maximum number of DaemonSet pods that can be unavailable during the update. Value can be an absolute number (ex: 5) or a percentage of total number of DaemonSet pods at the start of the update (ex: 10%). Absolute number is calculated from percentage by rounding up. This cannot be 0. Default value is 1. Example: when this is set to 30%, at most 30% of the total number of nodes that should be running the daemon pod (i.e. status.desiredNumberScheduled) can have their pods stopped for an update at any given time. The update starts by stopping at most 30% of those DaemonSet pods and then brings up new DaemonSet pods in their place. Once the new pods are available, it then proceeds onto other DaemonSet pods, thus ensuring that at least 70% of original number of DaemonSet pods are available at all times during the update.",
+							Description: "The maximum number of DaemonSet pods that can be unavailable during the update. Value can be an absolute number (ex: 5) or a percentage of total number of DaemonSet pods at the start of the update (ex: 10%). Absolute number is calculated from percentage by rounding up. This cannot be 0 if MaxSurge is 0 Default value is 1. Example: when this is set to 30%, at most 30% of the total number of nodes that should be running the daemon pod (i.e. status.desiredNumberScheduled) can have their pods stopped for an update at any given time. The update starts by stopping at most 30% of those DaemonSet pods and then brings up new DaemonSet pods in their place. Once the new pods are available, it then proceeds onto other DaemonSet pods, thus ensuring that at least 70% of original number of DaemonSet pods are available at all times during the update.",
+							Ref:         ref("k8s.io/apimachinery/pkg/util/intstr.IntOrString"),
+						},
+					},
+					"maxSurge": {
+						SchemaProps: spec.SchemaProps{
+							Description: "The maximum number of nodes with an existing available DaemonSet pod that can have an updated DaemonSet pod during during an update. Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). This can not be 0 if MaxUnavailable is 0. Absolute number is calculated from percentage by rounding up to a minimum of 1. Default value is 0. Example: when this is set to 30%, at most 30% of the total number of nodes that should be running the daemon pod (i.e. status.desiredNumberScheduled) can have their a new pod created before the old pod is marked as deleted. The update starts by launching new pods on 30% of nodes. Once an updated pod is available (Ready for at least minReadySeconds) the old DaemonSet pod on that node is marked deleted. If the old pod becomes unavailable for any reason (Ready transitions to false, is evicted, or is drained) an updated pod is immediatedly created on that node without considering surge limits. Allowing surge implies the possibility that the resources consumed by the daemonset on any given node can double if the readiness check fails, and so resource intensive daemonsets should take into account that they may cause evictions during disruption. This is beta field and enabled/disabled by DaemonSetUpdateSurge feature gate.",
 							Ref:         ref("k8s.io/apimachinery/pkg/util/intstr.IntOrString"),
 						},
 					},
@@ -3003,7 +3008,8 @@ func schema_k8sio_api_apps_v1_StatefulSet(ref common.ReferenceCallback) common.O
 					},
 					"metadata": {
 						SchemaProps: spec.SchemaProps{
-							Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"),
+							Description: "Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata",
+							Ref:         ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"),
 						},
 					},
 					"spec": {
@@ -3099,12 +3105,14 @@ func schema_k8sio_api_apps_v1_StatefulSetList(ref common.ReferenceCallback) comm
 					},
 					"metadata": {
 						SchemaProps: spec.SchemaProps{
-							Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"),
+							Description: "Standard list's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata",
+							Ref:         ref("k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"),
 						},
 					},
 					"items": {
 						SchemaProps: spec.SchemaProps{
-							Type: []string{"array"},
+							Description: "Items is the list of stateful sets.",
+							Type:        []string{"array"},
 							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
 									SchemaProps: spec.SchemaProps{
@@ -3189,6 +3197,13 @@ func schema_k8sio_api_apps_v1_StatefulSetSpec(ref common.ReferenceCallback) comm
 							Format:      "int32",
 						},
 					},
+					"minReadySeconds": {
+						SchemaProps: spec.SchemaProps{
+							Description: "Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to be considered available. Defaults to 0 (pod will be considered available as soon as it is ready) This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate.",
+							Type:        []string{"integer"},
+							Format:      "int32",
+						},
+					},
 				},
 				Required: []string{"selector", "template", "serviceName"},
 			},
@@ -3280,6 +3295,13 @@ func schema_k8sio_api_apps_v1_StatefulSetStatus(ref common.ReferenceCallback) co
 							},
 						},
 					},
+					"availableReplicas": {
+						SchemaProps: spec.SchemaProps{
+							Description: "Total number of available pods (ready for at least minReadySeconds) targeted by this statefulset. This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate. Remove omitempty when graduating to beta",
+							Type:        []string{"integer"},
+							Format:      "int32",
+						},
+					},
 				},
 				Required: []string{"replicas"},
 			},
@@ -4213,7 +4235,7 @@ func schema_k8sio_api_core_v1_ConfigMap(ref common.ReferenceCallback) common.Ope
 					},
 					"immutable": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Immutable, if set to true, ensures that data stored in the ConfigMap cannot be updated (only object metadata can be modified). If not set to true, the field can be modified at any time. Defaulted to nil. This is a beta field enabled by ImmutableEphemeralVolumes feature gate.",
+							Description: "Immutable, if set to true, ensures that data stored in the ConfigMap cannot be updated (only object metadata can be modified). If not set to true, the field can be modified at any time. Defaulted to nil.",
 							Type:        []string{"boolean"},
 							Format:      "",
 						},
@@ -4371,7 +4393,7 @@ func schema_k8sio_api_core_v1_ConfigMapNodeConfigSource(ref common.ReferenceCall
 	return common.OpenAPIDefinition{
 		Schema: spec.Schema{
 			SchemaProps: spec.SchemaProps{
-				Description: "ConfigMapNodeConfigSource contains the information to reference a ConfigMap as a config source for the Node.",
+				Description: "ConfigMapNodeConfigSource contains the information to reference a ConfigMap as a config source for the Node. This API is deprecated since 1.22: https://git.k8s.io/enhancements/keps/sig-node/281-dynamic-kubelet-configuration",
 				Type:        []string{"object"},
 				Properties: map[string]spec.Schema{
 					"namespace": {
@@ -4530,7 +4552,7 @@ func schema_k8sio_api_core_v1_Container(ref common.ReferenceCallback) common.Ope
 					},
 					"command": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
+							Description: "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
 							Type:        []string{"array"},
 							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
@@ -4544,7 +4566,7 @@ func schema_k8sio_api_core_v1_Container(ref common.ReferenceCallback) common.Ope
 					},
 					"args": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
+							Description: "Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
 							Type:        []string{"array"},
 							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
@@ -4621,7 +4643,7 @@ func schema_k8sio_api_core_v1_Container(ref common.ReferenceCallback) common.Ope
 					},
 					"resources": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/",
+							Description: "Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/",
 							Ref:         ref("k8s.io/api/core/v1.ResourceRequirements"),
 						},
 					},
@@ -4710,7 +4732,7 @@ func schema_k8sio_api_core_v1_Container(ref common.ReferenceCallback) common.Ope
 					},
 					"securityContext": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/",
+							Description: "SecurityContext defines the security options the container should be run with. If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/",
 							Ref:         ref("k8s.io/api/core/v1.SecurityContext"),
 						},
 					},
@@ -4773,7 +4795,6 @@ func schema_k8sio_api_core_v1_ContainerImage(ref common.ReferenceCallback) commo
 						},
 					},
 				},
-				Required: []string{"names"},
 			},
 		},
 	}
@@ -5276,7 +5297,7 @@ func schema_k8sio_api_core_v1_EndpointPort(ref common.ReferenceCallback) common.
 					},
 					"appProtocol": {
 						SchemaProps: spec.SchemaProps{
-							Description: "The application protocol for this port. This field follows standard Kubernetes label syntax. Un-prefixed names are reserved for IANA standard service names (as per RFC-6335 and http://www.iana.org/assignments/service-names). Non-standard protocols should use prefixed names such as mycompany.com/my-custom-protocol. This is a beta field that is guarded by the ServiceAppProtocol feature gate and enabled by default.",
+							Description: "The application protocol for this port. This field follows standard Kubernetes label syntax. Un-prefixed names are reserved for IANA standard service names (as per RFC-6335 and http://www.iana.org/assignments/service-names). Non-standard protocols should use prefixed names such as mycompany.com/my-custom-protocol.",
 							Type:        []string{"string"},
 							Format:      "",
 						},
@@ -5489,7 +5510,7 @@ func schema_k8sio_api_core_v1_EnvVar(ref common.ReferenceCallback) common.OpenAP
 					},
 					"value": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \"\".",
+							Description: "Variable references $(VAR_NAME) are expanded using the previously defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \"\".",
 							Type:        []string{"string"},
 							Format:      "",
 						},
@@ -5571,7 +5592,7 @@ func schema_k8sio_api_core_v1_EphemeralContainer(ref common.ReferenceCallback) c
 					},
 					"command": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
+							Description: "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
 							Type:        []string{"array"},
 							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
@@ -5585,7 +5606,7 @@ func schema_k8sio_api_core_v1_EphemeralContainer(ref common.ReferenceCallback) c
 					},
 					"args": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
+							Description: "Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
 							Type:        []string{"array"},
 							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
@@ -5740,7 +5761,7 @@ func schema_k8sio_api_core_v1_EphemeralContainer(ref common.ReferenceCallback) c
 					},
 					"securityContext": {
 						SchemaProps: spec.SchemaProps{
-							Description: "SecurityContext is not allowed for ephemeral containers.",
+							Description: "Optional: SecurityContext defines the security options the ephemeral container should be run with. If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext.",
 							Ref:         ref("k8s.io/api/core/v1.SecurityContext"),
 						},
 					},
@@ -5804,7 +5825,7 @@ func schema_k8sio_api_core_v1_EphemeralContainerCommon(ref common.ReferenceCallb
 					},
 					"command": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
+							Description: "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
 							Type:        []string{"array"},
 							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
@@ -5818,7 +5839,7 @@ func schema_k8sio_api_core_v1_EphemeralContainerCommon(ref common.ReferenceCallb
 					},
 					"args": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
+							Description: "Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
 							Type:        []string{"array"},
 							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
@@ -5973,7 +5994,7 @@ func schema_k8sio_api_core_v1_EphemeralContainerCommon(ref common.ReferenceCallb
 					},
 					"securityContext": {
 						SchemaProps: spec.SchemaProps{
-							Description: "SecurityContext is not allowed for ephemeral containers.",
+							Description: "Optional: SecurityContext defines the security options the ephemeral container should be run with. If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext.",
 							Ref:         ref("k8s.io/api/core/v1.SecurityContext"),
 						},
 					},
@@ -6007,60 +6028,6 @@ func schema_k8sio_api_core_v1_EphemeralContainerCommon(ref common.ReferenceCallb
 	}
 }
 
-func schema_k8sio_api_core_v1_EphemeralContainers(ref common.ReferenceCallback) common.OpenAPIDefinition {
-	return common.OpenAPIDefinition{
-		Schema: spec.Schema{
-			SchemaProps: spec.SchemaProps{
-				Description: "A list of ephemeral containers used with the Pod ephemeralcontainers subresource.",
-				Type:        []string{"object"},
-				Properties: map[string]spec.Schema{
-					"kind": {
-						SchemaProps: spec.SchemaProps{
-							Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds",
-							Type:        []string{"string"},
-							Format:      "",
-						},
-					},
-					"apiVersion": {
-						SchemaProps: spec.SchemaProps{
-							Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources",
-							Type:        []string{"string"},
-							Format:      "",
-						},
-					},
-					"metadata": {
-						SchemaProps: spec.SchemaProps{
-							Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"),
-						},
-					},
-					"ephemeralContainers": {
-						VendorExtensible: spec.VendorExtensible{
-							Extensions: spec.Extensions{
-								"x-kubernetes-patch-merge-key": "name",
-								"x-kubernetes-patch-strategy":  "merge",
-							},
-						},
-						SchemaProps: spec.SchemaProps{
-							Description: "A list of ephemeral containers associated with this pod. New ephemeral containers may be appended to this list, but existing ephemeral containers may not be removed or modified.",
-							Type:        []string{"array"},
-							Items: &spec.SchemaOrArray{
-								Schema: &spec.Schema{
-									SchemaProps: spec.SchemaProps{
-										Ref: ref("k8s.io/api/core/v1.EphemeralContainer"),
-									},
-								},
-							},
-						},
-					},
-				},
-				Required: []string{"ephemeralContainers"},
-			},
-		},
-		Dependencies: []string{
-			"k8s.io/api/core/v1.EphemeralContainer", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"},
-	}
-}
-
 func schema_k8sio_api_core_v1_EphemeralVolumeSource(ref common.ReferenceCallback) common.OpenAPIDefinition {
 	return common.OpenAPIDefinition{
 		Schema: spec.Schema{
@@ -6074,13 +6041,6 @@ func schema_k8sio_api_core_v1_EphemeralVolumeSource(ref common.ReferenceCallback
 							Ref:         ref("k8s.io/api/core/v1.PersistentVolumeClaimTemplate"),
 						},
 					},
-					"readOnly": {
-						SchemaProps: spec.SchemaProps{
-							Description: "Specifies a read-only configuration for the volume. Defaults to false (read/write).",
-							Type:        []string{"boolean"},
-							Format:      "",
-						},
-					},
 				},
 			},
 		},
@@ -7308,7 +7268,7 @@ func schema_k8sio_api_core_v1_LimitRangeList(ref common.ReferenceCallback) commo
 					},
 					"items": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Items is a list of LimitRange objects. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/",
+							Description: "Items is a list of LimitRange objects. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/",
 							Type:        []string{"array"},
 							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
@@ -7944,7 +7904,7 @@ func schema_k8sio_api_core_v1_NodeConfigSource(ref common.ReferenceCallback) com
 	return common.OpenAPIDefinition{
 		Schema: spec.Schema{
 			SchemaProps: spec.SchemaProps{
-				Description: "NodeConfigSource specifies a source of node configuration. Exactly one subfield (excluding metadata) must be non-nil.",
+				Description: "NodeConfigSource specifies a source of node configuration. Exactly one subfield (excluding metadata) must be non-nil. This API is deprecated since 1.22",
 				Type:        []string{"object"},
 				Properties: map[string]spec.Schema{
 					"configMap": {
@@ -8309,7 +8269,7 @@ func schema_k8sio_api_core_v1_NodeSpec(ref common.ReferenceCallback) common.Open
 					},
 					"configSource": {
 						SchemaProps: spec.SchemaProps{
-							Description: "If specified, the source to get node configuration from The DynamicKubeletConfig feature gate must be enabled for the Kubelet to use this field",
+							Description: "Deprecated. If specified, the source of the node's configuration. The DynamicKubeletConfig feature gate must be enabled for the Kubelet to use this field. This field is deprecated as of 1.22: https://git.k8s.io/enhancements/keps/sig-node/281-dynamic-kubelet-configuration",
 							Ref:         ref("k8s.io/api/core/v1.NodeConfigSource"),
 						},
 					},
@@ -8901,7 +8861,13 @@ func schema_k8sio_api_core_v1_PersistentVolumeClaimSpec(ref common.ReferenceCall
 					},
 					"dataSource": {
 						SchemaProps: spec.SchemaProps{
-							Description: "This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) * An existing PVC (PersistentVolumeClaim) * An existing custom resource that implements data population (Alpha) In order to use custom resource types that implement data population, the AnyVolumeDataSource feature gate must be enabled. If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source.",
+							Description: "This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) * An existing PVC (PersistentVolumeClaim) If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source. If the AnyVolumeDataSource feature gate is enabled, this field will always have the same contents as the DataSourceRef field.",
+							Ref:         ref("k8s.io/api/core/v1.TypedLocalObjectReference"),
+						},
+					},
+					"dataSourceRef": {
+						SchemaProps: spec.SchemaProps{
+							Description: "Specifies the object from which to populate the volume with data, if a non-empty volume is desired. This may be any local object from a non-empty API group (non core object) or a PersistentVolumeClaim object. When this field is specified, volume binding will only succeed if the type of the specified object matches some installed volume populator or dynamic provisioner. This field will replace the functionality of the DataSource field and as such if both fields are non-empty, they must have the same value. For backwards compatibility, both fields (DataSource and DataSourceRef) will be set to the same value automatically if one of them is empty and the other is non-empty. There are two important differences between DataSource and DataSourceRef: * While DataSource only allows two specific types of objects, DataSourceRef\n  allows any non-core object, as well as PersistentVolumeClaim objects.\n* While DataSource ignores disallowed values (dropping them), DataSourceRef\n  preserves all values, and generates an error if a disallowed value is\n  specified.\n(Alpha) Using this field requires the AnyVolumeDataSource feature gate to be enabled.",
 							Ref:         ref("k8s.io/api/core/v1.TypedLocalObjectReference"),
 						},
 					},
@@ -9621,7 +9587,7 @@ func schema_k8sio_api_core_v1_PodAffinityTerm(ref common.ReferenceCallback) comm
 					},
 					"namespaces": {
 						SchemaProps: spec.SchemaProps{
-							Description: "namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"",
+							Description: "namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means \"this pod's namespace\"",
 							Type:        []string{"array"},
 							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
@@ -9640,6 +9606,12 @@ func schema_k8sio_api_core_v1_PodAffinityTerm(ref common.ReferenceCallback) comm
 							Format:      "",
 						},
 					},
+					"namespaceSelector": {
+						SchemaProps: spec.SchemaProps{
+							Description: "A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means \"this pod's namespace\". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled.",
+							Ref:         ref("k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector"),
+						},
+					},
 				},
 				Required: []string{"topologyKey"},
 			},
@@ -10431,7 +10403,7 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA
 					},
 					"terminationGracePeriodSeconds": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. Value must be non-negative integer. The value zero indicates delete immediately. If this value is nil, the default grace period will be used instead. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. Defaults to 30 seconds.",
+							Description: "Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). If this value is nil, the default grace period will be used instead. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. Defaults to 30 seconds.",
 							Type:        []string{"integer"},
 							Format:      "int64",
 						},
@@ -10627,7 +10599,7 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA
 					},
 					"readinessGates": {
 						SchemaProps: spec.SchemaProps{
-							Description: "If specified, all readiness gates will be evaluated for pod readiness. A pod is ready when all its containers are ready AND all conditions specified in the readiness gates have status equal to \"True\" More info: https://git.k8s.io/enhancements/keps/sig-network/0007-pod-ready%2B%2B.md",
+							Description: "If specified, all readiness gates will be evaluated for pod readiness. A pod is ready when all its containers are ready AND all conditions specified in the readiness gates have status equal to \"True\" More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates",
 							Type:        []string{"array"},
 							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
@@ -10640,7 +10612,7 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA
 					},
 					"runtimeClassName": {
 						SchemaProps: spec.SchemaProps{
-							Description: "RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used to run this pod.  If no RuntimeClass resource matches the named class, the pod will not be run. If unset or empty, the \"legacy\" RuntimeClass will be used, which is an implicit class with an empty definition that uses the default runtime handler. More info: https://git.k8s.io/enhancements/keps/sig-node/runtime-class.md This is a beta feature as of Kubernetes v1.14.",
+							Description: "RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used to run this pod.  If no RuntimeClass resource matches the named class, the pod will not be run. If unset or empty, the \"legacy\" RuntimeClass will be used, which is an implicit class with an empty definition that uses the default runtime handler. More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class This is a beta feature as of Kubernetes v1.14.",
 							Type:        []string{"string"},
 							Format:      "",
 						},
@@ -10661,7 +10633,7 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA
 					},
 					"overhead": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. This field will be autopopulated at admission time by the RuntimeClass admission controller. If the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. The RuntimeClass admission controller will reject Pod create requests which have the overhead already set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. More info: https://git.k8s.io/enhancements/keps/sig-node/20190226-pod-overhead.md This field is alpha-level as of Kubernetes v1.16, and is only honored by servers that enable the PodOverhead feature.",
+							Description: "Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. This field will be autopopulated at admission time by the RuntimeClass admission controller. If the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. The RuntimeClass admission controller will reject Pod create requests which have the overhead already set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md This field is beta-level as of Kubernetes v1.18, and is only honored by servers that enable the PodOverhead feature.",
 							Type:        []string{"object"},
 							AdditionalProperties: &spec.SchemaOrBool{
 								Allows: true,
@@ -11218,6 +11190,13 @@ func schema_k8sio_api_core_v1_Probe(ref common.ReferenceCallback) common.OpenAPI
 							Format:      "int32",
 						},
 					},
+					"terminationGracePeriodSeconds": {
+						SchemaProps: spec.SchemaProps{
+							Description: "Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset.",
+							Type:        []string{"integer"},
+							Format:      "int64",
+						},
+					},
 				},
 			},
 		},
@@ -12022,7 +12001,7 @@ func schema_k8sio_api_core_v1_ResourceRequirements(ref common.ReferenceCallback)
 				Properties: map[string]spec.Schema{
 					"limits": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/",
+							Description: "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/",
 							Type:        []string{"object"},
 							AdditionalProperties: &spec.SchemaOrBool{
 								Allows: true,
@@ -12036,7 +12015,7 @@ func schema_k8sio_api_core_v1_ResourceRequirements(ref common.ReferenceCallback)
 					},
 					"requests": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/",
+							Description: "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/",
 							Type:        []string{"object"},
 							AdditionalProperties: &spec.SchemaOrBool{
 								Allows: true,
@@ -12406,7 +12385,7 @@ func schema_k8sio_api_core_v1_Secret(ref common.ReferenceCallback) common.OpenAP
 					},
 					"immutable": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Immutable, if set to true, ensures that data stored in the Secret cannot be updated (only object metadata can be modified). If not set to true, the field can be modified at any time. Defaulted to nil. This is a beta field enabled by ImmutableEphemeralVolumes feature gate.",
+							Description: "Immutable, if set to true, ensures that data stored in the Secret cannot be updated (only object metadata can be modified). If not set to true, the field can be modified at any time. Defaulted to nil.",
 							Type:        []string{"boolean"},
 							Format:      "",
 						},
@@ -12428,7 +12407,7 @@ func schema_k8sio_api_core_v1_Secret(ref common.ReferenceCallback) common.OpenAP
 					},
 					"stringData": {
 						SchemaProps: spec.SchemaProps{
-							Description: "stringData allows specifying non-binary secret data in string form. It is provided as a write-only convenience method. All keys and values are merged into the data field on write, overwriting any existing values. It is never output when reading from the API.",
+							Description: "stringData allows specifying non-binary secret data in string form. It is provided as a write-only input field for convenience. All keys and values are merged into the data field on write, overwriting any existing values. The stringData field is never output when reading from the API.",
 							Type:        []string{"object"},
 							AdditionalProperties: &spec.SchemaOrBool{
 								Allows: true,
@@ -13085,7 +13064,7 @@ func schema_k8sio_api_core_v1_ServicePort(ref common.ReferenceCallback) common.O
 					},
 					"appProtocol": {
 						SchemaProps: spec.SchemaProps{
-							Description: "The application protocol for this port. This field follows standard Kubernetes label syntax. Un-prefixed names are reserved for IANA standard service names (as per RFC-6335 and http://www.iana.org/assignments/service-names). Non-standard protocols should use prefixed names such as mycompany.com/my-custom-protocol. This is a beta field that is guarded by the ServiceAppProtocol feature gate and enabled by default.",
+							Description: "The application protocol for this port. This field follows standard Kubernetes label syntax. Un-prefixed names are reserved for IANA standard service names (as per RFC-6335 and http://www.iana.org/assignments/service-names). Non-standard protocols should use prefixed names such as mycompany.com/my-custom-protocol.",
 							Type:        []string{"string"},
 							Format:      "",
 						},
@@ -13262,7 +13241,7 @@ func schema_k8sio_api_core_v1_ServiceSpec(ref common.ReferenceCallback) common.O
 					},
 					"loadBalancerSourceRanges": {
 						SchemaProps: spec.SchemaProps{
-							Description: "If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/",
+							Description: "If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/",
 							Type:        []string{"array"},
 							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
@@ -13276,7 +13255,7 @@ func schema_k8sio_api_core_v1_ServiceSpec(ref common.ReferenceCallback) common.O
 					},
 					"externalName": {
 						SchemaProps: spec.SchemaProps{
-							Description: "externalName is the external reference that discovery mechanisms will return as an alias for this service (e.g. a DNS CNAME record). No proxying will be involved.  Must be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be",
+							Description: "externalName is the external reference that discovery mechanisms will return as an alias for this service (e.g. a DNS CNAME record). No proxying will be involved.  Must be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires `type` to be \"ExternalName\".",
 							Type:        []string{"string"},
 							Format:      "",
 						},
@@ -13308,20 +13287,6 @@ func schema_k8sio_api_core_v1_ServiceSpec(ref common.ReferenceCallback) common.O
 							Ref:         ref("k8s.io/api/core/v1.SessionAffinityConfig"),
 						},
 					},
-					"topologyKeys": {
-						SchemaProps: spec.SchemaProps{
-							Description: "topologyKeys is a preference-order list of topology keys which implementations of services should use to preferentially sort endpoints when accessing this Service, it can not be used at the same time as externalTrafficPolicy=Local. Topology keys must be valid label keys and at most 16 keys may be specified. Endpoints are chosen based on the first topology key with available backends. If this field is specified and all entries have no backends that match the topology of the client, the service has no backends for that client and connections should fail. The special value \"*\" may be used to mean \"any topology\". This catch-all value, if used, only makes sense as the last value in the list. If this is not specified or empty, no topology constraints will be applied. This field is alpha-level and is only honored by servers that enable the ServiceTopology feature.",
-							Type:        []string{"array"},
-							Items: &spec.SchemaOrArray{
-								Schema: &spec.Schema{
-									SchemaProps: spec.SchemaProps{
-										Type:   []string{"string"},
-										Format: "",
-									},
-								},
-							},
-						},
-					},
 					"ipFamilies": {
 						VendorExtensible: spec.VendorExtensible{
 							Extensions: spec.Extensions{
@@ -13350,11 +13315,25 @@ func schema_k8sio_api_core_v1_ServiceSpec(ref common.ReferenceCallback) common.O
 					},
 					"allocateLoadBalancerNodePorts": {
 						SchemaProps: spec.SchemaProps{
-							Description: "allocateLoadBalancerNodePorts defines if NodePorts will be automatically allocated for services with type LoadBalancer.  Default is \"true\". It may be set to \"false\" if the cluster load-balancer does not rely on NodePorts. allocateLoadBalancerNodePorts may only be set for services with type LoadBalancer and will be cleared if the type is changed to any other type. This field is alpha-level and is only honored by servers that enable the ServiceLBNodePortControl feature.",
+							Description: "allocateLoadBalancerNodePorts defines if NodePorts will be automatically allocated for services with type LoadBalancer.  Default is \"true\". It may be set to \"false\" if the cluster load-balancer does not rely on NodePorts.  If the caller requests specific NodePorts (by specifying a value), those requests will be respected, regardless of this field. This field may only be set for services with type LoadBalancer and will be cleared if the type is changed to any other type. This field is beta-level and is only honored by servers that enable the ServiceLBNodePortControl feature.",
 							Type:        []string{"boolean"},
 							Format:      "",
 						},
 					},
+					"loadBalancerClass": {
+						SchemaProps: spec.SchemaProps{
+							Description: "loadBalancerClass is the class of the load balancer implementation this Service belongs to. If specified, the value of this field must be a label-style identifier, with an optional prefix, e.g. \"internal-vip\" or \"example.com/internal-vip\". Unprefixed names are reserved for end-users. This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load balancer implementation is used, today this is typically done through the cloud provider integration, but should apply for any default implementation. If set, it is assumed that a load balancer implementation is watching for Services with a matching class. Any default load balancer implementation (e.g. cloud providers) should ignore Services that set this field. This field can only be set when creating or updating a Service to type 'LoadBalancer'. Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type.",
+							Type:        []string{"string"},
+							Format:      "",
+						},
+					},
+					"internalTrafficPolicy": {
+						SchemaProps: spec.SchemaProps{
+							Description: "InternalTrafficPolicy specifies if the cluster internal traffic should be routed to all endpoints or node-local endpoints only. \"Cluster\" routes internal traffic to a Service to all endpoints. \"Local\" routes traffic to node-local endpoints only, traffic is dropped if no node-local endpoints are ready. The default value is \"Cluster\".",
+							Type:        []string{"string"},
+							Format:      "",
+						},
+					},
 				},
 			},
 		},
@@ -13999,7 +13978,7 @@ func schema_k8sio_api_core_v1_Volume(ref common.ReferenceCallback) common.OpenAP
 					},
 					"ephemeral": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Ephemeral represents a volume that is handled by a cluster storage driver (Alpha feature). The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, and deleted when the pod is removed.\n\nUse this if: a) the volume is only needed while the pod runs, b) features of normal volumes like restoring from snapshot or capacity\n   tracking are needed,\nc) the storage driver is specified through a storage class, and d) the storage driver supports dynamic volume provisioning through\n   a PersistentVolumeClaim (see EphemeralVolumeSource for more\n   information on the connection between this volume type\n   and PersistentVolumeClaim).\n\nUse PersistentVolumeClaim or one of the vendor-specific APIs for volumes that persist for longer than the lifecycle of an individual pod.\n\nUse CSI for light-weight local ephemeral volumes if the CSI driver is meant to be used that way - see the documentation of the driver for more information.\n\nA pod can use both types of ephemeral volumes and persistent volumes at the same time.",
+							Description: "Ephemeral represents a volume that is handled by a cluster storage driver. The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, and deleted when the pod is removed.\n\nUse this if: a) the volume is only needed while the pod runs, b) features of normal volumes like restoring from snapshot or capacity\n   tracking are needed,\nc) the storage driver is specified through a storage class, and d) the storage driver supports dynamic volume provisioning through\n   a PersistentVolumeClaim (see EphemeralVolumeSource for more\n   information on the connection between this volume type\n   and PersistentVolumeClaim).\n\nUse PersistentVolumeClaim or one of the vendor-specific APIs for volumes that persist for longer than the lifecycle of an individual pod.\n\nUse CSI for light-weight local ephemeral volumes if the CSI driver is meant to be used that way - see the documentation of the driver for more information.\n\nA pod can use both types of ephemeral volumes and persistent volumes at the same time.\n\nThis is a beta feature and only available when the GenericEphemeralVolume feature gate is enabled.",
 							Ref:         ref("k8s.io/api/core/v1.EphemeralVolumeSource"),
 						},
 					},
@@ -14333,7 +14312,7 @@ func schema_k8sio_api_core_v1_VolumeSource(ref common.ReferenceCallback) common.
 					},
 					"ephemeral": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Ephemeral represents a volume that is handled by a cluster storage driver (Alpha feature). The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, and deleted when the pod is removed.\n\nUse this if: a) the volume is only needed while the pod runs, b) features of normal volumes like restoring from snapshot or capacity\n   tracking are needed,\nc) the storage driver is specified through a storage class, and d) the storage driver supports dynamic volume provisioning through\n   a PersistentVolumeClaim (see EphemeralVolumeSource for more\n   information on the connection between this volume type\n   and PersistentVolumeClaim).\n\nUse PersistentVolumeClaim or one of the vendor-specific APIs for volumes that persist for longer than the lifecycle of an individual pod.\n\nUse CSI for light-weight local ephemeral volumes if the CSI driver is meant to be used that way - see the documentation of the driver for more information.\n\nA pod can use both types of ephemeral volumes and persistent volumes at the same time.",
+							Description: "Ephemeral represents a volume that is handled by a cluster storage driver. The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, and deleted when the pod is removed.\n\nUse this if: a) the volume is only needed while the pod runs, b) features of normal volumes like restoring from snapshot or capacity\n   tracking are needed,\nc) the storage driver is specified through a storage class, and d) the storage driver supports dynamic volume provisioning through\n   a PersistentVolumeClaim (see EphemeralVolumeSource for more\n   information on the connection between this volume type\n   and PersistentVolumeClaim).\n\nUse PersistentVolumeClaim or one of the vendor-specific APIs for volumes that persist for longer than the lifecycle of an individual pod.\n\nUse CSI for light-weight local ephemeral volumes if the CSI driver is meant to be used that way - see the documentation of the driver for more information.\n\nA pod can use both types of ephemeral volumes and persistent volumes at the same time.\n\nThis is a beta feature and only available when the GenericEphemeralVolume feature gate is enabled.",
 							Ref:         ref("k8s.io/api/core/v1.EphemeralVolumeSource"),
 						},
 					},
@@ -14444,6 +14423,13 @@ func schema_k8sio_api_core_v1_WindowsSecurityContextOptions(ref common.Reference
 							Format:      "",
 						},
 					},
+					"hostProcess": {
+						SchemaProps: spec.SchemaProps{
+							Description: "HostProcess determines if a container should be run as a 'Host Process' container. This field is alpha-level and will only be honored by components that enable the WindowsHostProcessContainers feature flag. Setting this field without the feature flag will result in errors when validating the Pod. All of a Pod's containers must have the same effective HostProcess value (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers).  In addition, if HostProcess is true then HostNetwork must also be set to true.",
+							Type:        []string{"boolean"},
+							Format:      "",
+						},
+					},
 				},
 			},
 		},
@@ -14812,6 +14798,62 @@ func schema_pkg_apis_meta_v1_APIVersions(ref common.ReferenceCallback) common.Op
 	}
 }
 
+func schema_pkg_apis_meta_v1_ApplyOptions(ref common.ReferenceCallback) common.OpenAPIDefinition {
+	return common.OpenAPIDefinition{
+		Schema: spec.Schema{
+			SchemaProps: spec.SchemaProps{
+				Description: "ApplyOptions may be provided when applying an API object. FieldManager is required for apply requests. ApplyOptions is equivalent to PatchOptions. It is provided as a convenience with documentation that speaks specifically to how the options fields relate to apply.",
+				Type:        []string{"object"},
+				Properties: map[string]spec.Schema{
+					"kind": {
+						SchemaProps: spec.SchemaProps{
+							Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds",
+							Type:        []string{"string"},
+							Format:      "",
+						},
+					},
+					"apiVersion": {
+						SchemaProps: spec.SchemaProps{
+							Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources",
+							Type:        []string{"string"},
+							Format:      "",
+						},
+					},
+					"dryRun": {
+						SchemaProps: spec.SchemaProps{
+							Description: "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed",
+							Type:        []string{"array"},
+							Items: &spec.SchemaOrArray{
+								Schema: &spec.Schema{
+									SchemaProps: spec.SchemaProps{
+										Type:   []string{"string"},
+										Format: "",
+									},
+								},
+							},
+						},
+					},
+					"force": {
+						SchemaProps: spec.SchemaProps{
+							Description: "Force is going to \"force\" Apply requests. It means user will re-acquire conflicting fields owned by other people.",
+							Type:        []string{"boolean"},
+							Format:      "",
+						},
+					},
+					"fieldManager": {
+						SchemaProps: spec.SchemaProps{
+							Description: "fieldManager is a name associated with the actor or entity that is making these changes. The value must be less than or 128 characters long, and only contain printable characters, as defined by https://golang.org/pkg/unicode/#IsPrint. This field is required.",
+							Type:        []string{"string"},
+							Format:      "",
+						},
+					},
+				},
+				Required: []string{"force", "fieldManager"},
+			},
+		},
+	}
+}
+
 func schema_pkg_apis_meta_v1_Condition(ref common.ReferenceCallback) common.OpenAPIDefinition {
 	return common.OpenAPIDefinition{
 		Schema: spec.Schema{
@@ -14999,48 +15041,6 @@ func schema_pkg_apis_meta_v1_Duration(ref common.ReferenceCallback) common.OpenA
 	}
 }
 
-func schema_pkg_apis_meta_v1_ExportOptions(ref common.ReferenceCallback) common.OpenAPIDefinition {
-	return common.OpenAPIDefinition{
-		Schema: spec.Schema{
-			SchemaProps: spec.SchemaProps{
-				Description: "ExportOptions is the query options to the standard REST get call. Deprecated. Planned for removal in 1.18.",
-				Type:        []string{"object"},
-				Properties: map[string]spec.Schema{
-					"kind": {
-						SchemaProps: spec.SchemaProps{
-							Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds",
-							Type:        []string{"string"},
-							Format:      "",
-						},
-					},
-					"apiVersion": {
-						SchemaProps: spec.SchemaProps{
-							Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources",
-							Type:        []string{"string"},
-							Format:      "",
-						},
-					},
-					"export": {
-						SchemaProps: spec.SchemaProps{
-							Description: "Should this value be exported.  Export strips fields that a user can not specify. Deprecated. Planned for removal in 1.18.",
-							Type:        []string{"boolean"},
-							Format:      "",
-						},
-					},
-					"exact": {
-						SchemaProps: spec.SchemaProps{
-							Description: "Should the export be exact.  Exact export maintains cluster-specific fields like 'Namespace'. Deprecated. Planned for removal in 1.18.",
-							Type:        []string{"boolean"},
-							Format:      "",
-						},
-					},
-				},
-				Required: []string{"export", "exact"},
-			},
-		},
-	}
-}
-
 func schema_pkg_apis_meta_v1_FieldsV1(ref common.ReferenceCallback) common.OpenAPIDefinition {
 	return common.OpenAPIDefinition{
 		Schema: spec.Schema{
@@ -15509,7 +15509,7 @@ func schema_pkg_apis_meta_v1_ListOptions(ref common.ReferenceCallback) common.Op
 					},
 					"allowWatchBookmarks": {
 						SchemaProps: spec.SchemaProps{
-							Description: "allowWatchBookmarks requests watch events with type \"BOOKMARK\". Servers that do not implement bookmarks may ignore this flag and bookmarks are sent at the server's discretion. Clients should not assume bookmarks are returned at any specific interval, nor may they assume the server will send any BOOKMARK event during a session. If this is not a watch, this field is ignored. If the feature gate WatchBookmarks is not enabled in apiserver, this field is ignored.",
+							Description: "allowWatchBookmarks requests watch events with type \"BOOKMARK\". Servers that do not implement bookmarks may ignore this flag and bookmarks are sent at the server's discretion. Clients should not assume bookmarks are returned at any specific interval, nor may they assume the server will send any BOOKMARK event during a session. If this is not a watch, this field is ignored.",
 							Type:        []string{"boolean"},
 							Format:      "",
 						},
@@ -15602,6 +15602,13 @@ func schema_pkg_apis_meta_v1_ManagedFieldsEntry(ref common.ReferenceCallback) co
 							Ref:         ref("k8s.io/apimachinery/pkg/apis/meta/v1.FieldsV1"),
 						},
 					},
+					"subresource": {
+						SchemaProps: spec.SchemaProps{
+							Description: "Subresource is the name of the subresource used to update that object, or empty string if the object was updated through the main resource. The value of this field is used to distinguish between managers, even if they share the same name. For example, a status update will be distinct from a regular update using the same manager name. Note that the APIVersion field is not related to the Subresource field and it always corresponds to the version of the main resource.",
+							Type:        []string{"string"},
+							Format:      "",
+						},
+					},
 				},
 			},
 		},

From 7544871994f8bc30405e085fe92b90dbae80fc28 Mon Sep 17 00:00:00 2001
From: echken <echken@localhost.localdomain>
Date: Sun, 20 Mar 2022 23:21:13 -0400
Subject: [PATCH 04/32] fix er policyrule dereference error

Signed-off-by: echken <echken@localhost.localdomain>
---
 pkg/agent/datapath/multiBridgeDatapath.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/pkg/agent/datapath/multiBridgeDatapath.go b/pkg/agent/datapath/multiBridgeDatapath.go
index 095be2ef0..91dda07f8 100644
--- a/pkg/agent/datapath/multiBridgeDatapath.go
+++ b/pkg/agent/datapath/multiBridgeDatapath.go
@@ -835,7 +835,9 @@ func (datapathManager *DpManager) RemoveEveroutePolicyRule(ruleID string, ruleNa
 	// check and remove rule reference
 	if pRule.PolicyRuleReference.Has(ruleName) {
 		pRule.PolicyRuleReference.Delete(ruleName)
-		return nil
+		if pRule.PolicyRuleReference.Len() > 0 {
+			return nil
+		}
 	}
 
 	log.Infof("Received remove rule: %+v", ruleName)

From fc7f0165fd4ac9b8a4257a77a50f727ec7d3f732 Mon Sep 17 00:00:00 2001
From: echken <echken@localhost.localdomain>
Date: Mon, 21 Mar 2022 01:26:42 -0400
Subject: [PATCH 05/32] change ct commit table design

Signed-off-by: echken <echken@localhost.localdomain>
---
 pkg/agent/datapath/policyBridge.go | 175 +++++++++++++++--------------
 1 file changed, 93 insertions(+), 82 deletions(-)

diff --git a/pkg/agent/datapath/policyBridge.go b/pkg/agent/datapath/policyBridge.go
index 2d631e6f5..52d0b41bb 100644
--- a/pkg/agent/datapath/policyBridge.go
+++ b/pkg/agent/datapath/policyBridge.go
@@ -15,24 +15,25 @@ import (
 
 //nolint
 const (
-	INPUT_TABLE                 = 0
-	CT_STATE_TABLE              = 1
-	DIRECTION_SELECTION_TABLE   = 10
-	EGRESS_TIER1_TABLE          = 20
-	EGRESS_TIER2_MONITOR_TABLE  = 24
-	EGRESS_TIER2_TABLE          = 25
-	EGRESS_TIER3_MONITOR_TABLE  = 29
-	EGRESS_TIER3_TABLE          = 30
-	EGRESS_DROP_TABLR           = 31
+	INPUT_TABLE                = 0
+	CT_STATE_TABLE             = 1
+	DIRECTION_SELECTION_TABLE  = 10
+	EGRESS_TIER1_TABLE         = 20
+	EGRESS_TIER2_MONITOR_TABLE = 24
+	EGRESS_TIER2_TABLE         = 25
+	EGRESS_TIER3_MONITOR_TABLE = 29
+	EGRESS_TIER3_TABLE         = 30
+	// EGRESS_DROP_TABLR           = 31
 	INGRESS_TIER1_TABLE         = 50
 	INGRESS_TIER2_MONITOR_TABLE = 54
 	INGRESS_TIER2_TABLE         = 55
 	INGRESS_TIER3_MONITOR_TABLE = 59
 	INGRESS_TIER3_TABLE         = 60
-	INGRESS_DROP_TABLE          = 61
-	CT_COMMIT_TABLE             = 70
-	SFC_POLICY_TABLE            = 80
-	POLICY_FORWARDING_TABLE     = 90
+	// INGRESS_DROP_TABLE          = 61
+	CT_COMMIT_TABLE         = 70
+	CT_DROP_TABLE           = 71
+	SFC_POLICY_TABLE        = 80
+	POLICY_FORWARDING_TABLE = 90
 )
 
 type PolicyBridge struct {
@@ -40,24 +41,25 @@ type PolicyBridge struct {
 	OfSwitch        *ofctrl.OFSwitch
 	datapathManager *DpManager
 
-	inputTable                     *ofctrl.Table
-	ctStateTable                   *ofctrl.Table
-	directionSelectionTable        *ofctrl.Table
-	egressTier1PolicyTable         *ofctrl.Table
-	egressTier2PolicyMonitorTable  *ofctrl.Table
-	egressTier2PolicyTable         *ofctrl.Table
-	egressTier3PolicyMonitorTable  *ofctrl.Table
-	egressTier3PolicyTable         *ofctrl.Table
-	egressDropTable                *ofctrl.Table
+	inputTable                    *ofctrl.Table
+	ctStateTable                  *ofctrl.Table
+	directionSelectionTable       *ofctrl.Table
+	egressTier1PolicyTable        *ofctrl.Table
+	egressTier2PolicyMonitorTable *ofctrl.Table
+	egressTier2PolicyTable        *ofctrl.Table
+	egressTier3PolicyMonitorTable *ofctrl.Table
+	egressTier3PolicyTable        *ofctrl.Table
+	// egressDropTable                *ofctrl.Table
 	ingressTier1PolicyTable        *ofctrl.Table
 	ingressTier2PolicyMonitorTable *ofctrl.Table
 	ingressTier2PolicyTable        *ofctrl.Table
 	ingressTier3PolicyMonitorTable *ofctrl.Table
 	ingressTier3PolicyTable        *ofctrl.Table
-	ingressDropTable               *ofctrl.Table
-	ctCommitTable                  *ofctrl.Table
-	sfcPolicyTable                 *ofctrl.Table
-	policyForwardingTable          *ofctrl.Table
+	// ingressDropTable               *ofctrl.Table
+	ctCommitTable         *ofctrl.Table
+	ctDropTable           *ofctrl.Table
+	sfcPolicyTable        *ofctrl.Table
+	policyForwardingTable *ofctrl.Table
 
 	policySwitchStatusMutex sync.RWMutex
 	isPolicySwitchConnected bool
@@ -128,14 +130,15 @@ func (p *PolicyBridge) BridgeInit() {
 	p.ingressTier2PolicyTable, _ = sw.NewTable(INGRESS_TIER2_TABLE)
 	p.ingressTier3PolicyMonitorTable, _ = sw.NewTable(INGRESS_TIER3_MONITOR_TABLE)
 	p.ingressTier3PolicyTable, _ = sw.NewTable(INGRESS_TIER3_TABLE)
-	p.ingressDropTable, _ = sw.NewTable(INGRESS_DROP_TABLE)
+	// p.ingressDropTable, _ = sw.NewTable(INGRESS_DROP_TABLE)
 	p.egressTier1PolicyTable, _ = sw.NewTable(EGRESS_TIER1_TABLE)
 	p.egressTier2PolicyMonitorTable, _ = sw.NewTable(EGRESS_TIER2_MONITOR_TABLE)
 	p.egressTier2PolicyTable, _ = sw.NewTable(EGRESS_TIER2_TABLE)
 	p.egressTier3PolicyMonitorTable, _ = sw.NewTable(EGRESS_TIER3_MONITOR_TABLE)
 	p.egressTier3PolicyTable, _ = sw.NewTable(EGRESS_TIER3_TABLE)
-	p.egressDropTable, _ = sw.NewTable(EGRESS_DROP_TABLR)
+	// p.egressDropTable, _ = sw.NewTable(EGRESS_DROP_TABLR)
 	p.ctCommitTable, _ = sw.NewTable(CT_COMMIT_TABLE)
+	p.ctDropTable, _ = sw.NewTable(CT_DROP_TABLE)
 	p.sfcPolicyTable, _ = sw.NewTable(SFC_POLICY_TABLE)
 	p.policyForwardingTable, _ = sw.NewTable(POLICY_FORWARDING_TABLE)
 
@@ -255,7 +258,30 @@ func (p *PolicyBridge) initCTFlow(sw *ofctrl.OFSwitch) error {
 	}
 
 	// Table 70 conntrack commit table
-	ctByPassFlow1, _ := p.ctCommitTable.NewFlow(ofctrl.FlowMatch{
+	ctTrkState := openflow13.NewCTStates()
+	ctTrkState.SetNew()
+	ctTrkState.SetTrk()
+	ctCommitFlow, _ := p.ctCommitTable.NewFlow(ofctrl.FlowMatch{
+		Priority:  MID_MATCH_FLOW_PRIORITY,
+		Ethertype: PROTOCOL_IP,
+		CtStates:  ctTrkState,
+	})
+	var ctDropTable uint8 = CT_DROP_TABLE
+	srcField, _ := openflow13.FindFieldHeaderByName("nxm_nx_xxreg0", false)
+	dstField, _ := openflow13.FindFieldHeaderByName("nxm_nx_ct_label", false)
+	moveAct := openflow13.NewNXActionRegMove(128, 0, 0, srcField, dstField)
+	ctCommitAction := ofctrl.NewConntrackAction(true, false, &ctDropTable, &policyConntrackZone, moveAct)
+	_ = ctCommitFlow.SetConntrack(ctCommitAction)
+
+	ctCommitTableDefaultFlow, _ := p.ctCommitTable.NewFlow(ofctrl.FlowMatch{
+		Priority: DEFAULT_FLOW_MISS_PRIORITY,
+	})
+	if err := ctCommitTableDefaultFlow.Next(p.ctDropTable); err != nil {
+		return fmt.Errorf("failed to install ct commit flow, error: %v", err)
+	}
+
+	// ct drop table: 71
+	ctByPassFlow1, _ := p.ctDropTable.NewFlow(ofctrl.FlowMatch{
 		Priority: MID_MATCH_FLOW_PRIORITY + FLOW_MATCH_OFFSET,
 		Regs: []*ofctrl.NXRegister{
 			{
@@ -268,7 +294,7 @@ func (p *PolicyBridge) initCTFlow(sw *ofctrl.OFSwitch) error {
 	if err := ctByPassFlow1.Next(p.OfSwitch.DropAction()); err != nil {
 		return fmt.Errorf("failed to install ct drop flow, error: %v", err)
 	}
-	ctByPassFlow2, _ := p.ctCommitTable.NewFlow(ofctrl.FlowMatch{
+	ctByPassFlow2, _ := p.ctDropTable.NewFlow(ofctrl.FlowMatch{
 		Priority: MID_MATCH_FLOW_PRIORITY + FLOW_MATCH_OFFSET,
 		Regs: []*ofctrl.NXRegister{
 			{
@@ -285,26 +311,11 @@ func (p *PolicyBridge) initCTFlow(sw *ofctrl.OFSwitch) error {
 		return fmt.Errorf("failed to install ct bypass flow 2, error: %v", err)
 	}
 
-	ctTrkState := openflow13.NewCTStates()
-	ctTrkState.SetNew()
-	ctTrkState.SetTrk()
-	ctCommitFlow, _ := p.ctCommitTable.NewFlow(ofctrl.FlowMatch{
-		Priority:  MID_MATCH_FLOW_PRIORITY,
-		Ethertype: PROTOCOL_IP,
-		CtStates:  ctTrkState,
-	})
-	var sfcPolicyTable uint8 = SFC_POLICY_TABLE
-	srcField, _ := openflow13.FindFieldHeaderByName("nxm_nx_xxreg0", false)
-	dstField, _ := openflow13.FindFieldHeaderByName("nxm_nx_ct_label", false)
-	moveAct := openflow13.NewNXActionRegMove(128, 0, 0, srcField, dstField)
-	ctCommitAction := ofctrl.NewConntrackAction(true, false, &sfcPolicyTable, &policyConntrackZone, moveAct)
-	_ = ctCommitFlow.SetConntrack(ctCommitAction)
-
-	ctCommitTableDefaultFlow, _ := p.ctCommitTable.NewFlow(ofctrl.FlowMatch{
+	ctPassDefaultFlow, _ := p.ctDropTable.NewFlow(ofctrl.FlowMatch{
 		Priority: DEFAULT_FLOW_MISS_PRIORITY,
 	})
-	if err := ctCommitTableDefaultFlow.Next(p.sfcPolicyTable); err != nil {
-		return fmt.Errorf("failed to install ct commit flow, error: %v", err)
+	if err := ctPassDefaultFlow.Next(p.sfcPolicyTable); err != nil {
+		return fmt.Errorf("failed to install egress tier3 drop table flow, error: %v", err)
 	}
 
 	return nil
@@ -344,26 +355,26 @@ func (p *PolicyBridge) initPolicyTable() error {
 	}
 
 	// egress drop table
-	egressDropFlow, _ := p.egressDropTable.NewFlow(ofctrl.FlowMatch{
-		Priority: DEFAULT_FLOW_MISS_PRIORITY,
-	})
-	if err := egressDropFlow.Next(p.ctCommitTable); err != nil {
-		return fmt.Errorf("failed to install egress tier3 drop table flow, error: %v", err)
-	}
-	ctTrkState := openflow13.NewCTStates()
-	ctTrkState.SetNew()
-	ctTrkState.SetTrk()
-	egressDropTableCtCommitFlow, _ := p.egressDropTable.NewFlow(ofctrl.FlowMatch{
-		Priority:  NORMAL_MATCH_FLOW_PRIORITY,
-		Ethertype: PROTOCOL_IP,
-		CtStates:  ctTrkState,
-	})
-	var policyConntrackZone uint16 = 65520
-	srcField, _ := openflow13.FindFieldHeaderByName("nxm_nx_xxreg0", false)
-	dstField, _ := openflow13.FindFieldHeaderByName("nxm_nx_ct_label", false)
-	moveAct := openflow13.NewNXActionRegMove(128, 0, 0, srcField, dstField)
-	ctCommitAction := ofctrl.NewConntrackAction(true, false, &p.ctCommitTable.TableId, &policyConntrackZone, moveAct)
-	_ = egressDropTableCtCommitFlow.SetConntrack(ctCommitAction)
+	// egressDropFlow, _ := p.egressDropTable.NewFlow(ofctrl.FlowMatch{
+	// 	Priority: DEFAULT_FLOW_MISS_PRIORITY,
+	// })
+	// if err := egressDropFlow.Next(p.ctCommitTable); err != nil {
+	// 	return fmt.Errorf("failed to install egress tier3 drop table flow, error: %v", err)
+	// }
+	// ctTrkState := openflow13.NewCTStates()
+	// ctTrkState.SetNew()
+	// ctTrkState.SetTrk()
+	// egressDropTableCtCommitFlow, _ := p.egressDropTable.NewFlow(ofctrl.FlowMatch{
+	// 	Priority:  NORMAL_MATCH_FLOW_PRIORITY,
+	// 	Ethertype: PROTOCOL_IP,
+	// 	CtStates:  ctTrkState,
+	// })
+	// var policyConntrackZone uint16 = 65520
+	// srcField, _ := openflow13.FindFieldHeaderByName("nxm_nx_xxreg0", false)
+	// dstField, _ := openflow13.FindFieldHeaderByName("nxm_nx_ct_label", false)
+	// moveAct := openflow13.NewNXActionRegMove(128, 0, 0, srcField, dstField)
+	// ctCommitAction := ofctrl.NewConntrackAction(true, false, &p.ctCommitTable.TableId, &policyConntrackZone, moveAct)
+	// _ = egressDropTableCtCommitFlow.SetConntrack(ctCommitAction)
 
 	// ingress policy table
 	ingressTier1DefaultFlow, _ := p.ingressTier1PolicyTable.NewFlow(ofctrl.FlowMatch{
@@ -398,20 +409,20 @@ func (p *PolicyBridge) initPolicyTable() error {
 	}
 
 	// ingress tier3 drop table
-	ingressDropFlow, _ := p.ingressDropTable.NewFlow(ofctrl.FlowMatch{
-		Priority: DEFAULT_FLOW_MISS_PRIORITY,
-	})
-	if err := ingressDropFlow.Next(p.ctCommitTable); err != nil {
-		return fmt.Errorf("failed to install ingress tier3 drop table flow, error: %v", err)
-	}
-	ingressDropTableCtCommitFlow, _ := p.ingressDropTable.NewFlow(ofctrl.FlowMatch{
-		Priority:  NORMAL_MATCH_FLOW_PRIORITY,
-		Ethertype: PROTOCOL_IP,
-		CtStates:  ctTrkState,
-	})
-	moveAct = openflow13.NewNXActionRegMove(128, 0, 0, srcField, dstField)
-	ctCommitAction = ofctrl.NewConntrackAction(true, false, &p.ctCommitTable.TableId, &policyConntrackZone, moveAct)
-	_ = ingressDropTableCtCommitFlow.SetConntrack(ctCommitAction)
+	// ingressDropFlow, _ := p.ingressDropTable.NewFlow(ofctrl.FlowMatch{
+	// 	Priority: DEFAULT_FLOW_MISS_PRIORITY,
+	// })
+	// if err := ingressDropFlow.Next(p.ctCommitTable); err != nil {
+	// 	return fmt.Errorf("failed to install ingress tier3 drop table flow, error: %v", err)
+	// }
+	// ingressDropTableCtCommitFlow, _ := p.ingressDropTable.NewFlow(ofctrl.FlowMatch{
+	// 	Priority:  NORMAL_MATCH_FLOW_PRIORITY,
+	// 	Ethertype: PROTOCOL_IP,
+	// 	CtStates:  ctTrkState,
+	// })
+	// moveAct = openflow13.NewNXActionRegMove(128, 0, 0, srcField, dstField)
+	// ctCommitAction = ofctrl.NewConntrackAction(true, false, &p.ctCommitTable.TableId, &policyConntrackZone, moveAct)
+	// _ = ingressDropTableCtCommitFlow.SetConntrack(ctCommitAction)
 
 	// sfc policy table
 	sfcPolicyTableDefaultFlow, _ := p.sfcPolicyTable.NewFlow(ofctrl.FlowMatch{

From c2d2047186effc7a3dabed0dca387f118f1c9759 Mon Sep 17 00:00:00 2001
From: echken <echken@localhost.localdomain>
Date: Mon, 21 Mar 2022 22:53:57 -0400
Subject: [PATCH 06/32] refactor: update work mode default value etc

1. remove ingress and egress drop table
2. update securitypolicy crd with default work mode
3. constant default work mode in internel policy and agent test

Signed-off-by: echken <echken@localhost.localdomain>
---
 ...security.everoute.io_securitypolicies.yaml |   3 +-
 deploy/everoute.yaml                          |   3 +-
 pkg/agent/datapath/multiBridgeDatapath.go     |   4 +-
 .../datapath/multiBridgeDatapath_test.go      |   8 +-
 pkg/agent/datapath/policyBridge.go            |  95 ++---
 pkg/apis/security/v1alpha1/types.go           |   1 +
 pkg/openapi/openapi_generated.go              | 325 +++++++++---------
 7 files changed, 194 insertions(+), 245 deletions(-)

diff --git a/deploy/crds/security.everoute.io_securitypolicies.yaml b/deploy/crds/security.everoute.io_securitypolicies.yaml
index e5a2fdf08..2efc3952c 100644
--- a/deploy/crds/security.everoute.io_securitypolicies.yaml
+++ b/deploy/crds/security.everoute.io_securitypolicies.yaml
@@ -906,11 +906,12 @@ spec:
                   that include "Egress" (since such a policy would not include an
                   Egress section and would otherwise default to just [ "Ingress" ]).
                 items:
-                  description: PolicyType string describes the NetworkPolicy type
+                  description: Policy Type string describes the NetworkPolicy type
                     This type is beta-level in 1.8
                   type: string
                 type: array
               securityPolicyEnforcementMode:
+                default: work
                 description: 'Work mode specify the policy enforcement state: monitor
                   or work'
                 type: string
diff --git a/deploy/everoute.yaml b/deploy/everoute.yaml
index 082eec5de..c26d8d3bf 100644
--- a/deploy/everoute.yaml
+++ b/deploy/everoute.yaml
@@ -1643,11 +1643,12 @@ spec:
                   that include "Egress" (since such a policy would not include an
                   Egress section and would otherwise default to just [ "Ingress" ]).
                 items:
-                  description: PolicyType string describes the NetworkPolicy type
+                  description: Policy Type string describes the NetworkPolicy type
                     This type is beta-level in 1.8
                   type: string
                 type: array
               securityPolicyEnforcementMode:
+                default: work
                 description: 'Work mode specify the policy enforcement state: monitor
                   or work'
                 type: string
diff --git a/pkg/agent/datapath/multiBridgeDatapath.go b/pkg/agent/datapath/multiBridgeDatapath.go
index 91dda07f8..72f4cbe3a 100644
--- a/pkg/agent/datapath/multiBridgeDatapath.go
+++ b/pkg/agent/datapath/multiBridgeDatapath.go
@@ -305,12 +305,12 @@ func (datapathManager *DpManager) InitializeDatapath(stopChan <-chan struct{}) {
 	// add rules for internalIP
 	for _, internalIP := range datapathManager.datapathConfig.InternalIPs {
 		// internal ingress rule
-		err := datapathManager.AddEveroutePolicyRule(newInternalIngressRule(internalIP), InternalIngressRulePrefix, POLICY_DIRECTION_IN, POLICY_TIER3, "work")
+		err := datapathManager.AddEveroutePolicyRule(newInternalIngressRule(internalIP), InternalIngressRulePrefix, POLICY_DIRECTION_IN, POLICY_TIER3, DEFAULT_POLICY_ENFORCEMENT_MODE)
 		if err != nil {
 			log.Fatalf("Failed to add internal whitelist: %v", err)
 		}
 		// internal egress rule
-		err = datapathManager.AddEveroutePolicyRule(newInternalEgressRule(internalIP), InternalEgressRulePrefix, POLICY_DIRECTION_OUT, POLICY_TIER3, "work")
+		err = datapathManager.AddEveroutePolicyRule(newInternalEgressRule(internalIP), InternalEgressRulePrefix, POLICY_DIRECTION_OUT, POLICY_TIER3, DEFAULT_POLICY_ENFORCEMENT_MODE)
 		if err != nil {
 			log.Fatalf("Failed to add internal whitelist: %v", err)
 		}
diff --git a/pkg/agent/datapath/multiBridgeDatapath_test.go b/pkg/agent/datapath/multiBridgeDatapath_test.go
index 0f849a23b..691bef30c 100644
--- a/pkg/agent/datapath/multiBridgeDatapath_test.go
+++ b/pkg/agent/datapath/multiBridgeDatapath_test.go
@@ -142,7 +142,7 @@ func testLocalEndpoint(t *testing.T) {
 
 func testERPolicyRule(t *testing.T) {
 	t.Run("test ER policy rule", func(t *testing.T) {
-		if err := datapathManager.AddEveroutePolicyRule(rule1, "rule1", POLICY_DIRECTION_IN, POLICY_TIER2, "work"); err != nil {
+		if err := datapathManager.AddEveroutePolicyRule(rule1, "rule1", POLICY_DIRECTION_IN, POLICY_TIER2, DEFAULT_POLICY_ENFORCEMENT_MODE); err != nil {
 			t.Errorf("Failed to add ER policy rule: %v, error: %v", rule1, err)
 		}
 		if _, ok := datapathManager.Rules[rule1.RuleID]; !ok {
@@ -156,13 +156,13 @@ func testERPolicyRule(t *testing.T) {
 			t.Errorf("Failed to remove ER policy rule, rule %v in cache", rule1)
 		}
 
-		if err := datapathManager.AddEveroutePolicyRule(rule2, "rule2", POLICY_DIRECTION_OUT, POLICY_TIER1, "work"); err != nil {
+		if err := datapathManager.AddEveroutePolicyRule(rule2, "rule2", POLICY_DIRECTION_OUT, POLICY_TIER1, DEFAULT_POLICY_ENFORCEMENT_MODE); err != nil {
 			t.Errorf("Failed to add ER policy rule: %v, error: %v", rule2, err)
 		}
 		if _, ok := datapathManager.Rules[rule2.RuleID]; !ok {
 			t.Errorf("Failed to add ER policy rule, not found %v in cache", rule2)
 		}
-		if err := datapathManager.AddEveroutePolicyRule(rule2, "rule2", POLICY_DIRECTION_OUT, POLICY_TIER1, "work"); err != nil {
+		if err := datapathManager.AddEveroutePolicyRule(rule2, "rule2", POLICY_DIRECTION_OUT, POLICY_TIER1, DEFAULT_POLICY_ENFORCEMENT_MODE); err != nil {
 			t.Errorf("Failed to add ER policy rule: %v, error: %v", rule2, err)
 		}
 	})
@@ -177,7 +177,7 @@ func testFlowReplay(t *testing.T) {
 	t.Run("add ER policy rule", func(t *testing.T) {
 		Eventually(func() error {
 			log.Infof("add policy rule to datapath, tier: %d", POLICY_TIER3)
-			return datapathManager.AddEveroutePolicyRule(rule1, "rule1", POLICY_DIRECTION_IN, POLICY_TIER3, "work")
+			return datapathManager.AddEveroutePolicyRule(rule1, "rule1", POLICY_DIRECTION_IN, POLICY_TIER3, DEFAULT_POLICY_ENFORCEMENT_MODE)
 		}, timeout, interval).Should(Succeed())
 	})
 
diff --git a/pkg/agent/datapath/policyBridge.go b/pkg/agent/datapath/policyBridge.go
index 52d0b41bb..be6fb9d51 100644
--- a/pkg/agent/datapath/policyBridge.go
+++ b/pkg/agent/datapath/policyBridge.go
@@ -15,25 +15,23 @@ import (
 
 //nolint
 const (
-	INPUT_TABLE                = 0
-	CT_STATE_TABLE             = 1
-	DIRECTION_SELECTION_TABLE  = 10
-	EGRESS_TIER1_TABLE         = 20
-	EGRESS_TIER2_MONITOR_TABLE = 24
-	EGRESS_TIER2_TABLE         = 25
-	EGRESS_TIER3_MONITOR_TABLE = 29
-	EGRESS_TIER3_TABLE         = 30
-	// EGRESS_DROP_TABLR           = 31
+	INPUT_TABLE                 = 0
+	CT_STATE_TABLE              = 1
+	DIRECTION_SELECTION_TABLE   = 10
+	EGRESS_TIER1_TABLE          = 20
+	EGRESS_TIER2_MONITOR_TABLE  = 24
+	EGRESS_TIER2_TABLE          = 25
+	EGRESS_TIER3_MONITOR_TABLE  = 29
+	EGRESS_TIER3_TABLE          = 30
 	INGRESS_TIER1_TABLE         = 50
 	INGRESS_TIER2_MONITOR_TABLE = 54
 	INGRESS_TIER2_TABLE         = 55
 	INGRESS_TIER3_MONITOR_TABLE = 59
 	INGRESS_TIER3_TABLE         = 60
-	// INGRESS_DROP_TABLE          = 61
-	CT_COMMIT_TABLE         = 70
-	CT_DROP_TABLE           = 71
-	SFC_POLICY_TABLE        = 80
-	POLICY_FORWARDING_TABLE = 90
+	CT_COMMIT_TABLE             = 70
+	CT_DROP_TABLE               = 71
+	SFC_POLICY_TABLE            = 80
+	POLICY_FORWARDING_TABLE     = 90
 )
 
 type PolicyBridge struct {
@@ -41,25 +39,23 @@ type PolicyBridge struct {
 	OfSwitch        *ofctrl.OFSwitch
 	datapathManager *DpManager
 
-	inputTable                    *ofctrl.Table
-	ctStateTable                  *ofctrl.Table
-	directionSelectionTable       *ofctrl.Table
-	egressTier1PolicyTable        *ofctrl.Table
-	egressTier2PolicyMonitorTable *ofctrl.Table
-	egressTier2PolicyTable        *ofctrl.Table
-	egressTier3PolicyMonitorTable *ofctrl.Table
-	egressTier3PolicyTable        *ofctrl.Table
-	// egressDropTable                *ofctrl.Table
+	inputTable                     *ofctrl.Table
+	ctStateTable                   *ofctrl.Table
+	directionSelectionTable        *ofctrl.Table
+	egressTier1PolicyTable         *ofctrl.Table
+	egressTier2PolicyMonitorTable  *ofctrl.Table
+	egressTier2PolicyTable         *ofctrl.Table
+	egressTier3PolicyMonitorTable  *ofctrl.Table
+	egressTier3PolicyTable         *ofctrl.Table
 	ingressTier1PolicyTable        *ofctrl.Table
 	ingressTier2PolicyMonitorTable *ofctrl.Table
 	ingressTier2PolicyTable        *ofctrl.Table
 	ingressTier3PolicyMonitorTable *ofctrl.Table
 	ingressTier3PolicyTable        *ofctrl.Table
-	// ingressDropTable               *ofctrl.Table
-	ctCommitTable         *ofctrl.Table
-	ctDropTable           *ofctrl.Table
-	sfcPolicyTable        *ofctrl.Table
-	policyForwardingTable *ofctrl.Table
+	ctCommitTable                  *ofctrl.Table
+	ctDropTable                    *ofctrl.Table
+	sfcPolicyTable                 *ofctrl.Table
+	policyForwardingTable          *ofctrl.Table
 
 	policySwitchStatusMutex sync.RWMutex
 	isPolicySwitchConnected bool
@@ -130,13 +126,11 @@ func (p *PolicyBridge) BridgeInit() {
 	p.ingressTier2PolicyTable, _ = sw.NewTable(INGRESS_TIER2_TABLE)
 	p.ingressTier3PolicyMonitorTable, _ = sw.NewTable(INGRESS_TIER3_MONITOR_TABLE)
 	p.ingressTier3PolicyTable, _ = sw.NewTable(INGRESS_TIER3_TABLE)
-	// p.ingressDropTable, _ = sw.NewTable(INGRESS_DROP_TABLE)
 	p.egressTier1PolicyTable, _ = sw.NewTable(EGRESS_TIER1_TABLE)
 	p.egressTier2PolicyMonitorTable, _ = sw.NewTable(EGRESS_TIER2_MONITOR_TABLE)
 	p.egressTier2PolicyTable, _ = sw.NewTable(EGRESS_TIER2_TABLE)
 	p.egressTier3PolicyMonitorTable, _ = sw.NewTable(EGRESS_TIER3_MONITOR_TABLE)
 	p.egressTier3PolicyTable, _ = sw.NewTable(EGRESS_TIER3_TABLE)
-	// p.egressDropTable, _ = sw.NewTable(EGRESS_DROP_TABLR)
 	p.ctCommitTable, _ = sw.NewTable(CT_COMMIT_TABLE)
 	p.ctDropTable, _ = sw.NewTable(CT_DROP_TABLE)
 	p.sfcPolicyTable, _ = sw.NewTable(SFC_POLICY_TABLE)
@@ -354,28 +348,6 @@ func (p *PolicyBridge) initPolicyTable() error {
 		return fmt.Errorf("failed to install egress tier3 default flow, error: %v", err)
 	}
 
-	// egress drop table
-	// egressDropFlow, _ := p.egressDropTable.NewFlow(ofctrl.FlowMatch{
-	// 	Priority: DEFAULT_FLOW_MISS_PRIORITY,
-	// })
-	// if err := egressDropFlow.Next(p.ctCommitTable); err != nil {
-	// 	return fmt.Errorf("failed to install egress tier3 drop table flow, error: %v", err)
-	// }
-	// ctTrkState := openflow13.NewCTStates()
-	// ctTrkState.SetNew()
-	// ctTrkState.SetTrk()
-	// egressDropTableCtCommitFlow, _ := p.egressDropTable.NewFlow(ofctrl.FlowMatch{
-	// 	Priority:  NORMAL_MATCH_FLOW_PRIORITY,
-	// 	Ethertype: PROTOCOL_IP,
-	// 	CtStates:  ctTrkState,
-	// })
-	// var policyConntrackZone uint16 = 65520
-	// srcField, _ := openflow13.FindFieldHeaderByName("nxm_nx_xxreg0", false)
-	// dstField, _ := openflow13.FindFieldHeaderByName("nxm_nx_ct_label", false)
-	// moveAct := openflow13.NewNXActionRegMove(128, 0, 0, srcField, dstField)
-	// ctCommitAction := ofctrl.NewConntrackAction(true, false, &p.ctCommitTable.TableId, &policyConntrackZone, moveAct)
-	// _ = egressDropTableCtCommitFlow.SetConntrack(ctCommitAction)
-
 	// ingress policy table
 	ingressTier1DefaultFlow, _ := p.ingressTier1PolicyTable.NewFlow(ofctrl.FlowMatch{
 		Priority: DEFAULT_FLOW_MISS_PRIORITY,
@@ -408,22 +380,6 @@ func (p *PolicyBridge) initPolicyTable() error {
 		return fmt.Errorf("failed to install ingress tier3 default flow, error: %v", err)
 	}
 
-	// ingress tier3 drop table
-	// ingressDropFlow, _ := p.ingressDropTable.NewFlow(ofctrl.FlowMatch{
-	// 	Priority: DEFAULT_FLOW_MISS_PRIORITY,
-	// })
-	// if err := ingressDropFlow.Next(p.ctCommitTable); err != nil {
-	// 	return fmt.Errorf("failed to install ingress tier3 drop table flow, error: %v", err)
-	// }
-	// ingressDropTableCtCommitFlow, _ := p.ingressDropTable.NewFlow(ofctrl.FlowMatch{
-	// 	Priority:  NORMAL_MATCH_FLOW_PRIORITY,
-	// 	Ethertype: PROTOCOL_IP,
-	// 	CtStates:  ctTrkState,
-	// })
-	// moveAct = openflow13.NewNXActionRegMove(128, 0, 0, srcField, dstField)
-	// ctCommitAction = ofctrl.NewConntrackAction(true, false, &p.ctCommitTable.TableId, &policyConntrackZone, moveAct)
-	// _ = ingressDropTableCtCommitFlow.SetConntrack(ctCommitAction)
-
 	// sfc policy table
 	sfcPolicyTableDefaultFlow, _ := p.sfcPolicyTable.NewFlow(ofctrl.FlowMatch{
 		Priority: DEFAULT_FLOW_MISS_PRIORITY,
@@ -560,9 +516,6 @@ func (p *PolicyBridge) AddMicroSegmentRule(rule *EveroutePolicyRule, direction u
 	var ipSa *net.IP = nil
 	var ipSaMask *net.IP = nil
 	var err error
-	if mode == "" {
-		mode = DEFAULT_POLICY_ENFORCEMENT_MODE
-	}
 
 	// make sure switch is connected
 	if !p.IsSwitchConnected() {
diff --git a/pkg/apis/security/v1alpha1/types.go b/pkg/apis/security/v1alpha1/types.go
index fb30c05ec..507672879 100644
--- a/pkg/apis/security/v1alpha1/types.go
+++ b/pkg/apis/security/v1alpha1/types.go
@@ -71,6 +71,7 @@ type SecurityPolicySpec struct {
 	Tier string `json:"tier"`
 
 	// Work mode specify the policy enforcement state: monitor or work
+	// +kubebuilder:default=work
 	SecurityPolicyEnforcementMode PolicyMode `json:"securityPolicyEnforcementMode,omitempty"`
 
 	// SymmetricMode will generate symmetry rules for the policy.
diff --git a/pkg/openapi/openapi_generated.go b/pkg/openapi/openapi_generated.go
index 827f0ae17..069109900 100644
--- a/pkg/openapi/openapi_generated.go
+++ b/pkg/openapi/openapi_generated.go
@@ -144,6 +144,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA
 		"k8s.io/api/core/v1.EnvVarSource":                                            schema_k8sio_api_core_v1_EnvVarSource(ref),
 		"k8s.io/api/core/v1.EphemeralContainer":                                      schema_k8sio_api_core_v1_EphemeralContainer(ref),
 		"k8s.io/api/core/v1.EphemeralContainerCommon":                                schema_k8sio_api_core_v1_EphemeralContainerCommon(ref),
+		"k8s.io/api/core/v1.EphemeralContainers":                                     schema_k8sio_api_core_v1_EphemeralContainers(ref),
 		"k8s.io/api/core/v1.EphemeralVolumeSource":                                   schema_k8sio_api_core_v1_EphemeralVolumeSource(ref),
 		"k8s.io/api/core/v1.Event":                                                   schema_k8sio_api_core_v1_Event(ref),
 		"k8s.io/api/core/v1.EventList":                                               schema_k8sio_api_core_v1_EventList(ref),
@@ -308,11 +309,11 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA
 		"k8s.io/apimachinery/pkg/apis/meta/v1.APIResource":                           schema_pkg_apis_meta_v1_APIResource(ref),
 		"k8s.io/apimachinery/pkg/apis/meta/v1.APIResourceList":                       schema_pkg_apis_meta_v1_APIResourceList(ref),
 		"k8s.io/apimachinery/pkg/apis/meta/v1.APIVersions":                           schema_pkg_apis_meta_v1_APIVersions(ref),
-		"k8s.io/apimachinery/pkg/apis/meta/v1.ApplyOptions":                          schema_pkg_apis_meta_v1_ApplyOptions(ref),
 		"k8s.io/apimachinery/pkg/apis/meta/v1.Condition":                             schema_pkg_apis_meta_v1_Condition(ref),
 		"k8s.io/apimachinery/pkg/apis/meta/v1.CreateOptions":                         schema_pkg_apis_meta_v1_CreateOptions(ref),
 		"k8s.io/apimachinery/pkg/apis/meta/v1.DeleteOptions":                         schema_pkg_apis_meta_v1_DeleteOptions(ref),
 		"k8s.io/apimachinery/pkg/apis/meta/v1.Duration":                              schema_pkg_apis_meta_v1_Duration(ref),
+		"k8s.io/apimachinery/pkg/apis/meta/v1.ExportOptions":                         schema_pkg_apis_meta_v1_ExportOptions(ref),
 		"k8s.io/apimachinery/pkg/apis/meta/v1.FieldsV1":                              schema_pkg_apis_meta_v1_FieldsV1(ref),
 		"k8s.io/apimachinery/pkg/apis/meta/v1.GetOptions":                            schema_pkg_apis_meta_v1_GetOptions(ref),
 		"k8s.io/apimachinery/pkg/apis/meta/v1.GroupKind":                             schema_pkg_apis_meta_v1_GroupKind(ref),
@@ -2339,7 +2340,7 @@ func schema_k8sio_api_apps_v1_Deployment(ref common.ReferenceCallback) common.Op
 					},
 					"metadata": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata",
+							Description: "Standard object metadata.",
 							Ref:         ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"),
 						},
 					},
@@ -2920,13 +2921,7 @@ func schema_k8sio_api_apps_v1_RollingUpdateDaemonSet(ref common.ReferenceCallbac
 				Properties: map[string]spec.Schema{
 					"maxUnavailable": {
 						SchemaProps: spec.SchemaProps{
-							Description: "The maximum number of DaemonSet pods that can be unavailable during the update. Value can be an absolute number (ex: 5) or a percentage of total number of DaemonSet pods at the start of the update (ex: 10%). Absolute number is calculated from percentage by rounding up. This cannot be 0 if MaxSurge is 0 Default value is 1. Example: when this is set to 30%, at most 30% of the total number of nodes that should be running the daemon pod (i.e. status.desiredNumberScheduled) can have their pods stopped for an update at any given time. The update starts by stopping at most 30% of those DaemonSet pods and then brings up new DaemonSet pods in their place. Once the new pods are available, it then proceeds onto other DaemonSet pods, thus ensuring that at least 70% of original number of DaemonSet pods are available at all times during the update.",
-							Ref:         ref("k8s.io/apimachinery/pkg/util/intstr.IntOrString"),
-						},
-					},
-					"maxSurge": {
-						SchemaProps: spec.SchemaProps{
-							Description: "The maximum number of nodes with an existing available DaemonSet pod that can have an updated DaemonSet pod during during an update. Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). This can not be 0 if MaxUnavailable is 0. Absolute number is calculated from percentage by rounding up to a minimum of 1. Default value is 0. Example: when this is set to 30%, at most 30% of the total number of nodes that should be running the daemon pod (i.e. status.desiredNumberScheduled) can have their a new pod created before the old pod is marked as deleted. The update starts by launching new pods on 30% of nodes. Once an updated pod is available (Ready for at least minReadySeconds) the old DaemonSet pod on that node is marked deleted. If the old pod becomes unavailable for any reason (Ready transitions to false, is evicted, or is drained) an updated pod is immediatedly created on that node without considering surge limits. Allowing surge implies the possibility that the resources consumed by the daemonset on any given node can double if the readiness check fails, and so resource intensive daemonsets should take into account that they may cause evictions during disruption. This is beta field and enabled/disabled by DaemonSetUpdateSurge feature gate.",
+							Description: "The maximum number of DaemonSet pods that can be unavailable during the update. Value can be an absolute number (ex: 5) or a percentage of total number of DaemonSet pods at the start of the update (ex: 10%). Absolute number is calculated from percentage by rounding up. This cannot be 0. Default value is 1. Example: when this is set to 30%, at most 30% of the total number of nodes that should be running the daemon pod (i.e. status.desiredNumberScheduled) can have their pods stopped for an update at any given time. The update starts by stopping at most 30% of those DaemonSet pods and then brings up new DaemonSet pods in their place. Once the new pods are available, it then proceeds onto other DaemonSet pods, thus ensuring that at least 70% of original number of DaemonSet pods are available at all times during the update.",
 							Ref:         ref("k8s.io/apimachinery/pkg/util/intstr.IntOrString"),
 						},
 					},
@@ -3008,8 +3003,7 @@ func schema_k8sio_api_apps_v1_StatefulSet(ref common.ReferenceCallback) common.O
 					},
 					"metadata": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata",
-							Ref:         ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"),
+							Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"),
 						},
 					},
 					"spec": {
@@ -3105,14 +3099,12 @@ func schema_k8sio_api_apps_v1_StatefulSetList(ref common.ReferenceCallback) comm
 					},
 					"metadata": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Standard list's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata",
-							Ref:         ref("k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"),
+							Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"),
 						},
 					},
 					"items": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Items is the list of stateful sets.",
-							Type:        []string{"array"},
+							Type: []string{"array"},
 							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
 									SchemaProps: spec.SchemaProps{
@@ -3197,13 +3189,6 @@ func schema_k8sio_api_apps_v1_StatefulSetSpec(ref common.ReferenceCallback) comm
 							Format:      "int32",
 						},
 					},
-					"minReadySeconds": {
-						SchemaProps: spec.SchemaProps{
-							Description: "Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to be considered available. Defaults to 0 (pod will be considered available as soon as it is ready) This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate.",
-							Type:        []string{"integer"},
-							Format:      "int32",
-						},
-					},
 				},
 				Required: []string{"selector", "template", "serviceName"},
 			},
@@ -3295,13 +3280,6 @@ func schema_k8sio_api_apps_v1_StatefulSetStatus(ref common.ReferenceCallback) co
 							},
 						},
 					},
-					"availableReplicas": {
-						SchemaProps: spec.SchemaProps{
-							Description: "Total number of available pods (ready for at least minReadySeconds) targeted by this statefulset. This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate. Remove omitempty when graduating to beta",
-							Type:        []string{"integer"},
-							Format:      "int32",
-						},
-					},
 				},
 				Required: []string{"replicas"},
 			},
@@ -4235,7 +4213,7 @@ func schema_k8sio_api_core_v1_ConfigMap(ref common.ReferenceCallback) common.Ope
 					},
 					"immutable": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Immutable, if set to true, ensures that data stored in the ConfigMap cannot be updated (only object metadata can be modified). If not set to true, the field can be modified at any time. Defaulted to nil.",
+							Description: "Immutable, if set to true, ensures that data stored in the ConfigMap cannot be updated (only object metadata can be modified). If not set to true, the field can be modified at any time. Defaulted to nil. This is a beta field enabled by ImmutableEphemeralVolumes feature gate.",
 							Type:        []string{"boolean"},
 							Format:      "",
 						},
@@ -4393,7 +4371,7 @@ func schema_k8sio_api_core_v1_ConfigMapNodeConfigSource(ref common.ReferenceCall
 	return common.OpenAPIDefinition{
 		Schema: spec.Schema{
 			SchemaProps: spec.SchemaProps{
-				Description: "ConfigMapNodeConfigSource contains the information to reference a ConfigMap as a config source for the Node. This API is deprecated since 1.22: https://git.k8s.io/enhancements/keps/sig-node/281-dynamic-kubelet-configuration",
+				Description: "ConfigMapNodeConfigSource contains the information to reference a ConfigMap as a config source for the Node.",
 				Type:        []string{"object"},
 				Properties: map[string]spec.Schema{
 					"namespace": {
@@ -4552,7 +4530,7 @@ func schema_k8sio_api_core_v1_Container(ref common.ReferenceCallback) common.Ope
 					},
 					"command": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
+							Description: "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
 							Type:        []string{"array"},
 							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
@@ -4566,7 +4544,7 @@ func schema_k8sio_api_core_v1_Container(ref common.ReferenceCallback) common.Ope
 					},
 					"args": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
+							Description: "Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
 							Type:        []string{"array"},
 							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
@@ -4643,7 +4621,7 @@ func schema_k8sio_api_core_v1_Container(ref common.ReferenceCallback) common.Ope
 					},
 					"resources": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/",
+							Description: "Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/",
 							Ref:         ref("k8s.io/api/core/v1.ResourceRequirements"),
 						},
 					},
@@ -4732,7 +4710,7 @@ func schema_k8sio_api_core_v1_Container(ref common.ReferenceCallback) common.Ope
 					},
 					"securityContext": {
 						SchemaProps: spec.SchemaProps{
-							Description: "SecurityContext defines the security options the container should be run with. If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/",
+							Description: "Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/",
 							Ref:         ref("k8s.io/api/core/v1.SecurityContext"),
 						},
 					},
@@ -4795,6 +4773,7 @@ func schema_k8sio_api_core_v1_ContainerImage(ref common.ReferenceCallback) commo
 						},
 					},
 				},
+				Required: []string{"names"},
 			},
 		},
 	}
@@ -5297,7 +5276,7 @@ func schema_k8sio_api_core_v1_EndpointPort(ref common.ReferenceCallback) common.
 					},
 					"appProtocol": {
 						SchemaProps: spec.SchemaProps{
-							Description: "The application protocol for this port. This field follows standard Kubernetes label syntax. Un-prefixed names are reserved for IANA standard service names (as per RFC-6335 and http://www.iana.org/assignments/service-names). Non-standard protocols should use prefixed names such as mycompany.com/my-custom-protocol.",
+							Description: "The application protocol for this port. This field follows standard Kubernetes label syntax. Un-prefixed names are reserved for IANA standard service names (as per RFC-6335 and http://www.iana.org/assignments/service-names). Non-standard protocols should use prefixed names such as mycompany.com/my-custom-protocol. This is a beta field that is guarded by the ServiceAppProtocol feature gate and enabled by default.",
 							Type:        []string{"string"},
 							Format:      "",
 						},
@@ -5510,7 +5489,7 @@ func schema_k8sio_api_core_v1_EnvVar(ref common.ReferenceCallback) common.OpenAP
 					},
 					"value": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Variable references $(VAR_NAME) are expanded using the previously defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \"\".",
+							Description: "Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \"\".",
 							Type:        []string{"string"},
 							Format:      "",
 						},
@@ -5592,7 +5571,7 @@ func schema_k8sio_api_core_v1_EphemeralContainer(ref common.ReferenceCallback) c
 					},
 					"command": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
+							Description: "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
 							Type:        []string{"array"},
 							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
@@ -5606,7 +5585,7 @@ func schema_k8sio_api_core_v1_EphemeralContainer(ref common.ReferenceCallback) c
 					},
 					"args": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
+							Description: "Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
 							Type:        []string{"array"},
 							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
@@ -5761,7 +5740,7 @@ func schema_k8sio_api_core_v1_EphemeralContainer(ref common.ReferenceCallback) c
 					},
 					"securityContext": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Optional: SecurityContext defines the security options the ephemeral container should be run with. If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext.",
+							Description: "SecurityContext is not allowed for ephemeral containers.",
 							Ref:         ref("k8s.io/api/core/v1.SecurityContext"),
 						},
 					},
@@ -5825,7 +5804,7 @@ func schema_k8sio_api_core_v1_EphemeralContainerCommon(ref common.ReferenceCallb
 					},
 					"command": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
+							Description: "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
 							Type:        []string{"array"},
 							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
@@ -5839,7 +5818,7 @@ func schema_k8sio_api_core_v1_EphemeralContainerCommon(ref common.ReferenceCallb
 					},
 					"args": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
+							Description: "Arguments to the entrypoint. The docker image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell",
 							Type:        []string{"array"},
 							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
@@ -5994,7 +5973,7 @@ func schema_k8sio_api_core_v1_EphemeralContainerCommon(ref common.ReferenceCallb
 					},
 					"securityContext": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Optional: SecurityContext defines the security options the ephemeral container should be run with. If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext.",
+							Description: "SecurityContext is not allowed for ephemeral containers.",
 							Ref:         ref("k8s.io/api/core/v1.SecurityContext"),
 						},
 					},
@@ -6028,6 +6007,60 @@ func schema_k8sio_api_core_v1_EphemeralContainerCommon(ref common.ReferenceCallb
 	}
 }
 
+func schema_k8sio_api_core_v1_EphemeralContainers(ref common.ReferenceCallback) common.OpenAPIDefinition {
+	return common.OpenAPIDefinition{
+		Schema: spec.Schema{
+			SchemaProps: spec.SchemaProps{
+				Description: "A list of ephemeral containers used with the Pod ephemeralcontainers subresource.",
+				Type:        []string{"object"},
+				Properties: map[string]spec.Schema{
+					"kind": {
+						SchemaProps: spec.SchemaProps{
+							Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds",
+							Type:        []string{"string"},
+							Format:      "",
+						},
+					},
+					"apiVersion": {
+						SchemaProps: spec.SchemaProps{
+							Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources",
+							Type:        []string{"string"},
+							Format:      "",
+						},
+					},
+					"metadata": {
+						SchemaProps: spec.SchemaProps{
+							Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"),
+						},
+					},
+					"ephemeralContainers": {
+						VendorExtensible: spec.VendorExtensible{
+							Extensions: spec.Extensions{
+								"x-kubernetes-patch-merge-key": "name",
+								"x-kubernetes-patch-strategy":  "merge",
+							},
+						},
+						SchemaProps: spec.SchemaProps{
+							Description: "A list of ephemeral containers associated with this pod. New ephemeral containers may be appended to this list, but existing ephemeral containers may not be removed or modified.",
+							Type:        []string{"array"},
+							Items: &spec.SchemaOrArray{
+								Schema: &spec.Schema{
+									SchemaProps: spec.SchemaProps{
+										Ref: ref("k8s.io/api/core/v1.EphemeralContainer"),
+									},
+								},
+							},
+						},
+					},
+				},
+				Required: []string{"ephemeralContainers"},
+			},
+		},
+		Dependencies: []string{
+			"k8s.io/api/core/v1.EphemeralContainer", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"},
+	}
+}
+
 func schema_k8sio_api_core_v1_EphemeralVolumeSource(ref common.ReferenceCallback) common.OpenAPIDefinition {
 	return common.OpenAPIDefinition{
 		Schema: spec.Schema{
@@ -6041,6 +6074,13 @@ func schema_k8sio_api_core_v1_EphemeralVolumeSource(ref common.ReferenceCallback
 							Ref:         ref("k8s.io/api/core/v1.PersistentVolumeClaimTemplate"),
 						},
 					},
+					"readOnly": {
+						SchemaProps: spec.SchemaProps{
+							Description: "Specifies a read-only configuration for the volume. Defaults to false (read/write).",
+							Type:        []string{"boolean"},
+							Format:      "",
+						},
+					},
 				},
 			},
 		},
@@ -7268,7 +7308,7 @@ func schema_k8sio_api_core_v1_LimitRangeList(ref common.ReferenceCallback) commo
 					},
 					"items": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Items is a list of LimitRange objects. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/",
+							Description: "Items is a list of LimitRange objects. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/",
 							Type:        []string{"array"},
 							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
@@ -7904,7 +7944,7 @@ func schema_k8sio_api_core_v1_NodeConfigSource(ref common.ReferenceCallback) com
 	return common.OpenAPIDefinition{
 		Schema: spec.Schema{
 			SchemaProps: spec.SchemaProps{
-				Description: "NodeConfigSource specifies a source of node configuration. Exactly one subfield (excluding metadata) must be non-nil. This API is deprecated since 1.22",
+				Description: "NodeConfigSource specifies a source of node configuration. Exactly one subfield (excluding metadata) must be non-nil.",
 				Type:        []string{"object"},
 				Properties: map[string]spec.Schema{
 					"configMap": {
@@ -8269,7 +8309,7 @@ func schema_k8sio_api_core_v1_NodeSpec(ref common.ReferenceCallback) common.Open
 					},
 					"configSource": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Deprecated. If specified, the source of the node's configuration. The DynamicKubeletConfig feature gate must be enabled for the Kubelet to use this field. This field is deprecated as of 1.22: https://git.k8s.io/enhancements/keps/sig-node/281-dynamic-kubelet-configuration",
+							Description: "If specified, the source to get node configuration from The DynamicKubeletConfig feature gate must be enabled for the Kubelet to use this field",
 							Ref:         ref("k8s.io/api/core/v1.NodeConfigSource"),
 						},
 					},
@@ -8861,13 +8901,7 @@ func schema_k8sio_api_core_v1_PersistentVolumeClaimSpec(ref common.ReferenceCall
 					},
 					"dataSource": {
 						SchemaProps: spec.SchemaProps{
-							Description: "This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) * An existing PVC (PersistentVolumeClaim) If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source. If the AnyVolumeDataSource feature gate is enabled, this field will always have the same contents as the DataSourceRef field.",
-							Ref:         ref("k8s.io/api/core/v1.TypedLocalObjectReference"),
-						},
-					},
-					"dataSourceRef": {
-						SchemaProps: spec.SchemaProps{
-							Description: "Specifies the object from which to populate the volume with data, if a non-empty volume is desired. This may be any local object from a non-empty API group (non core object) or a PersistentVolumeClaim object. When this field is specified, volume binding will only succeed if the type of the specified object matches some installed volume populator or dynamic provisioner. This field will replace the functionality of the DataSource field and as such if both fields are non-empty, they must have the same value. For backwards compatibility, both fields (DataSource and DataSourceRef) will be set to the same value automatically if one of them is empty and the other is non-empty. There are two important differences between DataSource and DataSourceRef: * While DataSource only allows two specific types of objects, DataSourceRef\n  allows any non-core object, as well as PersistentVolumeClaim objects.\n* While DataSource ignores disallowed values (dropping them), DataSourceRef\n  preserves all values, and generates an error if a disallowed value is\n  specified.\n(Alpha) Using this field requires the AnyVolumeDataSource feature gate to be enabled.",
+							Description: "This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) * An existing PVC (PersistentVolumeClaim) * An existing custom resource that implements data population (Alpha) In order to use custom resource types that implement data population, the AnyVolumeDataSource feature gate must be enabled. If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source.",
 							Ref:         ref("k8s.io/api/core/v1.TypedLocalObjectReference"),
 						},
 					},
@@ -9587,7 +9621,7 @@ func schema_k8sio_api_core_v1_PodAffinityTerm(ref common.ReferenceCallback) comm
 					},
 					"namespaces": {
 						SchemaProps: spec.SchemaProps{
-							Description: "namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means \"this pod's namespace\"",
+							Description: "namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"",
 							Type:        []string{"array"},
 							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
@@ -9606,12 +9640,6 @@ func schema_k8sio_api_core_v1_PodAffinityTerm(ref common.ReferenceCallback) comm
 							Format:      "",
 						},
 					},
-					"namespaceSelector": {
-						SchemaProps: spec.SchemaProps{
-							Description: "A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means \"this pod's namespace\". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled.",
-							Ref:         ref("k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector"),
-						},
-					},
 				},
 				Required: []string{"topologyKey"},
 			},
@@ -10403,7 +10431,7 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA
 					},
 					"terminationGracePeriodSeconds": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). If this value is nil, the default grace period will be used instead. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. Defaults to 30 seconds.",
+							Description: "Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. Value must be non-negative integer. The value zero indicates delete immediately. If this value is nil, the default grace period will be used instead. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. Defaults to 30 seconds.",
 							Type:        []string{"integer"},
 							Format:      "int64",
 						},
@@ -10599,7 +10627,7 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA
 					},
 					"readinessGates": {
 						SchemaProps: spec.SchemaProps{
-							Description: "If specified, all readiness gates will be evaluated for pod readiness. A pod is ready when all its containers are ready AND all conditions specified in the readiness gates have status equal to \"True\" More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates",
+							Description: "If specified, all readiness gates will be evaluated for pod readiness. A pod is ready when all its containers are ready AND all conditions specified in the readiness gates have status equal to \"True\" More info: https://git.k8s.io/enhancements/keps/sig-network/0007-pod-ready%2B%2B.md",
 							Type:        []string{"array"},
 							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
@@ -10612,7 +10640,7 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA
 					},
 					"runtimeClassName": {
 						SchemaProps: spec.SchemaProps{
-							Description: "RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used to run this pod.  If no RuntimeClass resource matches the named class, the pod will not be run. If unset or empty, the \"legacy\" RuntimeClass will be used, which is an implicit class with an empty definition that uses the default runtime handler. More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class This is a beta feature as of Kubernetes v1.14.",
+							Description: "RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used to run this pod.  If no RuntimeClass resource matches the named class, the pod will not be run. If unset or empty, the \"legacy\" RuntimeClass will be used, which is an implicit class with an empty definition that uses the default runtime handler. More info: https://git.k8s.io/enhancements/keps/sig-node/runtime-class.md This is a beta feature as of Kubernetes v1.14.",
 							Type:        []string{"string"},
 							Format:      "",
 						},
@@ -10633,7 +10661,7 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA
 					},
 					"overhead": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. This field will be autopopulated at admission time by the RuntimeClass admission controller. If the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. The RuntimeClass admission controller will reject Pod create requests which have the overhead already set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md This field is beta-level as of Kubernetes v1.18, and is only honored by servers that enable the PodOverhead feature.",
+							Description: "Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. This field will be autopopulated at admission time by the RuntimeClass admission controller. If the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. The RuntimeClass admission controller will reject Pod create requests which have the overhead already set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. More info: https://git.k8s.io/enhancements/keps/sig-node/20190226-pod-overhead.md This field is alpha-level as of Kubernetes v1.16, and is only honored by servers that enable the PodOverhead feature.",
 							Type:        []string{"object"},
 							AdditionalProperties: &spec.SchemaOrBool{
 								Allows: true,
@@ -11190,13 +11218,6 @@ func schema_k8sio_api_core_v1_Probe(ref common.ReferenceCallback) common.OpenAPI
 							Format:      "int32",
 						},
 					},
-					"terminationGracePeriodSeconds": {
-						SchemaProps: spec.SchemaProps{
-							Description: "Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset.",
-							Type:        []string{"integer"},
-							Format:      "int64",
-						},
-					},
 				},
 			},
 		},
@@ -12001,7 +12022,7 @@ func schema_k8sio_api_core_v1_ResourceRequirements(ref common.ReferenceCallback)
 				Properties: map[string]spec.Schema{
 					"limits": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/",
+							Description: "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/",
 							Type:        []string{"object"},
 							AdditionalProperties: &spec.SchemaOrBool{
 								Allows: true,
@@ -12015,7 +12036,7 @@ func schema_k8sio_api_core_v1_ResourceRequirements(ref common.ReferenceCallback)
 					},
 					"requests": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/",
+							Description: "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/",
 							Type:        []string{"object"},
 							AdditionalProperties: &spec.SchemaOrBool{
 								Allows: true,
@@ -12385,7 +12406,7 @@ func schema_k8sio_api_core_v1_Secret(ref common.ReferenceCallback) common.OpenAP
 					},
 					"immutable": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Immutable, if set to true, ensures that data stored in the Secret cannot be updated (only object metadata can be modified). If not set to true, the field can be modified at any time. Defaulted to nil.",
+							Description: "Immutable, if set to true, ensures that data stored in the Secret cannot be updated (only object metadata can be modified). If not set to true, the field can be modified at any time. Defaulted to nil. This is a beta field enabled by ImmutableEphemeralVolumes feature gate.",
 							Type:        []string{"boolean"},
 							Format:      "",
 						},
@@ -12407,7 +12428,7 @@ func schema_k8sio_api_core_v1_Secret(ref common.ReferenceCallback) common.OpenAP
 					},
 					"stringData": {
 						SchemaProps: spec.SchemaProps{
-							Description: "stringData allows specifying non-binary secret data in string form. It is provided as a write-only input field for convenience. All keys and values are merged into the data field on write, overwriting any existing values. The stringData field is never output when reading from the API.",
+							Description: "stringData allows specifying non-binary secret data in string form. It is provided as a write-only convenience method. All keys and values are merged into the data field on write, overwriting any existing values. It is never output when reading from the API.",
 							Type:        []string{"object"},
 							AdditionalProperties: &spec.SchemaOrBool{
 								Allows: true,
@@ -13064,7 +13085,7 @@ func schema_k8sio_api_core_v1_ServicePort(ref common.ReferenceCallback) common.O
 					},
 					"appProtocol": {
 						SchemaProps: spec.SchemaProps{
-							Description: "The application protocol for this port. This field follows standard Kubernetes label syntax. Un-prefixed names are reserved for IANA standard service names (as per RFC-6335 and http://www.iana.org/assignments/service-names). Non-standard protocols should use prefixed names such as mycompany.com/my-custom-protocol.",
+							Description: "The application protocol for this port. This field follows standard Kubernetes label syntax. Un-prefixed names are reserved for IANA standard service names (as per RFC-6335 and http://www.iana.org/assignments/service-names). Non-standard protocols should use prefixed names such as mycompany.com/my-custom-protocol. This is a beta field that is guarded by the ServiceAppProtocol feature gate and enabled by default.",
 							Type:        []string{"string"},
 							Format:      "",
 						},
@@ -13241,7 +13262,7 @@ func schema_k8sio_api_core_v1_ServiceSpec(ref common.ReferenceCallback) common.O
 					},
 					"loadBalancerSourceRanges": {
 						SchemaProps: spec.SchemaProps{
-							Description: "If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/",
+							Description: "If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/",
 							Type:        []string{"array"},
 							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
@@ -13255,7 +13276,7 @@ func schema_k8sio_api_core_v1_ServiceSpec(ref common.ReferenceCallback) common.O
 					},
 					"externalName": {
 						SchemaProps: spec.SchemaProps{
-							Description: "externalName is the external reference that discovery mechanisms will return as an alias for this service (e.g. a DNS CNAME record). No proxying will be involved.  Must be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires `type` to be \"ExternalName\".",
+							Description: "externalName is the external reference that discovery mechanisms will return as an alias for this service (e.g. a DNS CNAME record). No proxying will be involved.  Must be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be",
 							Type:        []string{"string"},
 							Format:      "",
 						},
@@ -13287,6 +13308,20 @@ func schema_k8sio_api_core_v1_ServiceSpec(ref common.ReferenceCallback) common.O
 							Ref:         ref("k8s.io/api/core/v1.SessionAffinityConfig"),
 						},
 					},
+					"topologyKeys": {
+						SchemaProps: spec.SchemaProps{
+							Description: "topologyKeys is a preference-order list of topology keys which implementations of services should use to preferentially sort endpoints when accessing this Service, it can not be used at the same time as externalTrafficPolicy=Local. Topology keys must be valid label keys and at most 16 keys may be specified. Endpoints are chosen based on the first topology key with available backends. If this field is specified and all entries have no backends that match the topology of the client, the service has no backends for that client and connections should fail. The special value \"*\" may be used to mean \"any topology\". This catch-all value, if used, only makes sense as the last value in the list. If this is not specified or empty, no topology constraints will be applied. This field is alpha-level and is only honored by servers that enable the ServiceTopology feature.",
+							Type:        []string{"array"},
+							Items: &spec.SchemaOrArray{
+								Schema: &spec.Schema{
+									SchemaProps: spec.SchemaProps{
+										Type:   []string{"string"},
+										Format: "",
+									},
+								},
+							},
+						},
+					},
 					"ipFamilies": {
 						VendorExtensible: spec.VendorExtensible{
 							Extensions: spec.Extensions{
@@ -13315,25 +13350,11 @@ func schema_k8sio_api_core_v1_ServiceSpec(ref common.ReferenceCallback) common.O
 					},
 					"allocateLoadBalancerNodePorts": {
 						SchemaProps: spec.SchemaProps{
-							Description: "allocateLoadBalancerNodePorts defines if NodePorts will be automatically allocated for services with type LoadBalancer.  Default is \"true\". It may be set to \"false\" if the cluster load-balancer does not rely on NodePorts.  If the caller requests specific NodePorts (by specifying a value), those requests will be respected, regardless of this field. This field may only be set for services with type LoadBalancer and will be cleared if the type is changed to any other type. This field is beta-level and is only honored by servers that enable the ServiceLBNodePortControl feature.",
+							Description: "allocateLoadBalancerNodePorts defines if NodePorts will be automatically allocated for services with type LoadBalancer.  Default is \"true\". It may be set to \"false\" if the cluster load-balancer does not rely on NodePorts. allocateLoadBalancerNodePorts may only be set for services with type LoadBalancer and will be cleared if the type is changed to any other type. This field is alpha-level and is only honored by servers that enable the ServiceLBNodePortControl feature.",
 							Type:        []string{"boolean"},
 							Format:      "",
 						},
 					},
-					"loadBalancerClass": {
-						SchemaProps: spec.SchemaProps{
-							Description: "loadBalancerClass is the class of the load balancer implementation this Service belongs to. If specified, the value of this field must be a label-style identifier, with an optional prefix, e.g. \"internal-vip\" or \"example.com/internal-vip\". Unprefixed names are reserved for end-users. This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load balancer implementation is used, today this is typically done through the cloud provider integration, but should apply for any default implementation. If set, it is assumed that a load balancer implementation is watching for Services with a matching class. Any default load balancer implementation (e.g. cloud providers) should ignore Services that set this field. This field can only be set when creating or updating a Service to type 'LoadBalancer'. Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type.",
-							Type:        []string{"string"},
-							Format:      "",
-						},
-					},
-					"internalTrafficPolicy": {
-						SchemaProps: spec.SchemaProps{
-							Description: "InternalTrafficPolicy specifies if the cluster internal traffic should be routed to all endpoints or node-local endpoints only. \"Cluster\" routes internal traffic to a Service to all endpoints. \"Local\" routes traffic to node-local endpoints only, traffic is dropped if no node-local endpoints are ready. The default value is \"Cluster\".",
-							Type:        []string{"string"},
-							Format:      "",
-						},
-					},
 				},
 			},
 		},
@@ -13978,7 +13999,7 @@ func schema_k8sio_api_core_v1_Volume(ref common.ReferenceCallback) common.OpenAP
 					},
 					"ephemeral": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Ephemeral represents a volume that is handled by a cluster storage driver. The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, and deleted when the pod is removed.\n\nUse this if: a) the volume is only needed while the pod runs, b) features of normal volumes like restoring from snapshot or capacity\n   tracking are needed,\nc) the storage driver is specified through a storage class, and d) the storage driver supports dynamic volume provisioning through\n   a PersistentVolumeClaim (see EphemeralVolumeSource for more\n   information on the connection between this volume type\n   and PersistentVolumeClaim).\n\nUse PersistentVolumeClaim or one of the vendor-specific APIs for volumes that persist for longer than the lifecycle of an individual pod.\n\nUse CSI for light-weight local ephemeral volumes if the CSI driver is meant to be used that way - see the documentation of the driver for more information.\n\nA pod can use both types of ephemeral volumes and persistent volumes at the same time.\n\nThis is a beta feature and only available when the GenericEphemeralVolume feature gate is enabled.",
+							Description: "Ephemeral represents a volume that is handled by a cluster storage driver (Alpha feature). The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, and deleted when the pod is removed.\n\nUse this if: a) the volume is only needed while the pod runs, b) features of normal volumes like restoring from snapshot or capacity\n   tracking are needed,\nc) the storage driver is specified through a storage class, and d) the storage driver supports dynamic volume provisioning through\n   a PersistentVolumeClaim (see EphemeralVolumeSource for more\n   information on the connection between this volume type\n   and PersistentVolumeClaim).\n\nUse PersistentVolumeClaim or one of the vendor-specific APIs for volumes that persist for longer than the lifecycle of an individual pod.\n\nUse CSI for light-weight local ephemeral volumes if the CSI driver is meant to be used that way - see the documentation of the driver for more information.\n\nA pod can use both types of ephemeral volumes and persistent volumes at the same time.",
 							Ref:         ref("k8s.io/api/core/v1.EphemeralVolumeSource"),
 						},
 					},
@@ -14312,7 +14333,7 @@ func schema_k8sio_api_core_v1_VolumeSource(ref common.ReferenceCallback) common.
 					},
 					"ephemeral": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Ephemeral represents a volume that is handled by a cluster storage driver. The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, and deleted when the pod is removed.\n\nUse this if: a) the volume is only needed while the pod runs, b) features of normal volumes like restoring from snapshot or capacity\n   tracking are needed,\nc) the storage driver is specified through a storage class, and d) the storage driver supports dynamic volume provisioning through\n   a PersistentVolumeClaim (see EphemeralVolumeSource for more\n   information on the connection between this volume type\n   and PersistentVolumeClaim).\n\nUse PersistentVolumeClaim or one of the vendor-specific APIs for volumes that persist for longer than the lifecycle of an individual pod.\n\nUse CSI for light-weight local ephemeral volumes if the CSI driver is meant to be used that way - see the documentation of the driver for more information.\n\nA pod can use both types of ephemeral volumes and persistent volumes at the same time.\n\nThis is a beta feature and only available when the GenericEphemeralVolume feature gate is enabled.",
+							Description: "Ephemeral represents a volume that is handled by a cluster storage driver (Alpha feature). The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, and deleted when the pod is removed.\n\nUse this if: a) the volume is only needed while the pod runs, b) features of normal volumes like restoring from snapshot or capacity\n   tracking are needed,\nc) the storage driver is specified through a storage class, and d) the storage driver supports dynamic volume provisioning through\n   a PersistentVolumeClaim (see EphemeralVolumeSource for more\n   information on the connection between this volume type\n   and PersistentVolumeClaim).\n\nUse PersistentVolumeClaim or one of the vendor-specific APIs for volumes that persist for longer than the lifecycle of an individual pod.\n\nUse CSI for light-weight local ephemeral volumes if the CSI driver is meant to be used that way - see the documentation of the driver for more information.\n\nA pod can use both types of ephemeral volumes and persistent volumes at the same time.",
 							Ref:         ref("k8s.io/api/core/v1.EphemeralVolumeSource"),
 						},
 					},
@@ -14423,13 +14444,6 @@ func schema_k8sio_api_core_v1_WindowsSecurityContextOptions(ref common.Reference
 							Format:      "",
 						},
 					},
-					"hostProcess": {
-						SchemaProps: spec.SchemaProps{
-							Description: "HostProcess determines if a container should be run as a 'Host Process' container. This field is alpha-level and will only be honored by components that enable the WindowsHostProcessContainers feature flag. Setting this field without the feature flag will result in errors when validating the Pod. All of a Pod's containers must have the same effective HostProcess value (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers).  In addition, if HostProcess is true then HostNetwork must also be set to true.",
-							Type:        []string{"boolean"},
-							Format:      "",
-						},
-					},
 				},
 			},
 		},
@@ -14798,62 +14812,6 @@ func schema_pkg_apis_meta_v1_APIVersions(ref common.ReferenceCallback) common.Op
 	}
 }
 
-func schema_pkg_apis_meta_v1_ApplyOptions(ref common.ReferenceCallback) common.OpenAPIDefinition {
-	return common.OpenAPIDefinition{
-		Schema: spec.Schema{
-			SchemaProps: spec.SchemaProps{
-				Description: "ApplyOptions may be provided when applying an API object. FieldManager is required for apply requests. ApplyOptions is equivalent to PatchOptions. It is provided as a convenience with documentation that speaks specifically to how the options fields relate to apply.",
-				Type:        []string{"object"},
-				Properties: map[string]spec.Schema{
-					"kind": {
-						SchemaProps: spec.SchemaProps{
-							Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds",
-							Type:        []string{"string"},
-							Format:      "",
-						},
-					},
-					"apiVersion": {
-						SchemaProps: spec.SchemaProps{
-							Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources",
-							Type:        []string{"string"},
-							Format:      "",
-						},
-					},
-					"dryRun": {
-						SchemaProps: spec.SchemaProps{
-							Description: "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed",
-							Type:        []string{"array"},
-							Items: &spec.SchemaOrArray{
-								Schema: &spec.Schema{
-									SchemaProps: spec.SchemaProps{
-										Type:   []string{"string"},
-										Format: "",
-									},
-								},
-							},
-						},
-					},
-					"force": {
-						SchemaProps: spec.SchemaProps{
-							Description: "Force is going to \"force\" Apply requests. It means user will re-acquire conflicting fields owned by other people.",
-							Type:        []string{"boolean"},
-							Format:      "",
-						},
-					},
-					"fieldManager": {
-						SchemaProps: spec.SchemaProps{
-							Description: "fieldManager is a name associated with the actor or entity that is making these changes. The value must be less than or 128 characters long, and only contain printable characters, as defined by https://golang.org/pkg/unicode/#IsPrint. This field is required.",
-							Type:        []string{"string"},
-							Format:      "",
-						},
-					},
-				},
-				Required: []string{"force", "fieldManager"},
-			},
-		},
-	}
-}
-
 func schema_pkg_apis_meta_v1_Condition(ref common.ReferenceCallback) common.OpenAPIDefinition {
 	return common.OpenAPIDefinition{
 		Schema: spec.Schema{
@@ -15041,6 +14999,48 @@ func schema_pkg_apis_meta_v1_Duration(ref common.ReferenceCallback) common.OpenA
 	}
 }
 
+func schema_pkg_apis_meta_v1_ExportOptions(ref common.ReferenceCallback) common.OpenAPIDefinition {
+	return common.OpenAPIDefinition{
+		Schema: spec.Schema{
+			SchemaProps: spec.SchemaProps{
+				Description: "ExportOptions is the query options to the standard REST get call. Deprecated. Planned for removal in 1.18.",
+				Type:        []string{"object"},
+				Properties: map[string]spec.Schema{
+					"kind": {
+						SchemaProps: spec.SchemaProps{
+							Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds",
+							Type:        []string{"string"},
+							Format:      "",
+						},
+					},
+					"apiVersion": {
+						SchemaProps: spec.SchemaProps{
+							Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources",
+							Type:        []string{"string"},
+							Format:      "",
+						},
+					},
+					"export": {
+						SchemaProps: spec.SchemaProps{
+							Description: "Should this value be exported.  Export strips fields that a user can not specify. Deprecated. Planned for removal in 1.18.",
+							Type:        []string{"boolean"},
+							Format:      "",
+						},
+					},
+					"exact": {
+						SchemaProps: spec.SchemaProps{
+							Description: "Should the export be exact.  Exact export maintains cluster-specific fields like 'Namespace'. Deprecated. Planned for removal in 1.18.",
+							Type:        []string{"boolean"},
+							Format:      "",
+						},
+					},
+				},
+				Required: []string{"export", "exact"},
+			},
+		},
+	}
+}
+
 func schema_pkg_apis_meta_v1_FieldsV1(ref common.ReferenceCallback) common.OpenAPIDefinition {
 	return common.OpenAPIDefinition{
 		Schema: spec.Schema{
@@ -15509,7 +15509,7 @@ func schema_pkg_apis_meta_v1_ListOptions(ref common.ReferenceCallback) common.Op
 					},
 					"allowWatchBookmarks": {
 						SchemaProps: spec.SchemaProps{
-							Description: "allowWatchBookmarks requests watch events with type \"BOOKMARK\". Servers that do not implement bookmarks may ignore this flag and bookmarks are sent at the server's discretion. Clients should not assume bookmarks are returned at any specific interval, nor may they assume the server will send any BOOKMARK event during a session. If this is not a watch, this field is ignored.",
+							Description: "allowWatchBookmarks requests watch events with type \"BOOKMARK\". Servers that do not implement bookmarks may ignore this flag and bookmarks are sent at the server's discretion. Clients should not assume bookmarks are returned at any specific interval, nor may they assume the server will send any BOOKMARK event during a session. If this is not a watch, this field is ignored. If the feature gate WatchBookmarks is not enabled in apiserver, this field is ignored.",
 							Type:        []string{"boolean"},
 							Format:      "",
 						},
@@ -15602,13 +15602,6 @@ func schema_pkg_apis_meta_v1_ManagedFieldsEntry(ref common.ReferenceCallback) co
 							Ref:         ref("k8s.io/apimachinery/pkg/apis/meta/v1.FieldsV1"),
 						},
 					},
-					"subresource": {
-						SchemaProps: spec.SchemaProps{
-							Description: "Subresource is the name of the subresource used to update that object, or empty string if the object was updated through the main resource. The value of this field is used to distinguish between managers, even if they share the same name. For example, a status update will be distinct from a regular update using the same manager name. Note that the APIVersion field is not related to the Subresource field and it always corresponds to the version of the main resource.",
-							Type:        []string{"string"},
-							Format:      "",
-						},
-					},
 				},
 			},
 		},

From 117243be2d82748ef7f52eeb9c104929f2916661 Mon Sep 17 00:00:00 2001
From: echken <echken@localhost.localdomain>
Date: Wed, 23 Mar 2022 04:48:07 -0400
Subject: [PATCH 07/32] fix: tier0 policy next table

Signed-off-by: echken <echken@localhost.localdomain>
---
 pkg/agent/datapath/policyBridge.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/agent/datapath/policyBridge.go b/pkg/agent/datapath/policyBridge.go
index be6fb9d51..8413356c1 100644
--- a/pkg/agent/datapath/policyBridge.go
+++ b/pkg/agent/datapath/policyBridge.go
@@ -453,7 +453,7 @@ func (p *PolicyBridge) GetTierTable(direction uint8, tier uint8, mode string) (*
 			switch tier {
 			case POLICY_TIER1:
 				policyTable = p.egressTier1PolicyTable
-				nextTable = p.egressTier2PolicyTable
+				nextTable = p.ctCommitTable
 			case POLICY_TIER2:
 				policyTable = p.egressTier2PolicyTable
 				nextTable = p.ctCommitTable
@@ -467,7 +467,7 @@ func (p *PolicyBridge) GetTierTable(direction uint8, tier uint8, mode string) (*
 			switch tier {
 			case POLICY_TIER1:
 				policyTable = p.ingressTier1PolicyTable
-				nextTable = p.ingressTier2PolicyTable
+				nextTable = p.ctCommitTable
 			case POLICY_TIER2:
 				policyTable = p.ingressTier2PolicyTable
 				nextTable = p.ctCommitTable

From 357e29d992dd15fa8fe0c5ea723f7332e3b64987 Mon Sep 17 00:00:00 2001
From: Changliang Wu <changliang.wu@smartx.com>
Date: Tue, 22 Mar 2022 14:28:58 +0800
Subject: [PATCH 08/32] ci: add ci for all pull request branch

---
 .github/workflows/ci.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 349de1602..33b3045d0 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -2,7 +2,8 @@ name: ci
 
 on:
   pull_request:
-    branches: [ main, release-1.0 ]
+    branches:
+      - '**'
 
   push:
     branches: [ main, release-1.0 ]

From b2612f934b3463fdd5bf630115e0e0b0184b166d Mon Sep 17 00:00:00 2001
From: Changliang Wu <changliang.wu@smartx.com>
Date: Wed, 23 Mar 2022 16:52:20 +0800
Subject: [PATCH 09/32] test: fix test cases

Signed-off-by: zwtop <wang.zhan@smartx.com>
---
 .github/workflows/ci.yaml                     |  2 +-
 .../security.everoute.io_globalpolicies.yaml  |  3 +--
 deploy/everoute.yaml                          |  3 +--
 .../policy/global_policy_controller.go        | 27 ++++++++++---------
 .../policy/global_policy_controller_test.go   |  3 ++-
 .../policy/policy_controller_test.go          |  3 ++-
 .../datapath/multiBridgeDatapath_test.go      |  9 +++++--
 pkg/agent/datapath/policyBridge.go            | 12 +++++----
 pkg/apis/security/v1alpha1/types.go           |  3 ++-
 pkg/openapi/openapi_generated.go              |  1 -
 tests/e2e/cases/security_mode.go              | 12 ++++-----
 tests/e2e/framework/node/agent.go             |  8 +++++-
 12 files changed, 50 insertions(+), 36 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 33b3045d0..a3969618e 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -45,7 +45,7 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: install e2e environment dependency
-        run: sudo sh -c "apt update && apt install -y openvswitch-switch && systemctl start openvswitch-switch"
+        run: sudo sh -c "apt update && apt install -y openvswitch-switch=2.13.* && systemctl start openvswitch-switch"
 
       - name: allow ssh connect to localhost
         run: sudo -H sh -c "ssh-keygen -qN '' </dev/zero; cp ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys"
diff --git a/deploy/crds/security.everoute.io_globalpolicies.yaml b/deploy/crds/security.everoute.io_globalpolicies.yaml
index e39222972..7d19b7b50 100644
--- a/deploy/crds/security.everoute.io_globalpolicies.yaml
+++ b/deploy/crds/security.everoute.io_globalpolicies.yaml
@@ -49,10 +49,9 @@ spec:
                 - Drop
                 type: string
               globalPolicyEnforcementMode:
+                default: work
                 description: GlobalPolicy enforcement mode
                 type: string
-            required:
-            - globalPolicyEnforcementMode
             type: object
         type: object
     served: true
diff --git a/deploy/everoute.yaml b/deploy/everoute.yaml
index c26d8d3bf..f7d15e2e3 100644
--- a/deploy/everoute.yaml
+++ b/deploy/everoute.yaml
@@ -720,10 +720,9 @@ spec:
                 - Drop
                 type: string
               globalPolicyEnforcementMode:
+                default: work
                 description: GlobalPolicy enforcement mode
                 type: string
-            required:
-            - globalPolicyEnforcementMode
             type: object
         type: object
     served: true
diff --git a/pkg/agent/controller/policy/global_policy_controller.go b/pkg/agent/controller/policy/global_policy_controller.go
index f8f2c4b15..88ed99873 100644
--- a/pkg/agent/controller/policy/global_policy_controller.go
+++ b/pkg/agent/controller/policy/global_policy_controller.go
@@ -80,8 +80,7 @@ func (r *Reconciler) calculateExpectGlobalPolicyRules() ([]cache.PolicyRule, err
 
 	switch len(policyList.Items) {
 	case 1:
-		ruleList := newGlobalPolicyRulePair("", cache.RuleTypeGlobalDefaultRule,
-			cache.RuleAction(policyList.Items[0].Spec.DefaultAction))
+		ruleList := newGlobalPolicyRulePair(policyList.Items[0])
 		return ruleList, nil
 	case 0:
 		return []cache.PolicyRule{}, nil
@@ -90,24 +89,26 @@ func (r *Reconciler) calculateExpectGlobalPolicyRules() ([]cache.PolicyRule, err
 	}
 }
 
-func newGlobalPolicyRulePair(ipCIDR string, ruleType cache.RuleType, ruleAction cache.RuleAction) []cache.PolicyRule {
+func newGlobalPolicyRulePair(policy securityv1alpha1.GlobalPolicy) []cache.PolicyRule {
 	var ingressRule, egressRule cache.PolicyRule
 
 	ingressRule = cache.PolicyRule{
-		Direction: cache.RuleDirectionIn,
-		RuleType:  ruleType,
-		Tier:      constants.Tier2,
-		DstIPAddr: ipCIDR,
-		Action:    ruleAction,
+		Direction:       cache.RuleDirectionIn,
+		RuleType:        cache.RuleTypeGlobalDefaultRule,
+		Tier:            constants.Tier2,
+		DstIPAddr:       "",
+		Action:          cache.RuleAction(policy.Spec.DefaultAction),
+		EnforcementMode: string(policy.Spec.GlobalPolicyEnforcementMode),
 	}
 	ingressRule.Name = fmt.Sprintf("/%s/global.ingress/-%s", DefaultGlobalPolicyName, cache.GenerateFlowKey(ingressRule))
 
 	egressRule = cache.PolicyRule{
-		Direction: cache.RuleDirectionOut,
-		RuleType:  ruleType,
-		Tier:      constants.Tier2,
-		SrcIPAddr: ipCIDR,
-		Action:    ruleAction,
+		Direction:       cache.RuleDirectionOut,
+		RuleType:        cache.RuleTypeGlobalDefaultRule,
+		Tier:            constants.Tier2,
+		SrcIPAddr:       "",
+		Action:          cache.RuleAction(policy.Spec.DefaultAction),
+		EnforcementMode: string(policy.Spec.GlobalPolicyEnforcementMode),
 	}
 	egressRule.Name = fmt.Sprintf("/%s/global.egress/-%s", DefaultGlobalPolicyName, cache.GenerateFlowKey(egressRule))
 
diff --git a/pkg/agent/controller/policy/global_policy_controller_test.go b/pkg/agent/controller/policy/global_policy_controller_test.go
index cbee66847..d75738f6f 100644
--- a/pkg/agent/controller/policy/global_policy_controller_test.go
+++ b/pkg/agent/controller/policy/global_policy_controller_test.go
@@ -82,11 +82,12 @@ var _ = Describe("PolicyController", func() {
 	})
 })
 
-func newTestGlobalPolicy(defaultAction securityv1alpha1.GlobalDefaultAction, whitelist ...string) *securityv1alpha1.GlobalPolicy {
+func newTestGlobalPolicy(defaultAction securityv1alpha1.GlobalDefaultAction) *securityv1alpha1.GlobalPolicy {
 	var policy securityv1alpha1.GlobalPolicy
 
 	policy.Name = rand.String(6)
 	policy.Spec.DefaultAction = defaultAction
+	policy.Spec.GlobalPolicyEnforcementMode = securityv1alpha1.WorkMode
 
 	return &policy
 }
diff --git a/pkg/agent/controller/policy/policy_controller_test.go b/pkg/agent/controller/policy/policy_controller_test.go
index a4e2f4e27..236155606 100644
--- a/pkg/agent/controller/policy/policy_controller_test.go
+++ b/pkg/agent/controller/policy/policy_controller_test.go
@@ -1118,7 +1118,8 @@ func newTestPolicy(appliedTo, ingress, egress *testGroup, ingressPort, egressPor
 				networkingv1.PolicyTypeIngress,
 				networkingv1.PolicyTypeEgress,
 			},
-			Tier: constants.Tier2,
+			Tier:                          constants.Tier2,
+			SecurityPolicyEnforcementMode: securityv1alpha1.WorkMode,
 		},
 	}
 }
diff --git a/pkg/agent/datapath/multiBridgeDatapath_test.go b/pkg/agent/datapath/multiBridgeDatapath_test.go
index 691bef30c..7a2710e33 100644
--- a/pkg/agent/datapath/multiBridgeDatapath_test.go
+++ b/pkg/agent/datapath/multiBridgeDatapath_test.go
@@ -21,6 +21,7 @@ import (
 	"net"
 	"os"
 	"os/exec"
+	"regexp"
 	"strings"
 	"testing"
 	"time"
@@ -80,7 +81,8 @@ var (
 		Action:     "deny",
 	}
 
-	rule1Flow           = "table=60, priority=200,icmp,nw_src=10.100.100.1,nw_dst=10.100.100.2 actions=goto_table:70"
+	rule1Flow = `table=60, priority=200,icmp,nw_src=10.100.100.1,nw_dst=10.100.100.2 ` +
+		`actions=load:0x->NXM_NX_XXREG0[54..75],load:0x->NXM_NX_XXREG0[0..9],goto_table:70`
 	ep1VlanInputFlow    = "table=0, priority=200,in_port=11 actions=push_vlan:0x8100,set_field:4097->vlan_vid,resubmit(,10),resubmit(,15)"
 	ep1LocalToLocalFlow = "table=5, priority=200,dl_vlan=1,dl_src=00:00:aa:aa:aa:aa actions=load:0xb->NXM_OF_IN_PORT[],load:0->NXM_OF_VLAN_TCI[0..12],NORMAL"
 )
@@ -217,7 +219,10 @@ func flowValidator(expectedFlows []string) error {
 	for _, expectedFlow := range expectedFlows {
 		isExpectedFlowExists := false
 		for _, actualFlow := range currentFlowList {
-			if strings.Contains(expectedFlow, actualFlow) {
+			expr := `load:0x[0-9,a-f]+?->NXM_NX_XXREG0`
+			re, _ := regexp.Compile(expr)
+			actual := re.ReplaceAllString(actualFlow, "load:0x->NXM_NX_XXREG0")
+			if strings.Contains(expectedFlow, actual) {
 				isExpectedFlowExists = true
 			}
 		}
diff --git a/pkg/agent/datapath/policyBridge.go b/pkg/agent/datapath/policyBridge.go
index 8413356c1..0496b63e0 100644
--- a/pkg/agent/datapath/policyBridge.go
+++ b/pkg/agent/datapath/policyBridge.go
@@ -461,7 +461,7 @@ func (p *PolicyBridge) GetTierTable(direction uint8, tier uint8, mode string) (*
 				policyTable = p.egressTier3PolicyTable
 				nextTable = p.ctCommitTable
 			default:
-				return nil, nil, errors.New("unknow policy tier")
+				return nil, nil, errors.New("unknown policy tier")
 			}
 		case POLICY_DIRECTION_IN:
 			switch tier {
@@ -475,7 +475,7 @@ func (p *PolicyBridge) GetTierTable(direction uint8, tier uint8, mode string) (*
 				policyTable = p.ingressTier3PolicyTable
 				nextTable = p.ctCommitTable
 			default:
-				return nil, nil, errors.New("unknow policy tier")
+				return nil, nil, errors.New("unknown policy tier")
 			}
 		}
 	case "monitor":
@@ -490,7 +490,7 @@ func (p *PolicyBridge) GetTierTable(direction uint8, tier uint8, mode string) (*
 				policyTable = p.egressTier3PolicyMonitorTable
 				nextTable = p.egressTier3PolicyTable
 			default:
-				return nil, nil, errors.New("unknow policy tier")
+				return nil, nil, errors.New("unknown policy tier")
 			}
 		case POLICY_DIRECTION_IN:
 			switch tier {
@@ -502,9 +502,11 @@ func (p *PolicyBridge) GetTierTable(direction uint8, tier uint8, mode string) (*
 				policyTable = p.ingressTier3PolicyMonitorTable
 				nextTable = p.ingressTier3PolicyTable
 			default:
-				return nil, nil, errors.New("unknow policy tier")
+				return nil, nil, errors.New("unknown policy tier")
 			}
 		}
+	default:
+		return nil, nil, fmt.Errorf("unknown work mode (%s)", mode)
 	}
 
 	return policyTable, nextTable, nil
@@ -526,7 +528,7 @@ func (p *PolicyBridge) AddMicroSegmentRule(rule *EveroutePolicyRule, direction u
 	policyTable, nextTable, e := p.GetTierTable(direction, tier, mode)
 	if e != nil {
 		log.Errorf("Failed to get policy table tier %v", tier)
-		return nil, errors.New("failed get policy table")
+		return nil, fmt.Errorf("failed get policy table, err:%s", e)
 	}
 
 	// Parse dst ip
diff --git a/pkg/apis/security/v1alpha1/types.go b/pkg/apis/security/v1alpha1/types.go
index 507672879..5fe996571 100644
--- a/pkg/apis/security/v1alpha1/types.go
+++ b/pkg/apis/security/v1alpha1/types.go
@@ -354,7 +354,8 @@ type GlobalPolicySpec struct {
 	DefaultAction GlobalDefaultAction `json:"defaultAction,omitempty"`
 
 	// GlobalPolicy enforcement mode
-	GlobalPolicyEnforcementMode PolicyMode `json:"globalPolicyEnforcementMode"`
+	// +kubebuilder:default=work
+	GlobalPolicyEnforcementMode PolicyMode `json:"globalPolicyEnforcementMode,omitempty"`
 }
 
 // GlobalDefaultAction defines actions supported for GlobalPolicy.
diff --git a/pkg/openapi/openapi_generated.go b/pkg/openapi/openapi_generated.go
index 069109900..39e2d2458 100644
--- a/pkg/openapi/openapi_generated.go
+++ b/pkg/openapi/openapi_generated.go
@@ -1551,7 +1551,6 @@ func schema_pkg_apis_security_v1alpha1_GlobalPolicySpec(ref common.ReferenceCall
 						},
 					},
 				},
-				Required: []string{"globalPolicyEnforcementMode"},
 			},
 		},
 	}
diff --git a/tests/e2e/cases/security_mode.go b/tests/e2e/cases/security_mode.go
index 7bf4c69a9..7838c3c95 100644
--- a/tests/e2e/cases/security_mode.go
+++ b/tests/e2e/cases/security_mode.go
@@ -152,14 +152,14 @@ func computePolicyFlow(tier string, appliedToIPs, ingressIPs, egressIPs []string
 				protocol := strings.ToLower(string(ingressGroupPort.Protocol))
 
 				if ingressGroupPort.DstPort == 0 && ingressGroupPort.SrcPort == 0 {
-					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s actions=goto_table:%d",
+					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s actions=load:0x->NXM_NX_XXREG0[54..75],load:0x->NXM_NX_XXREG0[0..9],goto_table:%d",
 						*ingressTableID, priority, protocol, srcIP, appliedToIP, *ingressNextTableID)
 				} else if ingressGroupPort.DstPort != 0 {
-					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=%d actions=goto_table:%d",
+					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=%d actions=load:0x->NXM_NX_XXREG0[54..75],load:0x->NXM_NX_XXREG0[0..9],goto_table:%d",
 						*ingressTableID, priority, protocol, srcIP, appliedToIP, ingressGroupPort.DstPort,
 						*ingressNextTableID)
 					if ingressGroupPort.DstPort != 0 && ingressGroupPort.DstPortMask != 0xffff {
-						flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=0x%x/0x%x actions=goto_table:%d",
+						flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=0x%x/0x%x actions=load:0x->NXM_NX_XXREG0[54..75],load:0x->NXM_NX_XXREG0[0..9],goto_table:%d",
 							*ingressTableID, priority, protocol, srcIP, appliedToIP, ingressGroupPort.DstPort, ingressGroupPort.DstPortMask,
 							*ingressNextTableID)
 					}
@@ -183,13 +183,13 @@ func computePolicyFlow(tier string, appliedToIPs, ingressIPs, egressIPs []string
 				protocol := strings.ToLower(string(egressGroupPort.Protocol))
 
 				if egressGroupPort.DstPort == 0 && egressGroupPort.SrcPort == 0 {
-					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s actions=goto_table:%d",
+					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s actions=load:0x->NXM_NX_XXREG0[54..75],load:0x->NXM_NX_XXREG0[0..9],goto_table:%d",
 						*egressTableID, priority, protocol, appliedToIP, dstIP, *egressNextTableID)
 				} else if egressGroupPort.DstPort != 0 {
-					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=%d actions=goto_table:%d",
+					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=%d actions=load:0x->NXM_NX_XXREG0[54..75],load:0x->NXM_NX_XXREG0[0..9],goto_table:%d",
 						*egressTableID, priority, protocol, appliedToIP, dstIP, egressGroupPort.DstPort, *egressNextTableID)
 					if egressGroupPort.DstPort != 0 && egressGroupPort.DstPortMask != 0xffff {
-						flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=0x%x/0x%x actions=goto_table:%d",
+						flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=0x%x/0x%x actions=load:0x->NXM_NX_XXREG0[54..75],load:0x->NXM_NX_XXREG0[0..9],goto_table:%d",
 							*ingressTableID, priority, protocol, dstIP, appliedToIP, egressGroupPort.DstPort, egressGroupPort.DstPortMask,
 							*egressNextTableID)
 					}
diff --git a/tests/e2e/framework/node/agent.go b/tests/e2e/framework/node/agent.go
index 051ecebd1..2b52b2e13 100644
--- a/tests/e2e/framework/node/agent.go
+++ b/tests/e2e/framework/node/agent.go
@@ -18,6 +18,7 @@ package node
 
 import (
 	"fmt"
+	"regexp"
 	"strings"
 )
 
@@ -64,7 +65,12 @@ func (n *Agent) DumpFlow() ([]string, error) {
 		if len(felem) >= 5 {
 			felem = append([]string{felem[2]}, felem[5:]...)
 			fstr := strings.Join(felem, " ")
-			flowList = append(flowList, fstr)
+
+			// replace roundNum and sequenceNum with static format
+			expr := `load:0x[0-9,a-f]+?->NXM_NX_XXREG0`
+			re, _ := regexp.Compile(expr)
+
+			flowList = append(flowList, re.ReplaceAllString(fstr, "load:0x->NXM_NX_XXREG0"))
 		}
 	}
 

From 49a25676e40a1f9b2c9359783e8c6a485a6c9410 Mon Sep 17 00:00:00 2001
From: echken <echken@localhost.localdomain>
Date: Tue, 12 Apr 2022 02:59:46 -0400
Subject: [PATCH 10/32] fix: agent roundNum greater than maximum accepted value

update agent round num length and flowID coding mechnism:
using 4 bits round num and 28 bits flow sequence num

Signed-off-by: echken <echken@localhost.localdomain>
---
 go.mod                                        |  3 +--
 go.sum                                        |  5 ++--
 pkg/agent/datapath/multiBridgeDatapath.go     | 24 ++++++++++++-------
 .../datapath/multiBridgeDatapath_test.go      | 23 +++++++++++++++++-
 pkg/agent/datapath/policyBridge.go            |  8 +++----
 tests/e2e/cases/security_mode.go              | 12 +++++-----
 6 files changed, 51 insertions(+), 24 deletions(-)

diff --git a/go.mod b/go.mod
index d3a9e940c..71ac9fd2c 100644
--- a/go.mod
+++ b/go.mod
@@ -49,8 +49,7 @@ require (
 replace (
 	github.com/contiv/libOpenflow => github.com/everoute/libOpenflow v0.0.0-20210716071814-4ef09249fae5
 	github.com/contiv/libovsdb => github.com/everoute/libovsdb v0.0.0-20210326110222-6c508538aa65
-	//github.com/contiv/ofnet => github.com/everoute/ofnet v0.0.0-20211229083707-5cb1e875a38b
-	github.com/contiv/ofnet => github.com/echkenluo/ofnet v0.0.0-20220316073105-f24ad3ca4c99
+	github.com/contiv/ofnet => github.com/everoute/ofnet v0.0.0-20220412063200-c0360fe2ddcb
 	github.com/osrg/gobgp => github.com/everoute/gobgp v0.0.0-20210127101833-12edfc1f4514
 	k8s.io/api v0.22.2 => k8s.io/api v0.20.6
 	k8s.io/apimachinery v0.22.2 => k8s.io/apimachinery v0.20.6
diff --git a/go.sum b/go.sum
index 1868df722..80474b2a3 100644
--- a/go.sum
+++ b/go.sum
@@ -300,8 +300,6 @@ github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:Htrtb
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
-github.com/echkenluo/ofnet v0.0.0-20220316073105-f24ad3ca4c99 h1:WQWeCnqASk/iKFSPYjosYoonzaugeXqACRbINXfA7hc=
-github.com/echkenluo/ofnet v0.0.0-20220316073105-f24ad3ca4c99/go.mod h1:cJR/tu7mdjR0Rm4EXUqw7nJU9cMCoWg2lEO7jfwq7MM=
 github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -322,6 +320,8 @@ github.com/everoute/libOpenflow v0.0.0-20210716071814-4ef09249fae5 h1:+z3ZBux28V
 github.com/everoute/libOpenflow v0.0.0-20210716071814-4ef09249fae5/go.mod h1:xsbTUkWYuDqVv4jMxwinRGdjlYwj0aZS8Rzf6HoNLdg=
 github.com/everoute/libovsdb v0.0.0-20210326110222-6c508538aa65 h1:a3dwYQt/Y9Yfz1XKpUelQKS5YDgmG4NtVpBrRguQY9k=
 github.com/everoute/libovsdb v0.0.0-20210326110222-6c508538aa65/go.mod h1:N8fR1bYMqi4pQ7jL17QlBlpJ0hHmqRAXVKkBwuIjClw=
+github.com/everoute/ofnet v0.0.0-20220412063200-c0360fe2ddcb h1:9n+KfNXOHZQhmqbt0BQTsdgKxIib9SEuQYKCw22HyWU=
+github.com/everoute/ofnet v0.0.0-20220412063200-c0360fe2ddcb/go.mod h1:VI923BFjPJipkV0SbiOmQ1qXefJnBBkjGxA3F56fenM=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
@@ -996,7 +996,6 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
-golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
diff --git a/pkg/agent/datapath/multiBridgeDatapath.go b/pkg/agent/datapath/multiBridgeDatapath.go
index 72f4cbe3a..863d986f3 100644
--- a/pkg/agent/datapath/multiBridgeDatapath.go
+++ b/pkg/agent/datapath/multiBridgeDatapath.go
@@ -80,13 +80,12 @@ const (
 	LOOP_BACK_ADDR    = "127.0.0.1"
 )
 
-//nolink
+//nolint
 const (
-	FLOW_ROUND_NUM_LENGTH = 10
-	FLOW_SEQ_NUM_LENGTH   = 22
-	FLOW_ROUND_NUM_MASK   = 0xffc
-	FLOW_SEQ_NUM_MASK     = 0x003fffff
-
+	FLOW_ROUND_NUM_LENGTH           = 4
+	FLOW_SEQ_NUM_LENGTH             = 28
+	FLOW_ROUND_NUM_MASK             = 0xf0000000
+	FLOW_SEQ_NUM_MASK               = 0x0fffffff
 	DEFAULT_POLICY_ENFORCEMENT_MODE = "work"
 )
 
@@ -132,6 +131,8 @@ const (
 
 	InternalIngressRulePrefix = "/INTERNAL_INGRESS_POLICY/ingress/-"
 	InternalEgressRulePrefix  = "/INTERNAL_EGRESS_POLICY/egress/-"
+
+	MaxRoundNum = 15
 )
 
 type Bridge interface {
@@ -873,7 +874,7 @@ func DeepCopyMap(theMap interface{}) interface{} {
 }
 
 func getRoundInfo(ovsdbDriver *ovsdbDriver.OvsDriver) (*RoundInfo, error) {
-	var num uint64
+	var num, newRoundNum uint64
 	var err error
 
 	externalIds, err := ovsdbDriver.GetExternalIds()
@@ -901,9 +902,16 @@ func getRoundInfo(ovsdbDriver *ovsdbDriver.OvsDriver) (*RoundInfo, error) {
 		return nil, fmt.Errorf("bad format of round number: %+v, parse error: %+v", roundNum, err)
 	}
 
+	// Flipping current round num with minimum round num value while it equals with the maximum round num
+	if num >= MaxRoundNum {
+		newRoundNum = 1
+	} else {
+		newRoundNum = num + 1
+	}
+
 	return &RoundInfo{
 		previousRoundNum: num,
-		curRoundNum:      num + 1,
+		curRoundNum:      newRoundNum,
 	}, nil
 }
 
diff --git a/pkg/agent/datapath/multiBridgeDatapath_test.go b/pkg/agent/datapath/multiBridgeDatapath_test.go
index 7a2710e33..6a33abf8f 100644
--- a/pkg/agent/datapath/multiBridgeDatapath_test.go
+++ b/pkg/agent/datapath/multiBridgeDatapath_test.go
@@ -82,7 +82,7 @@ var (
 	}
 
 	rule1Flow = `table=60, priority=200,icmp,nw_src=10.100.100.1,nw_dst=10.100.100.2 ` +
-		`actions=load:0x->NXM_NX_XXREG0[54..75],load:0x->NXM_NX_XXREG0[0..9],goto_table:70`
+		`actions=load:0x->NXM_NX_XXREG0[60..87],load:0x->NXM_NX_XXREG0[0..3],goto_table:70`
 	ep1VlanInputFlow    = "table=0, priority=200,in_port=11 actions=push_vlan:0x8100,set_field:4097->vlan_vid,resubmit(,10),resubmit(,15)"
 	ep1LocalToLocalFlow = "table=5, priority=200,dl_vlan=1,dl_src=00:00:aa:aa:aa:aa actions=load:0xb->NXM_OF_IN_PORT[],load:0->NXM_OF_VLAN_TCI[0..12],NORMAL"
 )
@@ -111,6 +111,7 @@ func TestDpManager(t *testing.T) {
 	testLocalEndpoint(t)
 	testERPolicyRule(t)
 	testFlowReplay(t)
+	testRoundNumFlip(t)
 }
 
 func testLocalEndpoint(t *testing.T) {
@@ -209,6 +210,26 @@ func testFlowReplay(t *testing.T) {
 	})
 }
 
+func testRoundNumFlip(t *testing.T) {
+	roundInfo := RoundInfo{
+		curRoundNum:      MaxRoundNum,
+		previousRoundNum: MaxRoundNum - 1,
+	}
+
+	t.Run("persistentRoundInfo into local bridge", func(t *testing.T) {
+		Eventually(func() error {
+			return persistentRoundInfo(roundInfo.curRoundNum, datapathManager.OvsdbDriverMap["ovsbr0"][LOCAL_BRIDGE_KEYWORD])
+		}, timeout, interval).Should(Succeed())
+	})
+
+	t.Run("validate ER agent Round num flip", func(t *testing.T) {
+		Eventually(func() bool {
+			round, _ := getRoundInfo(datapathManager.OvsdbDriverMap["ovsbr0"][LOCAL_BRIDGE_KEYWORD])
+			return round.curRoundNum == 1
+		}, timeout, interval).Should(BeTrue())
+	})
+}
+
 func flowValidator(expectedFlows []string) error {
 	var currentFlowList []string
 	var err error
diff --git a/pkg/agent/datapath/policyBridge.go b/pkg/agent/datapath/policyBridge.go
index 0496b63e0..508fd56f5 100644
--- a/pkg/agent/datapath/policyBridge.go
+++ b/pkg/agent/datapath/policyBridge.go
@@ -578,10 +578,10 @@ func (p *PolicyBridge) AddMicroSegmentRule(rule *EveroutePolicyRule, direction u
 			return nil, fmt.Errorf("policy tier1 without monitor mode support")
 		}
 
-		if err := ruleFlow.LoadField("nxm_nx_xxreg0", ruleFlow.FlowID>>FLOW_SEQ_NUM_LENGTH, openflow13.NewNXRange(0, 9)); err != nil {
+		if err := ruleFlow.LoadField("nxm_nx_xxreg0", ruleFlow.FlowID>>FLOW_SEQ_NUM_LENGTH, openflow13.NewNXRange(0, 3)); err != nil {
 			return nil, err
 		}
-		if err := ruleFlow.LoadField("nxm_nx_xxreg0", ruleFlow.FlowID&FLOW_SEQ_NUM_MASK, openflow13.NewNXRange(32, 53)); err != nil {
+		if err := ruleFlow.LoadField("nxm_nx_xxreg0", ruleFlow.FlowID&FLOW_SEQ_NUM_MASK, openflow13.NewNXRange(32, 59)); err != nil {
 			return nil, err
 		}
 
@@ -604,10 +604,10 @@ func (p *PolicyBridge) AddMicroSegmentRule(rule *EveroutePolicyRule, direction u
 			return nil, fmt.Errorf("unknown action")
 		}
 
-		if err := ruleFlow.LoadField("nxm_nx_xxreg0", ruleFlow.FlowID>>FLOW_SEQ_NUM_LENGTH, openflow13.NewNXRange(0, 9)); err != nil {
+		if err := ruleFlow.LoadField("nxm_nx_xxreg0", ruleFlow.FlowID>>FLOW_SEQ_NUM_LENGTH, openflow13.NewNXRange(0, 3)); err != nil {
 			return nil, err
 		}
-		if err := ruleFlow.LoadField("nxm_nx_xxreg0", ruleFlow.FlowID&FLOW_SEQ_NUM_MASK, openflow13.NewNXRange(54, 75)); err != nil {
+		if err := ruleFlow.LoadField("nxm_nx_xxreg0", ruleFlow.FlowID&FLOW_SEQ_NUM_MASK, openflow13.NewNXRange(60, 87)); err != nil {
 			return nil, err
 		}
 
diff --git a/tests/e2e/cases/security_mode.go b/tests/e2e/cases/security_mode.go
index 7838c3c95..f2073bf7e 100644
--- a/tests/e2e/cases/security_mode.go
+++ b/tests/e2e/cases/security_mode.go
@@ -152,14 +152,14 @@ func computePolicyFlow(tier string, appliedToIPs, ingressIPs, egressIPs []string
 				protocol := strings.ToLower(string(ingressGroupPort.Protocol))
 
 				if ingressGroupPort.DstPort == 0 && ingressGroupPort.SrcPort == 0 {
-					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s actions=load:0x->NXM_NX_XXREG0[54..75],load:0x->NXM_NX_XXREG0[0..9],goto_table:%d",
+					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s actions=load:0x->NXM_NX_XXREG0[60..87],load:0x->NXM_NX_XXREG0[0..3],goto_table:%d",
 						*ingressTableID, priority, protocol, srcIP, appliedToIP, *ingressNextTableID)
 				} else if ingressGroupPort.DstPort != 0 {
-					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=%d actions=load:0x->NXM_NX_XXREG0[54..75],load:0x->NXM_NX_XXREG0[0..9],goto_table:%d",
+					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=%d actions=load:0x->NXM_NX_XXREG0[60..87],load:0x->NXM_NX_XXREG0[0..3],goto_table:%d",
 						*ingressTableID, priority, protocol, srcIP, appliedToIP, ingressGroupPort.DstPort,
 						*ingressNextTableID)
 					if ingressGroupPort.DstPort != 0 && ingressGroupPort.DstPortMask != 0xffff {
-						flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=0x%x/0x%x actions=load:0x->NXM_NX_XXREG0[54..75],load:0x->NXM_NX_XXREG0[0..9],goto_table:%d",
+						flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=0x%x/0x%x actions=load:0x->NXM_NX_XXREG0[60..87],load:0x->NXM_NX_XXREG0[0..3],goto_table:%d",
 							*ingressTableID, priority, protocol, srcIP, appliedToIP, ingressGroupPort.DstPort, ingressGroupPort.DstPortMask,
 							*ingressNextTableID)
 					}
@@ -183,13 +183,13 @@ func computePolicyFlow(tier string, appliedToIPs, ingressIPs, egressIPs []string
 				protocol := strings.ToLower(string(egressGroupPort.Protocol))
 
 				if egressGroupPort.DstPort == 0 && egressGroupPort.SrcPort == 0 {
-					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s actions=load:0x->NXM_NX_XXREG0[54..75],load:0x->NXM_NX_XXREG0[0..9],goto_table:%d",
+					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s actions=load:0x->NXM_NX_XXREG0[60..87],load:0x->NXM_NX_XXREG0[0..3],goto_table:%d",
 						*egressTableID, priority, protocol, appliedToIP, dstIP, *egressNextTableID)
 				} else if egressGroupPort.DstPort != 0 {
-					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=%d actions=load:0x->NXM_NX_XXREG0[54..75],load:0x->NXM_NX_XXREG0[0..9],goto_table:%d",
+					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=%d actions=load:0x->NXM_NX_XXREG0[60..87],load:0x->NXM_NX_XXREG0[0..3],goto_table:%d",
 						*egressTableID, priority, protocol, appliedToIP, dstIP, egressGroupPort.DstPort, *egressNextTableID)
 					if egressGroupPort.DstPort != 0 && egressGroupPort.DstPortMask != 0xffff {
-						flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=0x%x/0x%x actions=load:0x->NXM_NX_XXREG0[54..75],load:0x->NXM_NX_XXREG0[0..9],goto_table:%d",
+						flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=0x%x/0x%x actions=load:0x->NXM_NX_XXREG0[60..87],load:0x->NXM_NX_XXREG0[0..3],goto_table:%d",
 							*ingressTableID, priority, protocol, dstIP, appliedToIP, egressGroupPort.DstPort, egressGroupPort.DstPortMask,
 							*egressNextTableID)
 					}

From 3d5610a29e7f6001651484fac71aaa23ee125bdc Mon Sep 17 00:00:00 2001
From: Changliang Wu <changliang.wu@smartx.com>
Date: Thu, 24 Feb 2022 11:43:45 +0800
Subject: [PATCH 11/32] feat: fetch and upload ct flow related policies

---
 pkg/agent/datapath/multiBridgeDatapath.go | 43 +++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/pkg/agent/datapath/multiBridgeDatapath.go b/pkg/agent/datapath/multiBridgeDatapath.go
index 863d986f3..d0831607b 100644
--- a/pkg/agent/datapath/multiBridgeDatapath.go
+++ b/pkg/agent/datapath/multiBridgeDatapath.go
@@ -39,6 +39,7 @@ import (
 	"github.com/contiv/ofnet/ovsdbDriver"
 	"github.com/fsnotify/fsnotify"
 	cmap "github.com/streamrail/concurrent-map"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 
@@ -179,6 +180,7 @@ type DpManager struct {
 	ofPortIPAddressUpdateChan chan map[string]net.IP // map bridgename-ofport to endpoint ips
 	datapathConfig            *Config
 	Rules                     map[string]*EveroutePolicyRuleEntry // rules database
+	FlowIdToRules             map[uint64]*EveroutePolicyRuleEntry
 	flowReplayChan            chan struct{}
 	flowReplayMutex           sync.RWMutex
 	ovsdbReconnectChan        chan struct{}
@@ -254,6 +256,12 @@ type RoundInfo struct {
 	curRoundNum      uint64
 }
 
+type PolicyInfo struct {
+	Dir            uint8
+	Action         string
+	NamespacedName []types.NamespacedName
+}
+
 // Datapath manager act as openflow controller:
 // 1. event driven local endpoint info crud and related flow update,
 // 2. collect local endpoint ip learned from different ovsbr(1 per vds), and sync it to management plane
@@ -265,6 +273,7 @@ func NewDatapathManager(datapathConfig *Config, ofPortIPAddressUpdateChan chan m
 	datapathManager.ControllerMap = make(map[string]map[string]*ofctrl.Controller)
 	datapathManager.controllerIDSets = sets.NewString()
 	datapathManager.Rules = make(map[string]*EveroutePolicyRuleEntry)
+	datapathManager.FlowIdToRules = make(map[uint64]*EveroutePolicyRuleEntry)
 	datapathManager.datapathConfig = datapathConfig
 	datapathManager.localEndpointDB = cmap.New()
 	datapathManager.AgentInfo = new(AgentConf)
@@ -342,6 +351,32 @@ func (datapathManager *DpManager) InitializeDatapath(stopChan <-chan struct{}) {
 	}
 }
 
+func (datapathManager *DpManager) GetPolicyByFlowID(flowID ...uint64) []*PolicyInfo {
+	datapathManager.flowReplayMutex.RLock()
+	defer datapathManager.flowReplayMutex.RUnlock()
+
+	var policyInfoList []*PolicyInfo
+
+	for _, id := range flowID {
+		item := datapathManager.FlowIdToRules[id]
+		if item != nil {
+			policyInfo := &PolicyInfo{
+				Dir:    item.Direction,
+				Action: item.EveroutePolicyRule.Action,
+			}
+			for _, p := range item.PolicyRuleReference.List() {
+				policyInfo.NamespacedName = append(policyInfo.NamespacedName, types.NamespacedName{
+					Namespace: strings.Split(p, "/")[0],
+					Name:      strings.Split(p, "/")[1],
+				})
+			}
+			policyInfoList = append(policyInfoList, policyInfo)
+		}
+	}
+
+	return policyInfoList
+}
+
 func (datapathManager *DpManager) InitializeCNI() {
 	var wg sync.WaitGroup
 	for vdsID := range datapathManager.datapathConfig.ManagedVDSMap {
@@ -816,6 +851,12 @@ func (datapathManager *DpManager) AddEveroutePolicyRule(rule *EveroutePolicyRule
 	ruleEntry.EveroutePolicyRule = rule
 	ruleEntry.RuleFlowMap = ruleFlowMap
 
+	// save flowID reference
+	for _, v := range ruleEntry.RuleFlowMap {
+		datapathManager.FlowIdToRules[v.FlowID] = ruleEntry
+		log.Info(v.FlowID)
+	}
+
 	datapathManager.Rules[rule.RuleID] = ruleEntry
 
 	return nil
@@ -848,6 +889,8 @@ func (datapathManager *DpManager) RemoveEveroutePolicyRule(ruleID string, ruleNa
 			log.Errorf("Failed to delete flow for rule: %+v. Err: %v", ruleID, err)
 			return err
 		}
+		// remove flowID reference
+		delete(datapathManager.FlowIdToRules, pRule.RuleFlowMap[vdsID].FlowID)
 	}
 
 	if pRule.PolicyRuleReference.Len() == 0 {

From 5c8b39f5586238dbe5924c9b6d80db29f49fa31a Mon Sep 17 00:00:00 2001
From: Changliang Wu <changliang.wu@smartx.com>
Date: Tue, 1 Mar 2022 17:19:31 +0800
Subject: [PATCH 12/32] feat: add type for flow message

global policy or normal policy
---
 pkg/agent/controller/policy/cache/rule.go     |  9 +++++++--
 .../policy/global_policy_controller.go        |  4 ++--
 .../controller/policy/policy_controller.go    | 14 ++-----------
 pkg/agent/datapath/multiBridgeDatapath.go     | 20 ++++++++++++-------
 4 files changed, 24 insertions(+), 23 deletions(-)

diff --git a/pkg/agent/controller/policy/cache/rule.go b/pkg/agent/controller/policy/cache/rule.go
index fe65722cf..1cecacc42 100644
--- a/pkg/agent/controller/policy/cache/rule.go
+++ b/pkg/agent/controller/policy/cache/rule.go
@@ -32,6 +32,8 @@ type RuleType string
 type RuleAction string
 type RuleDirection string
 
+type PolicyType string
+
 const (
 	RuleTypeGlobalDefaultRule RuleType = "GlobalDefaultRule"
 	RuleTypeDefaultRule       RuleType = "DefaultRule"
@@ -42,10 +44,13 @@ const (
 
 	RuleDirectionIn  RuleDirection = "Ingress"
 	RuleDirectionOut RuleDirection = "Egress"
+
+	NormalPolicy PolicyType = "normal"
+	GlobalPolicy PolicyType = "global"
 )
 
 type PolicyRule struct {
-	// Name format policyNamespace/policyName/ruleName-flowKey
+	// Name format policyNamespace/policyName/policyType/ruleName-flowKey
 	Name   string     `json:"name"`
 	Action RuleAction `json:"action"`
 
@@ -95,7 +100,7 @@ func NewIPBlockItem() *IPBlockItem {
 type CompleteRule struct {
 	lock sync.RWMutex
 
-	// RuleID is a unique identifier of rule, it's always set to policyNamespace/policyName/ruleName.
+	// RuleID is a unique identifier of rule, it's always set to policyNamespace/policyName/policyType/ruleName.
 	RuleID string
 
 	Tier            string
diff --git a/pkg/agent/controller/policy/global_policy_controller.go b/pkg/agent/controller/policy/global_policy_controller.go
index 88ed99873..46be6a773 100644
--- a/pkg/agent/controller/policy/global_policy_controller.go
+++ b/pkg/agent/controller/policy/global_policy_controller.go
@@ -100,7 +100,7 @@ func newGlobalPolicyRulePair(policy securityv1alpha1.GlobalPolicy) []cache.Polic
 		Action:          cache.RuleAction(policy.Spec.DefaultAction),
 		EnforcementMode: string(policy.Spec.GlobalPolicyEnforcementMode),
 	}
-	ingressRule.Name = fmt.Sprintf("/%s/global.ingress/-%s", DefaultGlobalPolicyName, cache.GenerateFlowKey(ingressRule))
+	ingressRule.Name = fmt.Sprintf("/%s/%s/global.ingress/-%s", DefaultGlobalPolicyName, cache.GlobalPolicy, cache.GenerateFlowKey(ingressRule))
 
 	egressRule = cache.PolicyRule{
 		Direction:       cache.RuleDirectionOut,
@@ -110,7 +110,7 @@ func newGlobalPolicyRulePair(policy securityv1alpha1.GlobalPolicy) []cache.Polic
 		Action:          cache.RuleAction(policy.Spec.DefaultAction),
 		EnforcementMode: string(policy.Spec.GlobalPolicyEnforcementMode),
 	}
-	egressRule.Name = fmt.Sprintf("/%s/global.egress/-%s", DefaultGlobalPolicyName, cache.GenerateFlowKey(egressRule))
+	egressRule.Name = fmt.Sprintf("/%s/%s/global.egress/-%s", DefaultGlobalPolicyName, cache.GlobalPolicy, cache.GenerateFlowKey(egressRule))
 
 	return []cache.PolicyRule{ingressRule, egressRule}
 }
diff --git a/pkg/agent/controller/policy/policy_controller.go b/pkg/agent/controller/policy/policy_controller.go
index a1578f22c..52f655c46 100644
--- a/pkg/agent/controller/policy/policy_controller.go
+++ b/pkg/agent/controller/policy/policy_controller.go
@@ -349,7 +349,7 @@ func (r *Reconciler) completePolicy(policy *securityv1alpha1.SecurityPolicy) ([]
 
 		if policy.Spec.DefaultRule == securityv1alpha1.DefaultRuleDrop {
 			defaultIngressRule := &policycache.CompleteRule{
-				RuleID:            fmt.Sprintf("%s/%s/%s.%s", policy.Namespace, policy.Name, "default", "ingress"),
+				RuleID:            fmt.Sprintf("%s/%s/%s/%s.%s", policy.Namespace, policy.Name, policycache.NormalPolicy, "default", "ingress"),
 				Tier:              policy.Spec.Tier,
 				EnforcementMode:   policy.Spec.SecurityPolicyEnforcementMode.String(),
 				Action:            policycache.RuleActionDrop,
@@ -368,15 +368,6 @@ func (r *Reconciler) completePolicy(policy *securityv1alpha1.SecurityPolicy) ([]
 	if egressEnabled {
 		for _, rule := range policy.Spec.EgressRules {
 			egressRule := &policycache.CompleteRule{
-<<<<<<< HEAD
-				RuleID:        fmt.Sprintf("%s/%s/%s.%s", policy.Namespace, policy.Name, "egress", rule.Name),
-				Tier:          policy.Spec.Tier,
-				Action:        policycache.RuleActionAllow,
-				Direction:     policycache.RuleDirectionOut,
-				SymmetricMode: policy.Spec.SymmetricMode,
-				SrcGroups:     policycache.DeepCopyMap(appliedGroups).(map[string]int32),
-				SrcIPBlocks:   policycache.DeepCopyMap(appliedIPBlocks).(map[string]*policycache.IPBlockItem),
-=======
 				RuleID:          fmt.Sprintf("%s/%s/%s/%s.%s", policy.Namespace, policy.Name, policycache.NormalPolicy, "egress", rule.Name),
 				Tier:            policy.Spec.Tier,
 				EnforcementMode: policy.Spec.SecurityPolicyEnforcementMode.String(),
@@ -385,7 +376,6 @@ func (r *Reconciler) completePolicy(policy *securityv1alpha1.SecurityPolicy) ([]
 				SymmetricMode:   policy.Spec.SymmetricMode,
 				SrcGroups:       policycache.DeepCopyMap(appliedGroups).(map[string]int32),
 				SrcIPBlocks:     policycache.DeepCopyMap(appliedIPBlocks).(map[string]*policycache.IPBlockItem),
->>>>>>> 1f9a0f0 (add monitor mode support)
 			}
 
 			if len(rule.To) == 0 {
@@ -414,7 +404,7 @@ func (r *Reconciler) completePolicy(policy *securityv1alpha1.SecurityPolicy) ([]
 
 		if policy.Spec.DefaultRule == securityv1alpha1.DefaultRuleDrop {
 			defaultEgressRule := &policycache.CompleteRule{
-				RuleID:            fmt.Sprintf("%s/%s/%s.%s", policy.Namespace, policy.Name, "default", "egress"),
+				RuleID:            fmt.Sprintf("%s/%s/%s/%s.%s", policy.Namespace, policy.Name, policycache.NormalPolicy, "default", "egress"),
 				Tier:              policy.Spec.Tier,
 				EnforcementMode:   policy.Spec.SecurityPolicyEnforcementMode.String(),
 				Action:            policycache.RuleActionDrop,
diff --git a/pkg/agent/datapath/multiBridgeDatapath.go b/pkg/agent/datapath/multiBridgeDatapath.go
index d0831607b..e3db1af43 100644
--- a/pkg/agent/datapath/multiBridgeDatapath.go
+++ b/pkg/agent/datapath/multiBridgeDatapath.go
@@ -39,10 +39,10 @@ import (
 	"github.com/contiv/ofnet/ovsdbDriver"
 	"github.com/fsnotify/fsnotify"
 	cmap "github.com/streamrail/concurrent-map"
-	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 
+	policycache "github.com/everoute/everoute/pkg/agent/controller/policy/cache"
 	"github.com/everoute/everoute/pkg/constants"
 )
 
@@ -257,9 +257,14 @@ type RoundInfo struct {
 }
 
 type PolicyInfo struct {
-	Dir            uint8
-	Action         string
-	NamespacedName []types.NamespacedName
+	Dir    uint8
+	Action string
+	Item   []PolicyItem
+}
+type PolicyItem struct {
+	Name       string
+	Namespace  string
+	PolicyType policycache.PolicyType
 }
 
 // Datapath manager act as openflow controller:
@@ -365,9 +370,10 @@ func (datapathManager *DpManager) GetPolicyByFlowID(flowID ...uint64) []*PolicyI
 				Action: item.EveroutePolicyRule.Action,
 			}
 			for _, p := range item.PolicyRuleReference.List() {
-				policyInfo.NamespacedName = append(policyInfo.NamespacedName, types.NamespacedName{
-					Namespace: strings.Split(p, "/")[0],
-					Name:      strings.Split(p, "/")[1],
+				policyInfo.Item = append(policyInfo.Item, PolicyItem{
+					Name:       strings.Split(p, "/")[1],
+					Namespace:  strings.Split(p, "/")[0],
+					PolicyType: policycache.PolicyType(strings.Split(p, "/")[2]),
 				})
 			}
 			policyInfoList = append(policyInfoList, policyInfo)

From b2c87a4d1759b821fc5ffe4aea70e8acde361978 Mon Sep 17 00:00:00 2001
From: Changliang Wu <changliang.wu@smartx.com>
Date: Tue, 22 Mar 2022 12:53:42 +0800
Subject: [PATCH 13/32] feat(exporter): modify ct label decode policy for new
 enforce mode

---
 pkg/agent/datapath/multiBridgeDatapath.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/pkg/agent/datapath/multiBridgeDatapath.go b/pkg/agent/datapath/multiBridgeDatapath.go
index e3db1af43..37935c4d9 100644
--- a/pkg/agent/datapath/multiBridgeDatapath.go
+++ b/pkg/agent/datapath/multiBridgeDatapath.go
@@ -259,6 +259,7 @@ type RoundInfo struct {
 type PolicyInfo struct {
 	Dir    uint8
 	Action string
+	Mode   string
 	Item   []PolicyItem
 }
 type PolicyItem struct {
@@ -363,11 +364,15 @@ func (datapathManager *DpManager) GetPolicyByFlowID(flowID ...uint64) []*PolicyI
 	var policyInfoList []*PolicyInfo
 
 	for _, id := range flowID {
+		if id == 0 {
+			continue
+		}
 		item := datapathManager.FlowIdToRules[id]
 		if item != nil {
 			policyInfo := &PolicyInfo{
 				Dir:    item.Direction,
 				Action: item.EveroutePolicyRule.Action,
+				Mode:   item.Mode,
 			}
 			for _, p := range item.PolicyRuleReference.List() {
 				policyInfo.Item = append(policyInfo.Item, PolicyItem{

From 270e24e12dbd85d358c2f7d18315eae0f02475e4 Mon Sep 17 00:00:00 2001
From: Changliang Wu <changliang.wu@smartx.com>
Date: Wed, 13 Apr 2022 16:20:46 +0800
Subject: [PATCH 14/32] feat: add policy type for internal policy

Signed-off-by: Changliang Wu <changliang.wu@smartx.com>
---
 pkg/agent/controller/policy/cache/rule.go | 5 +++--
 pkg/agent/datapath/multiBridgeDatapath.go | 4 ++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/pkg/agent/controller/policy/cache/rule.go b/pkg/agent/controller/policy/cache/rule.go
index 1cecacc42..f424940bf 100644
--- a/pkg/agent/controller/policy/cache/rule.go
+++ b/pkg/agent/controller/policy/cache/rule.go
@@ -45,8 +45,9 @@ const (
 	RuleDirectionIn  RuleDirection = "Ingress"
 	RuleDirectionOut RuleDirection = "Egress"
 
-	NormalPolicy PolicyType = "normal"
-	GlobalPolicy PolicyType = "global"
+	NormalPolicy   PolicyType = "normal"
+	GlobalPolicy   PolicyType = "global"
+	InternalPolicy PolicyType = "internal"
 )
 
 type PolicyRule struct {
diff --git a/pkg/agent/datapath/multiBridgeDatapath.go b/pkg/agent/datapath/multiBridgeDatapath.go
index 37935c4d9..84e0de808 100644
--- a/pkg/agent/datapath/multiBridgeDatapath.go
+++ b/pkg/agent/datapath/multiBridgeDatapath.go
@@ -130,8 +130,8 @@ const (
 	ClsToUplinkSuffix   = "cls-to-uplink"
 	UplinkToClsSuffix   = "uplink-to-cls"
 
-	InternalIngressRulePrefix = "/INTERNAL_INGRESS_POLICY/ingress/-"
-	InternalEgressRulePrefix  = "/INTERNAL_EGRESS_POLICY/egress/-"
+	InternalIngressRulePrefix = "/INTERNAL_INGRESS_POLICY/internal/ingress/-"
+	InternalEgressRulePrefix  = "/INTERNAL_EGRESS_POLICY/internal/egress/-"
 
 	MaxRoundNum = 15
 )

From d14ddc6a95b02bfc3cce74ce6fb6d284d1b4a3b3 Mon Sep 17 00:00:00 2001
From: Changliang Wu <changliang.wu@smartx.com>
Date: Mon, 16 May 2022 16:17:28 +0800
Subject: [PATCH 15/32] style: fix lint error

Signed-off-by: Changliang Wu <changliang.wu@smartx.com>
---
 pkg/agent/datapath/multiBridgeDatapath.go | 19 ++++++++++---------
 pkg/agent/datapath/policyBridge.go        |  1 +
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/pkg/agent/datapath/multiBridgeDatapath.go b/pkg/agent/datapath/multiBridgeDatapath.go
index 84e0de808..96e38c939 100644
--- a/pkg/agent/datapath/multiBridgeDatapath.go
+++ b/pkg/agent/datapath/multiBridgeDatapath.go
@@ -180,7 +180,7 @@ type DpManager struct {
 	ofPortIPAddressUpdateChan chan map[string]net.IP // map bridgename-ofport to endpoint ips
 	datapathConfig            *Config
 	Rules                     map[string]*EveroutePolicyRuleEntry // rules database
-	FlowIdToRules             map[uint64]*EveroutePolicyRuleEntry
+	FlowIDToRules             map[uint64]*EveroutePolicyRuleEntry
 	flowReplayChan            chan struct{}
 	flowReplayMutex           sync.RWMutex
 	ovsdbReconnectChan        chan struct{}
@@ -279,7 +279,7 @@ func NewDatapathManager(datapathConfig *Config, ofPortIPAddressUpdateChan chan m
 	datapathManager.ControllerMap = make(map[string]map[string]*ofctrl.Controller)
 	datapathManager.controllerIDSets = sets.NewString()
 	datapathManager.Rules = make(map[string]*EveroutePolicyRuleEntry)
-	datapathManager.FlowIdToRules = make(map[uint64]*EveroutePolicyRuleEntry)
+	datapathManager.FlowIDToRules = make(map[uint64]*EveroutePolicyRuleEntry)
 	datapathManager.datapathConfig = datapathConfig
 	datapathManager.localEndpointDB = cmap.New()
 	datapathManager.AgentInfo = new(AgentConf)
@@ -321,12 +321,14 @@ func (datapathManager *DpManager) InitializeDatapath(stopChan <-chan struct{}) {
 	// add rules for internalIP
 	for _, internalIP := range datapathManager.datapathConfig.InternalIPs {
 		// internal ingress rule
-		err := datapathManager.AddEveroutePolicyRule(newInternalIngressRule(internalIP), InternalIngressRulePrefix, POLICY_DIRECTION_IN, POLICY_TIER3, DEFAULT_POLICY_ENFORCEMENT_MODE)
+		err := datapathManager.AddEveroutePolicyRule(newInternalIngressRule(internalIP),
+			InternalIngressRulePrefix, POLICY_DIRECTION_IN, POLICY_TIER3, DEFAULT_POLICY_ENFORCEMENT_MODE)
 		if err != nil {
 			log.Fatalf("Failed to add internal whitelist: %v", err)
 		}
 		// internal egress rule
-		err = datapathManager.AddEveroutePolicyRule(newInternalEgressRule(internalIP), InternalEgressRulePrefix, POLICY_DIRECTION_OUT, POLICY_TIER3, DEFAULT_POLICY_ENFORCEMENT_MODE)
+		err = datapathManager.AddEveroutePolicyRule(newInternalEgressRule(internalIP),
+			InternalEgressRulePrefix, POLICY_DIRECTION_OUT, POLICY_TIER3, DEFAULT_POLICY_ENFORCEMENT_MODE)
 		if err != nil {
 			log.Fatalf("Failed to add internal whitelist: %v", err)
 		}
@@ -367,7 +369,7 @@ func (datapathManager *DpManager) GetPolicyByFlowID(flowID ...uint64) []*PolicyI
 		if id == 0 {
 			continue
 		}
-		item := datapathManager.FlowIdToRules[id]
+		item := datapathManager.FlowIDToRules[id]
 		if item != nil {
 			policyInfo := &PolicyInfo{
 				Dir:    item.Direction,
@@ -833,9 +835,8 @@ func (datapathManager *DpManager) AddEveroutePolicyRule(rule *EveroutePolicyRule
 			datapathManager.Rules[rule.RuleID].PolicyRuleReference.Insert(ruleName)
 			log.Infof("Rule already exists. new rule: {%+v}, old rule: {%+v}", rule, ruleEntry.EveroutePolicyRule)
 			return nil
-		} else {
-			log.Infof("Rule already exists. update old rule: {%+v} to new rule: {%+v} ", ruleEntry.EveroutePolicyRule, rule)
 		}
+		log.Infof("Rule already exists. update old rule: {%+v} to new rule: {%+v} ", ruleEntry.EveroutePolicyRule, rule)
 	}
 
 	log.Infof("Received AddRule: %+v", rule)
@@ -864,7 +865,7 @@ func (datapathManager *DpManager) AddEveroutePolicyRule(rule *EveroutePolicyRule
 
 	// save flowID reference
 	for _, v := range ruleEntry.RuleFlowMap {
-		datapathManager.FlowIdToRules[v.FlowID] = ruleEntry
+		datapathManager.FlowIDToRules[v.FlowID] = ruleEntry
 		log.Info(v.FlowID)
 	}
 
@@ -901,7 +902,7 @@ func (datapathManager *DpManager) RemoveEveroutePolicyRule(ruleID string, ruleNa
 			return err
 		}
 		// remove flowID reference
-		delete(datapathManager.FlowIdToRules, pRule.RuleFlowMap[vdsID].FlowID)
+		delete(datapathManager.FlowIDToRules, pRule.RuleFlowMap[vdsID].FlowID)
 	}
 
 	if pRule.PolicyRuleReference.Len() == 0 {
diff --git a/pkg/agent/datapath/policyBridge.go b/pkg/agent/datapath/policyBridge.go
index 508fd56f5..92909942c 100644
--- a/pkg/agent/datapath/policyBridge.go
+++ b/pkg/agent/datapath/policyBridge.go
@@ -512,6 +512,7 @@ func (p *PolicyBridge) GetTierTable(direction uint8, tier uint8, mode string) (*
 	return policyTable, nextTable, nil
 }
 
+//nolint: funlen
 func (p *PolicyBridge) AddMicroSegmentRule(rule *EveroutePolicyRule, direction uint8, tier uint8, mode string) (*FlowEntry, error) {
 	var ipDa *net.IP = nil
 	var ipDaMask *net.IP = nil

From 3af4e7545817665f447529e43de8fc6dbd143c39 Mon Sep 17 00:00:00 2001
From: zwtop <wang.zhan@smartx.com>
Date: Fri, 1 Apr 2022 14:41:56 +0800
Subject: [PATCH 16/32] refactor(tower-plugin): add more types to support
 everoute-analyzer

Signed-off-by: zwtop <wang.zhan@smartx.com>
---
 plugin/tower/pkg/schema/policy_types.go       |   1 +
 plugin/tower/pkg/schema/policy_types.graphqls |   1 +
 plugin/tower/pkg/schema/types.go              |  23 +-
 plugin/tower/pkg/schema/types.graphqls        |  16 +-
 .../server/fake/graph/generated/generated.go  | 596 +++++++++++++++++-
 plugin/tower/pkg/utils/testing/builder.go     |   8 -
 6 files changed, 598 insertions(+), 47 deletions(-)

diff --git a/plugin/tower/pkg/schema/policy_types.go b/plugin/tower/pkg/schema/policy_types.go
index edaa673bb..bf990c800 100644
--- a/plugin/tower/pkg/schema/policy_types.go
+++ b/plugin/tower/pkg/schema/policy_types.go
@@ -19,6 +19,7 @@ package schema
 type SecurityPolicy struct {
 	ObjectMeta
 
+	Name            string                `json:"name"`
 	EverouteCluster ObjectReference       `json:"everoute_cluster"`
 	ApplyTo         []SecurityPolicyApply `json:"apply_to"`
 	Ingress         []NetworkPolicyRule   `json:"ingress,omitempty"`
diff --git a/plugin/tower/pkg/schema/policy_types.graphqls b/plugin/tower/pkg/schema/policy_types.graphqls
index 89db6a782..5dc92780a 100644
--- a/plugin/tower/pkg/schema/policy_types.graphqls
+++ b/plugin/tower/pkg/schema/policy_types.graphqls
@@ -1,5 +1,6 @@
 type SecurityPolicy {
     id: ID!
+    name: String!
     everoute_cluster: ObjectReference!
     apply_to: [SecurityPolicyApply!]!
     ingress: [NetworkPolicyRule!]
diff --git a/plugin/tower/pkg/schema/types.go b/plugin/tower/pkg/schema/types.go
index 946dc0c27..8a60c207b 100644
--- a/plugin/tower/pkg/schema/types.go
+++ b/plugin/tower/pkg/schema/types.go
@@ -45,6 +45,8 @@ type VMNic struct {
 	Enabled     bool       `json:"enabled,omitempty"`
 	Mirror      bool       `json:"mirror,omitempty"`
 	Model       VMNicModel `json:"model,omitempty"`
+	MacAddress  string     `json:"mac_address,omitempty"`
+	IPAddress   string     `json:"ip_address,omitempty"`
 	InterfaceID string     `json:"interface_id,omitempty"`
 }
 
@@ -59,9 +61,10 @@ const (
 type Vlan struct {
 	ObjectMeta
 
-	Name   string      `json:"name,omitempty"`
-	VlanID int         `json:"vlan_id"`
-	Type   NetworkType `json:"type,omitempty"`
+	VDS    ObjectReference `json:"vds"`
+	Name   string          `json:"name,omitempty"`
+	VlanID int             `json:"vlan_id"`
+	Type   NetworkType     `json:"type,omitempty"`
 }
 
 // NetworkType is enumeration of network types
@@ -98,6 +101,7 @@ type EverouteCluster struct {
 	ObjectMeta
 
 	AgentELFClusters    []ObjectReference            `json:"agent_elf_clusters"`
+	AgentELFVDSes       []ObjectReference            `json:"agent_elf_vdses,omitempty"`
 	ControllerInstances []EverouteControllerInstance `json:"controller_instances"`
 	GlobalDefaultAction GlobalPolicyAction           `json:"global_default_action"`
 	GlobalWhitelist     EverouteClusterWhitelist     `json:"global_whitelist,omitempty"`
@@ -124,6 +128,15 @@ const (
 type Host struct {
 	ObjectMeta
 
-	ManagementIP string          `json:"management_ip"`
-	Cluster      ObjectReference `json:"cluster"`
+	Name string `json:"name,omitempty"`
+	Nics []Nic  `json:"nics,omitempty"`
+}
+
+type Nic struct {
+	ObjectMeta
+
+	Physical   bool   `json:"physical"`
+	Name       string `json:"name,omitempty"`
+	MacAddress string `json:"mac_address,omitempty"`
+	IPAddress  string `json:"ip_address,omitempty"`
 }
diff --git a/plugin/tower/pkg/schema/types.graphqls b/plugin/tower/pkg/schema/types.graphqls
index 27f88b37b..1252f1e16 100644
--- a/plugin/tower/pkg/schema/types.graphqls
+++ b/plugin/tower/pkg/schema/types.graphqls
@@ -22,6 +22,8 @@ type VMNic {
     enabled: Boolean
     mirror: Boolean
     model: VMNicModel
+    mac_address: String
+    ip_address: String
     interface_id: String
 }
 
@@ -32,6 +34,7 @@ enum VMNicModel {
 
 type Vlan {
     id: ID!
+    vds: ObjectReference!
     name: String!
     vlan_id: Int!
     type: NetworkType!
@@ -55,6 +58,7 @@ type Label {
 type EverouteCluster {
     id: ID!
     agent_elf_clusters: [ObjectReference!]
+    agent_elf_vdses: [ObjectReference!]
     controller_instances: [EverouteControllerInstance!]!
     global_default_action: GlobalPolicyAction!
     global_whitelist: EverouteClusterWhitelist
@@ -77,6 +81,14 @@ enum GlobalPolicyAction {
 
 type Host {
     id: ID!
-    cluster: ObjectReference!
-    management_ip: String!
+    name: String!
+    nics: [Nic!]
+}
+
+type Nic {
+    id: ID!
+    physical: Boolean!
+    name: String!
+    mac_address: String!
+    ip_address: String
 }
diff --git a/plugin/tower/pkg/server/fake/graph/generated/generated.go b/plugin/tower/pkg/server/fake/graph/generated/generated.go
index 7d6661382..d4b30cf01 100644
--- a/plugin/tower/pkg/server/fake/graph/generated/generated.go
+++ b/plugin/tower/pkg/server/fake/graph/generated/generated.go
@@ -50,6 +50,7 @@ type DirectiveRoot struct {
 type ComplexityRoot struct {
 	EverouteCluster struct {
 		AgentELFClusters    func(childComplexity int) int
+		AgentELFVDSes       func(childComplexity int) int
 		ControllerInstances func(childComplexity int) int
 		GlobalDefaultAction func(childComplexity int) int
 		GlobalWhitelist     func(childComplexity int) int
@@ -73,9 +74,9 @@ type ComplexityRoot struct {
 	}
 
 	Host struct {
-		Cluster      func(childComplexity int) int
-		ID           func(childComplexity int) int
-		ManagementIP func(childComplexity int) int
+		ID   func(childComplexity int) int
+		Name func(childComplexity int) int
+		Nics func(childComplexity int) int
 	}
 
 	HostEvent struct {
@@ -149,6 +150,14 @@ type ComplexityRoot struct {
 		Protocol func(childComplexity int) int
 	}
 
+	Nic struct {
+		ID         func(childComplexity int) int
+		IPAddress  func(childComplexity int) int
+		MacAddress func(childComplexity int) int
+		Name       func(childComplexity int) int
+		Physical   func(childComplexity int) int
+	}
+
 	ObjectReference struct {
 		ID func(childComplexity int) int
 	}
@@ -184,6 +193,7 @@ type ComplexityRoot struct {
 		EverouteCluster func(childComplexity int) int
 		ID              func(childComplexity int) int
 		Ingress         func(childComplexity int) int
+		Name            func(childComplexity int) int
 	}
 
 	SecurityPolicyApply struct {
@@ -252,7 +262,9 @@ type ComplexityRoot struct {
 	VMNic struct {
 		Enabled     func(childComplexity int) int
 		ID          func(childComplexity int) int
+		IPAddress   func(childComplexity int) int
 		InterfaceID func(childComplexity int) int
+		MacAddress  func(childComplexity int) int
 		Mirror      func(childComplexity int) int
 		Model       func(childComplexity int) int
 		Vlan        func(childComplexity int) int
@@ -262,6 +274,7 @@ type ComplexityRoot struct {
 		ID     func(childComplexity int) int
 		Name   func(childComplexity int) int
 		Type   func(childComplexity int) int
+		VDS    func(childComplexity int) int
 		VlanID func(childComplexity int) int
 	}
 }
@@ -317,6 +330,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in
 
 		return e.complexity.EverouteCluster.AgentELFClusters(childComplexity), true
 
+	case "EverouteCluster.agent_elf_vdses":
+		if e.complexity.EverouteCluster.AgentELFVDSes == nil {
+			break
+		}
+
+		return e.complexity.EverouteCluster.AgentELFVDSes(childComplexity), true
+
 	case "EverouteCluster.controller_instances":
 		if e.complexity.EverouteCluster.ControllerInstances == nil {
 			break
@@ -394,26 +414,26 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in
 
 		return e.complexity.EverouteControllerInstance.IPAddr(childComplexity), true
 
-	case "Host.cluster":
-		if e.complexity.Host.Cluster == nil {
+	case "Host.id":
+		if e.complexity.Host.ID == nil {
 			break
 		}
 
-		return e.complexity.Host.Cluster(childComplexity), true
+		return e.complexity.Host.ID(childComplexity), true
 
-	case "Host.id":
-		if e.complexity.Host.ID == nil {
+	case "Host.name":
+		if e.complexity.Host.Name == nil {
 			break
 		}
 
-		return e.complexity.Host.ID(childComplexity), true
+		return e.complexity.Host.Name(childComplexity), true
 
-	case "Host.management_ip":
-		if e.complexity.Host.ManagementIP == nil {
+	case "Host.nics":
+		if e.complexity.Host.Nics == nil {
 			break
 		}
 
-		return e.complexity.Host.ManagementIP(childComplexity), true
+		return e.complexity.Host.Nics(childComplexity), true
 
 	case "HostEvent.mutation":
 		if e.complexity.HostEvent.Mutation == nil {
@@ -665,6 +685,41 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in
 
 		return e.complexity.NetworkPolicyRulePort.Protocol(childComplexity), true
 
+	case "Nic.id":
+		if e.complexity.Nic.ID == nil {
+			break
+		}
+
+		return e.complexity.Nic.ID(childComplexity), true
+
+	case "Nic.ip_address":
+		if e.complexity.Nic.IPAddress == nil {
+			break
+		}
+
+		return e.complexity.Nic.IPAddress(childComplexity), true
+
+	case "Nic.mac_address":
+		if e.complexity.Nic.MacAddress == nil {
+			break
+		}
+
+		return e.complexity.Nic.MacAddress(childComplexity), true
+
+	case "Nic.name":
+		if e.complexity.Nic.Name == nil {
+			break
+		}
+
+		return e.complexity.Nic.Name(childComplexity), true
+
+	case "Nic.physical":
+		if e.complexity.Nic.Physical == nil {
+			break
+		}
+
+		return e.complexity.Nic.Physical(childComplexity), true
+
 	case "ObjectReference.id":
 		if e.complexity.ObjectReference.ID == nil {
 			break
@@ -824,6 +879,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in
 
 		return e.complexity.SecurityPolicy.Ingress(childComplexity), true
 
+	case "SecurityPolicy.name":
+		if e.complexity.SecurityPolicy.Name == nil {
+			break
+		}
+
+		return e.complexity.SecurityPolicy.Name(childComplexity), true
+
 	case "SecurityPolicyApply.communicable":
 		if e.complexity.SecurityPolicyApply.Communicable == nil {
 			break
@@ -1111,6 +1173,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in
 
 		return e.complexity.VMNic.ID(childComplexity), true
 
+	case "VMNic.ip_address":
+		if e.complexity.VMNic.IPAddress == nil {
+			break
+		}
+
+		return e.complexity.VMNic.IPAddress(childComplexity), true
+
 	case "VMNic.interface_id":
 		if e.complexity.VMNic.InterfaceID == nil {
 			break
@@ -1118,6 +1187,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in
 
 		return e.complexity.VMNic.InterfaceID(childComplexity), true
 
+	case "VMNic.mac_address":
+		if e.complexity.VMNic.MacAddress == nil {
+			break
+		}
+
+		return e.complexity.VMNic.MacAddress(childComplexity), true
+
 	case "VMNic.mirror":
 		if e.complexity.VMNic.Mirror == nil {
 			break
@@ -1160,6 +1236,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in
 
 		return e.complexity.Vlan.Type(childComplexity), true
 
+	case "Vlan.vds":
+		if e.complexity.Vlan.VDS == nil {
+			break
+		}
+
+		return e.complexity.Vlan.VDS(childComplexity), true
+
 	case "Vlan.vlan_id":
 		if e.complexity.Vlan.VlanID == nil {
 			break
@@ -1359,6 +1442,7 @@ enum TaskOrderByInput {
 `, BuiltIn: false},
 	{Name: "../../schema/policy_types.graphqls", Input: `type SecurityPolicy {
     id: ID!
+    name: String!
     everoute_cluster: ObjectReference!
     apply_to: [SecurityPolicyApply!]!
     ingress: [NetworkPolicyRule!]
@@ -1488,6 +1572,8 @@ type VMNic {
     enabled: Boolean
     mirror: Boolean
     model: VMNicModel
+    mac_address: String
+    ip_address: String
     interface_id: String
 }
 
@@ -1498,6 +1584,7 @@ enum VMNicModel {
 
 type Vlan {
     id: ID!
+    vds: ObjectReference!
     name: String!
     vlan_id: Int!
     type: NetworkType!
@@ -1521,6 +1608,7 @@ type Label {
 type EverouteCluster {
     id: ID!
     agent_elf_clusters: [ObjectReference!]
+    agent_elf_vdses: [ObjectReference!]
     controller_instances: [EverouteControllerInstance!]!
     global_default_action: GlobalPolicyAction!
     global_whitelist: EverouteClusterWhitelist
@@ -1543,8 +1631,16 @@ enum GlobalPolicyAction {
 
 type Host {
     id: ID!
-    cluster: ObjectReference!
-    management_ip: String!
+    name: String!
+    nics: [Nic!]
+}
+
+type Nic {
+    id: ID!
+    physical: Boolean!
+    name: String!
+    mac_address: String!
+    ip_address: String
 }
 `, BuiltIn: false},
 }
@@ -1713,6 +1809,38 @@ func (ec *executionContext) _EverouteCluster_agent_elf_clusters(ctx context.Cont
 	return ec.marshalOObjectReference2ᚕgithubᚗcomᚋeverouteᚋeverouteᚋpluginᚋtowerᚋpkgᚋschemaᚐObjectReferenceᚄ(ctx, field.Selections, res)
 }
 
+func (ec *executionContext) _EverouteCluster_agent_elf_vdses(ctx context.Context, field graphql.CollectedField, obj *schema.EverouteCluster) (ret graphql.Marshaler) {
+	defer func() {
+		if r := recover(); r != nil {
+			ec.Error(ctx, ec.Recover(ctx, r))
+			ret = graphql.Null
+		}
+	}()
+	fc := &graphql.FieldContext{
+		Object:     "EverouteCluster",
+		Field:      field,
+		Args:       nil,
+		IsMethod:   false,
+		IsResolver: false,
+	}
+
+	ctx = graphql.WithFieldContext(ctx, fc)
+	resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
+		ctx = rctx // use context from middleware stack in children
+		return obj.AgentELFVDSes, nil
+	})
+	if err != nil {
+		ec.Error(ctx, err)
+		return graphql.Null
+	}
+	if resTmp == nil {
+		return graphql.Null
+	}
+	res := resTmp.([]schema.ObjectReference)
+	fc.Result = res
+	return ec.marshalOObjectReference2ᚕgithubᚗcomᚋeverouteᚋeverouteᚋpluginᚋtowerᚋpkgᚋschemaᚐObjectReferenceᚄ(ctx, field.Selections, res)
+}
+
 func (ec *executionContext) _EverouteCluster_controller_instances(ctx context.Context, field graphql.CollectedField, obj *schema.EverouteCluster) (ret graphql.Marshaler) {
 	defer func() {
 		if r := recover(); r != nil {
@@ -2086,7 +2214,7 @@ func (ec *executionContext) _Host_id(ctx context.Context, field graphql.Collecte
 	return ec.marshalNID2string(ctx, field.Selections, res)
 }
 
-func (ec *executionContext) _Host_cluster(ctx context.Context, field graphql.CollectedField, obj *schema.Host) (ret graphql.Marshaler) {
+func (ec *executionContext) _Host_name(ctx context.Context, field graphql.CollectedField, obj *schema.Host) (ret graphql.Marshaler) {
 	defer func() {
 		if r := recover(); r != nil {
 			ec.Error(ctx, ec.Recover(ctx, r))
@@ -2104,7 +2232,7 @@ func (ec *executionContext) _Host_cluster(ctx context.Context, field graphql.Col
 	ctx = graphql.WithFieldContext(ctx, fc)
 	resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
 		ctx = rctx // use context from middleware stack in children
-		return obj.Cluster, nil
+		return obj.Name, nil
 	})
 	if err != nil {
 		ec.Error(ctx, err)
@@ -2116,12 +2244,12 @@ func (ec *executionContext) _Host_cluster(ctx context.Context, field graphql.Col
 		}
 		return graphql.Null
 	}
-	res := resTmp.(schema.ObjectReference)
+	res := resTmp.(string)
 	fc.Result = res
-	return ec.marshalNObjectReference2githubᚗcomᚋeverouteᚋeverouteᚋpluginᚋtowerᚋpkgᚋschemaᚐObjectReference(ctx, field.Selections, res)
+	return ec.marshalNString2string(ctx, field.Selections, res)
 }
 
-func (ec *executionContext) _Host_management_ip(ctx context.Context, field graphql.CollectedField, obj *schema.Host) (ret graphql.Marshaler) {
+func (ec *executionContext) _Host_nics(ctx context.Context, field graphql.CollectedField, obj *schema.Host) (ret graphql.Marshaler) {
 	defer func() {
 		if r := recover(); r != nil {
 			ec.Error(ctx, ec.Recover(ctx, r))
@@ -2139,21 +2267,18 @@ func (ec *executionContext) _Host_management_ip(ctx context.Context, field graph
 	ctx = graphql.WithFieldContext(ctx, fc)
 	resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
 		ctx = rctx // use context from middleware stack in children
-		return obj.ManagementIP, nil
+		return obj.Nics, nil
 	})
 	if err != nil {
 		ec.Error(ctx, err)
 		return graphql.Null
 	}
 	if resTmp == nil {
-		if !graphql.HasFieldError(ctx, fc) {
-			ec.Errorf(ctx, "must not be null")
-		}
 		return graphql.Null
 	}
-	res := resTmp.(string)
+	res := resTmp.([]schema.Nic)
 	fc.Result = res
-	return ec.marshalNString2string(ctx, field.Selections, res)
+	return ec.marshalONic2ᚕgithubᚗcomᚋeverouteᚋeverouteᚋpluginᚋtowerᚋpkgᚋschemaᚐNicᚄ(ctx, field.Selections, res)
 }
 
 func (ec *executionContext) _HostEvent_mutation(ctx context.Context, field graphql.CollectedField, obj *model.HostEvent) (ret graphql.Marshaler) {
@@ -3346,6 +3471,178 @@ func (ec *executionContext) _NetworkPolicyRulePort_protocol(ctx context.Context,
 	return ec.marshalNNetworkPolicyRulePortProtocol2githubᚗcomᚋeverouteᚋeverouteᚋpluginᚋtowerᚋpkgᚋschemaᚐNetworkPolicyRulePortProtocol(ctx, field.Selections, res)
 }
 
+func (ec *executionContext) _Nic_id(ctx context.Context, field graphql.CollectedField, obj *schema.Nic) (ret graphql.Marshaler) {
+	defer func() {
+		if r := recover(); r != nil {
+			ec.Error(ctx, ec.Recover(ctx, r))
+			ret = graphql.Null
+		}
+	}()
+	fc := &graphql.FieldContext{
+		Object:     "Nic",
+		Field:      field,
+		Args:       nil,
+		IsMethod:   false,
+		IsResolver: false,
+	}
+
+	ctx = graphql.WithFieldContext(ctx, fc)
+	resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
+		ctx = rctx // use context from middleware stack in children
+		return obj.ID, nil
+	})
+	if err != nil {
+		ec.Error(ctx, err)
+		return graphql.Null
+	}
+	if resTmp == nil {
+		if !graphql.HasFieldError(ctx, fc) {
+			ec.Errorf(ctx, "must not be null")
+		}
+		return graphql.Null
+	}
+	res := resTmp.(string)
+	fc.Result = res
+	return ec.marshalNID2string(ctx, field.Selections, res)
+}
+
+func (ec *executionContext) _Nic_physical(ctx context.Context, field graphql.CollectedField, obj *schema.Nic) (ret graphql.Marshaler) {
+	defer func() {
+		if r := recover(); r != nil {
+			ec.Error(ctx, ec.Recover(ctx, r))
+			ret = graphql.Null
+		}
+	}()
+	fc := &graphql.FieldContext{
+		Object:     "Nic",
+		Field:      field,
+		Args:       nil,
+		IsMethod:   false,
+		IsResolver: false,
+	}
+
+	ctx = graphql.WithFieldContext(ctx, fc)
+	resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
+		ctx = rctx // use context from middleware stack in children
+		return obj.Physical, nil
+	})
+	if err != nil {
+		ec.Error(ctx, err)
+		return graphql.Null
+	}
+	if resTmp == nil {
+		if !graphql.HasFieldError(ctx, fc) {
+			ec.Errorf(ctx, "must not be null")
+		}
+		return graphql.Null
+	}
+	res := resTmp.(bool)
+	fc.Result = res
+	return ec.marshalNBoolean2bool(ctx, field.Selections, res)
+}
+
+func (ec *executionContext) _Nic_name(ctx context.Context, field graphql.CollectedField, obj *schema.Nic) (ret graphql.Marshaler) {
+	defer func() {
+		if r := recover(); r != nil {
+			ec.Error(ctx, ec.Recover(ctx, r))
+			ret = graphql.Null
+		}
+	}()
+	fc := &graphql.FieldContext{
+		Object:     "Nic",
+		Field:      field,
+		Args:       nil,
+		IsMethod:   false,
+		IsResolver: false,
+	}
+
+	ctx = graphql.WithFieldContext(ctx, fc)
+	resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
+		ctx = rctx // use context from middleware stack in children
+		return obj.Name, nil
+	})
+	if err != nil {
+		ec.Error(ctx, err)
+		return graphql.Null
+	}
+	if resTmp == nil {
+		if !graphql.HasFieldError(ctx, fc) {
+			ec.Errorf(ctx, "must not be null")
+		}
+		return graphql.Null
+	}
+	res := resTmp.(string)
+	fc.Result = res
+	return ec.marshalNString2string(ctx, field.Selections, res)
+}
+
+func (ec *executionContext) _Nic_mac_address(ctx context.Context, field graphql.CollectedField, obj *schema.Nic) (ret graphql.Marshaler) {
+	defer func() {
+		if r := recover(); r != nil {
+			ec.Error(ctx, ec.Recover(ctx, r))
+			ret = graphql.Null
+		}
+	}()
+	fc := &graphql.FieldContext{
+		Object:     "Nic",
+		Field:      field,
+		Args:       nil,
+		IsMethod:   false,
+		IsResolver: false,
+	}
+
+	ctx = graphql.WithFieldContext(ctx, fc)
+	resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
+		ctx = rctx // use context from middleware stack in children
+		return obj.MacAddress, nil
+	})
+	if err != nil {
+		ec.Error(ctx, err)
+		return graphql.Null
+	}
+	if resTmp == nil {
+		if !graphql.HasFieldError(ctx, fc) {
+			ec.Errorf(ctx, "must not be null")
+		}
+		return graphql.Null
+	}
+	res := resTmp.(string)
+	fc.Result = res
+	return ec.marshalNString2string(ctx, field.Selections, res)
+}
+
+func (ec *executionContext) _Nic_ip_address(ctx context.Context, field graphql.CollectedField, obj *schema.Nic) (ret graphql.Marshaler) {
+	defer func() {
+		if r := recover(); r != nil {
+			ec.Error(ctx, ec.Recover(ctx, r))
+			ret = graphql.Null
+		}
+	}()
+	fc := &graphql.FieldContext{
+		Object:     "Nic",
+		Field:      field,
+		Args:       nil,
+		IsMethod:   false,
+		IsResolver: false,
+	}
+
+	ctx = graphql.WithFieldContext(ctx, fc)
+	resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
+		ctx = rctx // use context from middleware stack in children
+		return obj.IPAddress, nil
+	})
+	if err != nil {
+		ec.Error(ctx, err)
+		return graphql.Null
+	}
+	if resTmp == nil {
+		return graphql.Null
+	}
+	res := resTmp.(string)
+	fc.Result = res
+	return ec.marshalOString2string(ctx, field.Selections, res)
+}
+
 func (ec *executionContext) _ObjectReference_id(ctx context.Context, field graphql.CollectedField, obj *schema.ObjectReference) (ret graphql.Marshaler) {
 	defer func() {
 		if r := recover(); r != nil {
@@ -4045,6 +4342,41 @@ func (ec *executionContext) _SecurityPolicy_id(ctx context.Context, field graphq
 	return ec.marshalNID2string(ctx, field.Selections, res)
 }
 
+func (ec *executionContext) _SecurityPolicy_name(ctx context.Context, field graphql.CollectedField, obj *schema.SecurityPolicy) (ret graphql.Marshaler) {
+	defer func() {
+		if r := recover(); r != nil {
+			ec.Error(ctx, ec.Recover(ctx, r))
+			ret = graphql.Null
+		}
+	}()
+	fc := &graphql.FieldContext{
+		Object:     "SecurityPolicy",
+		Field:      field,
+		Args:       nil,
+		IsMethod:   false,
+		IsResolver: false,
+	}
+
+	ctx = graphql.WithFieldContext(ctx, fc)
+	resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
+		ctx = rctx // use context from middleware stack in children
+		return obj.Name, nil
+	})
+	if err != nil {
+		ec.Error(ctx, err)
+		return graphql.Null
+	}
+	if resTmp == nil {
+		if !graphql.HasFieldError(ctx, fc) {
+			ec.Errorf(ctx, "must not be null")
+		}
+		return graphql.Null
+	}
+	res := resTmp.(string)
+	fc.Result = res
+	return ec.marshalNString2string(ctx, field.Selections, res)
+}
+
 func (ec *executionContext) _SecurityPolicy_everoute_cluster(ctx context.Context, field graphql.CollectedField, obj *schema.SecurityPolicy) (ret graphql.Marshaler) {
 	defer func() {
 		if r := recover(); r != nil {
@@ -5770,6 +6102,70 @@ func (ec *executionContext) _VMNic_model(ctx context.Context, field graphql.Coll
 	return ec.marshalOVMNicModel2githubᚗcomᚋeverouteᚋeverouteᚋpluginᚋtowerᚋpkgᚋschemaᚐVMNicModel(ctx, field.Selections, res)
 }
 
+func (ec *executionContext) _VMNic_mac_address(ctx context.Context, field graphql.CollectedField, obj *schema.VMNic) (ret graphql.Marshaler) {
+	defer func() {
+		if r := recover(); r != nil {
+			ec.Error(ctx, ec.Recover(ctx, r))
+			ret = graphql.Null
+		}
+	}()
+	fc := &graphql.FieldContext{
+		Object:     "VMNic",
+		Field:      field,
+		Args:       nil,
+		IsMethod:   false,
+		IsResolver: false,
+	}
+
+	ctx = graphql.WithFieldContext(ctx, fc)
+	resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
+		ctx = rctx // use context from middleware stack in children
+		return obj.MacAddress, nil
+	})
+	if err != nil {
+		ec.Error(ctx, err)
+		return graphql.Null
+	}
+	if resTmp == nil {
+		return graphql.Null
+	}
+	res := resTmp.(string)
+	fc.Result = res
+	return ec.marshalOString2string(ctx, field.Selections, res)
+}
+
+func (ec *executionContext) _VMNic_ip_address(ctx context.Context, field graphql.CollectedField, obj *schema.VMNic) (ret graphql.Marshaler) {
+	defer func() {
+		if r := recover(); r != nil {
+			ec.Error(ctx, ec.Recover(ctx, r))
+			ret = graphql.Null
+		}
+	}()
+	fc := &graphql.FieldContext{
+		Object:     "VMNic",
+		Field:      field,
+		Args:       nil,
+		IsMethod:   false,
+		IsResolver: false,
+	}
+
+	ctx = graphql.WithFieldContext(ctx, fc)
+	resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
+		ctx = rctx // use context from middleware stack in children
+		return obj.IPAddress, nil
+	})
+	if err != nil {
+		ec.Error(ctx, err)
+		return graphql.Null
+	}
+	if resTmp == nil {
+		return graphql.Null
+	}
+	res := resTmp.(string)
+	fc.Result = res
+	return ec.marshalOString2string(ctx, field.Selections, res)
+}
+
 func (ec *executionContext) _VMNic_interface_id(ctx context.Context, field graphql.CollectedField, obj *schema.VMNic) (ret graphql.Marshaler) {
 	defer func() {
 		if r := recover(); r != nil {
@@ -5837,6 +6233,41 @@ func (ec *executionContext) _Vlan_id(ctx context.Context, field graphql.Collecte
 	return ec.marshalNID2string(ctx, field.Selections, res)
 }
 
+func (ec *executionContext) _Vlan_vds(ctx context.Context, field graphql.CollectedField, obj *schema.Vlan) (ret graphql.Marshaler) {
+	defer func() {
+		if r := recover(); r != nil {
+			ec.Error(ctx, ec.Recover(ctx, r))
+			ret = graphql.Null
+		}
+	}()
+	fc := &graphql.FieldContext{
+		Object:     "Vlan",
+		Field:      field,
+		Args:       nil,
+		IsMethod:   false,
+		IsResolver: false,
+	}
+
+	ctx = graphql.WithFieldContext(ctx, fc)
+	resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
+		ctx = rctx // use context from middleware stack in children
+		return obj.VDS, nil
+	})
+	if err != nil {
+		ec.Error(ctx, err)
+		return graphql.Null
+	}
+	if resTmp == nil {
+		if !graphql.HasFieldError(ctx, fc) {
+			ec.Errorf(ctx, "must not be null")
+		}
+		return graphql.Null
+	}
+	res := resTmp.(schema.ObjectReference)
+	fc.Result = res
+	return ec.marshalNObjectReference2githubᚗcomᚋeverouteᚋeverouteᚋpluginᚋtowerᚋpkgᚋschemaᚐObjectReference(ctx, field.Selections, res)
+}
+
 func (ec *executionContext) _Vlan_name(ctx context.Context, field graphql.CollectedField, obj *schema.Vlan) (ret graphql.Marshaler) {
 	defer func() {
 		if r := recover(); r != nil {
@@ -7091,6 +7522,8 @@ func (ec *executionContext) _EverouteCluster(ctx context.Context, sel ast.Select
 			}
 		case "agent_elf_clusters":
 			out.Values[i] = ec._EverouteCluster_agent_elf_clusters(ctx, field, obj)
+		case "agent_elf_vdses":
+			out.Values[i] = ec._EverouteCluster_agent_elf_vdses(ctx, field, obj)
 		case "controller_instances":
 			out.Values[i] = ec._EverouteCluster_controller_instances(ctx, field, obj)
 			if out.Values[i] == graphql.Null {
@@ -7222,16 +7655,13 @@ func (ec *executionContext) _Host(ctx context.Context, sel ast.SelectionSet, obj
 			if out.Values[i] == graphql.Null {
 				invalids++
 			}
-		case "cluster":
-			out.Values[i] = ec._Host_cluster(ctx, field, obj)
-			if out.Values[i] == graphql.Null {
-				invalids++
-			}
-		case "management_ip":
-			out.Values[i] = ec._Host_management_ip(ctx, field, obj)
+		case "name":
+			out.Values[i] = ec._Host_name(ctx, field, obj)
 			if out.Values[i] == graphql.Null {
 				invalids++
 			}
+		case "nics":
+			out.Values[i] = ec._Host_nics(ctx, field, obj)
 		default:
 			panic("unknown field " + strconv.Quote(field.Name))
 		}
@@ -7653,6 +8083,50 @@ func (ec *executionContext) _NetworkPolicyRulePort(ctx context.Context, sel ast.
 	return out
 }
 
+var nicImplementors = []string{"Nic"}
+
+func (ec *executionContext) _Nic(ctx context.Context, sel ast.SelectionSet, obj *schema.Nic) graphql.Marshaler {
+	fields := graphql.CollectFields(ec.OperationContext, sel, nicImplementors)
+
+	out := graphql.NewFieldSet(fields)
+	var invalids uint32
+	for i, field := range fields {
+		switch field.Name {
+		case "__typename":
+			out.Values[i] = graphql.MarshalString("Nic")
+		case "id":
+			out.Values[i] = ec._Nic_id(ctx, field, obj)
+			if out.Values[i] == graphql.Null {
+				invalids++
+			}
+		case "physical":
+			out.Values[i] = ec._Nic_physical(ctx, field, obj)
+			if out.Values[i] == graphql.Null {
+				invalids++
+			}
+		case "name":
+			out.Values[i] = ec._Nic_name(ctx, field, obj)
+			if out.Values[i] == graphql.Null {
+				invalids++
+			}
+		case "mac_address":
+			out.Values[i] = ec._Nic_mac_address(ctx, field, obj)
+			if out.Values[i] == graphql.Null {
+				invalids++
+			}
+		case "ip_address":
+			out.Values[i] = ec._Nic_ip_address(ctx, field, obj)
+		default:
+			panic("unknown field " + strconv.Quote(field.Name))
+		}
+	}
+	out.Dispatch()
+	if invalids > 0 {
+		return graphql.Null
+	}
+	return out
+}
+
 var objectReferenceImplementors = []string{"ObjectReference"}
 
 func (ec *executionContext) _ObjectReference(ctx context.Context, sel ast.SelectionSet, obj *schema.ObjectReference) graphql.Marshaler {
@@ -7922,6 +8396,11 @@ func (ec *executionContext) _SecurityPolicy(ctx context.Context, sel ast.Selecti
 			if out.Values[i] == graphql.Null {
 				invalids++
 			}
+		case "name":
+			out.Values[i] = ec._SecurityPolicy_name(ctx, field, obj)
+			if out.Values[i] == graphql.Null {
+				invalids++
+			}
 		case "everoute_cluster":
 			out.Values[i] = ec._SecurityPolicy_everoute_cluster(ctx, field, obj)
 			if out.Values[i] == graphql.Null {
@@ -8284,6 +8763,10 @@ func (ec *executionContext) _VMNic(ctx context.Context, sel ast.SelectionSet, ob
 			out.Values[i] = ec._VMNic_mirror(ctx, field, obj)
 		case "model":
 			out.Values[i] = ec._VMNic_model(ctx, field, obj)
+		case "mac_address":
+			out.Values[i] = ec._VMNic_mac_address(ctx, field, obj)
+		case "ip_address":
+			out.Values[i] = ec._VMNic_ip_address(ctx, field, obj)
 		case "interface_id":
 			out.Values[i] = ec._VMNic_interface_id(ctx, field, obj)
 		default:
@@ -8313,6 +8796,11 @@ func (ec *executionContext) _Vlan(ctx context.Context, sel ast.SelectionSet, obj
 			if out.Values[i] == graphql.Null {
 				invalids++
 			}
+		case "vds":
+			out.Values[i] = ec._Vlan_vds(ctx, field, obj)
+			if out.Values[i] == graphql.Null {
+				invalids++
+			}
 		case "name":
 			out.Values[i] = ec._Vlan_name(ctx, field, obj)
 			if out.Values[i] == graphql.Null {
@@ -9074,6 +9562,10 @@ func (ec *executionContext) marshalNNetworkType2githubᚗcomᚋeverouteᚋeverou
 	return res
 }
 
+func (ec *executionContext) marshalNNic2githubᚗcomᚋeverouteᚋeverouteᚋpluginᚋtowerᚋpkgᚋschemaᚐNic(ctx context.Context, sel ast.SelectionSet, v schema.Nic) graphql.Marshaler {
+	return ec._Nic(ctx, sel, &v)
+}
+
 func (ec *executionContext) marshalNObjectReference2githubᚗcomᚋeverouteᚋeverouteᚋpluginᚋtowerᚋpkgᚋschemaᚐObjectReference(ctx context.Context, sel ast.SelectionSet, v schema.ObjectReference) graphql.Marshaler {
 	return ec._ObjectReference(ctx, sel, &v)
 }
@@ -9979,6 +10471,46 @@ func (ec *executionContext) marshalONetworkPolicyRulePort2ᚕgithubᚗcomᚋever
 	return ret
 }
 
+func (ec *executionContext) marshalONic2ᚕgithubᚗcomᚋeverouteᚋeverouteᚋpluginᚋtowerᚋpkgᚋschemaᚐNicᚄ(ctx context.Context, sel ast.SelectionSet, v []schema.Nic) graphql.Marshaler {
+	if v == nil {
+		return graphql.Null
+	}
+	ret := make(graphql.Array, len(v))
+	var wg sync.WaitGroup
+	isLen1 := len(v) == 1
+	if !isLen1 {
+		wg.Add(len(v))
+	}
+	for i := range v {
+		i := i
+		fc := &graphql.FieldContext{
+			Index:  &i,
+			Result: &v[i],
+		}
+		ctx := graphql.WithFieldContext(ctx, fc)
+		f := func(i int) {
+			defer func() {
+				if r := recover(); r != nil {
+					ec.Error(ctx, ec.Recover(ctx, r))
+					ret = nil
+				}
+			}()
+			if !isLen1 {
+				defer wg.Done()
+			}
+			ret[i] = ec.marshalNNic2githubᚗcomᚋeverouteᚋeverouteᚋpluginᚋtowerᚋpkgᚋschemaᚐNic(ctx, sel, v[i])
+		}
+		if isLen1 {
+			f(i)
+		} else {
+			go f(i)
+		}
+
+	}
+	wg.Wait()
+	return ret
+}
+
 func (ec *executionContext) marshalOObjectReference2ᚕgithubᚗcomᚋeverouteᚋeverouteᚋpluginᚋtowerᚋpkgᚋschemaᚐObjectReferenceᚄ(ctx context.Context, sel ast.SelectionSet, v []schema.ObjectReference) graphql.Marshaler {
 	if v == nil {
 		return graphql.Null
diff --git a/plugin/tower/pkg/utils/testing/builder.go b/plugin/tower/pkg/utils/testing/builder.go
index 6e2431c04..eab4c69fe 100644
--- a/plugin/tower/pkg/utils/testing/builder.go
+++ b/plugin/tower/pkg/utils/testing/builder.go
@@ -227,14 +227,6 @@ func LabelAsReference(labels ...*schema.Label) []schema.ObjectReference {
 	return labelRefs
 }
 
-func NewRandomHost(elfClusterID string) *schema.Host {
-	return &schema.Host{
-		ObjectMeta:   schema.ObjectMeta{ID: rand.String(10)},
-		ManagementIP: NewRandomIP().String(),
-		Cluster:      schema.ObjectReference{ID: elfClusterID},
-	}
-}
-
 func NewEverouteCluster(erClusterID string, defaultAction schema.GlobalPolicyAction) *schema.EverouteCluster {
 	return &schema.EverouteCluster{
 		ObjectMeta:          schema.ObjectMeta{ID: erClusterID},

From 652e9ac31ad0b30882ca293b8e216d61303dd884 Mon Sep 17 00:00:00 2001
From: zwtop <wang.zhan@smartx.com>
Date: Fri, 1 Apr 2022 15:24:03 +0800
Subject: [PATCH 17/32] feat(plugin-tower): support sync policy enforce mode
 from tower

Signed-off-by: zwtop <wang.zhan@smartx.com>
---
 .../tower/pkg/controller/policy/controller.go |  39 +++--
 .../pkg/controller/policy/controller_test.go  | 139 +++++++++++++-----
 plugin/tower/pkg/schema/policy_types.go       |   8 +
 plugin/tower/pkg/schema/policy_types.graphqls |   6 +
 .../server/fake/graph/generated/generated.go  |  58 ++++++++
 5 files changed, 199 insertions(+), 51 deletions(-)

diff --git a/plugin/tower/pkg/controller/policy/controller.go b/plugin/tower/pkg/controller/policy/controller.go
index b449d1222..d7e3a4da3 100644
--- a/plugin/tower/pkg/controller/policy/controller.go
+++ b/plugin/tower/pkg/controller/policy/controller.go
@@ -933,6 +933,7 @@ func (c *Controller) parseSystemEndpointsPolicy(systemEndpoints *schema.SystemEn
 // parseSecurityPolicy convert schema.SecurityPolicy to []v1alpha1.SecurityPolicy
 func (c *Controller) parseSecurityPolicy(securityPolicy *schema.SecurityPolicy) ([]v1alpha1.SecurityPolicy, error) {
 	var policyList []v1alpha1.SecurityPolicy
+	var policyMode = parseEnforcementMode(securityPolicy.PolicyMode)
 
 	applyToPeers, err := c.parseSecurityPolicyApplys(securityPolicy.ApplyTo)
 	if err != nil {
@@ -953,13 +954,14 @@ func (c *Controller) parseSecurityPolicy(securityPolicy *schema.SecurityPolicy)
 			Namespace: c.namespace,
 		},
 		Spec: v1alpha1.SecurityPolicySpec{
-			Tier:          constants.Tier2,
-			SymmetricMode: true,
-			AppliedTo:     applyToPeers,
-			IngressRules:  ingress,
-			EgressRules:   egress,
-			DefaultRule:   v1alpha1.DefaultRuleDrop,
-			PolicyTypes:   []networkingv1.PolicyType{networkingv1.PolicyTypeIngress, networkingv1.PolicyTypeEgress},
+			Tier:                          constants.Tier2,
+			SecurityPolicyEnforcementMode: policyMode,
+			SymmetricMode:                 true,
+			AppliedTo:                     applyToPeers,
+			IngressRules:                  ingress,
+			EgressRules:                   egress,
+			DefaultRule:                   v1alpha1.DefaultRuleDrop,
+			PolicyTypes:                   []networkingv1.PolicyType{networkingv1.PolicyTypeIngress, networkingv1.PolicyTypeEgress},
 		},
 	}
 	policyList = append(policyList, policy)
@@ -969,7 +971,7 @@ func (c *Controller) parseSecurityPolicy(securityPolicy *schema.SecurityPolicy)
 			continue
 		}
 		// generate intra group policy
-		policy, err := c.generateIntragroupPolicy(securityPolicy.GetID(), &securityPolicy.ApplyTo[item])
+		policy, err := c.generateIntragroupPolicy(securityPolicy.GetID(), policyMode, &securityPolicy.ApplyTo[item])
 		if err != nil || policy == nil {
 			return nil, err
 		}
@@ -1071,7 +1073,7 @@ func (c *Controller) generateIsolationPolicy(id string, mode schema.IsolationMod
 	return isolationPolices
 }
 
-func (c *Controller) generateIntragroupPolicy(securityPolicyID string, appliedPeer *schema.SecurityPolicyApply) (*v1alpha1.SecurityPolicy, error) {
+func (c *Controller) generateIntragroupPolicy(id string, policyMode v1alpha1.PolicyMode, appliedPeer *schema.SecurityPolicyApply) (*v1alpha1.SecurityPolicy, error) {
 	peerHash := nameutil.HashName(10, appliedPeer)
 
 	appliedPeers, err := c.parseSecurityPolicyApplys([]schema.SecurityPolicyApply{*appliedPeer})
@@ -1084,7 +1086,7 @@ func (c *Controller) generateIntragroupPolicy(securityPolicyID string, appliedPe
 
 	policy := v1alpha1.SecurityPolicy{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      SecurityPolicyCommunicablePrefix + peerHash + "-" + securityPolicyID,
+			Name:      SecurityPolicyCommunicablePrefix + peerHash + "-" + id,
 			Namespace: c.namespace,
 		},
 		Spec: v1alpha1.SecurityPolicySpec{
@@ -1098,8 +1100,9 @@ func (c *Controller) generateIntragroupPolicy(securityPolicyID string, appliedPe
 				Name: "egress",
 				To:   c.appliedPeersAsPolicyPeers(appliedPeers),
 			}},
-			DefaultRule: v1alpha1.DefaultRuleDrop,
-			PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress, networkingv1.PolicyTypeEgress},
+			SecurityPolicyEnforcementMode: policyMode,
+			DefaultRule:                   v1alpha1.DefaultRuleDrop,
+			PolicyTypes:                   []networkingv1.PolicyType{networkingv1.PolicyTypeIngress, networkingv1.PolicyTypeEgress},
 		},
 	}
 
@@ -1377,3 +1380,15 @@ func formatIPBlock(ipBlock string) (string, error) {
 
 	return "", fmt.Errorf("neither %s is cidr nor ipv4 nor ipv6", ipBlock)
 }
+
+func parseEnforcementMode(mode schema.PolicyMode) v1alpha1.PolicyMode {
+	switch mode {
+	case schema.PolicyModeWork:
+		return v1alpha1.WorkMode
+	case schema.PolicyModeMonitor:
+		return v1alpha1.MonitorMode
+	default:
+		// the default work mode is defined in the SecurityPolicy CRD
+		return ""
+	}
+}
diff --git a/plugin/tower/pkg/controller/policy/controller_test.go b/plugin/tower/pkg/controller/policy/controller_test.go
index 630822f49..d8c642df8 100644
--- a/plugin/tower/pkg/controller/policy/controller_test.go
+++ b/plugin/tower/pkg/controller/policy/controller_test.go
@@ -84,7 +84,7 @@ var _ = Describe("PolicyController", func() {
 				})
 				It("should generate expect policies", func() {
 					assertPoliciesNum(ctx, 1)
-					assertHasPolicy(ctx, constants.Tier2, true, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+					assertHasPolicy(ctx, constants.Tier2, true, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 						NewSecurityPolicyRuleIngress("tcp", "20-80", nil, labelB, labelC),
 						NewSecurityPolicyRuleEgress("udp", "123", nil, labelA, labelC),
 						NewSecurityPolicyApplyPeer("", labelA, labelB),
@@ -100,7 +100,7 @@ var _ = Describe("PolicyController", func() {
 					})
 					It("should update policy selector", func() {
 						assertPoliciesNum(ctx, 1)
-						assertHasPolicy(ctx, constants.Tier2, true, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+						assertHasPolicy(ctx, constants.Tier2, true, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 							NewSecurityPolicyRuleIngress("tcp", "20-80", nil, labelA),
 							NewSecurityPolicyRuleEgress("udp", "123", nil, labelB),
 							NewSecurityPolicyApplyPeer("", labelA, labelB),
@@ -116,7 +116,7 @@ var _ = Describe("PolicyController", func() {
 					})
 					It("should update policy selector value", func() {
 						assertPoliciesNum(ctx, 1)
-						assertHasPolicy(ctx, constants.Tier2, true, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+						assertHasPolicy(ctx, constants.Tier2, true, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 							NewSecurityPolicyRuleIngress("tcp", "20-80", nil, labelB, labelC),
 							NewSecurityPolicyRuleEgress("udp", "123", nil, labelA, labelC),
 							NewSecurityPolicyApplyPeer("", labelA, labelB),
@@ -132,7 +132,7 @@ var _ = Describe("PolicyController", func() {
 					})
 					It("should update policy without ingress", func() {
 						assertPoliciesNum(ctx, 1)
-						assertHasPolicy(ctx, constants.Tier2, true, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+						assertHasPolicy(ctx, constants.Tier2, true, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 							nil,
 							NewSecurityPolicyRuleEgress("udp", "123", nil, labelA, labelC),
 							NewSecurityPolicyApplyPeer("", labelA, labelB),
@@ -148,7 +148,7 @@ var _ = Describe("PolicyController", func() {
 					})
 					It("should update policy without egress", func() {
 						assertPoliciesNum(ctx, 1)
-						assertHasPolicy(ctx, constants.Tier2, true, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+						assertHasPolicy(ctx, constants.Tier2, true, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 							NewSecurityPolicyRuleIngress("tcp", "20-80", nil, labelB, labelC),
 							nil,
 							NewSecurityPolicyApplyPeer("", labelA, labelB),
@@ -164,12 +164,12 @@ var _ = Describe("PolicyController", func() {
 					})
 					It("should generate policy for intragroup", func() {
 						assertPoliciesNum(ctx, 2)
-						assertHasPolicy(ctx, constants.Tier2, true, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+						assertHasPolicy(ctx, constants.Tier2, true, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 							NewSecurityPolicyRuleIngress("tcp", "20-80", nil, labelB, labelC),
 							NewSecurityPolicyRuleEgress("udp", "123", nil, labelA, labelC),
 							NewSecurityPolicyApplyPeer("", labelA, labelB),
 						)
-						assertHasPolicy(ctx, constants.Tier2, false, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+						assertHasPolicy(ctx, constants.Tier2, false, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 							NewSecurityPolicyRuleIngress("", "", nil, labelA, labelB),
 							NewSecurityPolicyRuleEgress("", "", nil, labelA, labelB),
 							NewSecurityPolicyApplyPeer("", labelA, labelB),
@@ -208,7 +208,7 @@ var _ = Describe("PolicyController", func() {
 				})
 				It("should generate expect policies", func() {
 					assertPoliciesNum(ctx, 1)
-					assertHasPolicy(ctx, constants.Tier2, true, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+					assertHasPolicy(ctx, constants.Tier2, true, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 						NewSecurityPolicyRuleIngress("tcp", "20-80", &networkingv1.IPBlock{CIDR: "192.168.0.0/24", Except: []string{"192.168.0.1/32"}}),
 						NewSecurityPolicyRuleEgress("udp", "123", &networkingv1.IPBlock{CIDR: "192.168.1.0/24"}),
 						NewSecurityPolicyApplyPeer("", labelA, labelB),
@@ -229,7 +229,7 @@ var _ = Describe("PolicyController", func() {
 					})
 					It("should update policy ipBlock value", func() {
 						assertPoliciesNum(ctx, 1)
-						assertHasPolicy(ctx, constants.Tier2, true, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+						assertHasPolicy(ctx, constants.Tier2, true, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 							NewSecurityPolicyRuleIngress("tcp", "20-80", &networkingv1.IPBlock{CIDR: newIP + "/32"}),
 							NewSecurityPolicyRuleEgress("udp", "123", &networkingv1.IPBlock{CIDR: newIP + "/32"}),
 							NewSecurityPolicyApplyPeer("", labelA, labelB),
@@ -257,7 +257,7 @@ var _ = Describe("PolicyController", func() {
 				})
 				It("should create policy with allow all ports", func() {
 					assertPoliciesNum(ctx, 1)
-					assertHasPolicy(ctx, constants.Tier2, true, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+					assertHasPolicy(ctx, constants.Tier2, true, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 						NewSecurityPolicyRuleIngress("", "", nil, labelB, labelC),
 						NewSecurityPolicyRuleEgress("", "", nil, labelA, labelC),
 						NewSecurityPolicyApplyPeer("", labelA, labelB),
@@ -284,12 +284,12 @@ var _ = Describe("PolicyController", func() {
 				})
 				It("should generate expect policies", func() {
 					assertPoliciesNum(ctx, 2)
-					assertHasPolicy(ctx, constants.Tier2, true, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+					assertHasPolicy(ctx, constants.Tier2, true, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 						NewSecurityPolicyRuleIngress("tcp", "20-80", nil, labelB, labelC),
 						NewSecurityPolicyRuleEgress("udp", "123", nil, labelA, labelC),
 						NewSecurityPolicyApplyPeer("", labelA, labelB),
 					)
-					assertHasPolicy(ctx, constants.Tier2, false, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+					assertHasPolicy(ctx, constants.Tier2, false, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 						NewSecurityPolicyRuleIngress("", "", nil, labelA, labelB),
 						NewSecurityPolicyRuleEgress("", "", nil, labelA, labelB),
 						NewSecurityPolicyApplyPeer("", labelA, labelB),
@@ -304,13 +304,53 @@ var _ = Describe("PolicyController", func() {
 					})
 					It("should delete intragroup policy", func() {
 						assertPoliciesNum(ctx, 1)
-						assertHasPolicy(ctx, constants.Tier2, true, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+						assertHasPolicy(ctx, constants.Tier2, true, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 							NewSecurityPolicyRuleIngress("tcp", "20-80", nil, labelB, labelC),
 							NewSecurityPolicyRuleEgress("udp", "123", nil, labelA, labelC),
 							NewSecurityPolicyApplyPeer("", labelA, labelB),
 						)
 					})
 				})
+
+				When("update SecurityPolicy enforce mode to work", func() {
+					BeforeEach(func() {
+						policy.PolicyMode = schema.PolicyModeWork
+						By(fmt.Sprintf("update SecurityPolicy %+v", policy))
+						server.TrackerFactory().SecurityPolicy().CreateOrUpdate(policy)
+					})
+					It("should update intragroup policy", func() {
+						assertHasPolicy(ctx, constants.Tier2, true, v1alpha1.WorkMode, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+							NewSecurityPolicyRuleIngress("tcp", "20-80", nil, labelB, labelC),
+							NewSecurityPolicyRuleEgress("udp", "123", nil, labelA, labelC),
+							NewSecurityPolicyApplyPeer("", labelA, labelB),
+						)
+						assertHasPolicy(ctx, constants.Tier2, false, v1alpha1.WorkMode, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+							NewSecurityPolicyRuleIngress("", "", nil, labelA, labelB),
+							NewSecurityPolicyRuleEgress("", "", nil, labelA, labelB),
+							NewSecurityPolicyApplyPeer("", labelA, labelB),
+						)
+					})
+				})
+
+				When("update SecurityPolicy enforce mode to monitor", func() {
+					BeforeEach(func() {
+						policy.PolicyMode = schema.PolicyModeMonitor
+						By(fmt.Sprintf("update SecurityPolicy %+v", policy))
+						server.TrackerFactory().SecurityPolicy().CreateOrUpdate(policy)
+					})
+					It("should update intragroup policy", func() {
+						assertHasPolicy(ctx, constants.Tier2, true, v1alpha1.MonitorMode, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+							NewSecurityPolicyRuleIngress("tcp", "20-80", nil, labelB, labelC),
+							NewSecurityPolicyRuleEgress("udp", "123", nil, labelA, labelC),
+							NewSecurityPolicyApplyPeer("", labelA, labelB),
+						)
+						assertHasPolicy(ctx, constants.Tier2, false, v1alpha1.MonitorMode, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+							NewSecurityPolicyRuleIngress("", "", nil, labelA, labelB),
+							NewSecurityPolicyRuleEgress("", "", nil, labelA, labelB),
+							NewSecurityPolicyApplyPeer("", labelA, labelB),
+						)
+					})
+				})
 			})
 
 			When("create SecurityPolicy with allow all traffics", func() {
@@ -332,7 +372,7 @@ var _ = Describe("PolicyController", func() {
 				})
 				It("should create policy allow all traffics", func() {
 					assertPoliciesNum(ctx, 1)
-					assertHasPolicy(ctx, constants.Tier2, true, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+					assertHasPolicy(ctx, constants.Tier2, true, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 						NewSecurityPolicyRuleIngress("", "", nil),
 						NewSecurityPolicyRuleEgress("", "", nil),
 						NewSecurityPolicyApplyPeer("", labelA, labelB),
@@ -341,6 +381,26 @@ var _ = Describe("PolicyController", func() {
 			})
 		})
 
+		When("create SecurityPolicy with enforce mode", func() {
+			var policy *schema.SecurityPolicy
+
+			BeforeEach(func() {
+				policy = NewSecurityPolicy(everouteCluster, false, nil, labelA, labelB)
+				policy.PolicyMode = schema.PolicyModeMonitor
+				By(fmt.Sprintf("create SecurityPolicy %+v", policy))
+				server.TrackerFactory().SecurityPolicy().CreateOrUpdate(policy)
+			})
+
+			It("should create policy with enforce mode", func() {
+				assertPoliciesNum(ctx, 1)
+				assertHasPolicy(ctx, constants.Tier2, true, v1alpha1.MonitorMode, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+					nil,
+					nil,
+					NewSecurityPolicyApplyPeer("", labelA, labelB),
+				)
+			})
+		})
+
 		When("create SecurityPolicy out of the EverouteCluster", func() {
 			var policy *schema.SecurityPolicy
 			var randomEverouteCluster string
@@ -378,7 +438,7 @@ var _ = Describe("PolicyController", func() {
 			})
 			It("should create policy with ingress only", func() {
 				assertPoliciesNum(ctx, 1)
-				assertHasPolicy(ctx, constants.Tier2, true, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+				assertHasPolicy(ctx, constants.Tier2, true, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 					NewSecurityPolicyRuleIngress("tcp", "20-80", nil, labelB, labelC),
 					nil,
 					NewSecurityPolicyApplyPeer("", labelA, labelB),
@@ -403,7 +463,7 @@ var _ = Describe("PolicyController", func() {
 			})
 			It("should create policy with egress only", func() {
 				assertPoliciesNum(ctx, 1)
-				assertHasPolicy(ctx, constants.Tier2, true, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+				assertHasPolicy(ctx, constants.Tier2, true, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 					nil,
 					NewSecurityPolicyRuleEgress("udp", "123", nil, labelA, labelC),
 					NewSecurityPolicyApplyPeer("", labelA, labelB),
@@ -422,7 +482,7 @@ var _ = Describe("PolicyController", func() {
 
 			It("should create policy with no rules", func() {
 				assertPoliciesNum(ctx, 1)
-				assertHasPolicy(ctx, constants.Tier2, true, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+				assertHasPolicy(ctx, constants.Tier2, true, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 					nil,
 					nil,
 					NewSecurityPolicyApplyPeer("", labelA, labelB),
@@ -457,7 +517,7 @@ var _ = Describe("PolicyController", func() {
 
 			It("should generate expect policies", func() {
 				assertPoliciesNum(ctx, 1)
-				assertHasPolicy(ctx, constants.Tier0, true, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+				assertHasPolicy(ctx, constants.Tier0, true, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 					nil,
 					nil,
 					NewSecurityPolicyApplyPeer(vnicA.GetID()),
@@ -475,7 +535,7 @@ var _ = Describe("PolicyController", func() {
 				})
 				It("should update policy applied endpoints", func() {
 					assertPoliciesNum(ctx, 1)
-					assertHasPolicy(ctx, constants.Tier0, true, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+					assertHasPolicy(ctx, constants.Tier0, true, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 						nil,
 						nil,
 						NewSecurityPolicyApplyPeer(vnicA.GetID()),
@@ -493,7 +553,7 @@ var _ = Describe("PolicyController", func() {
 				})
 				It("should update policy applied endpoints", func() {
 					assertPoliciesNum(ctx, 1)
-					assertHasPolicy(ctx, constants.Tier0, true, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+					assertHasPolicy(ctx, constants.Tier0, true, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 						nil,
 						nil,
 						NewSecurityPolicyApplyPeer(vnicA.GetID()),
@@ -531,13 +591,13 @@ var _ = Describe("PolicyController", func() {
 
 			It("should generate expect policies", func() {
 				assertPoliciesNum(ctx, 2)
-				assertHasPolicy(ctx, constants.Tier0, true, v1alpha1.DefaultRuleDrop,
+				assertHasPolicy(ctx, constants.Tier0, true, "", v1alpha1.DefaultRuleDrop,
 					[]networkingv1.PolicyType{networkingv1.PolicyTypeEgress},
 					nil, nil,
 					NewSecurityPolicyApplyPeer(vnicA.GetID()),
 					NewSecurityPolicyApplyPeer(vnicB.GetID()),
 				)
-				assertHasPolicy(ctx, constants.Tier1, true, v1alpha1.DefaultRuleDrop,
+				assertHasPolicy(ctx, constants.Tier1, true, "", v1alpha1.DefaultRuleDrop,
 					[]networkingv1.PolicyType{networkingv1.PolicyTypeIngress},
 					NewSecurityPolicyRuleIngress("tcp", "22-80", nil, labelA, labelC),
 					nil,
@@ -562,13 +622,13 @@ var _ = Describe("PolicyController", func() {
 
 			It("should generate expect policies", func() {
 				assertPoliciesNum(ctx, 2)
-				assertHasPolicy(ctx, constants.Tier0, true, v1alpha1.DefaultRuleDrop,
+				assertHasPolicy(ctx, constants.Tier0, true, "", v1alpha1.DefaultRuleDrop,
 					[]networkingv1.PolicyType{networkingv1.PolicyTypeIngress},
 					nil, nil,
 					NewSecurityPolicyApplyPeer(vnicA.GetID()),
 					NewSecurityPolicyApplyPeer(vnicB.GetID()),
 				)
-				assertHasPolicy(ctx, constants.Tier1, true, v1alpha1.DefaultRuleDrop,
+				assertHasPolicy(ctx, constants.Tier1, true, "", v1alpha1.DefaultRuleDrop,
 					[]networkingv1.PolicyType{networkingv1.PolicyTypeEgress},
 					nil,
 					NewSecurityPolicyRuleEgress("udp", "123", nil, labelA, labelB),
@@ -590,7 +650,7 @@ var _ = Describe("PolicyController", func() {
 			})
 			It("should create security policy", func() {
 				assertPoliciesNum(ctx, 1)
-				assertHasPolicy(ctx, constants.Tier2, false, v1alpha1.DefaultRuleNone,
+				assertHasPolicy(ctx, constants.Tier2, false, "", v1alpha1.DefaultRuleNone,
 					[]networkingv1.PolicyType{networkingv1.PolicyTypeIngress, networkingv1.PolicyTypeEgress},
 					getEmptyIngress(), getEmptyEgress(),
 					NewSecurityPolicyApplyPeer(endpoint.GetSystemEndpointName(randomSystemEndpoints.IPPortEndpoints[0].Key)),
@@ -610,7 +670,7 @@ var _ = Describe("PolicyController", func() {
 				})
 				It("should update security policy", func() {
 					assertPoliciesNum(ctx, 1)
-					assertHasPolicy(ctx, constants.Tier2, false, v1alpha1.DefaultRuleNone,
+					assertHasPolicy(ctx, constants.Tier2, false, "", v1alpha1.DefaultRuleNone,
 						[]networkingv1.PolicyType{networkingv1.PolicyTypeIngress, networkingv1.PolicyTypeEgress},
 						getEmptyIngress(), getEmptyEgress(),
 						NewSecurityPolicyApplyPeer(endpoint.GetSystemEndpointName(randomSystemEndpoints.IPPortEndpoints[0].Key)),
@@ -641,7 +701,7 @@ var _ = Describe("PolicyController", func() {
 			})
 			It("should create security policy", func() {
 				assertPoliciesNum(ctx, 1)
-				assertHasPolicy(ctx, constants.Tier2, false, v1alpha1.DefaultRuleNone,
+				assertHasPolicy(ctx, constants.Tier2, false, "", v1alpha1.DefaultRuleNone,
 					[]networkingv1.PolicyType{networkingv1.PolicyTypeIngress, networkingv1.PolicyTypeEgress},
 					getEmptyIngress(), getEmptyEgress(),
 					NewSecurityPolicyApplyPeer(endpoint.GetCtrlEndpointName(cluster.GetID(), cluster.ControllerInstances[0])),
@@ -660,7 +720,7 @@ var _ = Describe("PolicyController", func() {
 				})
 				It("should update security policy", func() {
 					assertPoliciesNum(ctx, 1)
-					assertHasPolicy(ctx, constants.Tier2, false, v1alpha1.DefaultRuleNone,
+					assertHasPolicy(ctx, constants.Tier2, false, "", v1alpha1.DefaultRuleNone,
 						[]networkingv1.PolicyType{networkingv1.PolicyTypeIngress, networkingv1.PolicyTypeEgress},
 						getEmptyIngress(), getEmptyEgress(),
 						NewSecurityPolicyApplyPeer(endpoint.GetCtrlEndpointName(cluster.GetID(), cluster.ControllerInstances[0])),
@@ -703,7 +763,7 @@ var _ = Describe("PolicyController", func() {
 			})
 			It("should create security policy", func() {
 				assertPoliciesNum(ctx, 1)
-				assertHasPolicy(ctx, constants.Tier2, false, v1alpha1.DefaultRuleNone,
+				assertHasPolicy(ctx, constants.Tier2, false, "", v1alpha1.DefaultRuleNone,
 					[]networkingv1.PolicyType{networkingv1.PolicyTypeIngress, networkingv1.PolicyTypeEgress},
 					&v1alpha1.Rule{
 						Name: "ingress0",
@@ -739,7 +799,7 @@ var _ = Describe("PolicyController", func() {
 				})
 				It("should update security policy", func() {
 					assertPoliciesNum(ctx, 1)
-					assertHasPolicy(ctx, constants.Tier2, false, v1alpha1.DefaultRuleNone,
+					assertHasPolicy(ctx, constants.Tier2, false, "", v1alpha1.DefaultRuleNone,
 						[]networkingv1.PolicyType{networkingv1.PolicyTypeIngress, networkingv1.PolicyTypeEgress},
 						&v1alpha1.Rule{
 							Name: "ingress0",
@@ -759,7 +819,7 @@ var _ = Describe("PolicyController", func() {
 				})
 				It("should update security policy", func() {
 					assertPoliciesNum(ctx, 1)
-					assertHasPolicy(ctx, constants.Tier2, false, v1alpha1.DefaultRuleNone,
+					assertHasPolicy(ctx, constants.Tier2, false, "", v1alpha1.DefaultRuleNone,
 						[]networkingv1.PolicyType{networkingv1.PolicyTypeIngress, networkingv1.PolicyTypeEgress},
 						nil,
 						&v1alpha1.Rule{
@@ -785,7 +845,7 @@ var _ = Describe("PolicyController", func() {
 					})
 					It("should update security policy", func() {
 						assertPoliciesNum(ctx, 1)
-						assertHasPolicy(ctx, constants.Tier2, false, v1alpha1.DefaultRuleNone,
+						assertHasPolicy(ctx, constants.Tier2, false, "",  v1alpha1.DefaultRuleNone,
 							[]networkingv1.PolicyType{networkingv1.PolicyTypeIngress, networkingv1.PolicyTypeEgress},
 							&v1alpha1.Rule{
 								Name: "ingress0",
@@ -870,7 +930,7 @@ var _ = Describe("PolicyController", func() {
 
 			It("should create security policy with normal group", func() {
 				assertPoliciesNum(ctx, 1)
-				assertHasPolicy(ctx, constants.Tier2, true, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+				assertHasPolicy(ctx, constants.Tier2, true, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 					nil,
 					nil,
 					NewSecurityPolicyApplyPeer(vnicA.ID),
@@ -889,7 +949,7 @@ var _ = Describe("PolicyController", func() {
 
 				It("should update security policy with normal group", func() {
 					assertPoliciesNum(ctx, 1)
-					assertHasPolicy(ctx, constants.Tier2, true, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+					assertHasPolicy(ctx, constants.Tier2, true, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 						nil,
 						nil,
 						NewSecurityPolicyApplyPeer(vnicA.ID),
@@ -908,7 +968,7 @@ var _ = Describe("PolicyController", func() {
 
 				It("should update security policy with normal group", func() {
 					assertPoliciesNum(ctx, 1)
-					assertHasPolicy(ctx, constants.Tier2, true, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+					assertHasPolicy(ctx, constants.Tier2, true, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 						nil,
 						nil,
 						NewSecurityPolicyApplyPeer(vnicA.ID),
@@ -926,7 +986,7 @@ var _ = Describe("PolicyController", func() {
 
 				It("should update security policy with normal group", func() {
 					assertPoliciesNum(ctx, 1)
-					assertHasPolicy(ctx, constants.Tier2, true, v1alpha1.DefaultRuleDrop, allPolicyTypes(),
+					assertHasPolicy(ctx, constants.Tier2, true, "", v1alpha1.DefaultRuleDrop, allPolicyTypes(),
 						nil,
 						nil,
 						NewSecurityPolicyApplyPeer(vnicA.ID),
@@ -958,13 +1018,13 @@ func assertPoliciesNum(ctx context.Context, numOfPolicies int) {
 	}, timeout, interval).Should(Equal(numOfPolicies))
 }
 
-func assertHasPolicy(ctx context.Context, tier string, symmetricMode bool, defaultRule v1alpha1.DefaultRuleType,
+func assertHasPolicy(ctx context.Context, tier string, symmetricMode bool, enforceMode v1alpha1.PolicyMode, defaultRule v1alpha1.DefaultRuleType,
 	policyTypes []networkingv1.PolicyType, ingress, egress *v1alpha1.Rule, applyToPeers ...v1alpha1.ApplyToPeer) {
 	Eventually(func() bool {
 		policyList, err := crdClient.SecurityV1alpha1().SecurityPolicies(namespace).List(ctx, metav1.ListOptions{})
 		Expect(err).Should(Succeed())
 		for item := range policyList.Items {
-			if matchPolicy(&policyList.Items[item], tier, symmetricMode,
+			if matchPolicy(&policyList.Items[item], tier, symmetricMode, enforceMode,
 				defaultRule, policyTypes, ingress, egress, applyToPeers...) {
 				return true
 			}
@@ -973,7 +1033,7 @@ func assertHasPolicy(ctx context.Context, tier string, symmetricMode bool, defau
 	}, timeout, interval).Should(BeTrue())
 }
 
-func matchPolicy(policy *v1alpha1.SecurityPolicy, tier string, symmetricMode bool, defaultRule v1alpha1.DefaultRuleType,
+func matchPolicy(policy *v1alpha1.SecurityPolicy, tier string, symmetricMode bool, enforceMode v1alpha1.PolicyMode, defaultRule v1alpha1.DefaultRuleType,
 	policyTypes []networkingv1.PolicyType, ingress, egress *v1alpha1.Rule, applyToPeers ...v1alpha1.ApplyToPeer) bool {
 	matchAllPolicyTypes := func(policyTypes1 []networkingv1.PolicyType, policyTypes2 []networkingv1.PolicyType) bool {
 		if len(policyTypes1) != len(policyTypes2) {
@@ -1014,6 +1074,7 @@ func matchPolicy(policy *v1alpha1.SecurityPolicy, tier string, symmetricMode boo
 	return policy.Namespace == namespace &&
 		policy.Spec.Tier == tier &&
 		policy.Spec.SymmetricMode == symmetricMode &&
+		policy.Spec.SecurityPolicyEnforcementMode == enforceMode &&
 		policy.Spec.DefaultRule == defaultRule &&
 		matchAllPolicyTypes(policy.Spec.PolicyTypes, policyTypes) &&
 		matchRules(policy.Spec.IngressRules, ingress) &&
diff --git a/plugin/tower/pkg/schema/policy_types.go b/plugin/tower/pkg/schema/policy_types.go
index bf990c800..772382edb 100644
--- a/plugin/tower/pkg/schema/policy_types.go
+++ b/plugin/tower/pkg/schema/policy_types.go
@@ -24,8 +24,16 @@ type SecurityPolicy struct {
 	ApplyTo         []SecurityPolicyApply `json:"apply_to"`
 	Ingress         []NetworkPolicyRule   `json:"ingress,omitempty"`
 	Egress          []NetworkPolicyRule   `json:"egress,omitempty"`
+	PolicyMode      PolicyMode            `json:"policy_mode,omitempty"`
 }
 
+type PolicyMode string
+
+const (
+	PolicyModeMonitor = "MONITOR"
+	PolicyModeWork    = "WORK"
+)
+
 type IsolationPolicy struct {
 	ObjectMeta
 
diff --git a/plugin/tower/pkg/schema/policy_types.graphqls b/plugin/tower/pkg/schema/policy_types.graphqls
index 5dc92780a..2a9ca009b 100644
--- a/plugin/tower/pkg/schema/policy_types.graphqls
+++ b/plugin/tower/pkg/schema/policy_types.graphqls
@@ -5,6 +5,7 @@ type SecurityPolicy {
     apply_to: [SecurityPolicyApply!]!
     ingress: [NetworkPolicyRule!]
     egress: [NetworkPolicyRule!]
+    policy_mode: PolicyMode
 }
 
 type SecurityPolicyApply {
@@ -19,6 +20,11 @@ enum SecurityPolicyType {
     SELECTOR
 }
 
+enum PolicyMode {
+    MONITOR
+    WORK
+}
+
 type IsolationPolicy {
     id: ID!
     everoute_cluster: ObjectReference!
diff --git a/plugin/tower/pkg/server/fake/graph/generated/generated.go b/plugin/tower/pkg/server/fake/graph/generated/generated.go
index d4b30cf01..fd37dd17b 100644
--- a/plugin/tower/pkg/server/fake/graph/generated/generated.go
+++ b/plugin/tower/pkg/server/fake/graph/generated/generated.go
@@ -194,6 +194,7 @@ type ComplexityRoot struct {
 		ID              func(childComplexity int) int
 		Ingress         func(childComplexity int) int
 		Name            func(childComplexity int) int
+		PolicyMode      func(childComplexity int) int
 	}
 
 	SecurityPolicyApply struct {
@@ -886,6 +887,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in
 
 		return e.complexity.SecurityPolicy.Name(childComplexity), true
 
+	case "SecurityPolicy.policy_mode":
+		if e.complexity.SecurityPolicy.PolicyMode == nil {
+			break
+		}
+
+		return e.complexity.SecurityPolicy.PolicyMode(childComplexity), true
+
 	case "SecurityPolicyApply.communicable":
 		if e.complexity.SecurityPolicyApply.Communicable == nil {
 			break
@@ -1447,6 +1455,7 @@ enum TaskOrderByInput {
     apply_to: [SecurityPolicyApply!]!
     ingress: [NetworkPolicyRule!]
     egress: [NetworkPolicyRule!]
+    policy_mode: PolicyMode
 }
 
 type SecurityPolicyApply {
@@ -1461,6 +1470,11 @@ enum SecurityPolicyType {
     SELECTOR
 }
 
+enum PolicyMode {
+    MONITOR
+    WORK
+}
+
 type IsolationPolicy {
     id: ID!
     everoute_cluster: ObjectReference!
@@ -4511,6 +4525,38 @@ func (ec *executionContext) _SecurityPolicy_egress(ctx context.Context, field gr
 	return ec.marshalONetworkPolicyRule2ᚕgithubᚗcomᚋeverouteᚋeverouteᚋpluginᚋtowerᚋpkgᚋschemaᚐNetworkPolicyRuleᚄ(ctx, field.Selections, res)
 }
 
+func (ec *executionContext) _SecurityPolicy_policy_mode(ctx context.Context, field graphql.CollectedField, obj *schema.SecurityPolicy) (ret graphql.Marshaler) {
+	defer func() {
+		if r := recover(); r != nil {
+			ec.Error(ctx, ec.Recover(ctx, r))
+			ret = graphql.Null
+		}
+	}()
+	fc := &graphql.FieldContext{
+		Object:     "SecurityPolicy",
+		Field:      field,
+		Args:       nil,
+		IsMethod:   false,
+		IsResolver: false,
+	}
+
+	ctx = graphql.WithFieldContext(ctx, fc)
+	resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
+		ctx = rctx // use context from middleware stack in children
+		return obj.PolicyMode, nil
+	})
+	if err != nil {
+		ec.Error(ctx, err)
+		return graphql.Null
+	}
+	if resTmp == nil {
+		return graphql.Null
+	}
+	res := resTmp.(schema.PolicyMode)
+	fc.Result = res
+	return ec.marshalOPolicyMode2githubᚗcomᚋeverouteᚋeverouteᚋpluginᚋtowerᚋpkgᚋschemaᚐPolicyMode(ctx, field.Selections, res)
+}
+
 func (ec *executionContext) _SecurityPolicyApply_type(ctx context.Context, field graphql.CollectedField, obj *schema.SecurityPolicyApply) (ret graphql.Marshaler) {
 	defer func() {
 		if r := recover(); r != nil {
@@ -8415,6 +8461,8 @@ func (ec *executionContext) _SecurityPolicy(ctx context.Context, sel ast.Selecti
 			out.Values[i] = ec._SecurityPolicy_ingress(ctx, field, obj)
 		case "egress":
 			out.Values[i] = ec._SecurityPolicy_egress(ctx, field, obj)
+		case "policy_mode":
+			out.Values[i] = ec._SecurityPolicy_policy_mode(ctx, field, obj)
 		default:
 			panic("unknown field " + strconv.Quote(field.Name))
 		}
@@ -10558,6 +10606,16 @@ func (ec *executionContext) marshalOObjectReference2ᚖgithubᚗcomᚋeveroute
 	return ec._ObjectReference(ctx, sel, v)
 }
 
+func (ec *executionContext) unmarshalOPolicyMode2githubᚗcomᚋeverouteᚋeverouteᚋpluginᚋtowerᚋpkgᚋschemaᚐPolicyMode(ctx context.Context, v interface{}) (schema.PolicyMode, error) {
+	tmp, err := graphql.UnmarshalString(v)
+	res := schema.PolicyMode(tmp)
+	return res, graphql.ErrorOnPath(ctx, err)
+}
+
+func (ec *executionContext) marshalOPolicyMode2githubᚗcomᚋeverouteᚋeverouteᚋpluginᚋtowerᚋpkgᚋschemaᚐPolicyMode(ctx context.Context, sel ast.SelectionSet, v schema.PolicyMode) graphql.Marshaler {
+	return graphql.MarshalString(string(v))
+}
+
 func (ec *executionContext) unmarshalOString2string(ctx context.Context, v interface{}) (string, error) {
 	res, err := graphql.UnmarshalString(v)
 	return res, graphql.ErrorOnPath(ctx, err)

From fbb015732dce8a6aa12ad14e75c7125be4e7b656 Mon Sep 17 00:00:00 2001
From: zwtop <wang.zhan@smartx.com>
Date: Thu, 5 May 2022 13:51:50 +0800
Subject: [PATCH 18/32] test(plugin-tower): add testing host builder

Signed-off-by: zwtop <wang.zhan@smartx.com>
---
 plugin/tower/pkg/utils/testing/builder.go | 24 +++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/plugin/tower/pkg/utils/testing/builder.go b/plugin/tower/pkg/utils/testing/builder.go
index eab4c69fe..2b11abb97 100644
--- a/plugin/tower/pkg/utils/testing/builder.go
+++ b/plugin/tower/pkg/utils/testing/builder.go
@@ -289,3 +289,27 @@ func NewSecurityGroup(clusterID string) *schema.SecurityGroup {
 		EverouteCluster: schema.ObjectReference{ID: clusterID},
 	}
 }
+
+// NewRandomHost creates a random Host
+func NewRandomHost() *schema.Host {
+	return &schema.Host{
+		ObjectMeta: schema.ObjectMeta{ID: rand.String(10)},
+		Name:       rand.String(10),
+	}
+}
+
+// NewRandomNicAttachedTo creates a random NIC attached to the given Host
+func NewRandomNicAttachedTo(host *schema.Host, name string) *schema.Nic {
+	if name == "" {
+		name = rand.String(10)
+	}
+
+	nic := schema.Nic{
+		ObjectMeta: schema.ObjectMeta{ID: rand.String(10)},
+		Name:       name,
+		IPAddress:  NewRandomIP().String(),
+	}
+
+	host.Nics = append(host.Nics, nic)
+	return &nic
+}

From 71f735afeb54b85bfacbd57585472e9dc6ca57d0 Mon Sep 17 00:00:00 2001
From: Changliang Wu <changliang.wu@smartx.com>
Date: Tue, 17 May 2022 11:41:21 +0800
Subject: [PATCH 19/32] feat: record datapath controller id into
 /var/run/everoute

Signed-off-by: Changliang Wu <changliang.wu@smartx.com>
---
 go.mod                                    |  1 +
 go.sum                                    |  1 +
 pkg/agent/datapath/multiBridgeDatapath.go | 32 ++---------
 pkg/utils/utils.go                        | 66 +++++++++++++++++++++++
 4 files changed, 73 insertions(+), 27 deletions(-)

diff --git a/go.mod b/go.mod
index 71ac9fd2c..df258135d 100644
--- a/go.mod
+++ b/go.mod
@@ -6,6 +6,7 @@ require (
 	github.com/99designs/gqlgen v0.13.0
 	github.com/Sirupsen/logrus v0.8.8-0.20160119000032-f7f79f729e0f
 	github.com/alessio/shellescape v1.4.1
+	github.com/alexflint/go-filemutex v1.1.0
 	github.com/cenkalti/backoff v2.2.1+incompatible
 	github.com/containernetworking/cni v1.0.0
 	github.com/containernetworking/plugins v1.0.0
diff --git a/go.sum b/go.sum
index 80474b2a3..bbdc7c296 100644
--- a/go.sum
+++ b/go.sum
@@ -92,6 +92,7 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
 github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0=
 github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30=
 github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
+github.com/alexflint/go-filemutex v1.1.0 h1:IAWuUuRYL2hETx5b8vCgwnD+xSdlsTQY6s2JjBsqLdg=
 github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
diff --git a/pkg/agent/datapath/multiBridgeDatapath.go b/pkg/agent/datapath/multiBridgeDatapath.go
index 96e38c939..0e2daadd6 100644
--- a/pkg/agent/datapath/multiBridgeDatapath.go
+++ b/pkg/agent/datapath/multiBridgeDatapath.go
@@ -18,8 +18,6 @@ package datapath
 
 import (
 	"bytes"
-	"crypto/rand"
-	"encoding/binary"
 	"errors"
 	"fmt"
 	"net"
@@ -44,6 +42,7 @@ import (
 
 	policycache "github.com/everoute/everoute/pkg/agent/controller/policy/cache"
 	"github.com/everoute/everoute/pkg/constants"
+	"github.com/everoute/everoute/pkg/utils"
 )
 
 //nolint
@@ -175,7 +174,6 @@ type DpManager struct {
 	ControllerMap      map[string]map[string]*ofctrl.Controller
 	BridgeChainPortMap map[string]map[string]uint32 // map vds to patch port to ofport-num map
 
-	controllerIDSets          sets.String
 	localEndpointDB           cmap.ConcurrentMap     // list of local endpoint map
 	ofPortIPAddressUpdateChan chan map[string]net.IP // map bridgename-ofport to endpoint ips
 	datapathConfig            *Config
@@ -277,7 +275,6 @@ func NewDatapathManager(datapathConfig *Config, ofPortIPAddressUpdateChan chan m
 	datapathManager.BridgeChainPortMap = make(map[string]map[string]uint32)
 	datapathManager.OvsdbDriverMap = make(map[string]map[string]*ovsdbDriver.OvsDriver)
 	datapathManager.ControllerMap = make(map[string]map[string]*ofctrl.Controller)
-	datapathManager.controllerIDSets = sets.NewString()
 	datapathManager.Rules = make(map[string]*EveroutePolicyRuleEntry)
 	datapathManager.FlowIDToRules = make(map[uint64]*EveroutePolicyRuleEntry)
 	datapathManager.datapathConfig = datapathConfig
@@ -405,25 +402,6 @@ func (datapathManager *DpManager) InitializeCNI() {
 	wg.Wait()
 }
 
-func (datapathManager *DpManager) GenerateControllerID() uint16 {
-	datapathManager.DpManagerMutex.Lock()
-	defer datapathManager.DpManagerMutex.Unlock()
-
-	var ctrlID uint16
-	for {
-		err := binary.Read(rand.Reader, binary.LittleEndian, &ctrlID)
-		if err != nil {
-			log.Infof("get random ID from rand.Reader: %s", err)
-			continue
-		}
-		if datapathManager.controllerIDSets.Has(strconv.Itoa(int(ctrlID))) {
-			continue
-		}
-		datapathManager.controllerIDSets.Insert(strconv.Itoa(int(ctrlID)))
-		return ctrlID
-	}
-}
-
 func NewVDSForConfig(datapathManager *DpManager, vdsID, ovsbrname string) {
 	// initialize vds bridge chain
 	localBridge := NewLocalBridge(ovsbrname, datapathManager)
@@ -438,10 +416,10 @@ func NewVDSForConfig(datapathManager *DpManager, vdsID, ovsbrname string) {
 
 	// initialize of controller
 	vdsOfControllerMap := make(map[string]*ofctrl.Controller)
-	vdsOfControllerMap[LOCAL_BRIDGE_KEYWORD] = ofctrl.NewControllerAsOFClient(localBridge, datapathManager.GenerateControllerID())
-	vdsOfControllerMap[POLICY_BRIDGE_KEYWORD] = ofctrl.NewControllerAsOFClient(policyBridge, datapathManager.GenerateControllerID())
-	vdsOfControllerMap[CLS_BRIDGE_KEYWORD] = ofctrl.NewControllerAsOFClient(clsBridge, datapathManager.GenerateControllerID())
-	vdsOfControllerMap[UPLINK_BRIDGE_KEYWORD] = ofctrl.NewControllerAsOFClient(uplinkBridge, datapathManager.GenerateControllerID())
+	vdsOfControllerMap[LOCAL_BRIDGE_KEYWORD] = ofctrl.NewControllerAsOFClient(localBridge, utils.GenerateControllerID())
+	vdsOfControllerMap[POLICY_BRIDGE_KEYWORD] = ofctrl.NewControllerAsOFClient(policyBridge, utils.GenerateControllerID())
+	vdsOfControllerMap[CLS_BRIDGE_KEYWORD] = ofctrl.NewControllerAsOFClient(clsBridge, utils.GenerateControllerID())
+	vdsOfControllerMap[UPLINK_BRIDGE_KEYWORD] = ofctrl.NewControllerAsOFClient(uplinkBridge, utils.GenerateControllerID())
 
 	// initialize ovsdbDriver
 	vdsOvsdbDriverMap := make(map[string]*ovsdbDriver.OvsDriver)
diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go
index 9ac803bb8..33dac730b 100644
--- a/pkg/utils/utils.go
+++ b/pkg/utils/utils.go
@@ -1,15 +1,21 @@
 package utils
 
 import (
+	"crypto/rand"
 	"crypto/sha256"
 	"encoding/base64"
+	"encoding/binary"
 	"fmt"
 	"io/ioutil"
 	"net"
 	"os"
+	"path"
 	"sort"
+	"strconv"
 	"strings"
+	"sync"
 
+	"github.com/alexflint/go-filemutex"
 	"github.com/vishvananda/netlink"
 	"golang.org/x/sys/unix"
 	coretypes "k8s.io/apimachinery/pkg/types"
@@ -101,3 +107,63 @@ func CurrentAgentName() string {
 	klog.Infof("Current AgentName: %s", currentAgentName)
 	return currentAgentName
 }
+
+const controllerIDPath = "/var/run/everoute/controllerID"
+
+var _instance *filemutex.FileMutex
+var _once sync.Once
+
+func getCtrlPathMutex() *filemutex.FileMutex {
+	_once.Do(func() {
+		// create path
+		if _, err := os.Stat(controllerIDPath); os.IsNotExist(err) {
+			if err := os.MkdirAll(controllerIDPath, os.ModePerm); err != nil {
+				klog.Fatalf("fail to create %s", controllerIDPath)
+			}
+			if err := os.Chmod(controllerIDPath, os.ModePerm); err != nil {
+				klog.Fatalf("fail to chmod %s", controllerIDPath)
+			}
+		}
+
+		// use file mutex to ensure cocurrency
+		mutex, err := filemutex.New(path.Join(controllerIDPath, "lock"))
+		if err != nil {
+			klog.Fatal("Fail to create ControllerID file lock")
+		}
+		_instance = mutex
+	})
+	return _instance
+}
+
+func GenerateControllerID() uint16 {
+	mutex := getCtrlPathMutex()
+
+	_ = mutex.Lock()
+	defer func() {
+		_ = mutex.Unlock()
+	}()
+
+	var ctrlID uint16
+	for {
+		// genereate new ID
+		err := binary.Read(rand.Reader, binary.LittleEndian, &ctrlID)
+		if err != nil {
+			klog.Errorf("get random ID from rand.Reader: %s", err)
+			continue
+		}
+		targetFile := path.Join(controllerIDPath, strconv.Itoa(int(ctrlID)))
+
+		// check if id existed
+		if _, err := os.Stat(targetFile); err == nil {
+			continue
+		}
+
+		// record file in path
+		if _, err := os.Create(targetFile); err != nil {
+			klog.Errorf("create ctrlID file %s error: %s", targetFile, err)
+			continue
+		}
+
+		return ctrlID
+	}
+}

From f423952dc97b7684b6d70e2d57589d3b9aa41b43 Mon Sep 17 00:00:00 2001
From: Changliang Wu <changliang.wu@smartx.com>
Date: Tue, 17 May 2022 16:13:09 +0800
Subject: [PATCH 20/32] feat: add rpc server for everoute exporter

Signed-off-by: Changliang Wu <changliang.wu@smartx.com>
---
 Makefile                                  |   1 +
 cmd/everoute-agent/main.go                |   4 +
 pkg/agent/datapath/localBridge.go         |   5 +
 pkg/agent/datapath/multiBridgeDatapath.go |   6 +
 pkg/agent/rpcserver/collector.go          |  96 +++
 pkg/agent/rpcserver/server.go             |  90 +++
 pkg/apis/rpc/v1alpha1/collector.pb.go     | 726 ++++++++++++++++++++++
 pkg/apis/rpc/v1alpha1/collector.proto     |  46 ++
 8 files changed, 974 insertions(+)
 create mode 100644 pkg/agent/rpcserver/collector.go
 create mode 100644 pkg/agent/rpcserver/server.go
 create mode 100644 pkg/apis/rpc/v1alpha1/collector.pb.go
 create mode 100644 pkg/apis/rpc/v1alpha1/collector.proto

diff --git a/Makefile b/Makefile
index 1909373c2..23c0a9ccf 100644
--- a/Makefile
+++ b/Makefile
@@ -80,6 +80,7 @@ gqlgen:
 
 protopb:
 	protoc -I=. --go_out=plugins=grpc:.  pkg/apis/cni/v1alpha1/cni.proto
+	protoc -I=. --go_out=plugins=grpc:.  pkg/apis/rpc/v1alpha1/collector.proto
 
 apidocs-gen:
 	$(eval PATH := $$(PATH):$(shell go env GOPATH)/bin)
diff --git a/cmd/everoute-agent/main.go b/cmd/everoute-agent/main.go
index 68a132bb2..addb10d68 100644
--- a/cmd/everoute-agent/main.go
+++ b/cmd/everoute-agent/main.go
@@ -33,6 +33,7 @@ import (
 	"github.com/everoute/everoute/pkg/agent/controller/policy"
 	"github.com/everoute/everoute/pkg/agent/datapath"
 	"github.com/everoute/everoute/pkg/agent/proxy"
+	"github.com/everoute/everoute/pkg/agent/rpcserver"
 	clientsetscheme "github.com/everoute/everoute/pkg/client/clientset_generated/clientset/scheme"
 	"github.com/everoute/everoute/pkg/constants"
 	"github.com/everoute/everoute/pkg/monitor"
@@ -132,6 +133,9 @@ func main() {
 	agentmonitor := monitor.NewAgentMonitor(k8sClient, ovsdbMonitor, ofPortIPAddrMoniotorChan)
 	go agentmonitor.Run(stopChan)
 
+	rpcServer := rpcserver.Initialize(datapathManager)
+	go rpcServer.Run(stopChan)
+
 	<-stopChan
 }
 
diff --git a/pkg/agent/datapath/localBridge.go b/pkg/agent/datapath/localBridge.go
index f8be2bc84..9c3a7eff2 100644
--- a/pkg/agent/datapath/localBridge.go
+++ b/pkg/agent/datapath/localBridge.go
@@ -157,6 +157,11 @@ func (l *LocalBridge) processArp(pkt protocol.Ethernet, inPort uint32) {
 	case *protocol.ARP:
 		var arpIn protocol.ARP = *t
 
+		select {
+		case l.datapathManager.ArpChan <- arpIn:
+		default: // Non-block when arpChan is full
+		}
+
 		l.learnedIPAddressMapMutex.Lock()
 		defer l.learnedIPAddressMapMutex.Unlock()
 		l.setLocalEndpointIPAddr(arpIn, inPort)
diff --git a/pkg/agent/datapath/multiBridgeDatapath.go b/pkg/agent/datapath/multiBridgeDatapath.go
index 0e2daadd6..e34bf8492 100644
--- a/pkg/agent/datapath/multiBridgeDatapath.go
+++ b/pkg/agent/datapath/multiBridgeDatapath.go
@@ -32,6 +32,7 @@ import (
 	log "github.com/Sirupsen/logrus"
 	cnitypes "github.com/containernetworking/cni/pkg/types"
 	"github.com/contiv/libOpenflow/openflow13"
+	"github.com/contiv/libOpenflow/protocol"
 	"github.com/contiv/ofnet/ofctrl"
 	"github.com/contiv/ofnet/ofctrl/cookie"
 	"github.com/contiv/ofnet/ovsdbDriver"
@@ -133,6 +134,8 @@ const (
 	InternalEgressRulePrefix  = "/INTERNAL_EGRESS_POLICY/internal/egress/-"
 
 	MaxRoundNum = 15
+
+	MaxArpChanCache = 100
 )
 
 type Bridge interface {
@@ -183,6 +186,8 @@ type DpManager struct {
 	flowReplayMutex           sync.RWMutex
 	ovsdbReconnectChan        chan struct{}
 
+	ArpChan chan protocol.ARP
+
 	AgentInfo *AgentConf
 }
 
@@ -284,6 +289,7 @@ func NewDatapathManager(datapathConfig *Config, ofPortIPAddressUpdateChan chan m
 	datapathManager.flowReplayChan = make(chan struct{})
 	datapathManager.flowReplayMutex = sync.RWMutex{}
 	datapathManager.ovsdbReconnectChan = make(chan struct{})
+	datapathManager.ArpChan = make(chan protocol.ARP, MaxArpChanCache)
 
 	var wg sync.WaitGroup
 	for vdsID, ovsbrname := range datapathConfig.ManagedVDSMap {
diff --git a/pkg/agent/rpcserver/collector.go b/pkg/agent/rpcserver/collector.go
new file mode 100644
index 000000000..80ca69db3
--- /dev/null
+++ b/pkg/agent/rpcserver/collector.go
@@ -0,0 +1,96 @@
+/*
+Copyright 2021 The Everoute Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rpcserver
+
+import (
+	"context"
+
+	emptypb "google.golang.org/protobuf/types/known/emptypb"
+	"k8s.io/klog"
+
+	"github.com/everoute/everoute/pkg/agent/datapath"
+	pb "github.com/everoute/everoute/pkg/apis/rpc/v1alpha1"
+)
+
+type Collector struct {
+	dpManager *datapath.DpManager
+	stopChan  <-chan struct{}
+}
+
+func (c *Collector) ArpStream(req *emptypb.Empty, srv pb.Collector_ArpStreamServer) error {
+	klog.Info("receive collector client, start arp stream")
+	for {
+		select {
+		case arp := <-c.dpManager.ArpChan:
+			b, err := arp.MarshalBinary()
+			if err != nil {
+				continue
+			}
+			resp := pb.ArpResponse{
+				Pkt: b,
+			}
+			if err := srv.Send(&resp); err != nil {
+				klog.Infof("send error %v", err)
+				return nil
+			}
+
+		case <-c.stopChan:
+			return nil
+		}
+	}
+}
+
+func (c *Collector) GetChainBridge(ctx context.Context, req *emptypb.Empty) (*pb.ChainBridgeResp, error) {
+	resp := &pb.ChainBridgeResp{}
+
+	for br := range c.dpManager.BridgeChainMap {
+		resp.Bridge = append(resp.Bridge, br)
+	}
+	return resp, nil
+}
+
+func (c *Collector) Policy(ctx context.Context, req *pb.PolicyRequest) (*pb.PolicyResponse, error) {
+	policies := c.dpManager.GetPolicyByFlowID(req.FlowIDs...)
+	var policyList []*pb.PolicyList
+
+	for _, p := range policies {
+		policy := &pb.PolicyList{
+			Dir:    uint32(p.Dir),
+			Action: p.Action,
+			Mode:   p.Mode,
+		}
+		for _, item := range p.Item {
+			policy.Items = append(policy.Items, &pb.PolicyItem{
+				Name:       item.Name,
+				Namespace:  item.Namespace,
+				PolicyType: string(item.PolicyType),
+			})
+		}
+		policyList = append(policyList, policy)
+	}
+
+	return &pb.PolicyResponse{List: policyList}, nil
+}
+
+func NewCollectorServer(datapathManager *datapath.DpManager, stopChan <-chan struct{}) *Collector {
+	c := &Collector{
+		dpManager: datapathManager,
+		stopChan:  stopChan,
+	}
+
+	return c
+}
diff --git a/pkg/agent/rpcserver/server.go b/pkg/agent/rpcserver/server.go
new file mode 100644
index 000000000..52f341bf1
--- /dev/null
+++ b/pkg/agent/rpcserver/server.go
@@ -0,0 +1,90 @@
+/*
+Copyright 2021 The Everoute Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rpcserver
+
+import (
+	"net"
+	"os"
+
+	"google.golang.org/grpc"
+	"k8s.io/klog"
+
+	"github.com/everoute/everoute/pkg/agent/datapath"
+	pb "github.com/everoute/everoute/pkg/apis/rpc/v1alpha1"
+)
+
+const RPCSocketAddr = "/var/run/everoute/rpc.sock"
+const EverouteRunPath = "/var/run/everoute"
+
+type Server struct {
+	dpManager *datapath.DpManager
+	stopChan  <-chan struct{}
+}
+
+func Initialize(datapathManager *datapath.DpManager) *Server {
+	s := &Server{
+		dpManager: datapathManager,
+	}
+
+	return s
+}
+
+func (s *Server) Run(stopChan <-chan struct{}) {
+	klog.Info("Starting Everoute RPC Server")
+	s.stopChan = stopChan
+
+	// create path
+	if _, err := os.Stat(EverouteRunPath); os.IsNotExist(err) {
+		if err := os.MkdirAll(EverouteRunPath, os.ModePerm); err != nil {
+			klog.Fatalf("unable to create %s", EverouteRunPath)
+		}
+		if err := os.Chmod(EverouteRunPath, os.ModePerm); err != nil {
+			klog.Fatalf("unable to chmod %s", EverouteRunPath)
+		}
+	}
+
+	// remove the remaining sock file
+	_, err := os.Stat(RPCSocketAddr)
+	if err == nil {
+		err = os.Remove(RPCSocketAddr)
+		if err != nil {
+			klog.Fatalf("remove remaining sock file error, err:%s", err)
+			return
+		}
+	}
+
+	// listen socket
+	listener, err := net.Listen("unix", RPCSocketAddr)
+	if err != nil {
+		klog.Fatalf("Failed to bind on %s: %v", RPCSocketAddr, err)
+	}
+
+	rpcServer := grpc.NewServer()
+	// register collector service
+	collector := NewCollectorServer(s.dpManager, stopChan)
+	pb.RegisterCollectorServer(rpcServer, collector)
+
+	// start rpc Server
+	go func() {
+		if err = rpcServer.Serve(listener); err != nil {
+			klog.Fatalf("Failed to serve collectorServer connections: %v", err)
+		}
+	}()
+
+	klog.Info("RPC server is listening ...")
+	<-s.stopChan
+}
diff --git a/pkg/apis/rpc/v1alpha1/collector.pb.go b/pkg/apis/rpc/v1alpha1/collector.pb.go
new file mode 100644
index 000000000..592750732
--- /dev/null
+++ b/pkg/apis/rpc/v1alpha1/collector.pb.go
@@ -0,0 +1,726 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.26.0
+// 	protoc        v3.17.3
+// source: pkg/apis/rpc/v1alpha1/collector.proto
+
+package v1alpha1
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	emptypb "google.golang.org/protobuf/types/known/emptypb"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type ArpResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Pkt []byte `protobuf:"bytes,1,opt,name=pkt,proto3" json:"pkt,omitempty"`
+}
+
+func (x *ArpResponse) Reset() {
+	*x = ArpResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_pkg_apis_rpc_v1alpha1_collector_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ArpResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ArpResponse) ProtoMessage() {}
+
+func (x *ArpResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_pkg_apis_rpc_v1alpha1_collector_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ArpResponse.ProtoReflect.Descriptor instead.
+func (*ArpResponse) Descriptor() ([]byte, []int) {
+	return file_pkg_apis_rpc_v1alpha1_collector_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *ArpResponse) GetPkt() []byte {
+	if x != nil {
+		return x.Pkt
+	}
+	return nil
+}
+
+type PolicyRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	FlowIDs []uint64 `protobuf:"varint,1,rep,packed,name=flowIDs,proto3" json:"flowIDs,omitempty"`
+}
+
+func (x *PolicyRequest) Reset() {
+	*x = PolicyRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_pkg_apis_rpc_v1alpha1_collector_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *PolicyRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*PolicyRequest) ProtoMessage() {}
+
+func (x *PolicyRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_pkg_apis_rpc_v1alpha1_collector_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use PolicyRequest.ProtoReflect.Descriptor instead.
+func (*PolicyRequest) Descriptor() ([]byte, []int) {
+	return file_pkg_apis_rpc_v1alpha1_collector_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *PolicyRequest) GetFlowIDs() []uint64 {
+	if x != nil {
+		return x.FlowIDs
+	}
+	return nil
+}
+
+type PolicyResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	List []*PolicyList `protobuf:"bytes,1,rep,name=list,proto3" json:"list,omitempty"`
+}
+
+func (x *PolicyResponse) Reset() {
+	*x = PolicyResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_pkg_apis_rpc_v1alpha1_collector_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *PolicyResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*PolicyResponse) ProtoMessage() {}
+
+func (x *PolicyResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_pkg_apis_rpc_v1alpha1_collector_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use PolicyResponse.ProtoReflect.Descriptor instead.
+func (*PolicyResponse) Descriptor() ([]byte, []int) {
+	return file_pkg_apis_rpc_v1alpha1_collector_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *PolicyResponse) GetList() []*PolicyList {
+	if x != nil {
+		return x.List
+	}
+	return nil
+}
+
+type PolicyList struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Dir    uint32        `protobuf:"varint,1,opt,name=dir,proto3" json:"dir,omitempty"`
+	Action string        `protobuf:"bytes,2,opt,name=action,proto3" json:"action,omitempty"`
+	Mode   string        `protobuf:"bytes,3,opt,name=mode,proto3" json:"mode,omitempty"`
+	Items  []*PolicyItem `protobuf:"bytes,4,rep,name=items,proto3" json:"items,omitempty"`
+}
+
+func (x *PolicyList) Reset() {
+	*x = PolicyList{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_pkg_apis_rpc_v1alpha1_collector_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *PolicyList) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*PolicyList) ProtoMessage() {}
+
+func (x *PolicyList) ProtoReflect() protoreflect.Message {
+	mi := &file_pkg_apis_rpc_v1alpha1_collector_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use PolicyList.ProtoReflect.Descriptor instead.
+func (*PolicyList) Descriptor() ([]byte, []int) {
+	return file_pkg_apis_rpc_v1alpha1_collector_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *PolicyList) GetDir() uint32 {
+	if x != nil {
+		return x.Dir
+	}
+	return 0
+}
+
+func (x *PolicyList) GetAction() string {
+	if x != nil {
+		return x.Action
+	}
+	return ""
+}
+
+func (x *PolicyList) GetMode() string {
+	if x != nil {
+		return x.Mode
+	}
+	return ""
+}
+
+func (x *PolicyList) GetItems() []*PolicyItem {
+	if x != nil {
+		return x.Items
+	}
+	return nil
+}
+
+type PolicyItem struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Name       string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	Namespace  string `protobuf:"bytes,2,opt,name=namespace,proto3" json:"namespace,omitempty"`
+	PolicyType string `protobuf:"bytes,3,opt,name=policyType,proto3" json:"policyType,omitempty"`
+}
+
+func (x *PolicyItem) Reset() {
+	*x = PolicyItem{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_pkg_apis_rpc_v1alpha1_collector_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *PolicyItem) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*PolicyItem) ProtoMessage() {}
+
+func (x *PolicyItem) ProtoReflect() protoreflect.Message {
+	mi := &file_pkg_apis_rpc_v1alpha1_collector_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use PolicyItem.ProtoReflect.Descriptor instead.
+func (*PolicyItem) Descriptor() ([]byte, []int) {
+	return file_pkg_apis_rpc_v1alpha1_collector_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *PolicyItem) GetName() string {
+	if x != nil {
+		return x.Name
+	}
+	return ""
+}
+
+func (x *PolicyItem) GetNamespace() string {
+	if x != nil {
+		return x.Namespace
+	}
+	return ""
+}
+
+func (x *PolicyItem) GetPolicyType() string {
+	if x != nil {
+		return x.PolicyType
+	}
+	return ""
+}
+
+type ChainBridgeResp struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Bridge []string `protobuf:"bytes,1,rep,name=bridge,proto3" json:"bridge,omitempty"`
+}
+
+func (x *ChainBridgeResp) Reset() {
+	*x = ChainBridgeResp{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_pkg_apis_rpc_v1alpha1_collector_proto_msgTypes[5]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ChainBridgeResp) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ChainBridgeResp) ProtoMessage() {}
+
+func (x *ChainBridgeResp) ProtoReflect() protoreflect.Message {
+	mi := &file_pkg_apis_rpc_v1alpha1_collector_proto_msgTypes[5]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ChainBridgeResp.ProtoReflect.Descriptor instead.
+func (*ChainBridgeResp) Descriptor() ([]byte, []int) {
+	return file_pkg_apis_rpc_v1alpha1_collector_proto_rawDescGZIP(), []int{5}
+}
+
+func (x *ChainBridgeResp) GetBridge() []string {
+	if x != nil {
+		return x.Bridge
+	}
+	return nil
+}
+
+var File_pkg_apis_rpc_v1alpha1_collector_proto protoreflect.FileDescriptor
+
+var file_pkg_apis_rpc_v1alpha1_collector_proto_rawDesc = []byte{
+	0x0a, 0x25, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x72, 0x70, 0x63, 0x2f, 0x76,
+	0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f,
+	0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x21, 0x65, 0x76, 0x65, 0x72, 0x6f, 0x75, 0x74,
+	0x65, 0x5f, 0x69, 0x6f, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x72, 0x70,
+	0x63, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67,
+	0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74,
+	0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x1f, 0x0a, 0x0b, 0x41, 0x72, 0x70, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x6b, 0x74, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x0c, 0x52, 0x03, 0x70, 0x6b, 0x74, 0x22, 0x29, 0x0a, 0x0d, 0x50, 0x6f, 0x6c, 0x69,
+	0x63, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x66, 0x6c, 0x6f,
+	0x77, 0x49, 0x44, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x04, 0x52, 0x07, 0x66, 0x6c, 0x6f, 0x77,
+	0x49, 0x44, 0x73, 0x22, 0x53, 0x0a, 0x0e, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x04, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x01, 0x20,
+	0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x65, 0x76, 0x65, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x5f, 0x69,
+	0x6f, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x76,
+	0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4c, 0x69,
+	0x73, 0x74, 0x52, 0x04, 0x6c, 0x69, 0x73, 0x74, 0x22, 0x8f, 0x01, 0x0a, 0x0a, 0x50, 0x6f, 0x6c,
+	0x69, 0x63, 0x79, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x64, 0x69, 0x72, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x64, 0x69, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x63, 0x74,
+	0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f,
+	0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x04, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x43, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x04,
+	0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x65, 0x76, 0x65, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x5f,
+	0x69, 0x6f, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x72, 0x70, 0x63, 0x2e,
+	0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x49,
+	0x74, 0x65, 0x6d, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x5e, 0x0a, 0x0a, 0x50, 0x6f,
+	0x6c, 0x69, 0x63, 0x79, 0x49, 0x74, 0x65, 0x6d, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09,
+	0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x70, 0x6f,
+	0x6c, 0x69, 0x63, 0x79, 0x54, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a,
+	0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x54, 0x79, 0x70, 0x65, 0x22, 0x29, 0x0a, 0x0f, 0x43, 0x68,
+	0x61, 0x69, 0x6e, 0x42, 0x72, 0x69, 0x64, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a,
+	0x06, 0x62, 0x72, 0x69, 0x64, 0x67, 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x62,
+	0x72, 0x69, 0x64, 0x67, 0x65, 0x32, 0xb5, 0x02, 0x0a, 0x09, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63,
+	0x74, 0x6f, 0x72, 0x12, 0x57, 0x0a, 0x09, 0x41, 0x72, 0x70, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d,
+	0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62,
+	0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x2e, 0x2e, 0x65, 0x76, 0x65, 0x72, 0x6f,
+	0x75, 0x74, 0x65, 0x5f, 0x69, 0x6f, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e,
+	0x72, 0x70, 0x63, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x41, 0x72, 0x70,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x6f, 0x0a, 0x06,
+	0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x30, 0x2e, 0x65, 0x76, 0x65, 0x72, 0x6f, 0x75, 0x74,
+	0x65, 0x5f, 0x69, 0x6f, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x72, 0x70,
+	0x63, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x50, 0x6f, 0x6c, 0x69, 0x63,
+	0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x65, 0x76, 0x65, 0x72, 0x6f,
+	0x75, 0x74, 0x65, 0x5f, 0x69, 0x6f, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e,
+	0x72, 0x70, 0x63, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x50, 0x6f, 0x6c,
+	0x69, 0x63, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5e, 0x0a,
+	0x0e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x42, 0x72, 0x69, 0x64, 0x67, 0x65, 0x12,
+	0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
+	0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x32, 0x2e, 0x65, 0x76, 0x65, 0x72, 0x6f, 0x75,
+	0x74, 0x65, 0x5f, 0x69, 0x6f, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x72,
+	0x70, 0x63, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x43, 0x68, 0x61, 0x69,
+	0x6e, 0x42, 0x72, 0x69, 0x64, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x42, 0x17, 0x5a,
+	0x15, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x72, 0x70, 0x63, 0x2f, 0x76, 0x31,
+	0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_pkg_apis_rpc_v1alpha1_collector_proto_rawDescOnce sync.Once
+	file_pkg_apis_rpc_v1alpha1_collector_proto_rawDescData = file_pkg_apis_rpc_v1alpha1_collector_proto_rawDesc
+)
+
+func file_pkg_apis_rpc_v1alpha1_collector_proto_rawDescGZIP() []byte {
+	file_pkg_apis_rpc_v1alpha1_collector_proto_rawDescOnce.Do(func() {
+		file_pkg_apis_rpc_v1alpha1_collector_proto_rawDescData = protoimpl.X.CompressGZIP(file_pkg_apis_rpc_v1alpha1_collector_proto_rawDescData)
+	})
+	return file_pkg_apis_rpc_v1alpha1_collector_proto_rawDescData
+}
+
+var file_pkg_apis_rpc_v1alpha1_collector_proto_msgTypes = make([]protoimpl.MessageInfo, 6)
+var file_pkg_apis_rpc_v1alpha1_collector_proto_goTypes = []interface{}{
+	(*ArpResponse)(nil),     // 0: everoute_io.pkg.apis.rpc.v1alpha1.ArpResponse
+	(*PolicyRequest)(nil),   // 1: everoute_io.pkg.apis.rpc.v1alpha1.PolicyRequest
+	(*PolicyResponse)(nil),  // 2: everoute_io.pkg.apis.rpc.v1alpha1.PolicyResponse
+	(*PolicyList)(nil),      // 3: everoute_io.pkg.apis.rpc.v1alpha1.PolicyList
+	(*PolicyItem)(nil),      // 4: everoute_io.pkg.apis.rpc.v1alpha1.PolicyItem
+	(*ChainBridgeResp)(nil), // 5: everoute_io.pkg.apis.rpc.v1alpha1.ChainBridgeResp
+	(*emptypb.Empty)(nil),   // 6: google.protobuf.Empty
+}
+var file_pkg_apis_rpc_v1alpha1_collector_proto_depIdxs = []int32{
+	3, // 0: everoute_io.pkg.apis.rpc.v1alpha1.PolicyResponse.list:type_name -> everoute_io.pkg.apis.rpc.v1alpha1.PolicyList
+	4, // 1: everoute_io.pkg.apis.rpc.v1alpha1.PolicyList.items:type_name -> everoute_io.pkg.apis.rpc.v1alpha1.PolicyItem
+	6, // 2: everoute_io.pkg.apis.rpc.v1alpha1.Collector.ArpStream:input_type -> google.protobuf.Empty
+	1, // 3: everoute_io.pkg.apis.rpc.v1alpha1.Collector.Policy:input_type -> everoute_io.pkg.apis.rpc.v1alpha1.PolicyRequest
+	6, // 4: everoute_io.pkg.apis.rpc.v1alpha1.Collector.GetChainBridge:input_type -> google.protobuf.Empty
+	0, // 5: everoute_io.pkg.apis.rpc.v1alpha1.Collector.ArpStream:output_type -> everoute_io.pkg.apis.rpc.v1alpha1.ArpResponse
+	2, // 6: everoute_io.pkg.apis.rpc.v1alpha1.Collector.Policy:output_type -> everoute_io.pkg.apis.rpc.v1alpha1.PolicyResponse
+	5, // 7: everoute_io.pkg.apis.rpc.v1alpha1.Collector.GetChainBridge:output_type -> everoute_io.pkg.apis.rpc.v1alpha1.ChainBridgeResp
+	5, // [5:8] is the sub-list for method output_type
+	2, // [2:5] is the sub-list for method input_type
+	2, // [2:2] is the sub-list for extension type_name
+	2, // [2:2] is the sub-list for extension extendee
+	0, // [0:2] is the sub-list for field type_name
+}
+
+func init() { file_pkg_apis_rpc_v1alpha1_collector_proto_init() }
+func file_pkg_apis_rpc_v1alpha1_collector_proto_init() {
+	if File_pkg_apis_rpc_v1alpha1_collector_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_pkg_apis_rpc_v1alpha1_collector_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ArpResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_pkg_apis_rpc_v1alpha1_collector_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*PolicyRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_pkg_apis_rpc_v1alpha1_collector_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*PolicyResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_pkg_apis_rpc_v1alpha1_collector_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*PolicyList); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_pkg_apis_rpc_v1alpha1_collector_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*PolicyItem); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_pkg_apis_rpc_v1alpha1_collector_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ChainBridgeResp); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_pkg_apis_rpc_v1alpha1_collector_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   6,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_pkg_apis_rpc_v1alpha1_collector_proto_goTypes,
+		DependencyIndexes: file_pkg_apis_rpc_v1alpha1_collector_proto_depIdxs,
+		MessageInfos:      file_pkg_apis_rpc_v1alpha1_collector_proto_msgTypes,
+	}.Build()
+	File_pkg_apis_rpc_v1alpha1_collector_proto = out.File
+	file_pkg_apis_rpc_v1alpha1_collector_proto_rawDesc = nil
+	file_pkg_apis_rpc_v1alpha1_collector_proto_goTypes = nil
+	file_pkg_apis_rpc_v1alpha1_collector_proto_depIdxs = nil
+}
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConnInterface
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// CollectorClient is the client API for Collector service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
+type CollectorClient interface {
+	ArpStream(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (Collector_ArpStreamClient, error)
+	Policy(ctx context.Context, in *PolicyRequest, opts ...grpc.CallOption) (*PolicyResponse, error)
+	GetChainBridge(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ChainBridgeResp, error)
+}
+
+type collectorClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewCollectorClient(cc grpc.ClientConnInterface) CollectorClient {
+	return &collectorClient{cc}
+}
+
+func (c *collectorClient) ArpStream(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (Collector_ArpStreamClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_Collector_serviceDesc.Streams[0], "/everoute_io.pkg.apis.rpc.v1alpha1.Collector/ArpStream", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &collectorArpStreamClient{stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+type Collector_ArpStreamClient interface {
+	Recv() (*ArpResponse, error)
+	grpc.ClientStream
+}
+
+type collectorArpStreamClient struct {
+	grpc.ClientStream
+}
+
+func (x *collectorArpStreamClient) Recv() (*ArpResponse, error) {
+	m := new(ArpResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *collectorClient) Policy(ctx context.Context, in *PolicyRequest, opts ...grpc.CallOption) (*PolicyResponse, error) {
+	out := new(PolicyResponse)
+	err := c.cc.Invoke(ctx, "/everoute_io.pkg.apis.rpc.v1alpha1.Collector/Policy", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *collectorClient) GetChainBridge(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ChainBridgeResp, error) {
+	out := new(ChainBridgeResp)
+	err := c.cc.Invoke(ctx, "/everoute_io.pkg.apis.rpc.v1alpha1.Collector/GetChainBridge", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// CollectorServer is the server API for Collector service.
+type CollectorServer interface {
+	ArpStream(*emptypb.Empty, Collector_ArpStreamServer) error
+	Policy(context.Context, *PolicyRequest) (*PolicyResponse, error)
+	GetChainBridge(context.Context, *emptypb.Empty) (*ChainBridgeResp, error)
+}
+
+// UnimplementedCollectorServer can be embedded to have forward compatible implementations.
+type UnimplementedCollectorServer struct {
+}
+
+func (*UnimplementedCollectorServer) ArpStream(*emptypb.Empty, Collector_ArpStreamServer) error {
+	return status.Errorf(codes.Unimplemented, "method ArpStream not implemented")
+}
+func (*UnimplementedCollectorServer) Policy(context.Context, *PolicyRequest) (*PolicyResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Policy not implemented")
+}
+func (*UnimplementedCollectorServer) GetChainBridge(context.Context, *emptypb.Empty) (*ChainBridgeResp, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetChainBridge not implemented")
+}
+
+func RegisterCollectorServer(s *grpc.Server, srv CollectorServer) {
+	s.RegisterService(&_Collector_serviceDesc, srv)
+}
+
+func _Collector_ArpStream_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(emptypb.Empty)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return srv.(CollectorServer).ArpStream(m, &collectorArpStreamServer{stream})
+}
+
+type Collector_ArpStreamServer interface {
+	Send(*ArpResponse) error
+	grpc.ServerStream
+}
+
+type collectorArpStreamServer struct {
+	grpc.ServerStream
+}
+
+func (x *collectorArpStreamServer) Send(m *ArpResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func _Collector_Policy_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(PolicyRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(CollectorServer).Policy(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/everoute_io.pkg.apis.rpc.v1alpha1.Collector/Policy",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(CollectorServer).Policy(ctx, req.(*PolicyRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Collector_GetChainBridge_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(emptypb.Empty)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(CollectorServer).GetChainBridge(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/everoute_io.pkg.apis.rpc.v1alpha1.Collector/GetChainBridge",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(CollectorServer).GetChainBridge(ctx, req.(*emptypb.Empty))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _Collector_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "everoute_io.pkg.apis.rpc.v1alpha1.Collector",
+	HandlerType: (*CollectorServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Policy",
+			Handler:    _Collector_Policy_Handler,
+		},
+		{
+			MethodName: "GetChainBridge",
+			Handler:    _Collector_GetChainBridge_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "ArpStream",
+			Handler:       _Collector_ArpStream_Handler,
+			ServerStreams: true,
+		},
+	},
+	Metadata: "pkg/apis/rpc/v1alpha1/collector.proto",
+}
diff --git a/pkg/apis/rpc/v1alpha1/collector.proto b/pkg/apis/rpc/v1alpha1/collector.proto
new file mode 100644
index 000000000..5627d6f8a
--- /dev/null
+++ b/pkg/apis/rpc/v1alpha1/collector.proto
@@ -0,0 +1,46 @@
+syntax = "proto3";
+package everoute_io.pkg.apis.rpc.v1alpha1;
+option go_package = "pkg/apis/rpc/v1alpha1";
+
+import "google/protobuf/empty.proto";
+
+message ArpResponse {
+  bytes pkt = 1;
+}
+
+message PolicyRequest{
+  repeated uint64 flowIDs = 1;
+}
+
+message PolicyResponse{
+  repeated PolicyList list = 1;
+}
+
+message PolicyList{
+  uint32 dir = 1;
+  string action = 2;
+  string mode = 3; 
+  repeated PolicyItem items = 4;
+}
+
+message PolicyItem{
+  string name = 1;
+  string namespace = 2;
+  string policyType = 3;
+}
+
+
+message ChainBridgeResp{
+  repeated string bridge = 1;
+}
+
+service Collector {
+  rpc ArpStream (google.protobuf.Empty) returns (stream ArpResponse) {
+  }
+
+  rpc Policy (PolicyRequest) returns (PolicyResponse) {
+  }
+
+  rpc GetChainBridge (google.protobuf.Empty) returns (ChainBridgeResp){
+  }
+}

From 6999c1e283efc30dc778800fe0c982e4aa49563a Mon Sep 17 00:00:00 2001
From: Changliang Wu <changliang.wu@smartx.com>
Date: Thu, 16 Jun 2022 12:38:26 +0800
Subject: [PATCH 21/32] feat(collector): clear ct commit tables for collector
 if existed

---
 pkg/agent/datapath/localBridge.go | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/pkg/agent/datapath/localBridge.go b/pkg/agent/datapath/localBridge.go
index 9c3a7eff2..d2df99cce 100644
--- a/pkg/agent/datapath/localBridge.go
+++ b/pkg/agent/datapath/localBridge.go
@@ -39,6 +39,8 @@ const (
 	FROM_LOCAL_ARP_TO_CONTROLLER_TABLE = 25
 	CNI_CT_COMMIT_TABLE                = 100
 	CNI_CT_REDIRECT_TABLE              = 105
+	COLLECTOR_CT_COMMIT_TABLE          = 200
+	COLLECTOR_CT_REDIRECT_TABLE        = 205
 	FACK_MAC                           = "ee:ee:ee:ee:ee:ee"
 	P_NONE                             = 0xffff
 	CNI_CONNTRACK_ZONE                 = 65510
@@ -293,6 +295,21 @@ func (l *LocalBridge) BridgeInit() {
 	l.fromLocalArpPassTable, _ = sw.NewTable(FROM_LOCAL_ARP_PASS_TABLE)
 	l.fromLocalArpSendToCtrlTable, _ = sw.NewTable(FROM_LOCAL_ARP_TO_CONTROLLER_TABLE)
 
+	// clear collector flows if existed
+	var inputCtFlowPriority uint16 = NORMAL_MATCH_FLOW_PRIORITY
+	var matchFields []*openflow13.MatchField
+	protoField := openflow13.NewEthTypeField(protocol.IPv4_MSG)
+	matchFields = append(matchFields, protoField)
+	if err := sw.DeleteSpecTableFlows(VLAN_INPUT_TABLE, &inputCtFlowPriority, matchFields); err != nil {
+		log.Fatalf("Failed to delete single local bridge vlanInput table, error: %v", err)
+	}
+	if err := sw.DeleteSpecTableFlows(COLLECTOR_CT_COMMIT_TABLE, nil, nil); err != nil {
+		log.Fatalf("Failed to delete single local bridge ct commit table, error: %v", err)
+	}
+	if err := sw.DeleteSpecTableFlows(COLLECTOR_CT_REDIRECT_TABLE, nil, nil); err != nil {
+		log.Fatalf("Failed to delete single local bridge ct redirect table, error: %v", err)
+	}
+
 	if err := l.initVlanInputTable(sw); err != nil {
 		log.Fatalf("Failed to init local bridge vlanInput table, error: %v", err)
 	}

From 2a4b6e82e84cef5cfc4edf4600231d5b7085e39d Mon Sep 17 00:00:00 2001
From: Changliang Wu <changliang.wu@smartx.com>
Date: Mon, 20 Jun 2022 17:50:10 +0800
Subject: [PATCH 22/32] fix: move /var/run/everoute to /var/lib/everoute

containerd cannot mount to a director not existed
---
 deploy/everoute-agent/agent.yaml |  9 +--------
 deploy/everoute.yaml             |  9 +--------
 pkg/agent/cniserver/server.go    |  2 +-
 pkg/agent/rpcserver/server.go    | 14 +++++++-------
 pkg/cni/client.go                |  2 +-
 pkg/utils/utils.go               |  2 +-
 6 files changed, 12 insertions(+), 26 deletions(-)

diff --git a/deploy/everoute-agent/agent.yaml b/deploy/everoute-agent/agent.yaml
index c1b1c1dde..b401c1bbe 100644
--- a/deploy/everoute-agent/agent.yaml
+++ b/deploy/everoute-agent/agent.yaml
@@ -56,7 +56,7 @@ spec:
       serviceAccountName: everoute-agent
       containers:
         - name: init-agent
-          command: [ "init_agent" ]
+          command: ["init_agent"]
           image: everoute/release
           imagePullPolicy: IfNotPresent
           lifecycle:
@@ -70,8 +70,6 @@ spec:
               mountPath: /var/lib/everoute/
             - name: cni-bin
               mountPath: /opt/cni/bin/
-            - name: everoute-run
-              mountPath: /var/run/everoute/
             - name: cni-conf
               mountPath: /etc/cni/net.d
             - name: everoute-config
@@ -121,8 +119,6 @@ spec:
           volumeMounts:
             - name: everoute-agent
               mountPath: /var/lib/everoute/
-            - name: everoute-run
-              mountPath: /var/run/everoute/
             - name: everoute-config
               mountPath: /var/lib/everoute/agentconfig.yaml
               subPath: agentconfig.yaml
@@ -165,9 +161,6 @@ spec:
         - hostPath:
             path: /opt/cni/bin/
           name: cni-bin
-        - name: everoute-run
-          hostPath:
-            path: /var/run/everoute
         - hostPath:
             path: /etc/cni/net.d
           name: cni-conf
diff --git a/deploy/everoute.yaml b/deploy/everoute.yaml
index f7d15e2e3..808d4debc 100644
--- a/deploy/everoute.yaml
+++ b/deploy/everoute.yaml
@@ -1733,7 +1733,7 @@ spec:
       serviceAccountName: everoute-agent
       containers:
         - name: init-agent
-          command: [ "init_agent" ]
+          command: ["init_agent"]
           image: everoute/release
           imagePullPolicy: IfNotPresent
           lifecycle:
@@ -1747,8 +1747,6 @@ spec:
               mountPath: /var/lib/everoute/
             - name: cni-bin
               mountPath: /opt/cni/bin/
-            - name: everoute-run
-              mountPath: /var/run/everoute/
             - name: cni-conf
               mountPath: /etc/cni/net.d
             - name: everoute-config
@@ -1798,8 +1796,6 @@ spec:
           volumeMounts:
             - name: everoute-agent
               mountPath: /var/lib/everoute/
-            - name: everoute-run
-              mountPath: /var/run/everoute/
             - name: everoute-config
               mountPath: /var/lib/everoute/agentconfig.yaml
               subPath: agentconfig.yaml
@@ -1842,9 +1838,6 @@ spec:
         - hostPath:
             path: /opt/cni/bin/
           name: cni-bin
-        - name: everoute-run
-          hostPath:
-            path: /var/run/everoute
         - hostPath:
             path: /etc/cni/net.d
           name: cni-conf
diff --git a/pkg/agent/cniserver/server.go b/pkg/agent/cniserver/server.go
index aa44d6fcf..95acdee95 100644
--- a/pkg/agent/cniserver/server.go
+++ b/pkg/agent/cniserver/server.go
@@ -43,7 +43,7 @@ import (
 	"github.com/everoute/everoute/pkg/utils"
 )
 
-const CNISocketAddr = "/var/run/everoute/cni.sock"
+const CNISocketAddr = "/var/lib/everoute/cni.sock"
 
 type CNIServer struct {
 	k8sClient client.Client
diff --git a/pkg/agent/rpcserver/server.go b/pkg/agent/rpcserver/server.go
index 52f341bf1..b0bd679ed 100644
--- a/pkg/agent/rpcserver/server.go
+++ b/pkg/agent/rpcserver/server.go
@@ -27,8 +27,8 @@ import (
 	pb "github.com/everoute/everoute/pkg/apis/rpc/v1alpha1"
 )
 
-const RPCSocketAddr = "/var/run/everoute/rpc.sock"
-const EverouteRunPath = "/var/run/everoute"
+const RPCSocketAddr = "/var/lib/everoute/rpc.sock"
+const EverouteLibPath = "/var/lib/everoute"
 
 type Server struct {
 	dpManager *datapath.DpManager
@@ -48,12 +48,12 @@ func (s *Server) Run(stopChan <-chan struct{}) {
 	s.stopChan = stopChan
 
 	// create path
-	if _, err := os.Stat(EverouteRunPath); os.IsNotExist(err) {
-		if err := os.MkdirAll(EverouteRunPath, os.ModePerm); err != nil {
-			klog.Fatalf("unable to create %s", EverouteRunPath)
+	if _, err := os.Stat(EverouteLibPath); os.IsNotExist(err) {
+		if err := os.MkdirAll(EverouteLibPath, os.ModePerm); err != nil {
+			klog.Fatalf("unable to create %s", EverouteLibPath)
 		}
-		if err := os.Chmod(EverouteRunPath, os.ModePerm); err != nil {
-			klog.Fatalf("unable to chmod %s", EverouteRunPath)
+		if err := os.Chmod(EverouteLibPath, os.ModePerm); err != nil {
+			klog.Fatalf("unable to chmod %s", EverouteLibPath)
 		}
 	}
 
diff --git a/pkg/cni/client.go b/pkg/cni/client.go
index 833041a05..e12f921d1 100644
--- a/pkg/cni/client.go
+++ b/pkg/cni/client.go
@@ -14,7 +14,7 @@ import (
 	cnipb "github.com/everoute/everoute/pkg/apis/cni/v1alpha1"
 )
 
-const CNISocketAddr = "/var/run/everoute/cni.sock"
+const CNISocketAddr = "/var/lib/everoute/cni.sock"
 
 func rpcRequest(requestType string, arg *skel.CmdArgs) error {
 	conn, err := grpc.Dial(CNISocketAddr,
diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go
index 33dac730b..6d7cbbf9b 100644
--- a/pkg/utils/utils.go
+++ b/pkg/utils/utils.go
@@ -108,7 +108,7 @@ func CurrentAgentName() string {
 	return currentAgentName
 }
 
-const controllerIDPath = "/var/run/everoute/controllerID"
+const controllerIDPath = "/var/lib/everoute/controllerID"
 
 var _instance *filemutex.FileMutex
 var _once sync.Once

From dc7d84279adbb6860c80e9080077f45a3c2fe39c Mon Sep 17 00:00:00 2001
From: Changliang Wu <changliang.wu@smartx.com>
Date: Thu, 23 Jun 2022 13:09:33 +0800
Subject: [PATCH 23/32] fix(collector): retrieve bridge name not vdsID in
 GetChainBridge

---
 pkg/agent/datapath/multiBridgeDatapath.go | 12 ++++++++++++
 pkg/agent/rpcserver/collector.go          |  7 +++----
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/pkg/agent/datapath/multiBridgeDatapath.go b/pkg/agent/datapath/multiBridgeDatapath.go
index e34bf8492..c8276739b 100644
--- a/pkg/agent/datapath/multiBridgeDatapath.go
+++ b/pkg/agent/datapath/multiBridgeDatapath.go
@@ -362,6 +362,18 @@ func (datapathManager *DpManager) InitializeDatapath(stopChan <-chan struct{}) {
 	}
 }
 
+func (datapathManager *DpManager) GetChainBridge() []string {
+	datapathManager.flowReplayMutex.RLock()
+	defer datapathManager.flowReplayMutex.RUnlock()
+
+	var out []string
+	for _, br := range datapathManager.datapathConfig.ManagedVDSMap {
+		out = append(out, br)
+	}
+
+	return out
+}
+
 func (datapathManager *DpManager) GetPolicyByFlowID(flowID ...uint64) []*PolicyInfo {
 	datapathManager.flowReplayMutex.RLock()
 	defer datapathManager.flowReplayMutex.RUnlock()
diff --git a/pkg/agent/rpcserver/collector.go b/pkg/agent/rpcserver/collector.go
index 80ca69db3..699e6ee15 100644
--- a/pkg/agent/rpcserver/collector.go
+++ b/pkg/agent/rpcserver/collector.go
@@ -55,11 +55,10 @@ func (c *Collector) ArpStream(req *emptypb.Empty, srv pb.Collector_ArpStreamServ
 }
 
 func (c *Collector) GetChainBridge(ctx context.Context, req *emptypb.Empty) (*pb.ChainBridgeResp, error) {
-	resp := &pb.ChainBridgeResp{}
-
-	for br := range c.dpManager.BridgeChainMap {
-		resp.Bridge = append(resp.Bridge, br)
+	resp := &pb.ChainBridgeResp{
+		Bridge: c.dpManager.GetChainBridge(),
 	}
+
 	return resp, nil
 }
 

From 26e64451a1a2064436632e4da9661200ad3c210a Mon Sep 17 00:00:00 2001
From: Changliang Wu <changliang.wu@smartx.com>
Date: Mon, 27 Jun 2022 18:49:52 +0800
Subject: [PATCH 24/32] refactor: use component type and random seq as
 controller ID

---
 pkg/agent/datapath/multiBridgeDatapath.go |  8 +--
 pkg/constants/constants.go                |  2 +
 pkg/utils/utils.go                        | 68 +++++++++++------------
 3 files changed, 37 insertions(+), 41 deletions(-)

diff --git a/pkg/agent/datapath/multiBridgeDatapath.go b/pkg/agent/datapath/multiBridgeDatapath.go
index c8276739b..fc6576b01 100644
--- a/pkg/agent/datapath/multiBridgeDatapath.go
+++ b/pkg/agent/datapath/multiBridgeDatapath.go
@@ -434,10 +434,10 @@ func NewVDSForConfig(datapathManager *DpManager, vdsID, ovsbrname string) {
 
 	// initialize of controller
 	vdsOfControllerMap := make(map[string]*ofctrl.Controller)
-	vdsOfControllerMap[LOCAL_BRIDGE_KEYWORD] = ofctrl.NewControllerAsOFClient(localBridge, utils.GenerateControllerID())
-	vdsOfControllerMap[POLICY_BRIDGE_KEYWORD] = ofctrl.NewControllerAsOFClient(policyBridge, utils.GenerateControllerID())
-	vdsOfControllerMap[CLS_BRIDGE_KEYWORD] = ofctrl.NewControllerAsOFClient(clsBridge, utils.GenerateControllerID())
-	vdsOfControllerMap[UPLINK_BRIDGE_KEYWORD] = ofctrl.NewControllerAsOFClient(uplinkBridge, utils.GenerateControllerID())
+	vdsOfControllerMap[LOCAL_BRIDGE_KEYWORD] = ofctrl.NewControllerAsOFClient(localBridge, utils.GenerateControllerID(constants.EverouteComponentType))
+	vdsOfControllerMap[POLICY_BRIDGE_KEYWORD] = ofctrl.NewControllerAsOFClient(policyBridge, utils.GenerateControllerID(constants.EverouteComponentType))
+	vdsOfControllerMap[CLS_BRIDGE_KEYWORD] = ofctrl.NewControllerAsOFClient(clsBridge, utils.GenerateControllerID(constants.EverouteComponentType))
+	vdsOfControllerMap[UPLINK_BRIDGE_KEYWORD] = ofctrl.NewControllerAsOFClient(uplinkBridge, utils.GenerateControllerID(constants.EverouteComponentType))
 
 	// initialize ovsdbDriver
 	vdsOvsdbDriverMap := make(map[string]*ovsdbDriver.OvsDriver)
diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go
index db41b081e..d02150b50 100644
--- a/pkg/constants/constants.go
+++ b/pkg/constants/constants.go
@@ -49,4 +49,6 @@ const (
 
 	AgentNodeNameENV    = "NODE_NAME"
 	AgentNameConfigPath = "/var/lib/everoute/agent/name"
+
+	EverouteComponentType = 0x0
 )
diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go
index 6d7cbbf9b..4c824fcbd 100644
--- a/pkg/utils/utils.go
+++ b/pkg/utils/utils.go
@@ -9,13 +9,10 @@ import (
 	"io/ioutil"
 	"net"
 	"os"
-	"path"
 	"sort"
-	"strconv"
 	"strings"
 	"sync"
 
-	"github.com/alexflint/go-filemutex"
 	"github.com/vishvananda/netlink"
 	"golang.org/x/sys/unix"
 	coretypes "k8s.io/apimachinery/pkg/types"
@@ -108,40 +105,38 @@ func CurrentAgentName() string {
 	return currentAgentName
 }
 
-const controllerIDPath = "/var/lib/everoute/controllerID"
-
-var _instance *filemutex.FileMutex
+var _instance *ctrlID
 var _once sync.Once
 
-func getCtrlPathMutex() *filemutex.FileMutex {
-	_once.Do(func() {
-		// create path
-		if _, err := os.Stat(controllerIDPath); os.IsNotExist(err) {
-			if err := os.MkdirAll(controllerIDPath, os.ModePerm); err != nil {
-				klog.Fatalf("fail to create %s", controllerIDPath)
-			}
-			if err := os.Chmod(controllerIDPath, os.ModePerm); err != nil {
-				klog.Fatalf("fail to chmod %s", controllerIDPath)
-			}
-		}
+type ctrlID struct {
+	mutex sync.Mutex
+	ids   map[uint16]bool
+}
 
-		// use file mutex to ensure cocurrency
-		mutex, err := filemutex.New(path.Join(controllerIDPath, "lock"))
-		if err != nil {
-			klog.Fatal("Fail to create ControllerID file lock")
+func (c *ctrlID) AddID(id uint16) bool {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+	if _, ok := c.ids[id]; ok {
+		return false
+	}
+	c.ids[id] = true
+
+	return true
+}
+
+func getCtrlIDMap() *ctrlID {
+	_once.Do(func() {
+		// create map
+		_instance = &ctrlID{
+			mutex: sync.Mutex{},
+			ids:   map[uint16]bool{},
 		}
-		_instance = mutex
 	})
 	return _instance
 }
 
-func GenerateControllerID() uint16 {
-	mutex := getCtrlPathMutex()
-
-	_ = mutex.Lock()
-	defer func() {
-		_ = mutex.Unlock()
-	}()
+func GenerateControllerID(typeID uint16) uint16 {
+	ctrlIDs := getCtrlIDMap()
 
 	var ctrlID uint16
 	for {
@@ -151,19 +146,18 @@ func GenerateControllerID() uint16 {
 			klog.Errorf("get random ID from rand.Reader: %s", err)
 			continue
 		}
-		targetFile := path.Join(controllerIDPath, strconv.Itoa(int(ctrlID)))
 
-		// check if id existed
-		if _, err := os.Stat(targetFile); err == nil {
-			continue
-		}
+		// set component type
+		// controller id:
+		// | 4 bits component type | 12 bits random ID |
+		ctrlID >>= 4
+		ctrlID |= typeID << 12
 
-		// record file in path
-		if _, err := os.Create(targetFile); err != nil {
-			klog.Errorf("create ctrlID file %s error: %s", targetFile, err)
+		if !ctrlIDs.AddID(ctrlID) {
 			continue
 		}
 
+		klog.Infof("generate controller ID: %x", ctrlID)
 		return ctrlID
 	}
 }

From c3f86c091680840f890ec7cb49d7a04eb9f196fa Mon Sep 17 00:00:00 2001
From: Changliang Wu <changliang.wu@smartx.com>
Date: Tue, 5 Jul 2022 12:42:48 +0800
Subject: [PATCH 25/32] feat(agent): clear related CT flows while policy
 modified

---
 .github/workflows/ci.yaml                 |   2 +-
 build/images/release/Dockerfile           |   3 +-
 pkg/agent/datapath/multiBridgeDatapath.go |  51 ++++++++++-
 tests/e2e/cases/e2e_test.go               |   9 ++
 tests/e2e/cases/security_test.go          | 100 ++++++++++++++++++++++
 tests/e2e/framework/framework.go          |   2 +
 tests/e2e/framework/model/endpoint.go     |   5 ++
 7 files changed, 168 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index a3969618e..8e0d6d7cf 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -45,7 +45,7 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: install e2e environment dependency
-        run: sudo sh -c "apt update && apt install -y openvswitch-switch=2.13.* && systemctl start openvswitch-switch"
+        run: sudo sh -c "apt update && apt install -y openvswitch-switch=2.13.* conntrack && systemctl start openvswitch-switch"
 
       - name: allow ssh connect to localhost
         run: sudo -H sh -c "ssh-keygen -qN '' </dev/zero; cp ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys"
diff --git a/build/images/release/Dockerfile b/build/images/release/Dockerfile
index c13effc27..293d633dd 100644
--- a/build/images/release/Dockerfile
+++ b/build/images/release/Dockerfile
@@ -27,7 +27,8 @@ RUN chmod 755 /opt/everoute/bin/*
 FROM ubuntu:20.04
 
 #RUN apk update && apk add openvswitch
-RUN apt update && apt install -y openvswitch-switch=2.13.* iptables iproute2 tcpdump && rm -rf /var/lib/apt/lists/*
+RUN apt update && apt install -y openvswitch-switch=2.13.* iptables \
+    conntrack iproute2 tcpdump && rm -rf /var/lib/apt/lists/*
 
 # Automatically detect iptables mode (legacy or nf_tables) from baseOS
 RUN ln -s -f /opt/everoute/bin/iptables-wrapper /etc/alternatives/iptables
diff --git a/pkg/agent/datapath/multiBridgeDatapath.go b/pkg/agent/datapath/multiBridgeDatapath.go
index fc6576b01..fc156bc1d 100644
--- a/pkg/agent/datapath/multiBridgeDatapath.go
+++ b/pkg/agent/datapath/multiBridgeDatapath.go
@@ -40,6 +40,7 @@ import (
 	cmap "github.com/streamrail/concurrent-map"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/klog"
 
 	policycache "github.com/everoute/everoute/pkg/agent/controller/policy/cache"
 	"github.com/everoute/everoute/pkg/constants"
@@ -239,6 +240,11 @@ type EveroutePolicyRule struct {
 	Action      string // rule action: 'allow' or 'deny'
 }
 
+const (
+	EveroutePolicyAllow string = "allow"
+	EveroutePolicyDeny  string = "deny"
+)
+
 type FlowEntry struct {
 	Table    *ofctrl.Table
 	Priority uint16
@@ -833,6 +839,11 @@ func (datapathManager *DpManager) AddEveroutePolicyRule(rule *EveroutePolicyRule
 			return nil
 		}
 		log.Infof("Rule already exists. update old rule: {%+v} to new rule: {%+v} ", ruleEntry.EveroutePolicyRule, rule)
+
+		// clear CT flow while updating from "allow" to "deny"
+		if ruleEntry.EveroutePolicyRule.Action == EveroutePolicyAllow && rule.Action == EveroutePolicyDeny {
+			CleanConntrackFlow(rule)
+		}
 	}
 
 	log.Infof("Received AddRule: %+v", rule)
@@ -847,6 +858,11 @@ func (datapathManager *DpManager) AddEveroutePolicyRule(rule *EveroutePolicyRule
 		ruleFlowMap[vdsID] = flowEntry
 	}
 
+	// clean related CT flows only for "deny" action while adding
+	if rule.Action == EveroutePolicyDeny {
+		CleanConntrackFlow(rule)
+	}
+
 	// save the rule. ruleFlowMap need deepcopy, NOTE
 	if ruleEntry == nil {
 		ruleEntry = &EveroutePolicyRuleEntry{
@@ -901,6 +917,11 @@ func (datapathManager *DpManager) RemoveEveroutePolicyRule(ruleID string, ruleNa
 		delete(datapathManager.FlowIDToRules, pRule.RuleFlowMap[vdsID].FlowID)
 	}
 
+	// clean related CT flows only for "allow" action while deleting
+	if datapathManager.Rules[ruleID].EveroutePolicyRule.Action == EveroutePolicyAllow {
+		CleanConntrackFlow(datapathManager.Rules[ruleID].EveroutePolicyRule)
+	}
+
 	if pRule.PolicyRuleReference.Len() == 0 {
 		delete(datapathManager.Rules, ruleID)
 	}
@@ -908,6 +929,32 @@ func (datapathManager *DpManager) RemoveEveroutePolicyRule(ruleID string, ruleNa
 	return nil
 }
 
+func CleanConntrackFlow(rule *EveroutePolicyRule) {
+	args := []string{"-D"}
+	if rule.SrcIPAddr != "" {
+		args = append(args, "-s", rule.SrcIPAddr)
+	}
+	if rule.DstIPAddr != "" {
+		args = append(args, "-d", rule.DstIPAddr)
+	}
+	if rule.IPProtocol != 0 {
+		args = append(args, "-p", strconv.Itoa(int(rule.IPProtocol)))
+	}
+	if rule.IPProtocol == protocol.Type_TCP || rule.IPProtocol == protocol.Type_UDP {
+		if rule.SrcPort != 0 {
+			args = append(args, "--sport", strconv.Itoa(int(rule.SrcPort))+"/"+strconv.Itoa(int(rule.SrcPortMask)))
+		}
+		if rule.DstPort != 0 {
+			args = append(args, "--dport", strconv.Itoa(int(rule.DstPort))+"/"+strconv.Itoa(int(rule.DstPortMask)))
+		}
+	}
+	klog.Infof("clear conntrack for rule: %+v, conntrack args: conntrack %s", rule, args)
+	err := exec.Command("conntrack", args...).Run()
+	if err != nil {
+		klog.Errorf("clear conntrack error, rule: %+v, err: %s", rule, err)
+	}
+}
+
 func RuleIsSame(r1, r2 *EveroutePolicyRule) bool {
 	return reflect.DeepEqual(*r1, *r2)
 }
@@ -1099,7 +1146,7 @@ func newInternalIngressRule(internalIP string) *EveroutePolicyRule {
 		RuleID:    fmt.Sprintf("internal.ingress.%s", internalIP),
 		Priority:  constants.InternalWhitelistPriority,
 		DstIPAddr: internalIP,
-		Action:    "allow",
+		Action:    EveroutePolicyAllow,
 	}
 }
 
@@ -1109,6 +1156,6 @@ func newInternalEgressRule(internalIP string) *EveroutePolicyRule {
 		RuleID:    fmt.Sprintf("internal.egress.%s", internalIP),
 		Priority:  constants.InternalWhitelistPriority,
 		SrcIPAddr: internalIP,
-		Action:    "allow",
+		Action:    EveroutePolicyAllow,
 	}
 }
diff --git a/tests/e2e/cases/e2e_test.go b/tests/e2e/cases/e2e_test.go
index 388cb0323..5dfdf6d8f 100644
--- a/tests/e2e/cases/e2e_test.go
+++ b/tests/e2e/cases/e2e_test.go
@@ -25,6 +25,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/containernetworking/plugins/pkg/utils/sysctl"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -59,11 +60,19 @@ var _ = BeforeSuite(func() {
 	// reset resource before start e2e
 	Expect(e2eEnv.ResetResource(ctx)).ToNot(HaveOccurred())
 
+	timeoutSec := fmt.Sprintf("%1.0f", e2eEnv.Timeout().Seconds()*2)
+	resp, err := sysctl.Sysctl("net/netfilter/nf_conntrack_tcp_timeout_close", timeoutSec)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(resp).To(Equal(timeoutSec))
+
 	restarter := e2eEnv.NodeManager().ServiceRestarter(10, 30)
 	go restarter.Run(make(chan struct{}))
 })
 
 var _ = AfterSuite(func() {
+	// reset tcp close timeout
+	sysctl.Sysctl("net/netfilter/nf_conntrack_tcp_timeout_close", "10")
+
 	klog.Infof("complete all e2e test cases use %s", time.Since(startTime))
 	klog.Infof("run e2e-reset.sh to clean test environment")
 })
diff --git a/tests/e2e/cases/security_test.go b/tests/e2e/cases/security_test.go
index 13e8643db..c17a9d834 100644
--- a/tests/e2e/cases/security_test.go
+++ b/tests/e2e/cases/security_test.go
@@ -17,8 +17,11 @@ limitations under the License.
 package cases
 
 import (
+	"bytes"
 	"context"
 	"fmt"
+	"os/exec"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -29,6 +32,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/rand"
+	"k8s.io/klog"
 
 	securityv1alpha1 "github.com/everoute/everoute/pkg/apis/security/v1alpha1"
 	"github.com/everoute/everoute/pkg/constants"
@@ -40,6 +44,7 @@ import (
 var _ = Describe("SecurityPolicy", func() {
 	AfterEach(func() {
 		Expect(e2eEnv.ResetResource(ctx)).Should(Succeed())
+		Expect(cleanConntrack()).Should(Succeed())
 	})
 
 	// This case test policy with tcp and icmp can works. We setup three groups of vms (nginx/webserver/database), create
@@ -72,6 +77,23 @@ var _ = Describe("SecurityPolicy", func() {
 			Expect(e2eEnv.EndpointManager().SetupMany(ctx, nginx, server01, server02, db01, db02, client)).Should(Succeed())
 		})
 
+		It("should clean exist connection after adding drop policy", func() {
+			assertReachable([]*model.Endpoint{nginx}, []*model.Endpoint{db01}, "TCP", true)
+
+			nginxPolicy := newPolicy("nginx-policy", constants.Tier2, securityv1alpha1.DefaultRuleDrop, nginxSelector)
+			addEngressRule(nginxPolicy, "TCP", serverPort, serverSelector)
+
+			Eventually(func() bool {
+				return checkConntrackExist("TCP", nginx.Status.GetIP(), db01.Status.GetIP(), 0, 0)
+			}, e2eEnv.Timeout(), e2eEnv.Interval()).Should(BeTrue())
+
+			Expect(e2eEnv.SetupObjects(ctx, nginxPolicy)).Should(Succeed())
+
+			Eventually(func() bool {
+				return checkConntrackExist("TCP", nginx.Status.GetIP(), db01.Status.GetIP(), 0, 0)
+			}, e2eEnv.Timeout(), e2eEnv.Interval()).Should(BeFalse())
+		})
+
 		When("limits tcp packets between components", func() {
 			var nginxPolicy, serverPolicy, dbPolicy *securityv1alpha1.SecurityPolicy
 
@@ -105,6 +127,29 @@ var _ = Describe("SecurityPolicy", func() {
 				assertReachable([]*model.Endpoint{server01, server02, db01, db02}, []*model.Endpoint{db01, db02}, "TCP", true)
 			})
 
+			It("should clean exist allow connection after deleting policy", func() {
+				assertReachable([]*model.Endpoint{nginx}, []*model.Endpoint{server01}, "TCP", true)
+
+				Expect(e2eEnv.ResetResource(ctx)).Should(Succeed())
+
+				Eventually(func() bool {
+					return checkConntrackExist("TCP", nginx.Status.GetIP(), server01.Status.GetIP(), 0, 0)
+				}, e2eEnv.Timeout(), e2eEnv.Interval()).Should(BeFalse())
+			})
+
+			It("should keep exist allow connection after adding new allow policy", func() {
+				assertReachable([]*model.Endpoint{nginx}, []*model.Endpoint{server01}, "TCP", true)
+
+				ngxinDBPolicy := newPolicy("nginx-db-policy", constants.Tier2, securityv1alpha1.DefaultRuleDrop, nginxSelector)
+				addEngressRule(ngxinDBPolicy, "TCP", dbPort, dbSelector)
+
+				Expect(e2eEnv.SetupObjects(ctx, ngxinDBPolicy)).Should(Succeed())
+
+				Eventually(func() bool {
+					return checkConntrackExist("TCP", nginx.Status.GetIP(), server01.Status.GetIP(), 0, 0)
+				}, e2eEnv.Timeout(), e2eEnv.Interval()).Should(BeTrue())
+			})
+
 			When("add endpoint into the database group", func() {
 				var db03 *model.Endpoint
 
@@ -648,6 +693,25 @@ var _ = Describe("GlobalPolicy", func() {
 			assertMatchReachTable("TCP", tcpPort, expectedTruthTable)
 		})
 
+		It("should clean exist allow connection add global drop policy", func() {
+			securityModel := &SecurityModel{
+				Endpoints: []*model.Endpoint{endpointA, endpointB, endpointC},
+			}
+			By("verify reachable between endpoints")
+			expectedTruthTable := securityModel.NewEmptyTruthTable(true)
+			assertMatchReachTable("TCP", tcpPort, expectedTruthTable)
+
+			Eventually(func() bool {
+				return checkConntrackExist("TCP", endpointA.Status.GetIP(), endpointB.Status.GetIP(), 0, 0)
+			}, e2eEnv.Timeout(), e2eEnv.Interval()).Should(BeTrue())
+
+			Expect(e2eEnv.GlobalPolicyProvider().SetDefaultAction(ctx, securityv1alpha1.GlobalDefaultActionDrop)).Should(Succeed())
+
+			Eventually(func() bool {
+				return checkConntrackExist("TCP", endpointA.Status.GetIP(), endpointB.Status.GetIP(), 0, 0)
+			}, e2eEnv.Timeout(), e2eEnv.Interval()).Should(BeFalse())
+		})
+
 		When("update global default action to drop", func() {
 			BeforeEach(func() {
 				// drop all traffics between endpoints
@@ -763,6 +827,42 @@ var _ = Describe("GlobalPolicy", func() {
 	})
 })
 
+func checkConntrackExist(proto, srcIP, dstIP string, srcPort, dstPort uint16) bool {
+	args := []string{"-L"}
+
+	if srcIP != "" {
+		args = append(args, "-s", srcIP)
+	}
+	if dstIP != "" {
+		args = append(args, "-d", dstIP)
+	}
+	args = append(args, "-p", proto)
+
+	if proto == "TCP" || proto == "UDP" {
+		if srcPort != 0 {
+			args = append(args, "--sport", strconv.Itoa(int(srcPort)))
+		}
+		if dstPort != 0 {
+			args = append(args, "--dport", strconv.Itoa(int(dstPort)))
+		}
+	}
+
+	var b bytes.Buffer
+	cmd := exec.Command("conntrack", args...)
+	cmd.Stderr = &b
+	cmd.Run()
+	out := strings.TrimSpace(b.String())
+
+	reg, _ := regexp.Compile(": (.+?) flow entries")
+	flowCount, _ := strconv.Atoi(strings.TrimSpace(reg.FindStringSubmatch(out)[1]))
+	klog.Infof("checkConntrackExist find %d flows with %s", flowCount, args)
+	return flowCount != 0
+}
+
+func cleanConntrack() error {
+	return exec.Command("conntrack", "-F").Run()
+}
+
 func newSelector(selector map[string][]string) *labels.Selector {
 	return &labels.Selector{
 		ExtendMatchLabels: selector,
diff --git a/tests/e2e/framework/framework.go b/tests/e2e/framework/framework.go
index f5e430495..04fdfdc96 100644
--- a/tests/e2e/framework/framework.go
+++ b/tests/e2e/framework/framework.go
@@ -21,6 +21,8 @@ import (
 	"fmt"
 	"time"
 
+	"k8s.io/klog"
+
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
diff --git a/tests/e2e/framework/model/endpoint.go b/tests/e2e/framework/model/endpoint.go
index 18bb83d10..586db73a0 100644
--- a/tests/e2e/framework/model/endpoint.go
+++ b/tests/e2e/framework/model/endpoint.go
@@ -19,6 +19,7 @@ package model
 import (
 	"context"
 	"fmt"
+	"strings"
 )
 
 // Endpoint is a network communication entity. It's provided by the endpoint provider,
@@ -53,6 +54,10 @@ type EndpointStatus struct {
 	LocalID string
 }
 
+func (es *EndpointStatus) GetIP() string {
+	return strings.Split(es.IPAddr, "/")[0]
+}
+
 func (es *EndpointStatus) String() string {
 	if es != nil {
 		return fmt.Sprintf("%+v", *es)

From 0f838bcf260ee684e41a40818596e95441840cbd Mon Sep 17 00:00:00 2001
From: Changliang Wu <changliang.wu@smartx.com>
Date: Mon, 8 Aug 2022 16:09:50 +0800
Subject: [PATCH 26/32] test(e2e): reduce test log

---
 .github/workflows/ci.yaml                |  2 +-
 hack/0001-test-e2e-reduce-test-log.patch | 38 ++++++++++++++++++++++++
 2 files changed, 39 insertions(+), 1 deletion(-)
 create mode 100644 hack/0001-test-e2e-reduce-test-log.patch

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 8e0d6d7cf..41ec4caba 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -88,7 +88,7 @@ jobs:
           path: "./kubernetes"
 
       - name: apply e2e patch
-        run: cd kubernetes && git apply ../hack/0001-test-e2e-add-sleep-before-cannot-conntect-test.patch
+        run: cd kubernetes && git apply ../hack/0001-test-e2e-add-sleep-before-cannot-conntect-test.patch ../hack/0001-test-e2e-reduce-test-log.patch
 
       - name: build e2e
         run: cd kubernetes && make all WHAT=test/e2e/e2e.test && make all WHAT=vendor/github.com/onsi/ginkgo/ginkgo
diff --git a/hack/0001-test-e2e-reduce-test-log.patch b/hack/0001-test-e2e-reduce-test-log.patch
new file mode 100644
index 000000000..52599c6f9
--- /dev/null
+++ b/hack/0001-test-e2e-reduce-test-log.patch
@@ -0,0 +1,38 @@
+From e59688cefabaf068eab5bc6132b2d25f1c182e91 Mon Sep 17 00:00:00 2001
+From: Changliang Wu <changliang.wu@smartx.com>
+Date: Mon, 8 Aug 2022 16:06:48 +0800
+Subject: [PATCH] test(e2e): reduce test log
+
+---
+ test/e2e/framework/util.go             | 2 +-
+ test/e2e/network/netpol/kubemanager.go | 1 +
+ 2 files changed, 2 insertions(+), 1 deletion(-)
+
+diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go
+index 2040604a..2ed45fae 100644
+--- a/test/e2e/framework/util.go
++++ b/test/e2e/framework/util.go
+@@ -439,7 +439,7 @@ func countEndpointsNum(e *v1.Endpoints) int {
+ 
+ // restclientConfig returns a config holds the information needed to build connection to kubernetes clusters.
+ func restclientConfig(kubeContext string) (*clientcmdapi.Config, error) {
+-	Logf(">>> kubeConfig: %s", TestContext.KubeConfig)
++	//Logf(">>> kubeConfig: %s", TestContext.KubeConfig)
+ 	if TestContext.KubeConfig == "" {
+ 		return nil, fmt.Errorf("KubeConfig must be specified to load client config")
+ 	}
+diff --git a/test/e2e/network/netpol/kubemanager.go b/test/e2e/network/netpol/kubemanager.go
+index 08dad13e..6fa47cc9 100644
+--- a/test/e2e/network/netpol/kubemanager.go
++++ b/test/e2e/network/netpol/kubemanager.go
+@@ -141,6 +141,7 @@ func (k *kubeManager) executeRemoteCommand(namespace string, pod string, contain
+ 		CaptureStdout:      true,
+ 		CaptureStderr:      true,
+ 		PreserveWhitespace: false,
++		Quiet:              true,
+ 	})
+ }
+ 
+-- 
+2.31.1
+

From ce1888fb3bd24fd9dd065afe36d6946cb91988e8 Mon Sep 17 00:00:00 2001
From: Changliang Wu <changliang.wu@smartx.com>
Date: Tue, 9 Aug 2022 15:06:09 +0800
Subject: [PATCH 27/32] test: fix rebase errors

---
 pkg/agent/datapath/policyBridge.go    | 2 +-
 pkg/apis/rpc/v1alpha1/collector.pb.go | 5 +++--
 tests/e2e/framework/framework.go      | 2 --
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/pkg/agent/datapath/policyBridge.go b/pkg/agent/datapath/policyBridge.go
index 92909942c..6b0f05c2c 100644
--- a/pkg/agent/datapath/policyBridge.go
+++ b/pkg/agent/datapath/policyBridge.go
@@ -512,7 +512,7 @@ func (p *PolicyBridge) GetTierTable(direction uint8, tier uint8, mode string) (*
 	return policyTable, nextTable, nil
 }
 
-//nolint: funlen
+//nolint:funlen
 func (p *PolicyBridge) AddMicroSegmentRule(rule *EveroutePolicyRule, direction uint8, tier uint8, mode string) (*FlowEntry, error) {
 	var ipDa *net.IP = nil
 	var ipDaMask *net.IP = nil
diff --git a/pkg/apis/rpc/v1alpha1/collector.pb.go b/pkg/apis/rpc/v1alpha1/collector.pb.go
index 592750732..890b1d1ed 100644
--- a/pkg/apis/rpc/v1alpha1/collector.pb.go
+++ b/pkg/apis/rpc/v1alpha1/collector.pb.go
@@ -8,14 +8,15 @@ package v1alpha1
 
 import (
 	context "context"
+	reflect "reflect"
+	sync "sync"
+
 	grpc "google.golang.org/grpc"
 	codes "google.golang.org/grpc/codes"
 	status "google.golang.org/grpc/status"
 	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
 	emptypb "google.golang.org/protobuf/types/known/emptypb"
-	reflect "reflect"
-	sync "sync"
 )
 
 const (
diff --git a/tests/e2e/framework/framework.go b/tests/e2e/framework/framework.go
index 04fdfdc96..f5e430495 100644
--- a/tests/e2e/framework/framework.go
+++ b/tests/e2e/framework/framework.go
@@ -21,8 +21,6 @@ import (
 	"fmt"
 	"time"
 
-	"k8s.io/klog"
-
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"

From 74ea09f729641217df61b72bbd9b610a9a8d8b9f Mon Sep 17 00:00:00 2001
From: Changliang Wu <changliang.wu@smartx.com>
Date: Fri, 5 Aug 2022 10:47:29 +0800
Subject: [PATCH 28/32] refactor(cni): move pkt_mark for flow redirect from pos
 '0' to pos '29'

---
 pkg/agent/datapath/localBridge.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/pkg/agent/datapath/localBridge.go b/pkg/agent/datapath/localBridge.go
index d2df99cce..622e8748b 100644
--- a/pkg/agent/datapath/localBridge.go
+++ b/pkg/agent/datapath/localBridge.go
@@ -433,18 +433,18 @@ func (l *LocalBridge) initToLocalGwFlow(sw *ofctrl.OFSwitch) error {
 	_ = localToLocalGw.LoadField("nxm_of_eth_dst", ParseMacToUint64(l.datapathManager.AgentInfo.LocalGwMac),
 		openflow13.NewNXRange(0, 47))
 	_ = localToLocalGw.LoadField("nxm_nx_pkt_mark", 0x1,
-		openflow13.NewNXRange(0, 0))
+		openflow13.NewNXRange(29, 29))
 	outputPortLocalGateWay, _ := sw.OutputPort(LOCAL_GATEWAY_PORT)
 	if err := localToLocalGw.Next(outputPortLocalGateWay); err != nil {
 		return fmt.Errorf("failed to install from localToLocalGw flow, error: %v", err)
 	}
 
-	pktMarkMask := uint32(0x01)
+	pktMarkMask := uint32(0x20000000)
 	outToLocalGwBypassLocal, _ := l.vlanInputTable.NewFlow(ofctrl.FlowMatch{
 		Priority:    HIGH_MATCH_FLOW_PRIORITY + FLOW_MATCH_OFFSET,
 		Ethertype:   PROTOCOL_IP,
 		InputPort:   uint32(l.datapathManager.BridgeChainPortMap[l.name][LocalToPolicySuffix]),
-		PktMark:     0x01,
+		PktMark:     0x20000000,
 		PktMarkMask: &pktMarkMask,
 	})
 	if err := outToLocalGwBypassLocal.Resubmit(nil, &l.localEndpointL2ForwardingTable.TableId); err != nil {
@@ -504,12 +504,12 @@ func (l *LocalBridge) initToLocalGwFlow(sw *ofctrl.OFSwitch) error {
 }
 
 func (l *LocalBridge) initFromLocalGwFlow(sw *ofctrl.OFSwitch) error {
-	pktMarkMask := uint32(0x01)
+	pktMarkMask := uint32(0x20000000)
 	localGwToPolicy, _ := l.vlanInputTable.NewFlow(ofctrl.FlowMatch{
 		Priority:    HIGH_MATCH_FLOW_PRIORITY,
 		Ethertype:   PROTOCOL_IP,
 		InputPort:   uint32(LOCAL_GATEWAY_PORT),
-		PktMark:     0x01,
+		PktMark:     0x20000000,
 		PktMarkMask: &pktMarkMask,
 	})
 	if err := localGwToPolicy.LoadField("nxm_of_eth_src", ParseMacToUint64(l.datapathManager.AgentInfo.LocalGwMac),

From f5c7a02ad8c8703d54ffc3ff3ed264d6885451b7 Mon Sep 17 00:00:00 2001
From: Changliang Wu <changliang.wu@smartx.com>
Date: Tue, 9 Aug 2022 18:45:24 +0800
Subject: [PATCH 29/32] ci: update glangci-lint to v1.48.0

---
 .github/workflows/ci.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 41ec4caba..1bcfd9590 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -33,7 +33,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: golangci/golangci-lint-action@v2
         with:
-          version: v1.45
+          version: v1.48.0
           only-new-issues: true
           # skip cache because of flaky behaviors
           skip-build-cache: true

From 3fac17ad4fc98efc0734909e4f7fce0c3ecd34b5 Mon Sep 17 00:00:00 2001
From: Changliang Wu <changliang.wu@smartx.com>
Date: Wed, 10 Aug 2022 17:22:18 +0800
Subject: [PATCH 30/32] ci: update e2e harbor to registry.smtx.io

---
 .github/workflows/ci.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 1bcfd9590..f7bf04ce6 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -62,7 +62,7 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: clean old eveorute
-        run: sudo skaffold run -d=harbor.smartx.com/everoute -f skaffold-clean.yaml
+        run: sudo skaffold run -d=registry.smtx.io/everoute -f skaffold-clean.yaml
 
       - name: wait clean process ready
         run: kubectl wait po -n kube-system --for=condition=Ready=True -l app=everoute -l component=everoute-clean --timeout=3m
@@ -76,7 +76,7 @@ jobs:
         run: kubectl delete -f hack/clean.yaml
 
       - name: build everoute and deploy
-        run: sudo skaffold run -d=harbor.smartx.com/everoute
+        run: sudo skaffold run -d=registry.smtx.io/everoute
 
       - name: wait everoute ready
         run: bash hack/check_ready.sh

From ec597d9c19caaef0b856222b03f7d1fe6241d24e Mon Sep 17 00:00:00 2001
From: Changliang Wu <changliang.wu@smartx.com>
Date: Fri, 5 Aug 2022 16:31:09 +0800
Subject: [PATCH 31/32] feat(dp): store local ofport into pkt_mark

---
 pkg/agent/datapath/localBridge.go             | 45 ++++++++++++-------
 .../datapath/multiBridgeDatapath_test.go      |  2 +-
 2 files changed, 31 insertions(+), 16 deletions(-)

diff --git a/pkg/agent/datapath/localBridge.go b/pkg/agent/datapath/localBridge.go
index 622e8748b..9ed7e1b7a 100644
--- a/pkg/agent/datapath/localBridge.go
+++ b/pkg/agent/datapath/localBridge.go
@@ -20,6 +20,7 @@ import (
 	"encoding/binary"
 	"fmt"
 	"net"
+	"strings"
 	"sync"
 	"time"
 
@@ -720,28 +721,42 @@ func (l *LocalBridge) BridgeReset() {
 }
 
 func (l *LocalBridge) AddLocalEndpoint(endpoint *Endpoint) error {
+	// skip ovs patch port
+	if strings.HasSuffix(endpoint.InterfaceName, LocalToPolicySuffix) {
+		return nil
+	}
+
+	// skip cni gateway
+	if l.datapathManager.AgentInfo.LocalGwName == endpoint.InterfaceName {
+		return nil
+	}
+
 	// Table 0, from local endpoint
 	var vlanIDMask uint16 = 0x1fff
+	vlanInputTableFromLocalFlow, _ := l.vlanInputTable.NewFlow(ofctrl.FlowMatch{
+		Priority:  MID_MATCH_FLOW_PRIORITY,
+		InputPort: endpoint.PortNo,
+	})
 	if endpoint.VlanID != 0 {
-		vlanInputTableFromLocalFlow, _ := l.vlanInputTable.NewFlow(ofctrl.FlowMatch{
-			Priority:  MID_MATCH_FLOW_PRIORITY,
-			InputPort: endpoint.PortNo,
-		})
 		if err := vlanInputTableFromLocalFlow.SetVlan(endpoint.VlanID); err != nil {
 			return err
 		}
-		if err := vlanInputTableFromLocalFlow.Resubmit(nil, &l.localEndpointL2LearningTable.TableId); err != nil {
-			return err
-		}
-		if err := vlanInputTableFromLocalFlow.Resubmit(nil, &l.fromLocalRedirectTable.TableId); err != nil {
-			return err
-		}
-		if err := vlanInputTableFromLocalFlow.Next(ofctrl.NewEmptyElem()); err != nil {
-			return err
-		}
-		log.Infof("add from local endpoint flow: %v", vlanInputTableFromLocalFlow)
-		l.fromLocalEndpointFlow[endpoint.PortNo] = vlanInputTableFromLocalFlow
 	}
+	if err := vlanInputTableFromLocalFlow.LoadField("nxm_nx_pkt_mark", uint64(endpoint.PortNo),
+		openflow13.NewNXRange(0, 15)); err != nil {
+		return err
+	}
+	if err := vlanInputTableFromLocalFlow.Resubmit(nil, &l.localEndpointL2LearningTable.TableId); err != nil {
+		return err
+	}
+	if err := vlanInputTableFromLocalFlow.Resubmit(nil, &l.fromLocalRedirectTable.TableId); err != nil {
+		return err
+	}
+	if err := vlanInputTableFromLocalFlow.Next(ofctrl.NewEmptyElem()); err != nil {
+		return err
+	}
+	log.Infof("add from local endpoint flow: %v", vlanInputTableFromLocalFlow)
+	l.fromLocalEndpointFlow[endpoint.PortNo] = vlanInputTableFromLocalFlow
 
 	// Table 1, from local to local bum redirect flow
 	endpointMac, _ := net.ParseMAC(endpoint.MacAddrStr)
diff --git a/pkg/agent/datapath/multiBridgeDatapath_test.go b/pkg/agent/datapath/multiBridgeDatapath_test.go
index 6a33abf8f..fa13ac7df 100644
--- a/pkg/agent/datapath/multiBridgeDatapath_test.go
+++ b/pkg/agent/datapath/multiBridgeDatapath_test.go
@@ -83,7 +83,7 @@ var (
 
 	rule1Flow = `table=60, priority=200,icmp,nw_src=10.100.100.1,nw_dst=10.100.100.2 ` +
 		`actions=load:0x->NXM_NX_XXREG0[60..87],load:0x->NXM_NX_XXREG0[0..3],goto_table:70`
-	ep1VlanInputFlow    = "table=0, priority=200,in_port=11 actions=push_vlan:0x8100,set_field:4097->vlan_vid,resubmit(,10),resubmit(,15)"
+	ep1VlanInputFlow    = "table=0, priority=200,in_port=11 actions=load:0xb->NXM_NX_PKT_MARK[0..15],push_vlan:0x8100,set_field:4097->vlan_vid,resubmit(,10),resubmit(,15)"
 	ep1LocalToLocalFlow = "table=5, priority=200,dl_vlan=1,dl_src=00:00:aa:aa:aa:aa actions=load:0xb->NXM_OF_IN_PORT[],load:0->NXM_OF_VLAN_TCI[0..12],NORMAL"
 )
 

From b00aaed424f85f956cfa22f7cf31cdfd5328d3e7 Mon Sep 17 00:00:00 2001
From: Changliang Wu <changliang.wu@smartx.com>
Date: Mon, 15 Aug 2022 17:32:02 +0800
Subject: [PATCH 32/32] test: add test cases for monitor mode

---
 .../datapath/multiBridgeDatapath_test.go      | 31 ++++++++
 tests/e2e/cases/security_mode.go              | 70 ++++++++++++-------
 tests/e2e/cases/security_test.go              | 39 +++++++++++
 3 files changed, 116 insertions(+), 24 deletions(-)

diff --git a/pkg/agent/datapath/multiBridgeDatapath_test.go b/pkg/agent/datapath/multiBridgeDatapath_test.go
index fa13ac7df..98b681853 100644
--- a/pkg/agent/datapath/multiBridgeDatapath_test.go
+++ b/pkg/agent/datapath/multiBridgeDatapath_test.go
@@ -28,6 +28,8 @@ import (
 
 	log "github.com/Sirupsen/logrus"
 	. "github.com/onsi/gomega"
+
+	"github.com/everoute/everoute/pkg/apis/security/v1alpha1"
 )
 
 const (
@@ -110,6 +112,7 @@ func TestDpManager(t *testing.T) {
 
 	testLocalEndpoint(t)
 	testERPolicyRule(t)
+	testMonitorRule(t)
 	testFlowReplay(t)
 	testRoundNumFlip(t)
 }
@@ -171,6 +174,34 @@ func testERPolicyRule(t *testing.T) {
 	})
 }
 
+func testMonitorRule(t *testing.T) {
+	t.Run("test ER policy rule with monitor mode", func(t *testing.T) {
+		if err := datapathManager.AddEveroutePolicyRule(rule1, "rule1", POLICY_DIRECTION_IN, POLICY_TIER2, v1alpha1.MonitorMode.String()); err != nil {
+			t.Errorf("Failed to add ER policy rule: %v, error: %v", rule1, err)
+		}
+		if _, ok := datapathManager.Rules[rule1.RuleID]; !ok {
+			t.Errorf("Failed to add ER policy rule, not found %v in cache", rule1)
+		}
+
+		if err := datapathManager.RemoveEveroutePolicyRule(rule1.RuleID, "rule1"); err != nil {
+			t.Errorf("Failed to remove ER policy rule: %v, error: %v", rule1, err)
+		}
+		if _, ok := datapathManager.Rules[rule1.RuleID]; ok {
+			t.Errorf("Failed to remove ER policy rule, rule %v in cache", rule1)
+		}
+
+		if err := datapathManager.AddEveroutePolicyRule(rule2, "rule2", POLICY_DIRECTION_OUT, POLICY_TIER1, v1alpha1.MonitorMode.String()); err != nil {
+			t.Errorf("Failed to add ER policy rule: %v, error: %v", rule2, err)
+		}
+		if _, ok := datapathManager.Rules[rule2.RuleID]; !ok {
+			t.Errorf("Failed to add ER policy rule, not found %v in cache", rule2)
+		}
+		if err := datapathManager.AddEveroutePolicyRule(rule2, "rule2", POLICY_DIRECTION_OUT, POLICY_TIER1, v1alpha1.MonitorMode.String()); err != nil {
+			t.Errorf("Failed to add ER policy rule: %v, error: %v", rule2, err)
+		}
+	})
+}
+
 func testFlowReplay(t *testing.T) {
 	RegisterTestingT(t)
 
diff --git a/tests/e2e/cases/security_mode.go b/tests/e2e/cases/security_mode.go
index f2073bf7e..facc29efd 100644
--- a/tests/e2e/cases/security_mode.go
+++ b/tests/e2e/cases/security_mode.go
@@ -100,7 +100,8 @@ func (m *SecurityModel) collectPolicyFlows(policy *securityv1alpha1.SecurityPoli
 		})...)
 	}
 
-	return computePolicyFlow(policy.Spec.Tier, appliedIPs, ingressIPs, egressIPs, ingressPorts, egressPorts)
+	return computePolicyFlow(policy.Spec.Tier, policy.Spec.SecurityPolicyEnforcementMode,
+		appliedIPs, ingressIPs, egressIPs, ingressPorts, egressPorts)
 }
 
 func (m *SecurityModel) getPeerIPs(peer *securityv1alpha1.SecurityPolicyPeer) []string {
@@ -132,15 +133,22 @@ func matchEndpoint(peer *securityv1alpha1.SecurityPolicyPeer, endpoints []*model
 	return matchEp
 }
 
-func computePolicyFlow(tier string, appliedToIPs, ingressIPs, egressIPs []string, ingressPorts, egressGroupPorts []cache.RulePort) []string {
+func computePolicyFlow(tier string, mode securityv1alpha1.PolicyMode, appliedToIPs, ingressIPs, egressIPs []string, ingressPorts, egressGroupPorts []cache.RulePort) []string {
 	var flows []string
 	priority := constants.NormalPolicyRulePriority
-	ingressTableID, ingressNextTableID, egressTableID, egressNextTableID, err := getTableIds(tier)
+	ingressTableID, ingressNextTableID, egressTableID, egressNextTableID, err := getTableIds(tier, mode)
 	if err != nil {
 		klog.Infof("Failed to computePolicyFlow, error: %v", err)
 		return nil
 	}
 
+	ctLableRange := ""
+	if mode == securityv1alpha1.MonitorMode {
+		ctLableRange = "32..59"
+	} else {
+		ctLableRange = "60..87"
+	}
+
 	for _, appliedToIP := range appliedToIPs {
 		for _, srcIP := range ingressIPs {
 			if appliedToIP != "" && srcIP != "" && appliedToIP == srcIP {
@@ -152,15 +160,15 @@ func computePolicyFlow(tier string, appliedToIPs, ingressIPs, egressIPs []string
 				protocol := strings.ToLower(string(ingressGroupPort.Protocol))
 
 				if ingressGroupPort.DstPort == 0 && ingressGroupPort.SrcPort == 0 {
-					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s actions=load:0x->NXM_NX_XXREG0[60..87],load:0x->NXM_NX_XXREG0[0..3],goto_table:%d",
-						*ingressTableID, priority, protocol, srcIP, appliedToIP, *ingressNextTableID)
+					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s actions=load:0x->NXM_NX_XXREG0[%s],load:0x->NXM_NX_XXREG0[0..3],goto_table:%d",
+						*ingressTableID, priority, protocol, srcIP, appliedToIP, ctLableRange, *ingressNextTableID)
 				} else if ingressGroupPort.DstPort != 0 {
-					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=%d actions=load:0x->NXM_NX_XXREG0[60..87],load:0x->NXM_NX_XXREG0[0..3],goto_table:%d",
-						*ingressTableID, priority, protocol, srcIP, appliedToIP, ingressGroupPort.DstPort,
+					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=%d actions=load:0x->NXM_NX_XXREG0[%s],load:0x->NXM_NX_XXREG0[0..3],goto_table:%d",
+						*ingressTableID, priority, protocol, srcIP, appliedToIP, ingressGroupPort.DstPort, ctLableRange,
 						*ingressNextTableID)
 					if ingressGroupPort.DstPort != 0 && ingressGroupPort.DstPortMask != 0xffff {
-						flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=0x%x/0x%x actions=load:0x->NXM_NX_XXREG0[60..87],load:0x->NXM_NX_XXREG0[0..3],goto_table:%d",
-							*ingressTableID, priority, protocol, srcIP, appliedToIP, ingressGroupPort.DstPort, ingressGroupPort.DstPortMask,
+						flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=0x%x/0x%x actions=load:0x->NXM_NX_XXREG0[%s],load:0x->NXM_NX_XXREG0[0..3],goto_table:%d",
+							*ingressTableID, priority, protocol, srcIP, appliedToIP, ingressGroupPort.DstPort, ingressGroupPort.DstPortMask, ctLableRange,
 							*ingressNextTableID)
 					}
 				}
@@ -183,14 +191,14 @@ func computePolicyFlow(tier string, appliedToIPs, ingressIPs, egressIPs []string
 				protocol := strings.ToLower(string(egressGroupPort.Protocol))
 
 				if egressGroupPort.DstPort == 0 && egressGroupPort.SrcPort == 0 {
-					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s actions=load:0x->NXM_NX_XXREG0[60..87],load:0x->NXM_NX_XXREG0[0..3],goto_table:%d",
-						*egressTableID, priority, protocol, appliedToIP, dstIP, *egressNextTableID)
+					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s actions=load:0x->NXM_NX_XXREG0[%s],load:0x->NXM_NX_XXREG0[0..3],goto_table:%d",
+						*egressTableID, priority, protocol, appliedToIP, dstIP, ctLableRange, *egressNextTableID)
 				} else if egressGroupPort.DstPort != 0 {
-					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=%d actions=load:0x->NXM_NX_XXREG0[60..87],load:0x->NXM_NX_XXREG0[0..3],goto_table:%d",
-						*egressTableID, priority, protocol, appliedToIP, dstIP, egressGroupPort.DstPort, *egressNextTableID)
+					flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=%d actions=load:0x->NXM_NX_XXREG0[%s],load:0x->NXM_NX_XXREG0[0..3],goto_table:%d",
+						*egressTableID, priority, protocol, appliedToIP, dstIP, egressGroupPort.DstPort, ctLableRange, *egressNextTableID)
 					if egressGroupPort.DstPort != 0 && egressGroupPort.DstPortMask != 0xffff {
-						flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=0x%x/0x%x actions=load:0x->NXM_NX_XXREG0[60..87],load:0x->NXM_NX_XXREG0[0..3],goto_table:%d",
-							*ingressTableID, priority, protocol, dstIP, appliedToIP, egressGroupPort.DstPort, egressGroupPort.DstPortMask,
+						flow = fmt.Sprintf("table=%d, priority=%d,%s,nw_src=%s,nw_dst=%s,tp_dst=0x%x/0x%x actions=load:0x->NXM_NX_XXREG0[%s],load:0x->NXM_NX_XXREG0[0..3],goto_table:%d",
+							*ingressTableID, priority, protocol, dstIP, appliedToIP, egressGroupPort.DstPort, egressGroupPort.DstPortMask, ctLableRange,
 							*egressNextTableID)
 					}
 				}
@@ -208,7 +216,7 @@ func computePolicyFlow(tier string, appliedToIPs, ingressIPs, egressIPs []string
 	return flows
 }
 
-func getTableIds(tier string) (*int, *int, *int, *int, error) {
+func getTableIds(tier string, mode securityv1alpha1.PolicyMode) (*int, *int, *int, *int, error) {
 	var ingressTableID, ingressNextTableID, egressTableID, egressNextTableID int
 	switch tier {
 	case "tier0":
@@ -217,15 +225,29 @@ func getTableIds(tier string) (*int, *int, *int, *int, error) {
 		ingressTableID = 50
 		ingressNextTableID = 70
 	case "tier1":
-		egressTableID = 25
-		egressNextTableID = 70
-		ingressTableID = 55
-		ingressNextTableID = 70
+		if mode == securityv1alpha1.MonitorMode {
+			egressTableID = 24
+			egressNextTableID = 25
+			ingressTableID = 54
+			ingressNextTableID = 55
+		} else {
+			egressTableID = 25
+			egressNextTableID = 70
+			ingressTableID = 55
+			ingressNextTableID = 70
+		}
 	case "tier2":
-		egressTableID = 30
-		egressNextTableID = 70
-		ingressTableID = 60
-		ingressNextTableID = 70
+		if mode == securityv1alpha1.MonitorMode {
+			egressTableID = 29
+			egressNextTableID = 30
+			ingressTableID = 59
+			ingressNextTableID = 60
+		} else {
+			egressTableID = 30
+			egressNextTableID = 70
+			ingressTableID = 60
+			ingressNextTableID = 70
+		}
 	default:
 		return nil, nil, nil, nil, fmt.Errorf("failed to get tableId")
 	}
diff --git a/tests/e2e/cases/security_test.go b/tests/e2e/cases/security_test.go
index c17a9d834..d11a38c6e 100644
--- a/tests/e2e/cases/security_test.go
+++ b/tests/e2e/cases/security_test.go
@@ -224,6 +224,45 @@ var _ = Describe("SecurityPolicy", func() {
 			})
 		})
 
+		When("create monitor mode security policies", func() {
+			var nginxPolicy, serverPolicy, dbPolicy *securityv1alpha1.SecurityPolicy
+
+			BeforeEach(func() {
+				nginxPolicy = newPolicy("nginx-policy", constants.Tier2, securityv1alpha1.DefaultRuleDrop, nginxSelector)
+				nginxPolicy.Spec.SecurityPolicyEnforcementMode = securityv1alpha1.MonitorMode
+				addIngressRule(nginxPolicy, "TCP", nginxPort) // allow all connection with nginx port
+				addEngressRule(nginxPolicy, "TCP", serverPort, serverSelector)
+
+				serverPolicy = newPolicy("server-policy", constants.Tier2, securityv1alpha1.DefaultRuleDrop, serverSelector)
+				serverPolicy.Spec.SecurityPolicyEnforcementMode = securityv1alpha1.MonitorMode
+				addIngressRule(serverPolicy, "TCP", serverPort, nginxSelector)
+				addEngressRule(serverPolicy, "TCP", dbPort, dbSelector)
+
+				dbPolicy = newPolicy("db-policy", constants.Tier2, securityv1alpha1.DefaultRuleDrop, dbSelector)
+				dbPolicy.Spec.SecurityPolicyEnforcementMode = securityv1alpha1.MonitorMode
+				addIngressRule(dbPolicy, "TCP", dbPort, dbSelector, serverSelector)
+				addEngressRule(dbPolicy, "TCP", dbPort, dbSelector)
+
+				Expect(e2eEnv.SetupObjects(ctx, nginxPolicy, serverPolicy, dbPolicy)).Should(Succeed())
+			})
+
+			It("should allow all packets", func() {
+				assertFlowMatches(&SecurityModel{
+					Policies:  []*securityv1alpha1.SecurityPolicy{nginxPolicy, serverPolicy, dbPolicy},
+					Endpoints: []*model.Endpoint{nginx, server01, server02, db01, db02, client},
+				})
+
+				assertReachable([]*model.Endpoint{nginx},
+					[]*model.Endpoint{server01, server02, db01, db02}, "TCP", true)
+				assertReachable([]*model.Endpoint{server01},
+					[]*model.Endpoint{nginx, db01, db02}, "TCP", true)
+				assertReachable([]*model.Endpoint{db01},
+					[]*model.Endpoint{nginx, server01, server02}, "TCP", true)
+
+			})
+
+		})
+
 		When("limits icmp packets between components", func() {
 			var icmpAllowPolicy, icmpDropPolicy *securityv1alpha1.SecurityPolicy