From 218f51fc9be1a84e07b8ba8340326eb0f97cbf90 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Thu, 2 Jul 2020 17:51:32 -0700 Subject: [PATCH 01/47] Changed all compute/network/resources api versions, fixed availabilityzones, resourceskus, loadbalancers --- cloud/converters/image.go | 2 +- cloud/converters/vm.go | 2 +- cloud/converters/vmss.go | 18 ++++++++++------- cloud/converters/vmss_test.go | 20 +++++++++---------- cloud/services/agentpools/agentpools_test.go | 2 +- .../availabilityzones/availabilityzones.go | 7 +++---- .../availabilityzones_test.go | 14 ++++++------- cloud/services/availabilityzones/client.go | 8 ++++---- .../availabilityzones_mock.go | 9 +++++---- cloud/services/disks/client.go | 2 +- cloud/services/disks/disks_test.go | 2 +- cloud/services/groups/client.go | 2 +- cloud/services/groups/groups.go | 3 ++- cloud/services/groups/groups_test.go | 2 +- .../groups/mock_groups/groups_mock.go | 5 +++-- cloud/services/inboundnatrules/client.go | 2 +- .../inboundnatrules_mock.go | 2 +- .../services/internalloadbalancers/client.go | 2 +- .../internalloadbalancers.go | 6 +++--- .../internalloadbalancers_test.go | 2 +- .../internalloadbalancers_mock.go | 2 +- cloud/services/networkinterfaces/client.go | 2 +- .../networkinterfaces_mock.go | 5 +++-- .../networkinterfaces/networkinterfaces.go | 3 ++- .../networkinterfaces_test.go | 2 +- cloud/services/publicips/client.go | 2 +- .../publicips/mock_publicips/client_mock.go | 5 +++-- cloud/services/publicips/publicips.go | 2 +- cloud/services/publicips/publicips_test.go | 2 +- cloud/services/publicloadbalancers/client.go | 2 +- .../publicloadbalancers_mock.go | 2 +- .../publicloadbalancers.go | 10 ++++------ .../publicloadbalancers_test.go | 16 ++++++--------- cloud/services/resourceskus/client.go | 10 +++++----- .../mock_resourceskus/resourceskus_mock.go | 9 +++++---- cloud/services/routetables/client.go | 2 +- .../mock_routetables/routetables_mock.go | 2 +- cloud/services/routetables/routetables.go | 2 +- .../services/routetables/routetables_test.go | 2 +- cloud/services/scalesets/client.go | 4 ++-- .../mock_scalesets/scalesets_mock.go | 4 ++-- cloud/services/scalesets/vmss.go | 2 +- cloud/services/scalesets/vmss_test.go | 4 ++-- cloud/services/securitygroups/client.go | 2 +- .../securitygroups_mock.go | 5 +++-- .../services/securitygroups/securitygroups.go | 2 +- .../securitygroups/securitygroups_test.go | 2 +- cloud/services/subnets/client.go | 2 +- .../subnets/mock_subnets/subnets_mock.go | 5 +++-- cloud/services/subnets/subnets.go | 2 +- cloud/services/subnets/subnets_test.go | 2 +- .../virtualmachineextensions/client.go | 2 +- .../virtualmachineextensions_mock.go | 2 +- cloud/services/virtualmachines/client.go | 2 +- .../virtualmachines_mock.go | 2 +- .../virtualmachines/virtualmachines.go | 2 +- .../virtualmachines/virtualmachines_test.go | 4 ++-- cloud/services/virtualnetworks/client.go | 2 +- .../virtualnetworks_mock.go | 5 +++-- .../virtualnetworks/virtualnetworks.go | 2 +- .../virtualnetworks/virtualnetworks_test.go | 2 +- .../azuremanagedmachinepool_reconciler.go | 2 +- 62 files changed, 132 insertions(+), 125 deletions(-) diff --git a/cloud/converters/image.go b/cloud/converters/image.go index 2e871bc1b3d..10c7994e7f1 100644 --- a/cloud/converters/image.go +++ b/cloud/converters/image.go @@ -19,7 +19,7 @@ package converters import ( "fmt" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/pkg/errors" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" diff --git a/cloud/converters/vm.go b/cloud/converters/vm.go index d9c3b7425a3..ee28ed98391 100644 --- a/cloud/converters/vm.go +++ b/cloud/converters/vm.go @@ -17,7 +17,7 @@ limitations under the License. package converters import ( - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest/to" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" ) diff --git a/cloud/converters/vmss.go b/cloud/converters/vmss.go index e73ad59db04..7611b0d8428 100644 --- a/cloud/converters/vmss.go +++ b/cloud/converters/vmss.go @@ -17,7 +17,7 @@ limitations under the License. package converters import ( - "github.com/Azure/azure-sdk-for-go/profiles/latest/compute/mgmt/compute" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest/to" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" @@ -37,9 +37,11 @@ func SDKToVMSS(sdkvmss compute.VirtualMachineScaleSet, sdkinstances []compute.Vi vmss.Capacity = to.Int64(sdkvmss.Sku.Capacity) } - if sdkvmss.Zones != nil && len(*sdkvmss.Zones) > 0 { - vmss.Zones = to.StringSlice(sdkvmss.Zones) - } + /* + if sdkvmss.Zones != nil && len(*sdkvmss.Zones) > 0 { + vmss.Zones = to.StringSlice(sdkvmss.Zones) + } + */ if len(sdkvmss.Tags) > 0 { vmss.Tags = MapToTags(sdkvmss.Tags) @@ -55,9 +57,11 @@ func SDKToVMSS(sdkvmss compute.VirtualMachineScaleSet, sdkinstances []compute.Vi State: infrav1.VMState(to.String(vm.ProvisioningState)), } - if vm.Zones != nil && len(*vm.Zones) > 0 { - instance.AvailabilityZone = to.StringSlice(vm.Zones)[0] - } + /* + if vm.Zones != nil && len(*vm.Zones) > 0 { + instance.AvailabilityZone = to.StringSlice(vm.Zones)[0] + } + */ vmss.Instances[i] = instance } } diff --git a/cloud/converters/vmss_test.go b/cloud/converters/vmss_test.go index 00757199d5e..be509ef98df 100644 --- a/cloud/converters/vmss_test.go +++ b/cloud/converters/vmss_test.go @@ -20,7 +20,7 @@ import ( "fmt" "testing" - "github.com/Azure/azure-sdk-for-go/profiles/latest/compute/mgmt/compute" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest/to" "github.com/onsi/gomega" @@ -40,20 +40,20 @@ func Test_SDKToVMSS(t *testing.T) { tags := map[string]*string{ "foo": to.StringPtr("bazz"), } - zones := []string{"zone0", "zone1"} + // zones := []string{"zone0", "zone1"} return compute.VirtualMachineScaleSet{ Sku: &compute.Sku{ Name: to.StringPtr("skuName"), Tier: to.StringPtr("skuTier"), Capacity: to.Int64Ptr(2), }, - Zones: to.StringSlicePtr(zones), + // Zones: to.StringSlicePtr(zones), ID: to.StringPtr("vmssID"), Name: to.StringPtr("vmssName"), Location: to.StringPtr("westus2"), Tags: tags, VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - ProvisioningState: to.StringPtr(string(compute.ProvisioningState1Succeeded)), + ProvisioningState: to.StringPtr("Succeded"), }, }, []compute.VirtualMachineScaleSetVM{ @@ -61,18 +61,18 @@ func Test_SDKToVMSS(t *testing.T) { InstanceID: to.StringPtr("0"), ID: to.StringPtr("vm/0"), Name: to.StringPtr("vm0"), - Zones: to.StringSlicePtr([]string{"zone0"}), + // Zones: to.StringSlicePtr([]string{"zone0"}), VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ - ProvisioningState: to.StringPtr(string(compute.ProvisioningState1Succeeded)), + ProvisioningState: to.StringPtr("Succeeded"), }, }, { InstanceID: to.StringPtr("1"), ID: to.StringPtr("vm/1"), Name: to.StringPtr("vm1"), - Zones: to.StringSlicePtr([]string{"zone1"}), + // Zones: to.StringSlicePtr([]string{"zone1"}), VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ - ProvisioningState: to.StringPtr(string(compute.ProvisioningState1Succeeded)), + ProvisioningState: to.StringPtr("Succeeded"), }, }, } @@ -83,8 +83,8 @@ func Test_SDKToVMSS(t *testing.T) { Name: "vmssName", Sku: "skuName", Capacity: 2, - Zones: []string{"zone0", "zone1"}, - State: "Succeeded", + // Zones: []string{"zone0", "zone1"}, + State: "Succeeded", Tags: map[string]string{ "foo": "bazz", }, diff --git a/cloud/services/agentpools/agentpools_test.go b/cloud/services/agentpools/agentpools_test.go index 43b378aec77..001541f1702 100644 --- a/cloud/services/agentpools/agentpools_test.go +++ b/cloud/services/agentpools/agentpools_test.go @@ -22,7 +22,7 @@ import ( "testing" "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2020-02-01/containerservice" - network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" "github.com/golang/mock/gomock" diff --git a/cloud/services/availabilityzones/availabilityzones.go b/cloud/services/availabilityzones/availabilityzones.go index ee01e303c0b..eb4a979c9cb 100644 --- a/cloud/services/availabilityzones/availabilityzones.go +++ b/cloud/services/availabilityzones/availabilityzones.go @@ -18,11 +18,10 @@ package availabilityzones import ( "context" - "fmt" "sort" "strings" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/pkg/errors" ) @@ -39,10 +38,10 @@ func (s *Service) Get(ctx context.Context, spec interface{}) (interface{}, error return zones, errors.New("invalid availability zones specification") } - filter := fmt.Sprintf("location eq '%s'", s.Scope.Location()) + // filter := fmt.Sprintf("location eq '%s'", s.Scope.Location()) // Prefer ListComplete() over List() to automatically traverse pages via iterator. - res, err := s.Client.ListComplete(ctx, filter) + res, err := s.Client.ListComplete(ctx) if err != nil { return zones, err } diff --git a/cloud/services/availabilityzones/availabilityzones_test.go b/cloud/services/availabilityzones/availabilityzones_test.go index f58b58a4932..d46bb487cce 100644 --- a/cloud/services/availabilityzones/availabilityzones_test.go +++ b/cloud/services/availabilityzones/availabilityzones_test.go @@ -24,12 +24,12 @@ import ( . "github.com/onsi/gomega" "sigs.k8s.io/cluster-api-provider-azure/cloud/services/availabilityzones/mock_availabilityzones" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" "github.com/golang/mock/gomock" - network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" @@ -77,7 +77,7 @@ func TestGetAvailabilityZones(t *testing.T) { availabilityZoneSpec: Spec{VMSize: to.StringPtr("Standard_B2ms")}, expectedError: "", expect: func(m *mock_availabilityzones.MockClientMockRecorder) { - m.ListComplete(context.TODO(), "location eq 'centralus'").Return(compute.ResourceSkusResultIterator{}, nil) + m.ListComplete(context.TODO()).Return(compute.ResourceSkusResultIterator{}, nil) }, }, { @@ -85,7 +85,7 @@ func TestGetAvailabilityZones(t *testing.T) { availabilityZoneSpec: Spec{VMSize: to.StringPtr("Standard_B2ms")}, expectedError: "#: Internal Server Error: StatusCode=500", expect: func(m *mock_availabilityzones.MockClientMockRecorder) { - m.ListComplete(context.TODO(), "location eq 'centralus'").Return(compute.ResourceSkusResultIterator{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 500}, "Internal Server Error")) + m.ListComplete(context.TODO()).Return(compute.ResourceSkusResultIterator{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 500}, "Internal Server Error")) }, }, { @@ -93,7 +93,7 @@ func TestGetAvailabilityZones(t *testing.T) { availabilityZoneSpec: Spec{VMSize: to.StringPtr("Standard_B2ms")}, expectedError: "", expect: func(m *mock_availabilityzones.MockClientMockRecorder) { - m.ListComplete(context.TODO(), "location eq 'centralus'").Return(compute.NewResourceSkusResultIterator(compute.ResourceSkusResultPage{}), nil) + m.ListComplete(context.TODO()).Return(compute.NewResourceSkusResultIterator(compute.ResourceSkusResultPage{}), nil) }, }, { @@ -101,7 +101,7 @@ func TestGetAvailabilityZones(t *testing.T) { availabilityZoneSpec: Spec{}, expectedError: "", expect: func(m *mock_availabilityzones.MockClientMockRecorder) { - m.ListComplete(context.TODO(), "location eq 'centralus'").Return(compute.NewResourceSkusResultIterator(compute.ResourceSkusResultPage{}), nil) + m.ListComplete(context.TODO()).Return(compute.NewResourceSkusResultIterator(compute.ResourceSkusResultPage{}), nil) }, }, { @@ -109,7 +109,7 @@ func TestGetAvailabilityZones(t *testing.T) { availabilityZoneSpec: Spec{}, expectedError: "", expect: func(m *mock_availabilityzones.MockClientMockRecorder) { - m.ListComplete(context.TODO(), "location eq 'centralus'").Return(compute.NewResourceSkusResultIterator(compute.ResourceSkusResultPage{}), nil) + m.ListComplete(context.TODO()).Return(compute.NewResourceSkusResultIterator(compute.ResourceSkusResultPage{}), nil) }, }, } diff --git a/cloud/services/availabilityzones/client.go b/cloud/services/availabilityzones/client.go index 087205a2e1f..4c502f29330 100644 --- a/cloud/services/availabilityzones/client.go +++ b/cloud/services/availabilityzones/client.go @@ -19,14 +19,14 @@ package availabilityzones import ( "context" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk type Client interface { - ListComplete(context.Context, string) (compute.ResourceSkusResultIterator, error) + ListComplete(context.Context) (compute.ResourceSkusResultIterator, error) } // AzureClient contains the Azure go-sdk Client @@ -51,6 +51,6 @@ func newResourceSkusClient(subscriptionID string, baseURI string, authorizer aut } // ListComplete enumerates all values, automatically crossing page boundaries as required. -func (ac *AzureClient) ListComplete(ctx context.Context, filter string) (compute.ResourceSkusResultIterator, error) { - return ac.resourceSkus.ListComplete(ctx, filter) +func (ac *AzureClient) ListComplete(ctx context.Context) (compute.ResourceSkusResultIterator, error) { + return ac.resourceSkus.ListComplete(ctx) } diff --git a/cloud/services/availabilityzones/mock_availabilityzones/availabilityzones_mock.go b/cloud/services/availabilityzones/mock_availabilityzones/availabilityzones_mock.go index bd1757f3855..34e6569d702 100644 --- a/cloud/services/availabilityzones/mock_availabilityzones/availabilityzones_mock.go +++ b/cloud/services/availabilityzones/mock_availabilityzones/availabilityzones_mock.go @@ -22,9 +22,10 @@ package mock_availabilityzones import ( context "context" - compute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" - gomock "github.com/golang/mock/gomock" reflect "reflect" + + compute "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" + gomock "github.com/golang/mock/gomock" ) // MockClient is a mock of Client interface. @@ -51,9 +52,9 @@ func (m *MockClient) EXPECT() *MockClientMockRecorder { } // ListComplete mocks base method. -func (m *MockClient) ListComplete(arg0 context.Context, arg1 string) (compute.ResourceSkusResultIterator, error) { +func (m *MockClient) ListComplete(arg0 context.Context) (compute.ResourceSkusResultIterator, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListComplete", arg0, arg1) + ret := m.ctrl.Call(m, "ListComplete", arg0) ret0, _ := ret[0].(compute.ResourceSkusResultIterator) ret1, _ := ret[1].(error) return ret0, ret1 diff --git a/cloud/services/disks/client.go b/cloud/services/disks/client.go index 895297d5105..eb0664e68c3 100644 --- a/cloud/services/disks/client.go +++ b/cloud/services/disks/client.go @@ -19,7 +19,7 @@ package disks import ( "context" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) diff --git a/cloud/services/disks/disks_test.go b/cloud/services/disks/disks_test.go index b68061d1db0..0d144436b52 100644 --- a/cloud/services/disks/disks_test.go +++ b/cloud/services/disks/disks_test.go @@ -27,7 +27,7 @@ import ( "github.com/Azure/go-autorest/autorest" "github.com/golang/mock/gomock" - network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" diff --git a/cloud/services/groups/client.go b/cloud/services/groups/client.go index 81993728524..ded75d2dafd 100644 --- a/cloud/services/groups/client.go +++ b/cloud/services/groups/client.go @@ -19,7 +19,7 @@ package groups import ( "context" - "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-05-01/resources" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/resources/mgmt/resources" "github.com/Azure/go-autorest/autorest" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) diff --git a/cloud/services/groups/groups.go b/cloud/services/groups/groups.go index 7f227d46a8e..e1041d70438 100644 --- a/cloud/services/groups/groups.go +++ b/cloud/services/groups/groups.go @@ -19,7 +19,7 @@ package groups import ( "context" - "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-05-01/resources" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/resources/mgmt/resources" "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" @@ -30,6 +30,7 @@ import ( // Reconcile gets/creates/updates a resource group. func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { + klog.V(2).Infof("HI CHRISTINA") if _, err := s.Client.Get(ctx, s.Scope.ResourceGroup()); err == nil { // resource group already exists, skip creation return nil diff --git a/cloud/services/groups/groups_test.go b/cloud/services/groups/groups_test.go index b5fea9e71ed..aa766e005a1 100644 --- a/cloud/services/groups/groups_test.go +++ b/cloud/services/groups/groups_test.go @@ -27,7 +27,7 @@ import ( "github.com/Azure/go-autorest/autorest" "github.com/golang/mock/gomock" - "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-05-01/resources" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/resources/mgmt/resources" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" diff --git a/cloud/services/groups/mock_groups/groups_mock.go b/cloud/services/groups/mock_groups/groups_mock.go index a36a3873e56..29b500af8a9 100644 --- a/cloud/services/groups/mock_groups/groups_mock.go +++ b/cloud/services/groups/mock_groups/groups_mock.go @@ -22,9 +22,10 @@ package mock_groups import ( context "context" - resources "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-05-01/resources" - gomock "github.com/golang/mock/gomock" reflect "reflect" + + resources "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/resources/mgmt/resources" + gomock "github.com/golang/mock/gomock" ) // MockClient is a mock of Client interface. diff --git a/cloud/services/inboundnatrules/client.go b/cloud/services/inboundnatrules/client.go index 39961140dfb..4cc64d8107b 100644 --- a/cloud/services/inboundnatrules/client.go +++ b/cloud/services/inboundnatrules/client.go @@ -19,7 +19,7 @@ package inboundnatrules import ( "context" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) diff --git a/cloud/services/inboundnatrules/mock_inboundnatrules/inboundnatrules_mock.go b/cloud/services/inboundnatrules/mock_inboundnatrules/inboundnatrules_mock.go index 938174c7526..ca7d92bf11e 100644 --- a/cloud/services/inboundnatrules/mock_inboundnatrules/inboundnatrules_mock.go +++ b/cloud/services/inboundnatrules/mock_inboundnatrules/inboundnatrules_mock.go @@ -22,7 +22,7 @@ package mock_inboundnatrules import ( context "context" - network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" gomock "github.com/golang/mock/gomock" reflect "reflect" ) diff --git a/cloud/services/internalloadbalancers/client.go b/cloud/services/internalloadbalancers/client.go index 0f26585a581..c07dd0d582d 100644 --- a/cloud/services/internalloadbalancers/client.go +++ b/cloud/services/internalloadbalancers/client.go @@ -19,7 +19,7 @@ package internalloadbalancers import ( "context" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) diff --git a/cloud/services/internalloadbalancers/internalloadbalancers.go b/cloud/services/internalloadbalancers/internalloadbalancers.go index 6012849d2d5..c4ce8ff3c07 100644 --- a/cloud/services/internalloadbalancers/internalloadbalancers.go +++ b/cloud/services/internalloadbalancers/internalloadbalancers.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" @@ -103,7 +103,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { { Name: &probeName, ProbePropertiesFormat: &network.ProbePropertiesFormat{ - Protocol: network.ProbeProtocolHTTPS, + Protocol: "Https", RequestPath: to.StringPtr("/healthz"), Port: to.Int32Ptr(s.Scope.APIServerPort()), IntervalInSeconds: to.Int32Ptr(15), @@ -120,7 +120,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { BackendPort: to.Int32Ptr(s.Scope.APIServerPort()), IdleTimeoutInMinutes: to.Int32Ptr(4), EnableFloatingIP: to.BoolPtr(false), - LoadDistribution: network.LoadDistributionDefault, + LoadDistribution: "Default", FrontendIPConfiguration: &network.SubResource{ ID: to.StringPtr(fmt.Sprintf("/%s/%s/frontendIPConfigurations/%s", idPrefix, lbName, frontEndIPConfigName)), }, diff --git a/cloud/services/internalloadbalancers/internalloadbalancers_test.go b/cloud/services/internalloadbalancers/internalloadbalancers_test.go index 5d3bde9bc40..fa5598802ab 100644 --- a/cloud/services/internalloadbalancers/internalloadbalancers_test.go +++ b/cloud/services/internalloadbalancers/internalloadbalancers_test.go @@ -30,7 +30,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/golang/mock/gomock" - network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" diff --git a/cloud/services/internalloadbalancers/mock_internalloadbalancers/internalloadbalancers_mock.go b/cloud/services/internalloadbalancers/mock_internalloadbalancers/internalloadbalancers_mock.go index e9360824702..ebaee0fbad0 100644 --- a/cloud/services/internalloadbalancers/mock_internalloadbalancers/internalloadbalancers_mock.go +++ b/cloud/services/internalloadbalancers/mock_internalloadbalancers/internalloadbalancers_mock.go @@ -22,7 +22,7 @@ package mock_internalloadbalancers import ( context "context" - network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" gomock "github.com/golang/mock/gomock" reflect "reflect" ) diff --git a/cloud/services/networkinterfaces/client.go b/cloud/services/networkinterfaces/client.go index dbf871d4f21..77b83bb27e3 100644 --- a/cloud/services/networkinterfaces/client.go +++ b/cloud/services/networkinterfaces/client.go @@ -19,7 +19,7 @@ package networkinterfaces import ( "context" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) diff --git a/cloud/services/networkinterfaces/mock_networkinterfaces/networkinterfaces_mock.go b/cloud/services/networkinterfaces/mock_networkinterfaces/networkinterfaces_mock.go index 6e0727efab6..3025c49d7ca 100644 --- a/cloud/services/networkinterfaces/mock_networkinterfaces/networkinterfaces_mock.go +++ b/cloud/services/networkinterfaces/mock_networkinterfaces/networkinterfaces_mock.go @@ -22,9 +22,10 @@ package mock_networkinterfaces import ( context "context" - network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" - gomock "github.com/golang/mock/gomock" reflect "reflect" + + network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" + gomock "github.com/golang/mock/gomock" ) // MockClient is a mock of Client interface. diff --git a/cloud/services/networkinterfaces/networkinterfaces.go b/cloud/services/networkinterfaces/networkinterfaces.go index 870bc46e861..b328f3cabc5 100644 --- a/cloud/services/networkinterfaces/networkinterfaces.go +++ b/cloud/services/networkinterfaces/networkinterfaces.go @@ -19,7 +19,8 @@ package networkinterfaces import ( "context" "fmt" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" diff --git a/cloud/services/networkinterfaces/networkinterfaces_test.go b/cloud/services/networkinterfaces/networkinterfaces_test.go index de9e8baeec6..bb2dd0f88f3 100644 --- a/cloud/services/networkinterfaces/networkinterfaces_test.go +++ b/cloud/services/networkinterfaces/networkinterfaces_test.go @@ -35,7 +35,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/golang/mock/gomock" - network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/pointer" diff --git a/cloud/services/publicips/client.go b/cloud/services/publicips/client.go index b9ec5edcd14..aa92ccffef1 100644 --- a/cloud/services/publicips/client.go +++ b/cloud/services/publicips/client.go @@ -19,7 +19,7 @@ package publicips import ( "context" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) diff --git a/cloud/services/publicips/mock_publicips/client_mock.go b/cloud/services/publicips/mock_publicips/client_mock.go index ebbca6c293b..77d14c49a09 100644 --- a/cloud/services/publicips/mock_publicips/client_mock.go +++ b/cloud/services/publicips/mock_publicips/client_mock.go @@ -22,9 +22,10 @@ package mock_publicips import ( context "context" - network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" - gomock "github.com/golang/mock/gomock" reflect "reflect" + + network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" + gomock "github.com/golang/mock/gomock" ) // MockClient is a mock of Client interface. diff --git a/cloud/services/publicips/publicips.go b/cloud/services/publicips/publicips.go index 27343da9528..299620e9a1d 100644 --- a/cloud/services/publicips/publicips.go +++ b/cloud/services/publicips/publicips.go @@ -20,7 +20,7 @@ import ( "context" "strings" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" diff --git a/cloud/services/publicips/publicips_test.go b/cloud/services/publicips/publicips_test.go index 65f2fbc74d7..d11237a27c7 100644 --- a/cloud/services/publicips/publicips_test.go +++ b/cloud/services/publicips/publicips_test.go @@ -29,7 +29,7 @@ import ( "github.com/Azure/go-autorest/autorest" "github.com/golang/mock/gomock" - network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "k8s.io/client-go/kubernetes/scheme" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" ) diff --git a/cloud/services/publicloadbalancers/client.go b/cloud/services/publicloadbalancers/client.go index a0a77b5733d..7405d932441 100644 --- a/cloud/services/publicloadbalancers/client.go +++ b/cloud/services/publicloadbalancers/client.go @@ -19,7 +19,7 @@ package publicloadbalancers import ( "context" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) diff --git a/cloud/services/publicloadbalancers/mock_publicloadbalancers/publicloadbalancers_mock.go b/cloud/services/publicloadbalancers/mock_publicloadbalancers/publicloadbalancers_mock.go index f2e11033525..f3696f2a222 100644 --- a/cloud/services/publicloadbalancers/mock_publicloadbalancers/publicloadbalancers_mock.go +++ b/cloud/services/publicloadbalancers/mock_publicloadbalancers/publicloadbalancers_mock.go @@ -22,7 +22,7 @@ package mock_publicloadbalancers import ( context "context" - network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" gomock "github.com/golang/mock/gomock" reflect "reflect" ) diff --git a/cloud/services/publicloadbalancers/publicloadbalancers.go b/cloud/services/publicloadbalancers/publicloadbalancers.go index 8559df42909..f154843eee8 100644 --- a/cloud/services/publicloadbalancers/publicloadbalancers.go +++ b/cloud/services/publicloadbalancers/publicloadbalancers.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" @@ -85,12 +85,10 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { Name: &backEndAddressPoolName, }, }, - OutboundRules: &[]network.OutboundRule{ + OutboundNatRules: &[]network.OutboundNatRule{ { Name: to.StringPtr("OutboundNATAllProtocols"), - OutboundRulePropertiesFormat: &network.OutboundRulePropertiesFormat{ - Protocol: network.LoadBalancerOutboundRuleProtocolAll, - IdleTimeoutInMinutes: to.Int32Ptr(4), + OutboundNatRulePropertiesFormat: &network.OutboundNatRulePropertiesFormat{ FrontendIPConfigurations: &[]network.SubResource{ { ID: to.StringPtr(fmt.Sprintf("/%s/%s/frontendIPConfigurations/%s", idPrefix, lbName, frontEndIPConfigName)), @@ -130,7 +128,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { BackendPort: to.Int32Ptr(s.Scope.APIServerPort()), IdleTimeoutInMinutes: to.Int32Ptr(4), EnableFloatingIP: to.BoolPtr(false), - LoadDistribution: network.LoadDistributionDefault, + LoadDistribution: "Default", FrontendIPConfiguration: &network.SubResource{ ID: to.StringPtr(fmt.Sprintf("/%s/%s/frontendIPConfigurations/%s", idPrefix, lbName, frontEndIPConfigName)), }, diff --git a/cloud/services/publicloadbalancers/publicloadbalancers_test.go b/cloud/services/publicloadbalancers/publicloadbalancers_test.go index 45d78eaaf25..6f793001684 100644 --- a/cloud/services/publicloadbalancers/publicloadbalancers_test.go +++ b/cloud/services/publicloadbalancers/publicloadbalancers_test.go @@ -30,7 +30,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/golang/mock/gomock" - network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" @@ -202,7 +202,7 @@ func TestReconcilePublicLoadBalancer(t *testing.T) { BackendPort: to.Int32Ptr(6443), IdleTimeoutInMinutes: to.Int32Ptr(4), EnableFloatingIP: to.BoolPtr(false), - LoadDistribution: network.LoadDistributionDefault, + LoadDistribution: "Default", FrontendIPConfiguration: &network.SubResource{ ID: to.StringPtr("//subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-publiclb/frontendIPConfigurations/my-publiclb-frontEnd"), }, @@ -226,18 +226,16 @@ func TestReconcilePublicLoadBalancer(t *testing.T) { }, }, }, - OutboundRules: &[]network.OutboundRule{ + OutboundNatRules: &[]network.OutboundNatRule{ { Name: to.StringPtr("OutboundNATAllProtocols"), - OutboundRulePropertiesFormat: &network.OutboundRulePropertiesFormat{ + OutboundNatRulePropertiesFormat: &network.OutboundNatRulePropertiesFormat{ FrontendIPConfigurations: &[]network.SubResource{ {ID: to.StringPtr("//subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-publiclb/frontendIPConfigurations/my-publiclb-frontEnd")}, }, BackendAddressPool: &network.SubResource{ ID: to.StringPtr("//subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-publiclb/backendAddressPools/my-publiclb-backendPool"), }, - Protocol: network.LoadBalancerOutboundRuleProtocolAll, - IdleTimeoutInMinutes: to.Int32Ptr(4), }, }, }, @@ -279,18 +277,16 @@ func TestReconcilePublicLoadBalancer(t *testing.T) { Name: to.StringPtr("cluster-name-outboundBackendPool"), }, }, - OutboundRules: &[]network.OutboundRule{ + OutboundNatRules: &[]network.OutboundNatRule{ { Name: to.StringPtr("OutboundNATAllProtocols"), - OutboundRulePropertiesFormat: &network.OutboundRulePropertiesFormat{ + OutboundNatRulePropertiesFormat: &network.OutboundNatRulePropertiesFormat{ FrontendIPConfigurations: &[]network.SubResource{ {ID: to.StringPtr("//subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/cluster-name/frontendIPConfigurations/cluster-name-frontEnd")}, }, BackendAddressPool: &network.SubResource{ ID: to.StringPtr("//subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/cluster-name/backendAddressPools/cluster-name-outboundBackendPool"), }, - Protocol: network.LoadBalancerOutboundRuleProtocolAll, - IdleTimeoutInMinutes: to.Int32Ptr(4), }, }, }, diff --git a/cloud/services/resourceskus/client.go b/cloud/services/resourceskus/client.go index f5a19bda48f..f5d5f7e2a8c 100644 --- a/cloud/services/resourceskus/client.go +++ b/cloud/services/resourceskus/client.go @@ -20,7 +20,7 @@ import ( "context" "strings" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest" "github.com/pkg/errors" @@ -29,7 +29,7 @@ import ( // Client wraps go-sdk type Client interface { - List(context.Context, string) ([]compute.ResourceSku, error) + List(context.Context) ([]compute.ResourceSku, error) HasAcceleratedNetworking(context.Context, string) (bool, error) } @@ -56,8 +56,8 @@ func newResourceSkusClient(subscriptionID string, baseURI string, authorizer aut } // List returns all Resource SKUs available to the subscription. -func (ac *AzureClient) List(ctx context.Context, filter string) ([]compute.ResourceSku, error) { - iter, err := ac.skus.ListComplete(ctx, filter) +func (ac *AzureClient) List(ctx context.Context) ([]compute.ResourceSku, error) { + iter, err := ac.skus.ListComplete(ctx) if err != nil { return nil, errors.Wrap(err, "could not list resource skus") } @@ -78,7 +78,7 @@ func (ac *AzureClient) HasAcceleratedNetworking(ctx context.Context, name string if name == "" { return false, nil } - skus, err := ac.List(ctx, "") // "filter" argument only works for location, so filter in code + skus, err := ac.List(ctx) // "filter" argument only works for location, so filter in code if err != nil { return false, err } diff --git a/cloud/services/resourceskus/mock_resourceskus/resourceskus_mock.go b/cloud/services/resourceskus/mock_resourceskus/resourceskus_mock.go index 726876062e1..3d6f65d7bc4 100644 --- a/cloud/services/resourceskus/mock_resourceskus/resourceskus_mock.go +++ b/cloud/services/resourceskus/mock_resourceskus/resourceskus_mock.go @@ -22,9 +22,10 @@ package mock_resourceskus import ( context "context" - compute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" - gomock "github.com/golang/mock/gomock" reflect "reflect" + + compute "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" + gomock "github.com/golang/mock/gomock" ) // MockClient is a mock of Client interface. @@ -51,9 +52,9 @@ func (m *MockClient) EXPECT() *MockClientMockRecorder { } // List mocks base method. -func (m *MockClient) List(arg0 context.Context, arg1 string) ([]compute.ResourceSku, error) { +func (m *MockClient) List(arg0 context.Context) ([]compute.ResourceSku, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "List", arg0, arg1) + ret := m.ctrl.Call(m, "List", arg0) ret0, _ := ret[0].([]compute.ResourceSku) ret1, _ := ret[1].(error) return ret0, ret1 diff --git a/cloud/services/routetables/client.go b/cloud/services/routetables/client.go index ac3b7a082ea..3f1c9e49576 100644 --- a/cloud/services/routetables/client.go +++ b/cloud/services/routetables/client.go @@ -19,7 +19,7 @@ package routetables import ( "context" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) diff --git a/cloud/services/routetables/mock_routetables/routetables_mock.go b/cloud/services/routetables/mock_routetables/routetables_mock.go index 8fc01540d8b..4d884476f36 100644 --- a/cloud/services/routetables/mock_routetables/routetables_mock.go +++ b/cloud/services/routetables/mock_routetables/routetables_mock.go @@ -22,7 +22,7 @@ package mock_routetables import ( context "context" - network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" gomock "github.com/golang/mock/gomock" reflect "reflect" ) diff --git a/cloud/services/routetables/routetables.go b/cloud/services/routetables/routetables.go index 755c3ca154f..6b97e0d4b4a 100644 --- a/cloud/services/routetables/routetables.go +++ b/cloud/services/routetables/routetables.go @@ -18,7 +18,7 @@ package routetables import ( "context" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" diff --git a/cloud/services/routetables/routetables_test.go b/cloud/services/routetables/routetables_test.go index 3efd3f4f855..dab3ed6a7b4 100644 --- a/cloud/services/routetables/routetables_test.go +++ b/cloud/services/routetables/routetables_test.go @@ -28,7 +28,7 @@ import ( "github.com/Azure/go-autorest/autorest" "github.com/golang/mock/gomock" - network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" diff --git a/cloud/services/scalesets/client.go b/cloud/services/scalesets/client.go index 07acaf1473e..edda4e5ca7f 100644 --- a/cloud/services/scalesets/client.go +++ b/cloud/services/scalesets/client.go @@ -20,8 +20,8 @@ import ( "context" "fmt" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-11-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" diff --git a/cloud/services/scalesets/mock_scalesets/scalesets_mock.go b/cloud/services/scalesets/mock_scalesets/scalesets_mock.go index 4fa11ded64c..f6181def296 100644 --- a/cloud/services/scalesets/mock_scalesets/scalesets_mock.go +++ b/cloud/services/scalesets/mock_scalesets/scalesets_mock.go @@ -22,8 +22,8 @@ package mock_scalesets import ( context "context" - compute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" - network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-11-01/network" + compute "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" + network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" gomock "github.com/golang/mock/gomock" reflect "reflect" ) diff --git a/cloud/services/scalesets/vmss.go b/cloud/services/scalesets/vmss.go index b9dd0526c69..767df74c9bc 100644 --- a/cloud/services/scalesets/vmss.go +++ b/cloud/services/scalesets/vmss.go @@ -21,7 +21,7 @@ import ( "fmt" infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" diff --git a/cloud/services/scalesets/vmss_test.go b/cloud/services/scalesets/vmss_test.go index d2dc84fcd19..5db23360331 100644 --- a/cloud/services/scalesets/vmss_test.go +++ b/cloud/services/scalesets/vmss_test.go @@ -22,8 +22,8 @@ import ( "net/http" "testing" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" "github.com/golang/mock/gomock" diff --git a/cloud/services/securitygroups/client.go b/cloud/services/securitygroups/client.go index bb7c32050de..5b9b84d47c0 100644 --- a/cloud/services/securitygroups/client.go +++ b/cloud/services/securitygroups/client.go @@ -19,7 +19,7 @@ package securitygroups import ( "context" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) diff --git a/cloud/services/securitygroups/mock_securitygroups/securitygroups_mock.go b/cloud/services/securitygroups/mock_securitygroups/securitygroups_mock.go index c05f59dd3d4..8510e3adff6 100644 --- a/cloud/services/securitygroups/mock_securitygroups/securitygroups_mock.go +++ b/cloud/services/securitygroups/mock_securitygroups/securitygroups_mock.go @@ -22,9 +22,10 @@ package mock_securitygroups import ( context "context" - network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" - gomock "github.com/golang/mock/gomock" reflect "reflect" + + network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" + gomock "github.com/golang/mock/gomock" ) // MockClient is a mock of Client interface. diff --git a/cloud/services/securitygroups/securitygroups.go b/cloud/services/securitygroups/securitygroups.go index 16df6762d89..5af8aae53d0 100644 --- a/cloud/services/securitygroups/securitygroups.go +++ b/cloud/services/securitygroups/securitygroups.go @@ -21,7 +21,7 @@ import ( "strconv" "strings" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" diff --git a/cloud/services/securitygroups/securitygroups_test.go b/cloud/services/securitygroups/securitygroups_test.go index 8c52722db2b..b8972089f28 100644 --- a/cloud/services/securitygroups/securitygroups_test.go +++ b/cloud/services/securitygroups/securitygroups_test.go @@ -27,7 +27,7 @@ import ( "github.com/Azure/go-autorest/autorest" "github.com/golang/mock/gomock" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" diff --git a/cloud/services/subnets/client.go b/cloud/services/subnets/client.go index 71fcfd60044..57cfae82e87 100644 --- a/cloud/services/subnets/client.go +++ b/cloud/services/subnets/client.go @@ -19,7 +19,7 @@ package subnets import ( "context" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) diff --git a/cloud/services/subnets/mock_subnets/subnets_mock.go b/cloud/services/subnets/mock_subnets/subnets_mock.go index e155d84ed56..c88d0170b19 100644 --- a/cloud/services/subnets/mock_subnets/subnets_mock.go +++ b/cloud/services/subnets/mock_subnets/subnets_mock.go @@ -22,9 +22,10 @@ package mock_subnets import ( context "context" - network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" - gomock "github.com/golang/mock/gomock" reflect "reflect" + + network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" + gomock "github.com/golang/mock/gomock" ) // MockClient is a mock of Client interface. diff --git a/cloud/services/subnets/subnets.go b/cloud/services/subnets/subnets.go index 933fe31cc5e..223dde154e2 100644 --- a/cloud/services/subnets/subnets.go +++ b/cloud/services/subnets/subnets.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" diff --git a/cloud/services/subnets/subnets_test.go b/cloud/services/subnets/subnets_test.go index 20076798d5f..0fa9572aa3d 100644 --- a/cloud/services/subnets/subnets_test.go +++ b/cloud/services/subnets/subnets_test.go @@ -30,7 +30,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/golang/mock/gomock" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" diff --git a/cloud/services/virtualmachineextensions/client.go b/cloud/services/virtualmachineextensions/client.go index fe9605a2887..356328798fc 100644 --- a/cloud/services/virtualmachineextensions/client.go +++ b/cloud/services/virtualmachineextensions/client.go @@ -19,7 +19,7 @@ package virtualmachineextensions import ( "context" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) diff --git a/cloud/services/virtualmachineextensions/mock_virtualmachineextensions/virtualmachineextensions_mock.go b/cloud/services/virtualmachineextensions/mock_virtualmachineextensions/virtualmachineextensions_mock.go index 28202ddf115..4604ade3e8e 100644 --- a/cloud/services/virtualmachineextensions/mock_virtualmachineextensions/virtualmachineextensions_mock.go +++ b/cloud/services/virtualmachineextensions/mock_virtualmachineextensions/virtualmachineextensions_mock.go @@ -22,7 +22,7 @@ package mock_virtualmachineextensions import ( context "context" - compute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" + compute "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" gomock "github.com/golang/mock/gomock" reflect "reflect" ) diff --git a/cloud/services/virtualmachines/client.go b/cloud/services/virtualmachines/client.go index bb13a97553f..fdc70fd436d 100644 --- a/cloud/services/virtualmachines/client.go +++ b/cloud/services/virtualmachines/client.go @@ -19,7 +19,7 @@ package virtualmachines import ( "context" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) diff --git a/cloud/services/virtualmachines/mock_virtualmachines/virtualmachines_mock.go b/cloud/services/virtualmachines/mock_virtualmachines/virtualmachines_mock.go index af47f5dd6cd..f30772aed66 100644 --- a/cloud/services/virtualmachines/mock_virtualmachines/virtualmachines_mock.go +++ b/cloud/services/virtualmachines/mock_virtualmachines/virtualmachines_mock.go @@ -22,7 +22,7 @@ package mock_virtualmachines import ( context "context" - compute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" + compute "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" gomock "github.com/golang/mock/gomock" reflect "reflect" ) diff --git a/cloud/services/virtualmachines/virtualmachines.go b/cloud/services/virtualmachines/virtualmachines.go index 499d4857196..2a3e6200638 100644 --- a/cloud/services/virtualmachines/virtualmachines.go +++ b/cloud/services/virtualmachines/virtualmachines.go @@ -25,7 +25,7 @@ import ( "strings" "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/authorization/mgmt/authorization" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" diff --git a/cloud/services/virtualmachines/virtualmachines_test.go b/cloud/services/virtualmachines/virtualmachines_test.go index 93599c7d95a..ab4fdf749b3 100644 --- a/cloud/services/virtualmachines/virtualmachines_test.go +++ b/cloud/services/virtualmachines/virtualmachines_test.go @@ -31,8 +31,8 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/golang/mock/gomock" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" - network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" + network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" diff --git a/cloud/services/virtualnetworks/client.go b/cloud/services/virtualnetworks/client.go index 76dd211e9b6..0d79c926df6 100644 --- a/cloud/services/virtualnetworks/client.go +++ b/cloud/services/virtualnetworks/client.go @@ -19,7 +19,7 @@ package virtualnetworks import ( "context" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) diff --git a/cloud/services/virtualnetworks/mock_virtualnetworks/virtualnetworks_mock.go b/cloud/services/virtualnetworks/mock_virtualnetworks/virtualnetworks_mock.go index cd86079bc4e..acd179f1d24 100644 --- a/cloud/services/virtualnetworks/mock_virtualnetworks/virtualnetworks_mock.go +++ b/cloud/services/virtualnetworks/mock_virtualnetworks/virtualnetworks_mock.go @@ -22,9 +22,10 @@ package mock_virtualnetworks import ( context "context" - network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" - gomock "github.com/golang/mock/gomock" reflect "reflect" + + network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" + gomock "github.com/golang/mock/gomock" ) // MockClient is a mock of Client interface. diff --git a/cloud/services/virtualnetworks/virtualnetworks.go b/cloud/services/virtualnetworks/virtualnetworks.go index 28e3eb30cd2..43a116201ce 100644 --- a/cloud/services/virtualnetworks/virtualnetworks.go +++ b/cloud/services/virtualnetworks/virtualnetworks.go @@ -19,7 +19,7 @@ package virtualnetworks import ( "context" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" diff --git a/cloud/services/virtualnetworks/virtualnetworks_test.go b/cloud/services/virtualnetworks/virtualnetworks_test.go index d5d6398fa95..25e6181b71f 100644 --- a/cloud/services/virtualnetworks/virtualnetworks_test.go +++ b/cloud/services/virtualnetworks/virtualnetworks_test.go @@ -30,7 +30,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/golang/mock/gomock" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" diff --git a/exp/controllers/azuremanagedmachinepool_reconciler.go b/exp/controllers/azuremanagedmachinepool_reconciler.go index 6eac9cd1075..c933f1346f2 100644 --- a/exp/controllers/azuremanagedmachinepool_reconciler.go +++ b/exp/controllers/azuremanagedmachinepool_reconciler.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/pkg/errors" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" From 4a10bd65cfaf0c5fe1e63dc5d629cd2f01585f84 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Thu, 2 Jul 2020 21:47:58 -0700 Subject: [PATCH 02/47] Fixed virtualmachines service --- .../virtualmachines/virtualmachines.go | 38 +++++++++++++------ .../virtualmachines/virtualmachines_test.go | 3 -- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/cloud/services/virtualmachines/virtualmachines.go b/cloud/services/virtualmachines/virtualmachines.go index 2a3e6200638..a619c34adcc 100644 --- a/cloud/services/virtualmachines/virtualmachines.go +++ b/cloud/services/virtualmachines/virtualmachines.go @@ -21,7 +21,6 @@ import ( "crypto/rand" "encoding/base64" "fmt" - "strconv" "strings" "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/authorization/mgmt/authorization" @@ -103,10 +102,12 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { // Set the cloud provider tag additionalTags[infrav1.ClusterAzureCloudProviderTagKey(s.MachineScope.Name())] = string(infrav1.ResourceLifecycleOwned) - priority, evictionPolicy, billingProfile, err := getSpotVMOptions(vmSpec.SpotVMOptions) - if err != nil { - return errors.Wrapf(err, "failed to get Spot VM options") - } + /* + priority, evictionPolicy, billingProfile, err := getSpotVMOptions(vmSpec.SpotVMOptions) + if err != nil { + return errors.Wrapf(err, "failed to get Spot VM options") + } + */ virtualMachine := compute.VirtualMachine{ Location: to.StringPtr(s.Scope.Location()), @@ -148,9 +149,6 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { }, }, }, - Priority: priority, - EvictionPolicy: evictionPolicy, - BillingProfile: billingProfile, }, } @@ -172,18 +170,32 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { // UserAssignedIdentities - The list of user identities associated with the Virtual Machine. // The user identity dictionary key references will be ARM resource ids in the form: // '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}'. - userIdentitiesMap := make(map[string]*compute.VirtualMachineIdentityUserAssignedIdentitiesValue, len(vmSpec.UserAssignedIdentities)) + userIdentities := make([]string, len(vmSpec.UserAssignedIdentities)) for _, id := range vmSpec.UserAssignedIdentities { key := id.ProviderID if strings.HasPrefix(id.ProviderID, "azure:///") { key = strings.TrimPrefix(key, "azure:///") } - userIdentitiesMap[key] = &compute.VirtualMachineIdentityUserAssignedIdentitiesValue{} + userIdentities = append(userIdentities, key) } virtualMachine.Identity = &compute.VirtualMachineIdentity{ - Type: compute.ResourceIdentityTypeUserAssigned, - UserAssignedIdentities: userIdentitiesMap, + Type: compute.ResourceIdentityTypeUserAssigned, + IdentityIds: &userIdentities, } + /* + userIdentitiesMap := make(map[string]*compute.VirtualMachineIdentityUserAssignedIdentitiesValue, len(vmSpec.UserAssignedIdentities)) + for _, id := range vmSpec.UserAssignedIdentities { + key := id.ProviderID + if strings.HasPrefix(id.ProviderID, "azure:///") { + key = strings.TrimPrefix(key, "azure:///") + } + userIdentitiesMap[key] = &compute.VirtualMachineIdentityUserAssignedIdentitiesValue{} + } + virtualMachine.Identity = &compute.VirtualMachineIdentity{ + Type: compute.ResourceIdentityTypeUserAssigned, + UserAssignedIdentities: userIdentitiesMap, + } + */ } err = s.Client.CreateOrUpdate( @@ -349,6 +361,7 @@ func generateStorageProfile(vmSpec Spec) (*compute.StorageProfile, error) { return storageProfile, nil } +/* func getSpotVMOptions(spotVMOptions *infrav1.SpotVMOptions) (compute.VirtualMachinePriorityTypes, compute.VirtualMachineEvictionPolicyTypes, *compute.BillingProfile, error) { // Spot VM not requested, return zero values to apply defaults if spotVMOptions == nil { @@ -366,6 +379,7 @@ func getSpotVMOptions(spotVMOptions *infrav1.SpotVMOptions) (compute.VirtualMach } return compute.Spot, compute.Deallocate, billingProfile, nil } +*/ // GenerateRandomString returns a URL-safe, base64 encoded // securely generated random string. diff --git a/cloud/services/virtualmachines/virtualmachines_test.go b/cloud/services/virtualmachines/virtualmachines_test.go index ab4fdf749b3..0f1f424a000 100644 --- a/cloud/services/virtualmachines/virtualmachines_test.go +++ b/cloud/services/virtualmachines/virtualmachines_test.go @@ -548,9 +548,6 @@ func TestReconcileVM(t *testing.T) { expect: func(g *WithT, m *mock_virtualmachines.MockClientMockRecorder, mnic *mock_networkinterfaces.MockClientMockRecorder, mpip *mock_publicips.MockClientMockRecorder, mra *mock_roleassignments.MockClientMockRecorder) { mnic.Get(gomock.Any(), gomock.Any(), gomock.Any()) m.CreateOrUpdate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Do(func(_, _, _ interface{}, vm compute.VirtualMachine) { - g.Expect(vm.Priority).To(Equal(compute.Spot)) - g.Expect(vm.EvictionPolicy).To(Equal(compute.Deallocate)) - g.Expect(vm.BillingProfile).To(BeNil()) }) }, expectedError: "", From b79875e4fd02687dc6e34d24919af90909ad0acf Mon Sep 17 00:00:00 2001 From: chlau-az Date: Thu, 2 Jul 2020 21:53:18 -0700 Subject: [PATCH 03/47] Fixed scalesets service --- cloud/services/scalesets/vmss.go | 3 ++- cloud/services/scalesets/vmss_test.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cloud/services/scalesets/vmss.go b/cloud/services/scalesets/vmss.go index 767df74c9bc..eaa5250740b 100644 --- a/cloud/services/scalesets/vmss.go +++ b/cloud/services/scalesets/vmss.go @@ -19,6 +19,7 @@ package scalesets import ( "context" "fmt" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" @@ -224,7 +225,7 @@ func generateStorageProfile(vmssSpec Spec) (*compute.VirtualMachineScaleSetStora OsDisk: &compute.VirtualMachineScaleSetOSDisk{ OsType: compute.OperatingSystemTypes(vmssSpec.OSDisk.OSType), CreateOption: compute.DiskCreateOptionTypesFromImage, - DiskSizeGB: to.Int32Ptr(vmssSpec.OSDisk.DiskSizeGB), + // DiskSizeGB: to.Int32Ptr(vmssSpec.OSDisk.DiskSizeGB), ManagedDisk: &compute.VirtualMachineScaleSetManagedDiskParameters{ StorageAccountType: compute.StorageAccountTypes(vmssSpec.OSDisk.ManagedDisk.StorageAccountType), }, diff --git a/cloud/services/scalesets/vmss_test.go b/cloud/services/scalesets/vmss_test.go index 5db23360331..17307702fbc 100644 --- a/cloud/services/scalesets/vmss_test.go +++ b/cloud/services/scalesets/vmss_test.go @@ -601,7 +601,7 @@ func TestService_Reconcile(t *testing.T) { StorageProfile: &compute.VirtualMachineScaleSetUpdateStorageProfile{ ImageReference: &compute.ImageReference{ID: to.StringPtr("image")}, OsDisk: &compute.VirtualMachineScaleSetUpdateOSDisk{ - DiskSizeGB: to.Int32Ptr(120), + // DiskSizeGB: to.Int32Ptr(120), ManagedDisk: &compute.VirtualMachineScaleSetManagedDiskParameters{StorageAccountType: "accountType"}, }, }, From 76cbb6d0b702062e42b5727ea0f71b7561e5f2ae Mon Sep 17 00:00:00 2001 From: chlau-az Date: Thu, 2 Jul 2020 22:23:12 -0700 Subject: [PATCH 04/47] Fixed machine validation api version --- api/v1alpha3/azuremachine_validation.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/v1alpha3/azuremachine_validation.go b/api/v1alpha3/azuremachine_validation.go index 36ccc01c36a..86894e04b06 100644 --- a/api/v1alpha3/azuremachine_validation.go +++ b/api/v1alpha3/azuremachine_validation.go @@ -20,7 +20,6 @@ import ( "encoding/base64" "fmt" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" "golang.org/x/crypto/ssh" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -80,11 +79,12 @@ func validateStorageAccountType(storageAccountType string, fieldPath *field.Path return allErrs } - for _, possibleStorageAccountType := range compute.PossibleDiskStorageAccountTypesValues() { + possibleDiskStorageAccountTypesValues := []string{"Premium_LRS", "Standard_LRS", "StandardSSD_LRS", "UltraSSD_LRS"} + for _, possibleStorageAccountType := range possibleDiskStorageAccountTypesValues { if string(possibleStorageAccountType) == storageAccountType { return allErrs } } - allErrs = append(allErrs, field.Invalid(storageAccTypeChildPath, "", fmt.Sprintf("allowed values are %v", compute.PossibleDiskStorageAccountTypesValues()))) + allErrs = append(allErrs, field.Invalid(storageAccTypeChildPath, "", fmt.Sprintf("allowed values are %v", possibleDiskStorageAccountTypesValues))) return allErrs } From 48f644887b443b515a0f094fafcd73047dcc0f13 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Mon, 6 Jul 2020 16:43:32 -0700 Subject: [PATCH 05/47] Changed azure stack authorization --- api/v1alpha2/azurecluster_conversion.go | 2 +- api/v1alpha2/azuremachine_conversion.go | 2 +- .../azuremachinetemplate_conversion.go | 2 +- api/v1alpha2/conversion_test.go | 2 +- api/v1alpha2/doc.go | 2 +- api/v1alpha2/zz_generated.conversion.go | 2 +- cloud/converters/image.go | 2 +- cloud/converters/tags.go | 2 +- cloud/converters/vm.go | 2 +- cloud/converters/vmss.go | 4 +- cloud/converters/vmss_test.go | 4 +- cloud/defaults.go | 4 +- cloud/interfaces.go | 2 +- cloud/mocks/service_mock.go | 2 +- cloud/scope/clients.go | 48 ++++++++++++++++++- cloud/scope/cluster.go | 7 ++- cloud/scope/machine.go | 4 +- cloud/scope/machinepool.go | 6 +-- cloud/scope/managedcontrolplane.go | 2 +- cloud/services/agentpools/agentpools.go | 2 +- cloud/services/agentpools/agentpools_test.go | 2 +- cloud/services/agentpools/client.go | 2 +- cloud/services/agentpools/service.go | 2 +- .../availabilityzones_test.go | 6 +-- cloud/services/availabilityzones/client.go | 2 +- cloud/services/availabilityzones/service.go | 2 +- cloud/services/disks/client.go | 2 +- cloud/services/disks/disks.go | 2 +- cloud/services/disks/disks_test.go | 6 +-- cloud/services/disks/service.go | 2 +- cloud/services/groups/client.go | 4 +- cloud/services/groups/groups.go | 6 +-- cloud/services/groups/groups_test.go | 8 ++-- cloud/services/groups/service.go | 2 +- cloud/services/inboundnatrules/client.go | 2 +- .../services/internalloadbalancers/client.go | 2 +- .../internalloadbalancers.go | 4 +- .../internalloadbalancers_test.go | 10 ++-- .../services/internalloadbalancers/service.go | 6 +-- cloud/services/managedclusters/client.go | 2 +- .../managedclusters/managedclusters.go | 2 +- .../managedclusters/managedclusters_test.go | 2 +- cloud/services/managedclusters/service.go | 2 +- cloud/services/networkinterfaces/client.go | 2 +- .../networkinterfaces/networkinterfaces.go | 4 +- .../networkinterfaces_test.go | 20 ++++---- cloud/services/networkinterfaces/service.go | 14 +++--- cloud/services/publicips/client.go | 2 +- .../mock_publicips/publicips_mock.go | 4 +- cloud/services/publicips/publicips.go | 2 +- cloud/services/publicips/publicips_test.go | 4 +- cloud/services/publicips/service.go | 2 +- cloud/services/publicloadbalancers/client.go | 2 +- .../publicloadbalancers.go | 10 ++-- .../publicloadbalancers_test.go | 10 ++-- cloud/services/publicloadbalancers/service.go | 4 +- cloud/services/resourceskus/client.go | 2 +- cloud/services/roleassignments/client.go | 2 +- cloud/services/routetables/client.go | 2 +- cloud/services/routetables/routetables.go | 2 +- .../services/routetables/routetables_test.go | 6 +-- cloud/services/routetables/service.go | 2 +- cloud/services/scalesets/client.go | 2 +- cloud/services/scalesets/service.go | 6 +-- cloud/services/scalesets/vmss.go | 8 ++-- cloud/services/scalesets/vmss_test.go | 16 +++---- cloud/services/securitygroups/client.go | 2 +- .../services/securitygroups/securitygroups.go | 2 +- .../securitygroups/securitygroups_test.go | 6 +-- cloud/services/securitygroups/service.go | 2 +- cloud/services/subnets/client.go | 2 +- cloud/services/subnets/service.go | 6 +-- cloud/services/subnets/subnets.go | 4 +- cloud/services/subnets/subnets_test.go | 10 ++-- .../virtualmachineextensions/client.go | 2 +- .../virtualmachineextensions/service.go | 2 +- cloud/services/virtualmachines/client.go | 2 +- cloud/services/virtualmachines/service.go | 8 ++-- .../virtualmachines/virtualmachines.go | 6 +-- .../virtualmachines/virtualmachines_test.go | 12 ++--- cloud/services/virtualnetworks/client.go | 2 +- cloud/services/virtualnetworks/service.go | 2 +- .../virtualnetworks/virtualnetworks.go | 6 +-- .../virtualnetworks/virtualnetworks_test.go | 6 +-- config/default/manager_cloud_env_patch.yaml | 4 +- config/default/manager_credentials_patch.yaml | 4 ++ controllers/azurecluster_controller.go | 6 +-- controllers/azurecluster_controller_test.go | 8 ++-- controllers/azurecluster_reconciler.go | 24 +++++----- controllers/azuremachine_annotations.go | 2 +- controllers/azuremachine_controller.go | 6 +-- controllers/azuremachine_controller_test.go | 6 +-- controllers/azuremachine_reconciler.go | 16 +++---- controllers/azuremachine_reconciler_test.go | 6 +-- controllers/azuremachine_tags.go | 4 +- controllers/helpers.go | 4 +- controllers/helpers_test.go | 4 +- controllers/suite_test.go | 2 +- docs/development.md | 2 +- exp/PROJECT | 2 +- exp/api/v1alpha3/azuremachinepool_test.go | 4 +- exp/api/v1alpha3/azuremachinepool_types.go | 2 +- exp/api/v1alpha3/azuremachinepool_webhook.go | 2 +- exp/api/v1alpha3/types.go | 2 +- exp/api/v1alpha3/zz_generated.deepcopy.go | 2 +- .../azuremachinepool_controller.go | 14 +++--- .../azuremachinepool_controller_test.go | 2 +- .../azuremachinepool_controller_unit_test.go | 12 ++--- .../azuremanagedcluster_controller.go | 4 +- .../azuremanagedmachinepool_controller.go | 8 ++-- .../azuremanagedmachinepool_reconciler.go | 8 ++-- .../azuremangedcontrolplane_controller.go | 8 ++-- .../azuremangedcontrolplane_reconciler.go | 8 ++-- exp/controllers/helpers.go | 8 ++-- exp/controllers/helpers_test.go | 6 +-- exp/controllers/suite_test.go | 4 +- go.mod | 3 +- gosdk.log | 0 hack/boilerplate/test/BUILD | 2 +- hack/version.sh | 2 +- internal/test/env/env.go | 8 ++-- main.go | 16 +++---- test/e2e/e2e_suite_test.go | 2 +- util/reconciler/defaults_test.go | 2 +- 124 files changed, 342 insertions(+), 280 deletions(-) create mode 100644 gosdk.log diff --git a/api/v1alpha2/azurecluster_conversion.go b/api/v1alpha2/azurecluster_conversion.go index f3c583240e1..e5e27655b18 100644 --- a/api/v1alpha2/azurecluster_conversion.go +++ b/api/v1alpha2/azurecluster_conversion.go @@ -18,7 +18,7 @@ package v1alpha2 import ( apiconversion "k8s.io/apimachinery/pkg/conversion" - infrav1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + infrav1alpha3 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" utilconversion "sigs.k8s.io/cluster-api/util/conversion" "sigs.k8s.io/controller-runtime/pkg/conversion" ) diff --git a/api/v1alpha2/azuremachine_conversion.go b/api/v1alpha2/azuremachine_conversion.go index fb7e39db297..191cb3adc67 100644 --- a/api/v1alpha2/azuremachine_conversion.go +++ b/api/v1alpha2/azuremachine_conversion.go @@ -18,7 +18,7 @@ package v1alpha2 import ( apiconversion "k8s.io/apimachinery/pkg/conversion" - infrav1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + infrav1alpha3 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" utilconversion "sigs.k8s.io/cluster-api/util/conversion" "sigs.k8s.io/controller-runtime/pkg/conversion" ) diff --git a/api/v1alpha2/azuremachinetemplate_conversion.go b/api/v1alpha2/azuremachinetemplate_conversion.go index 5887c26eea7..c95b050c54a 100644 --- a/api/v1alpha2/azuremachinetemplate_conversion.go +++ b/api/v1alpha2/azuremachinetemplate_conversion.go @@ -17,7 +17,7 @@ limitations under the License. package v1alpha2 import ( - infrav1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + infrav1alpha3 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" utilconversion "sigs.k8s.io/cluster-api/util/conversion" "sigs.k8s.io/controller-runtime/pkg/conversion" ) diff --git a/api/v1alpha2/conversion_test.go b/api/v1alpha2/conversion_test.go index 1d6e0d721ed..1efbb5adaef 100644 --- a/api/v1alpha2/conversion_test.go +++ b/api/v1alpha2/conversion_test.go @@ -24,7 +24,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" - v1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + v1alpha3 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" utilconversion "sigs.k8s.io/cluster-api/util/conversion" ) diff --git a/api/v1alpha2/doc.go b/api/v1alpha2/doc.go index 3147930676a..c613f85957c 100644 --- a/api/v1alpha2/doc.go +++ b/api/v1alpha2/doc.go @@ -16,4 +16,4 @@ limitations under the License. package v1alpha2 -// +k8s:conversion-gen=sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3 +// +k8s:conversion-gen=github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3 diff --git a/api/v1alpha2/zz_generated.conversion.go b/api/v1alpha2/zz_generated.conversion.go index 3691f87db88..f5fa211a379 100644 --- a/api/v1alpha2/zz_generated.conversion.go +++ b/api/v1alpha2/zz_generated.conversion.go @@ -23,10 +23,10 @@ package v1alpha2 import ( unsafe "unsafe" + v1alpha3 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" v1 "k8s.io/api/core/v1" conversion "k8s.io/apimachinery/pkg/conversion" runtime "k8s.io/apimachinery/pkg/runtime" - v1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" ) func init() { diff --git a/cloud/converters/image.go b/cloud/converters/image.go index 10c7994e7f1..bcc8a298015 100644 --- a/cloud/converters/image.go +++ b/cloud/converters/image.go @@ -22,7 +22,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/pkg/errors" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" ) // ImageToSDK converts a CAPZ Image (as RawExtension) to a Azure SDK Image Reference. diff --git a/cloud/converters/tags.go b/cloud/converters/tags.go index a62cc4b0448..442c0491f36 100644 --- a/cloud/converters/tags.go +++ b/cloud/converters/tags.go @@ -18,7 +18,7 @@ package converters import ( "github.com/Azure/go-autorest/autorest/to" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" ) // MapToTags converts a map[string]*string into a infrav1.Tags. diff --git a/cloud/converters/vm.go b/cloud/converters/vm.go index ee28ed98391..1a1d9c96b23 100644 --- a/cloud/converters/vm.go +++ b/cloud/converters/vm.go @@ -19,7 +19,7 @@ package converters import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest/to" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" ) // SDKToVM converts an Azure SDK VirtualMachine to the CAPZ VM type. diff --git a/cloud/converters/vmss.go b/cloud/converters/vmss.go index 7611b0d8428..5ea59c93a41 100644 --- a/cloud/converters/vmss.go +++ b/cloud/converters/vmss.go @@ -20,8 +20,8 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest/to" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" ) // SDKToVMSS converts an Azure SDK VirtualMachineScaleSet to the AzureMachinePool type. diff --git a/cloud/converters/vmss_test.go b/cloud/converters/vmss_test.go index be509ef98df..49850a7d0b4 100644 --- a/cloud/converters/vmss_test.go +++ b/cloud/converters/vmss_test.go @@ -24,8 +24,8 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/onsi/gomega" - "sigs.k8s.io/cluster-api-provider-azure/cloud/converters" - infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/cloud/converters" + infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" ) func Test_SDKToVMSS(t *testing.T) { diff --git a/cloud/defaults.go b/cloud/defaults.go index fc654519df7..281d3cce1a3 100644 --- a/cloud/defaults.go +++ b/cloud/defaults.go @@ -21,8 +21,8 @@ import ( "github.com/blang/semver" "github.com/pkg/errors" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/version" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/version" ) const ( diff --git a/cloud/interfaces.go b/cloud/interfaces.go index d6be0728377..fbe099fbb3f 100644 --- a/cloud/interfaces.go +++ b/cloud/interfaces.go @@ -19,7 +19,7 @@ package azure import ( "context" "github.com/Azure/go-autorest/autorest" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" ) // Service is a generic interface used by components offering a type of service. diff --git a/cloud/mocks/service_mock.go b/cloud/mocks/service_mock.go index ef9a4110bd9..5ee10358619 100644 --- a/cloud/mocks/service_mock.go +++ b/cloud/mocks/service_mock.go @@ -15,7 +15,7 @@ limitations under the License. */ // Code generated by MockGen. DO NOT EDIT. -// Source: sigs.k8s.io/cluster-api-provider-azure/cloud (interfaces: Service,GetterService) +// Source: github.com/chlau-az/cluster-api-provider-azure/cloud (interfaces: Service,GetterService) // Package mocks is a generated GoMock package. package mocks diff --git a/cloud/scope/clients.go b/cloud/scope/clients.go index a39fb8f512b..fce33db7029 100644 --- a/cloud/scope/clients.go +++ b/cloud/scope/clients.go @@ -17,9 +17,12 @@ limitations under the License. package scope import ( + "log" "os" "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/adal" + "github.com/Azure/go-autorest/autorest/azure" "github.com/Azure/go-autorest/autorest/azure/auth" "github.com/pkg/errors" ) @@ -38,6 +41,9 @@ const ( // AzureClients contains all the Azure clients used by the scopes. type AzureClients struct { SubscriptionID string + ClientID string + ClientSecret string + TenantID string ResourceManagerEndpoint string ResourceManagerVMDNSSuffix string Authorizer autorest.Authorizer @@ -49,14 +55,27 @@ func (c *AzureClients) setCredentials(subscriptionID string) error { return err } c.SubscriptionID = subID + c.ClientID = os.Getenv("AZURE_CLIENT_ID") + c.ClientSecret = os.Getenv("AZURE_CLIENT_SECRET") + c.TenantID = os.Getenv("AZURE_TENANT_ID") settings, err := auth.GetSettingsFromEnvironment() if err != nil { return err } + + // To do: get arm endpoint in helper method + log.Println("HERE changing environment") + armEndpoint := os.Getenv("AZURE_ARM_ENDPOINT") + log.Println("HERE armEndpoint: ", armEndpoint) + settings.Environment, _ = azure.EnvironmentFromURL(armEndpoint) + c.ResourceManagerEndpoint = settings.Environment.ResourceManagerEndpoint + log.Println("HERE resource manager endpoint: ", c.ResourceManagerEndpoint) c.ResourceManagerVMDNSSuffix = GetAzureDNSZoneForEnvironment(settings.Environment.Name) settings.Values[auth.SubscriptionID] = subscriptionID - c.Authorizer, err = settings.GetAuthorizer() + // c.Authorizer, err = settings.GetAuthorizer() + c.Authorizer, err = c.getAuthorizerForResource(settings.Environment) + log.Println("HERE c.Authorizer: ", c.Authorizer, "err: ", err) return err } @@ -88,3 +107,30 @@ func GetAzureDNSZoneForEnvironment(environmentName string) string { return "cloudapp.azure.com" } } + +// getAuthorizerForResource gets an OAuthTokenAuthorizer for Azure Resource Manager +func (c *AzureClients) getAuthorizerForResource(env azure.Environment) (autorest.Authorizer, error) { + var a autorest.Authorizer + var err error + var oauthConfig *adal.OAuthConfig + + tokenAudience := env.TokenAudience + log.Println("HERE TokenAudience: ", env.TokenAudience) + log.Println("HERE ActiveDirectoryEndpoint: ", env.ActiveDirectoryEndpoint) + oauthConfig, err = adal.NewOAuthConfig( + env.ActiveDirectoryEndpoint, c.TenantID) + + if err != nil { + return nil, err + } + token, err := adal.NewServicePrincipalToken( + *oauthConfig, + c.ClientID, + c.ClientSecret, + tokenAudience) + + log.Println("HERE generated token") + a = autorest.NewBearerAuthorizer(token) + log.Println("HERE generated authorizer") + return a, err +} diff --git a/cloud/scope/cluster.go b/cloud/scope/cluster.go index dbde00c99fb..a89d42cfb3c 100644 --- a/cloud/scope/cluster.go +++ b/cloud/scope/cluster.go @@ -19,12 +19,14 @@ package scope import ( "context" "fmt" + "log" + "github.com/Azure/go-autorest/autorest" "github.com/go-logr/logr" "github.com/pkg/errors" "k8s.io/klog/klogr" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/controller-runtime/pkg/client" @@ -91,6 +93,7 @@ func (s *ClusterScope) SubscriptionID() string { // BaseURI returns the Azure ResourceManagerEndpoint. func (s *ClusterScope) BaseURI() string { + log.Println("HERE BaseURI s.ResourceManagerEndpoint: ", s.ResourceManagerEndpoint) return s.ResourceManagerEndpoint } diff --git a/cloud/scope/machine.go b/cloud/scope/machine.go index a846b879c07..76cb28af473 100644 --- a/cloud/scope/machine.go +++ b/cloud/scope/machine.go @@ -26,8 +26,8 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/klog/klogr" "k8s.io/utils/pointer" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/cluster-api/controllers/noderefutil" capierrors "sigs.k8s.io/cluster-api/errors" diff --git a/cloud/scope/machinepool.go b/cloud/scope/machinepool.go index 9798f861f06..a80a8946f41 100644 --- a/cloud/scope/machinepool.go +++ b/cloud/scope/machinepool.go @@ -26,14 +26,14 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/klog/klogr" "k8s.io/utils/pointer" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" "sigs.k8s.io/cluster-api/controllers/noderefutil" capierrors "sigs.k8s.io/cluster-api/errors" capiv1exp "sigs.k8s.io/cluster-api/exp/api/v1alpha3" "sigs.k8s.io/cluster-api/util/patch" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/cloud/scope/managedcontrolplane.go b/cloud/scope/managedcontrolplane.go index f3d98f6bedd..1437063a25b 100644 --- a/cloud/scope/managedcontrolplane.go +++ b/cloud/scope/managedcontrolplane.go @@ -23,7 +23,7 @@ import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/klog/klogr" - infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" + infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" expv1 "sigs.k8s.io/cluster-api/exp/api/v1alpha3" diff --git a/cloud/services/agentpools/agentpools.go b/cloud/services/agentpools/agentpools.go index 4b3187dec00..29692f8ed97 100644 --- a/cloud/services/agentpools/agentpools.go +++ b/cloud/services/agentpools/agentpools.go @@ -23,7 +23,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/pkg/errors" "k8s.io/klog" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Spec contains properties to create a agent pool. diff --git a/cloud/services/agentpools/agentpools_test.go b/cloud/services/agentpools/agentpools_test.go index 001541f1702..47a7a0935e1 100644 --- a/cloud/services/agentpools/agentpools_test.go +++ b/cloud/services/agentpools/agentpools_test.go @@ -27,7 +27,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/golang/mock/gomock" . "github.com/onsi/gomega" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/agentpools/mock_agentpools" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/agentpools/mock_agentpools" ) const ( diff --git a/cloud/services/agentpools/client.go b/cloud/services/agentpools/client.go index f680ebe0511..0293d29e997 100644 --- a/cloud/services/agentpools/client.go +++ b/cloud/services/agentpools/client.go @@ -22,7 +22,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2020-02-01/containerservice" "github.com/Azure/go-autorest/autorest" "github.com/pkg/errors" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/agentpools/service.go b/cloud/services/agentpools/service.go index 9e3ef623b4c..5448560ff6d 100644 --- a/cloud/services/agentpools/service.go +++ b/cloud/services/agentpools/service.go @@ -17,7 +17,7 @@ limitations under the License. package agentpools import ( - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Service provides operations on azure resources diff --git a/cloud/services/availabilityzones/availabilityzones_test.go b/cloud/services/availabilityzones/availabilityzones_test.go index d46bb487cce..602165609b0 100644 --- a/cloud/services/availabilityzones/availabilityzones_test.go +++ b/cloud/services/availabilityzones/availabilityzones_test.go @@ -22,7 +22,7 @@ import ( "testing" . "github.com/onsi/gomega" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/availabilityzones/mock_availabilityzones" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/availabilityzones/mock_availabilityzones" "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest" @@ -32,8 +32,8 @@ import ( network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) diff --git a/cloud/services/availabilityzones/client.go b/cloud/services/availabilityzones/client.go index 4c502f29330..4f391e156a3 100644 --- a/cloud/services/availabilityzones/client.go +++ b/cloud/services/availabilityzones/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/availabilityzones/service.go b/cloud/services/availabilityzones/service.go index 5a3aa546f81..c119e1bb2e0 100644 --- a/cloud/services/availabilityzones/service.go +++ b/cloud/services/availabilityzones/service.go @@ -17,7 +17,7 @@ limitations under the License. package availabilityzones import ( - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" ) // Service provides operations on azure resources diff --git a/cloud/services/disks/client.go b/cloud/services/disks/client.go index eb0664e68c3..d32ca128425 100644 --- a/cloud/services/disks/client.go +++ b/cloud/services/disks/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/disks/disks.go b/cloud/services/disks/disks.go index 8240ef6870f..9f51db77183 100644 --- a/cloud/services/disks/disks.go +++ b/cloud/services/disks/disks.go @@ -21,7 +21,7 @@ import ( "github.com/pkg/errors" "k8s.io/klog" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Spec specification for disk diff --git a/cloud/services/disks/disks_test.go b/cloud/services/disks/disks_test.go index 0d144436b52..95f84377863 100644 --- a/cloud/services/disks/disks_test.go +++ b/cloud/services/disks/disks_test.go @@ -22,7 +22,7 @@ import ( "testing" . "github.com/onsi/gomega" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/disks/mock_disks" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/disks/mock_disks" "github.com/Azure/go-autorest/autorest" "github.com/golang/mock/gomock" @@ -30,8 +30,8 @@ import ( network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) diff --git a/cloud/services/disks/service.go b/cloud/services/disks/service.go index e3114864528..e9643293f64 100644 --- a/cloud/services/disks/service.go +++ b/cloud/services/disks/service.go @@ -17,7 +17,7 @@ limitations under the License. package disks import ( - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" ) // Service provides operations on azure resources diff --git a/cloud/services/groups/client.go b/cloud/services/groups/client.go index ded75d2dafd..4bbdefc1f8b 100644 --- a/cloud/services/groups/client.go +++ b/cloud/services/groups/client.go @@ -18,10 +18,11 @@ package groups import ( "context" + "log" "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/resources/mgmt/resources" "github.com/Azure/go-autorest/autorest" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk @@ -46,6 +47,7 @@ func NewClient(auth azure.Authorizer) *AzureClient { // newGroupsClient creates a new groups client from subscription ID. func newGroupsClient(subscriptionID string, baseURI string, authorizer autorest.Authorizer) resources.GroupsClient { + log.Println("HERE creating new group client with baseuri") groupsClient := resources.NewGroupsClientWithBaseURI(baseURI, subscriptionID) groupsClient.Authorizer = authorizer groupsClient.AddToUserAgent(azure.UserAgent()) diff --git a/cloud/services/groups/groups.go b/cloud/services/groups/groups.go index e1041d70438..978b8ab7092 100644 --- a/cloud/services/groups/groups.go +++ b/cloud/services/groups/groups.go @@ -23,9 +23,9 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" - "sigs.k8s.io/cluster-api-provider-azure/cloud/converters" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + "github.com/chlau-az/cluster-api-provider-azure/cloud/converters" ) // Reconcile gets/creates/updates a resource group. diff --git a/cloud/services/groups/groups_test.go b/cloud/services/groups/groups_test.go index aa766e005a1..6e69d70d23c 100644 --- a/cloud/services/groups/groups_test.go +++ b/cloud/services/groups/groups_test.go @@ -22,7 +22,7 @@ import ( "testing" . "github.com/onsi/gomega" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/groups/mock_groups" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/groups/mock_groups" "github.com/Azure/go-autorest/autorest" "github.com/golang/mock/gomock" @@ -30,9 +30,9 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/resources/mgmt/resources" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/cloud/converters" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/cloud/converters" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) diff --git a/cloud/services/groups/service.go b/cloud/services/groups/service.go index b57544dd445..a8233fbd564 100644 --- a/cloud/services/groups/service.go +++ b/cloud/services/groups/service.go @@ -17,7 +17,7 @@ limitations under the License. package groups import ( - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" ) // Service provides operations on azure resources diff --git a/cloud/services/inboundnatrules/client.go b/cloud/services/inboundnatrules/client.go index 4cc64d8107b..9af1a46a6d9 100644 --- a/cloud/services/inboundnatrules/client.go +++ b/cloud/services/inboundnatrules/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/internalloadbalancers/client.go b/cloud/services/internalloadbalancers/client.go index c07dd0d582d..2949d66599d 100644 --- a/cloud/services/internalloadbalancers/client.go +++ b/cloud/services/internalloadbalancers/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/internalloadbalancers/internalloadbalancers.go b/cloud/services/internalloadbalancers/internalloadbalancers.go index c4ce8ff3c07..e090ae30bd4 100644 --- a/cloud/services/internalloadbalancers/internalloadbalancers.go +++ b/cloud/services/internalloadbalancers/internalloadbalancers.go @@ -24,8 +24,8 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Spec specification for internal load balancer diff --git a/cloud/services/internalloadbalancers/internalloadbalancers_test.go b/cloud/services/internalloadbalancers/internalloadbalancers_test.go index fa5598802ab..b30c71c9935 100644 --- a/cloud/services/internalloadbalancers/internalloadbalancers_test.go +++ b/cloud/services/internalloadbalancers/internalloadbalancers_test.go @@ -22,9 +22,9 @@ import ( "testing" . "github.com/onsi/gomega" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/internalloadbalancers/mock_internalloadbalancers" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/subnets/mock_subnets" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/virtualnetworks/mock_virtualnetworks" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/internalloadbalancers/mock_internalloadbalancers" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/subnets/mock_subnets" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/virtualnetworks/mock_virtualnetworks" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" @@ -33,8 +33,8 @@ import ( network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) diff --git a/cloud/services/internalloadbalancers/service.go b/cloud/services/internalloadbalancers/service.go index 7c85d4bc3f4..db06b4a6555 100644 --- a/cloud/services/internalloadbalancers/service.go +++ b/cloud/services/internalloadbalancers/service.go @@ -17,9 +17,9 @@ limitations under the License. package internalloadbalancers import ( - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/subnets" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/virtualnetworks" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/subnets" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/virtualnetworks" ) // Service provides operations on azure resources diff --git a/cloud/services/managedclusters/client.go b/cloud/services/managedclusters/client.go index 93648055614..b5e2bcda77b 100644 --- a/cloud/services/managedclusters/client.go +++ b/cloud/services/managedclusters/client.go @@ -22,7 +22,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2020-02-01/containerservice" "github.com/Azure/go-autorest/autorest" "github.com/pkg/errors" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/managedclusters/managedclusters.go b/cloud/services/managedclusters/managedclusters.go index ce17b130536..ebd2edb3088 100644 --- a/cloud/services/managedclusters/managedclusters.go +++ b/cloud/services/managedclusters/managedclusters.go @@ -25,7 +25,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2020-02-01/containerservice" "github.com/pkg/errors" "k8s.io/klog" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) var ( diff --git a/cloud/services/managedclusters/managedclusters_test.go b/cloud/services/managedclusters/managedclusters_test.go index 5ea90817141..e130d348b49 100644 --- a/cloud/services/managedclusters/managedclusters_test.go +++ b/cloud/services/managedclusters/managedclusters_test.go @@ -25,7 +25,7 @@ import ( "github.com/Azure/go-autorest/autorest" "github.com/golang/mock/gomock" . "github.com/onsi/gomega" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/managedclusters/mock_managedclusters" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/managedclusters/mock_managedclusters" ) func TestReconcile(t *testing.T) { diff --git a/cloud/services/managedclusters/service.go b/cloud/services/managedclusters/service.go index dfc744d6f1a..13300870981 100644 --- a/cloud/services/managedclusters/service.go +++ b/cloud/services/managedclusters/service.go @@ -17,7 +17,7 @@ limitations under the License. package managedclusters import ( - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Service provides operations on azure resources diff --git a/cloud/services/networkinterfaces/client.go b/cloud/services/networkinterfaces/client.go index 77b83bb27e3..b05095fa1f9 100644 --- a/cloud/services/networkinterfaces/client.go +++ b/cloud/services/networkinterfaces/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/networkinterfaces/networkinterfaces.go b/cloud/services/networkinterfaces/networkinterfaces.go index b328f3cabc5..ccf1678d8bc 100644 --- a/cloud/services/networkinterfaces/networkinterfaces.go +++ b/cloud/services/networkinterfaces/networkinterfaces.go @@ -24,8 +24,8 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Spec specification for routetable diff --git a/cloud/services/networkinterfaces/networkinterfaces_test.go b/cloud/services/networkinterfaces/networkinterfaces_test.go index bb2dd0f88f3..5cb585497d9 100644 --- a/cloud/services/networkinterfaces/networkinterfaces_test.go +++ b/cloud/services/networkinterfaces/networkinterfaces_test.go @@ -22,14 +22,14 @@ import ( "testing" . "github.com/onsi/gomega" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/inboundnatrules/mock_inboundnatrules" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/internalloadbalancers/mock_internalloadbalancers" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/networkinterfaces/mock_networkinterfaces" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicips/mock_publicips" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicloadbalancers/mock_publicloadbalancers" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/resourceskus/mock_resourceskus" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/subnets/mock_subnets" - "sigs.k8s.io/cluster-api-provider-azure/internal/test/matchers" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/inboundnatrules/mock_inboundnatrules" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/internalloadbalancers/mock_internalloadbalancers" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/networkinterfaces/mock_networkinterfaces" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicips/mock_publicips" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicloadbalancers/mock_publicloadbalancers" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/resourceskus/mock_resourceskus" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/subnets/mock_subnets" + "github.com/chlau-az/cluster-api-provider-azure/internal/test/matchers" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" @@ -39,8 +39,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/pointer" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) diff --git a/cloud/services/networkinterfaces/service.go b/cloud/services/networkinterfaces/service.go index 7f443abac25..2ed2b6c00a5 100644 --- a/cloud/services/networkinterfaces/service.go +++ b/cloud/services/networkinterfaces/service.go @@ -17,13 +17,13 @@ limitations under the License. package networkinterfaces import ( - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/inboundnatrules" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/internalloadbalancers" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicips" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicloadbalancers" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/resourceskus" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/subnets" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/inboundnatrules" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/internalloadbalancers" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicips" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicloadbalancers" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/resourceskus" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/subnets" ) // Service provides operations on azure resources diff --git a/cloud/services/publicips/client.go b/cloud/services/publicips/client.go index aa92ccffef1..41b3aa79f53 100644 --- a/cloud/services/publicips/client.go +++ b/cloud/services/publicips/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/publicips/mock_publicips/publicips_mock.go b/cloud/services/publicips/mock_publicips/publicips_mock.go index 948a1b4e768..bdb49340ee9 100644 --- a/cloud/services/publicips/mock_publicips/publicips_mock.go +++ b/cloud/services/publicips/mock_publicips/publicips_mock.go @@ -24,8 +24,8 @@ import ( autorest "github.com/Azure/go-autorest/autorest" gomock "github.com/golang/mock/gomock" reflect "reflect" - v1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + v1alpha3 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // MockPublicIPScope is a mock of PublicIPScope interface. diff --git a/cloud/services/publicips/publicips.go b/cloud/services/publicips/publicips.go index 299620e9a1d..9c9f83d2704 100644 --- a/cloud/services/publicips/publicips.go +++ b/cloud/services/publicips/publicips.go @@ -24,7 +24,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Reconcile gets/creates/updates a public ip. diff --git a/cloud/services/publicips/publicips_test.go b/cloud/services/publicips/publicips_test.go index d11237a27c7..efcb7d917dc 100644 --- a/cloud/services/publicips/publicips_test.go +++ b/cloud/services/publicips/publicips_test.go @@ -21,10 +21,10 @@ import ( "net/http" "testing" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" . "github.com/onsi/gomega" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicips/mock_publicips" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicips/mock_publicips" "github.com/Azure/go-autorest/autorest" "github.com/golang/mock/gomock" diff --git a/cloud/services/publicips/service.go b/cloud/services/publicips/service.go index 73898c8bdf8..d52f5369a99 100644 --- a/cloud/services/publicips/service.go +++ b/cloud/services/publicips/service.go @@ -17,7 +17,7 @@ limitations under the License. package publicips import ( - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // PublicIPScope defines the scope interface for a public IP service. diff --git a/cloud/services/publicloadbalancers/client.go b/cloud/services/publicloadbalancers/client.go index 7405d932441..a7dd31bcb61 100644 --- a/cloud/services/publicloadbalancers/client.go +++ b/cloud/services/publicloadbalancers/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/publicloadbalancers/publicloadbalancers.go b/cloud/services/publicloadbalancers/publicloadbalancers.go index f154843eee8..bce585c08ff 100644 --- a/cloud/services/publicloadbalancers/publicloadbalancers.go +++ b/cloud/services/publicloadbalancers/publicloadbalancers.go @@ -19,14 +19,15 @@ package publicloadbalancers import ( "context" "fmt" + "log" "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" - "sigs.k8s.io/cluster-api-provider-azure/cloud/converters" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + "github.com/chlau-az/cluster-api-provider-azure/cloud/converters" ) // Spec specification for public load balancer @@ -60,6 +61,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { return errors.Wrap(err, "failed to look for existing public IP") } klog.V(2).Infof("successfully got public ip %s", publicLBSpec.PublicIPName) + log.Println("HI HERE RECONCILING") lb := network.LoadBalancer{ Sku: &network.LoadBalancerSku{Name: network.LoadBalancerSkuNameStandard}, @@ -146,10 +148,12 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { err = s.Client.CreateOrUpdate(ctx, s.Scope.ResourceGroup(), lbName, lb) if err != nil { + log.Println("cannot create public load balancer") return errors.Wrap(err, "cannot create public load balancer") } klog.V(2).Infof("successfully created public load balancer %s", lbName) + log.Println("sucessfully created public load balancer") return nil } diff --git a/cloud/services/publicloadbalancers/publicloadbalancers_test.go b/cloud/services/publicloadbalancers/publicloadbalancers_test.go index 6f793001684..966c3241b61 100644 --- a/cloud/services/publicloadbalancers/publicloadbalancers_test.go +++ b/cloud/services/publicloadbalancers/publicloadbalancers_test.go @@ -21,20 +21,20 @@ import ( "net/http" "testing" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicips/mock_publicips" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicloadbalancers/mock_publicloadbalancers" + "github.com/chlau-az/cluster-api-provider-azure/internal/test/matchers" . "github.com/onsi/gomega" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicips/mock_publicips" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicloadbalancers/mock_publicloadbalancers" - "sigs.k8s.io/cluster-api-provider-azure/internal/test/matchers" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" "github.com/golang/mock/gomock" network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) diff --git a/cloud/services/publicloadbalancers/service.go b/cloud/services/publicloadbalancers/service.go index ee5b22b5cac..ea8283675f5 100644 --- a/cloud/services/publicloadbalancers/service.go +++ b/cloud/services/publicloadbalancers/service.go @@ -17,8 +17,8 @@ limitations under the License. package publicloadbalancers import ( - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicips" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicips" ) // Service provides operations on azure resources diff --git a/cloud/services/resourceskus/client.go b/cloud/services/resourceskus/client.go index f5d5f7e2a8c..bcbfb030dff 100644 --- a/cloud/services/resourceskus/client.go +++ b/cloud/services/resourceskus/client.go @@ -24,7 +24,7 @@ import ( "github.com/Azure/go-autorest/autorest" "github.com/pkg/errors" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/roleassignments/client.go b/cloud/services/roleassignments/client.go index 09a61d0f21b..aed3c78b522 100644 --- a/cloud/services/roleassignments/client.go +++ b/cloud/services/roleassignments/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/authorization/mgmt/authorization" "github.com/Azure/go-autorest/autorest" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/routetables/client.go b/cloud/services/routetables/client.go index 3f1c9e49576..d85e4e033c9 100644 --- a/cloud/services/routetables/client.go +++ b/cloud/services/routetables/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/routetables/routetables.go b/cloud/services/routetables/routetables.go index 6b97e0d4b4a..652ec865139 100644 --- a/cloud/services/routetables/routetables.go +++ b/cloud/services/routetables/routetables.go @@ -22,7 +22,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Spec specification for route table. diff --git a/cloud/services/routetables/routetables_test.go b/cloud/services/routetables/routetables_test.go index dab3ed6a7b4..4c3b7ab7843 100644 --- a/cloud/services/routetables/routetables_test.go +++ b/cloud/services/routetables/routetables_test.go @@ -23,7 +23,7 @@ import ( "testing" . "github.com/onsi/gomega" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/routetables/mock_routetables" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/routetables/mock_routetables" "github.com/Azure/go-autorest/autorest" "github.com/golang/mock/gomock" @@ -31,8 +31,8 @@ import ( network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) diff --git a/cloud/services/routetables/service.go b/cloud/services/routetables/service.go index 14b8d91a18c..94ee500b16b 100644 --- a/cloud/services/routetables/service.go +++ b/cloud/services/routetables/service.go @@ -17,7 +17,7 @@ limitations under the License. package routetables import ( - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" ) // Service provides operations on azure resources diff --git a/cloud/services/scalesets/client.go b/cloud/services/scalesets/client.go index edda4e5ca7f..b914c6f02ec 100644 --- a/cloud/services/scalesets/client.go +++ b/cloud/services/scalesets/client.go @@ -24,7 +24,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/scalesets/service.go b/cloud/services/scalesets/service.go index 992e659fb4a..1d09828885c 100644 --- a/cloud/services/scalesets/service.go +++ b/cloud/services/scalesets/service.go @@ -16,9 +16,9 @@ limitations under the License. package scalesets import ( - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicloadbalancers" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/resourceskus" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicloadbalancers" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/resourceskus" ) // Service provides operations on azure resources diff --git a/cloud/services/scalesets/vmss.go b/cloud/services/scalesets/vmss.go index eaa5250740b..ec0684a27ae 100644 --- a/cloud/services/scalesets/vmss.go +++ b/cloud/services/scalesets/vmss.go @@ -20,16 +20,16 @@ import ( "context" "fmt" - infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" + infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" - "sigs.k8s.io/cluster-api-provider-azure/cloud/converters" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + "github.com/chlau-az/cluster-api-provider-azure/cloud/converters" ) // Spec contains properties to create a managed cluster. diff --git a/cloud/services/scalesets/vmss_test.go b/cloud/services/scalesets/vmss_test.go index 17307702fbc..94748b4f42f 100644 --- a/cloud/services/scalesets/vmss_test.go +++ b/cloud/services/scalesets/vmss_test.go @@ -35,14 +35,14 @@ import ( clusterv1exp "sigs.k8s.io/cluster-api/exp/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicloadbalancers/mock_publicloadbalancers" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/resourceskus/mock_resourceskus" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/scalesets/mock_scalesets" - infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/internal/test/matchers" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicloadbalancers/mock_publicloadbalancers" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/resourceskus/mock_resourceskus" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/scalesets/mock_scalesets" + infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/internal/test/matchers" ) func init() { diff --git a/cloud/services/securitygroups/client.go b/cloud/services/securitygroups/client.go index 5b9b84d47c0..52bdcacb96c 100644 --- a/cloud/services/securitygroups/client.go +++ b/cloud/services/securitygroups/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/securitygroups/securitygroups.go b/cloud/services/securitygroups/securitygroups.go index 5af8aae53d0..53a1aedb62e 100644 --- a/cloud/services/securitygroups/securitygroups.go +++ b/cloud/services/securitygroups/securitygroups.go @@ -25,7 +25,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) const ( diff --git a/cloud/services/securitygroups/securitygroups_test.go b/cloud/services/securitygroups/securitygroups_test.go index b8972089f28..24f66ecc8c9 100644 --- a/cloud/services/securitygroups/securitygroups_test.go +++ b/cloud/services/securitygroups/securitygroups_test.go @@ -22,7 +22,7 @@ import ( "testing" . "github.com/onsi/gomega" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/securitygroups/mock_securitygroups" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/securitygroups/mock_securitygroups" "github.com/Azure/go-autorest/autorest" "github.com/golang/mock/gomock" @@ -30,8 +30,8 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) diff --git a/cloud/services/securitygroups/service.go b/cloud/services/securitygroups/service.go index 6a4285f8465..827dde02c5b 100644 --- a/cloud/services/securitygroups/service.go +++ b/cloud/services/securitygroups/service.go @@ -17,7 +17,7 @@ limitations under the License. package securitygroups import ( - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" ) // Service provides operations on azure resources diff --git a/cloud/services/subnets/client.go b/cloud/services/subnets/client.go index 57cfae82e87..74bb4802b81 100644 --- a/cloud/services/subnets/client.go +++ b/cloud/services/subnets/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/subnets/service.go b/cloud/services/subnets/service.go index 229aae45b47..b0f830efcd0 100644 --- a/cloud/services/subnets/service.go +++ b/cloud/services/subnets/service.go @@ -17,9 +17,9 @@ limitations under the License. package subnets import ( - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/routetables" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/securitygroups" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/routetables" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/securitygroups" ) // Service provides operations on azure resources diff --git a/cloud/services/subnets/subnets.go b/cloud/services/subnets/subnets.go index 223dde154e2..dd950a8deef 100644 --- a/cloud/services/subnets/subnets.go +++ b/cloud/services/subnets/subnets.go @@ -25,8 +25,8 @@ import ( "github.com/pkg/errors" "k8s.io/klog" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Spec input specification for Get/CreateOrUpdate/Delete calls diff --git a/cloud/services/subnets/subnets_test.go b/cloud/services/subnets/subnets_test.go index 0fa9572aa3d..dbd0578d213 100644 --- a/cloud/services/subnets/subnets_test.go +++ b/cloud/services/subnets/subnets_test.go @@ -22,9 +22,9 @@ import ( "testing" . "github.com/onsi/gomega" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/routetables/mock_routetables" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/securitygroups/mock_securitygroups" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/subnets/mock_subnets" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/routetables/mock_routetables" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/securitygroups/mock_securitygroups" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/subnets/mock_subnets" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" @@ -33,8 +33,8 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) diff --git a/cloud/services/virtualmachineextensions/client.go b/cloud/services/virtualmachineextensions/client.go index 356328798fc..34d33e9481c 100644 --- a/cloud/services/virtualmachineextensions/client.go +++ b/cloud/services/virtualmachineextensions/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/virtualmachineextensions/service.go b/cloud/services/virtualmachineextensions/service.go index cc438f7b0b6..9a8d81613d5 100644 --- a/cloud/services/virtualmachineextensions/service.go +++ b/cloud/services/virtualmachineextensions/service.go @@ -17,7 +17,7 @@ limitations under the License. package virtualmachineextensions import ( - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" ) // Service provides operations on azure resources diff --git a/cloud/services/virtualmachines/client.go b/cloud/services/virtualmachines/client.go index fdc70fd436d..f5cc229b90f 100644 --- a/cloud/services/virtualmachines/client.go +++ b/cloud/services/virtualmachines/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/virtualmachines/service.go b/cloud/services/virtualmachines/service.go index 0cbd3fef03d..248ccb209d6 100644 --- a/cloud/services/virtualmachines/service.go +++ b/cloud/services/virtualmachines/service.go @@ -17,10 +17,10 @@ limitations under the License. package virtualmachines import ( - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/networkinterfaces" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicips" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/roleassignments" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/networkinterfaces" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicips" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/roleassignments" ) // Service provides operations on azure resources diff --git a/cloud/services/virtualmachines/virtualmachines.go b/cloud/services/virtualmachines/virtualmachines.go index a619c34adcc..28a8bb3fb61 100644 --- a/cloud/services/virtualmachines/virtualmachines.go +++ b/cloud/services/virtualmachines/virtualmachines.go @@ -30,9 +30,9 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/klog" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" - "sigs.k8s.io/cluster-api-provider-azure/cloud/converters" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + "github.com/chlau-az/cluster-api-provider-azure/cloud/converters" ) const azureBuiltInContributorID = "b24988ac-6180-42a0-ab88-20f7382dd24c" diff --git a/cloud/services/virtualmachines/virtualmachines_test.go b/cloud/services/virtualmachines/virtualmachines_test.go index 0f1f424a000..e38a3035ca6 100644 --- a/cloud/services/virtualmachines/virtualmachines_test.go +++ b/cloud/services/virtualmachines/virtualmachines_test.go @@ -22,10 +22,10 @@ import ( "testing" . "github.com/onsi/gomega" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/networkinterfaces/mock_networkinterfaces" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicips/mock_publicips" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/roleassignments/mock_roleassignments" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/virtualmachines/mock_virtualmachines" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/networkinterfaces/mock_networkinterfaces" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicips/mock_publicips" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/roleassignments/mock_roleassignments" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/virtualmachines/mock_virtualmachines" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" @@ -36,8 +36,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) diff --git a/cloud/services/virtualnetworks/client.go b/cloud/services/virtualnetworks/client.go index 0d79c926df6..f7299fa70c8 100644 --- a/cloud/services/virtualnetworks/client.go +++ b/cloud/services/virtualnetworks/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/virtualnetworks/service.go b/cloud/services/virtualnetworks/service.go index abddc3b7304..ca5e4381ac6 100644 --- a/cloud/services/virtualnetworks/service.go +++ b/cloud/services/virtualnetworks/service.go @@ -17,7 +17,7 @@ limitations under the License. package virtualnetworks import ( - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" ) // Service provides operations on azure resources diff --git a/cloud/services/virtualnetworks/virtualnetworks.go b/cloud/services/virtualnetworks/virtualnetworks.go index 43a116201ce..dbb07157f6d 100644 --- a/cloud/services/virtualnetworks/virtualnetworks.go +++ b/cloud/services/virtualnetworks/virtualnetworks.go @@ -23,9 +23,9 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" - "sigs.k8s.io/cluster-api-provider-azure/cloud/converters" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + "github.com/chlau-az/cluster-api-provider-azure/cloud/converters" ) // Spec input specification for Get/CreateOrUpdate/Delete calls diff --git a/cloud/services/virtualnetworks/virtualnetworks_test.go b/cloud/services/virtualnetworks/virtualnetworks_test.go index 25e6181b71f..36d0d5707e6 100644 --- a/cloud/services/virtualnetworks/virtualnetworks_test.go +++ b/cloud/services/virtualnetworks/virtualnetworks_test.go @@ -24,7 +24,7 @@ import ( "testing" . "github.com/onsi/gomega" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/virtualnetworks/mock_virtualnetworks" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/virtualnetworks/mock_virtualnetworks" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" @@ -33,8 +33,8 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) diff --git a/config/default/manager_cloud_env_patch.yaml b/config/default/manager_cloud_env_patch.yaml index c42925ec22f..ee16064b6fa 100644 --- a/config/default/manager_cloud_env_patch.yaml +++ b/config/default/manager_cloud_env_patch.yaml @@ -13,4 +13,6 @@ spec: valueFrom: configMapKeyRef: name: manager-bootstrap-cloud-env - key: cloud-env \ No newline at end of file + key: cloud-env + - name: AZURE_ARM_ENDPOINT + value: "https://management.ppe2.stackpoc.com" \ No newline at end of file diff --git a/config/default/manager_credentials_patch.yaml b/config/default/manager_credentials_patch.yaml index 4388a83d351..e7d9c80d8df 100644 --- a/config/default/manager_credentials_patch.yaml +++ b/config/default/manager_credentials_patch.yaml @@ -29,3 +29,7 @@ spec: secretKeyRef: name: manager-bootstrap-credentials key: client-secret + - name: AZURE_GO_SDK_LOG_LEVEL + value: DEBUG + - name: AZURE_GO_SDK_LOG_FILE + value: /home/chlau/go/src/github.com/chlau-az/cluster-api-provider-azure/gosdk.log diff --git a/controllers/azurecluster_controller.go b/controllers/azurecluster_controller.go index 980ddb88390..807def5dbaa 100644 --- a/controllers/azurecluster_controller.go +++ b/controllers/azurecluster_controller.go @@ -36,9 +36,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" - "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/util/reconciler" ) // AzureClusterReconciler reconciles a AzureCluster object diff --git a/controllers/azurecluster_controller_test.go b/controllers/azurecluster_controller_test.go index 529c339b69c..ec505f6be83 100644 --- a/controllers/azurecluster_controller_test.go +++ b/controllers/azurecluster_controller_test.go @@ -29,10 +29,10 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/internal/test" - "sigs.k8s.io/cluster-api-provider-azure/internal/test/mock_log" - "sigs.k8s.io/cluster-api-provider-azure/internal/test/record" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/internal/test" + "github.com/chlau-az/cluster-api-provider-azure/internal/test/mock_log" + "github.com/chlau-az/cluster-api-provider-azure/internal/test/record" ) var _ = Describe("AzureClusterReconciler", func() { diff --git a/controllers/azurecluster_reconciler.go b/controllers/azurecluster_reconciler.go index 68378e04c33..45c4fd2aa03 100644 --- a/controllers/azurecluster_reconciler.go +++ b/controllers/azurecluster_reconciler.go @@ -26,18 +26,18 @@ import ( clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/availabilityzones" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/groups" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/internalloadbalancers" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicips" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicloadbalancers" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/routetables" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/securitygroups" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/subnets" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/virtualnetworks" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/availabilityzones" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/groups" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/internalloadbalancers" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicips" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicloadbalancers" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/routetables" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/securitygroups" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/subnets" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/virtualnetworks" ) // azureClusterReconciler is the reconciler called by the AzureCluster controller diff --git a/controllers/azuremachine_annotations.go b/controllers/azuremachine_annotations.go index e70192830f3..9ca8f91b9d6 100644 --- a/controllers/azuremachine_annotations.go +++ b/controllers/azuremachine_annotations.go @@ -19,7 +19,7 @@ package controllers import ( "encoding/json" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" ) // updateMachineAnnotationJSON updates the `annotation` on `machine` with diff --git a/controllers/azuremachine_controller.go b/controllers/azuremachine_controller.go index 90e1742cef9..b9503eeef3f 100644 --- a/controllers/azuremachine_controller.go +++ b/controllers/azuremachine_controller.go @@ -39,9 +39,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" - "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/util/reconciler" ) // AzureMachineReconciler reconciles a AzureMachine object diff --git a/controllers/azuremachine_controller_test.go b/controllers/azuremachine_controller_test.go index 97d59601100..5587fec7a73 100644 --- a/controllers/azuremachine_controller_test.go +++ b/controllers/azuremachine_controller_test.go @@ -28,9 +28,9 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/internal/test" - "sigs.k8s.io/cluster-api-provider-azure/internal/test/record" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/internal/test" + "github.com/chlau-az/cluster-api-provider-azure/internal/test/record" ) var _ = Describe("AzureMachineReconciler", func() { diff --git a/controllers/azuremachine_reconciler.go b/controllers/azuremachine_reconciler.go index 7a1c8bd9230..ed61f97933e 100644 --- a/controllers/azuremachine_reconciler.go +++ b/controllers/azuremachine_reconciler.go @@ -19,19 +19,19 @@ package controllers import ( "context" "encoding/base64" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicips" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicips" "time" "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/availabilityzones" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/disks" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/networkinterfaces" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/virtualmachines" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/availabilityzones" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/disks" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/networkinterfaces" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/virtualmachines" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/cluster-api/util" ) diff --git a/controllers/azuremachine_reconciler_test.go b/controllers/azuremachine_reconciler_test.go index a1cc4bd38b1..074dcfcdb20 100644 --- a/controllers/azuremachine_reconciler_test.go +++ b/controllers/azuremachine_reconciler_test.go @@ -17,14 +17,14 @@ limitations under the License. package controllers import ( - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" "testing" . "github.com/onsi/gomega" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/log" ) diff --git a/controllers/azuremachine_tags.go b/controllers/azuremachine_tags.go index 10ca8577484..99353abeb14 100644 --- a/controllers/azuremachine_tags.go +++ b/controllers/azuremachine_tags.go @@ -21,8 +21,8 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/virtualmachines" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/virtualmachines" ) const ( diff --git a/controllers/helpers.go b/controllers/helpers.go index 6d06636ad75..b9d3689affc 100644 --- a/controllers/helpers.go +++ b/controllers/helpers.go @@ -31,8 +31,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/handler" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/util/reconciler" ) // AzureClusterToAzureMachinesMapper creates a mapping handler to transform AzureClusters into AzureMachines. The transform diff --git a/controllers/helpers_test.go b/controllers/helpers_test.go index 4139a170fc8..7c0c49be11b 100644 --- a/controllers/helpers_test.go +++ b/controllers/helpers_test.go @@ -29,8 +29,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/handler" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/internal/test/mock_log" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/internal/test/mock_log" ) func TestAzureClusterToAzureMachinesMapper(t *testing.T) { diff --git a/controllers/suite_test.go b/controllers/suite_test.go index c2a56266e13..bec13255a3e 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/envtest/printer" - "sigs.k8s.io/cluster-api-provider-azure/internal/test/env" + "github.com/chlau-az/cluster-api-provider-azure/internal/test/env" // +kubebuilder:scaffold:imports ) diff --git a/docs/development.md b/docs/development.md index 97fbd86f436..b7dbaa697fd 100644 --- a/docs/development.md +++ b/docs/development.md @@ -338,7 +338,7 @@ You can optionally set the following variables: | Variable | Description | Default | |----------------------------|---------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------| -| `E2E_CONF_FILE` | The path of the [E2E configuration file](https://cluster-api.sigs.k8s.io/developer/e2e.html#defining-an-e2e-config-file). | `${GOPATH}/src/sigs.k8s.io/cluster-api-provider-azure/test/e2e/config/azure-dev.yaml` | +| `E2E_CONF_FILE` | The path of the [E2E configuration file](https://cluster-api.sigs.k8s.io/developer/e2e.html#defining-an-e2e-config-file). | `${GOPATH}/src/github.com/chlau-az/cluster-api-provider-azure/test/e2e/config/azure-dev.yaml` | | `SKIP_CLEANUP` | Set to `true` if you do not want the bootstrap and workload clusters to be cleaned up after running E2E tests. | `false` | | `SKIP_CREATE_MGMT_CLUSTER` | Skip management cluster creation. | `false` | diff --git a/exp/PROJECT b/exp/PROJECT index 1e99aa6fcd8..c1d2793e927 100644 --- a/exp/PROJECT +++ b/exp/PROJECT @@ -1,5 +1,5 @@ domain: exp.infrastructure.x-k8s.io -repo: sigs.k8s.io/cluster-api-provider-azure/exp +repo: github.com/chlau-az/cluster-api-provider-azure/exp version: "2" resources: - group: exp.infrastructure diff --git a/exp/api/v1alpha3/azuremachinepool_test.go b/exp/api/v1alpha3/azuremachinepool_test.go index 92c6afb82c0..0badbfe4395 100644 --- a/exp/api/v1alpha3/azuremachinepool_test.go +++ b/exp/api/v1alpha3/azuremachinepool_test.go @@ -21,8 +21,8 @@ import ( "github.com/onsi/gomega" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" ) func TestAzureMachinePool_Validate(t *testing.T) { diff --git a/exp/api/v1alpha3/azuremachinepool_types.go b/exp/api/v1alpha3/azuremachinepool_types.go index ca49e5f492b..ca574721fdf 100644 --- a/exp/api/v1alpha3/azuremachinepool_types.go +++ b/exp/api/v1alpha3/azuremachinepool_types.go @@ -20,7 +20,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/cluster-api/errors" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" ) type ( diff --git a/exp/api/v1alpha3/azuremachinepool_webhook.go b/exp/api/v1alpha3/azuremachinepool_webhook.go index 218a321f788..bb1d0485a52 100644 --- a/exp/api/v1alpha3/azuremachinepool_webhook.go +++ b/exp/api/v1alpha3/azuremachinepool_webhook.go @@ -24,7 +24,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" "sigs.k8s.io/controller-runtime/pkg/webhook" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" ) // log is for logging in this package. diff --git a/exp/api/v1alpha3/types.go b/exp/api/v1alpha3/types.go index 67fff704148..69f70d9c277 100644 --- a/exp/api/v1alpha3/types.go +++ b/exp/api/v1alpha3/types.go @@ -17,7 +17,7 @@ limitations under the License. package v1alpha3 import ( - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" ) type ( diff --git a/exp/api/v1alpha3/zz_generated.deepcopy.go b/exp/api/v1alpha3/zz_generated.deepcopy.go index 11a9ce1c844..9a1e8ff80fa 100644 --- a/exp/api/v1alpha3/zz_generated.deepcopy.go +++ b/exp/api/v1alpha3/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ package v1alpha3 import ( "k8s.io/apimachinery/pkg/runtime" - apiv1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + apiv1alpha3 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" "sigs.k8s.io/cluster-api/errors" ) diff --git a/exp/controllers/azuremachinepool_controller.go b/exp/controllers/azuremachinepool_controller.go index 05e069932a3..545ddc90a64 100644 --- a/exp/controllers/azuremachinepool_controller.go +++ b/exp/controllers/azuremachinepool_controller.go @@ -45,13 +45,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/scalesets" - "sigs.k8s.io/cluster-api-provider-azure/controllers" - infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/scalesets" + "github.com/chlau-az/cluster-api-provider-azure/controllers" + infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/util/reconciler" ) type ( diff --git a/exp/controllers/azuremachinepool_controller_test.go b/exp/controllers/azuremachinepool_controller_test.go index 8a8cc776971..8a8d0eb7562 100644 --- a/exp/controllers/azuremachinepool_controller_test.go +++ b/exp/controllers/azuremachinepool_controller_test.go @@ -25,7 +25,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" - infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" + infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" ) var _ = Describe("AzureMachinePoolReconciler", func() { diff --git a/exp/controllers/azuremachinepool_controller_unit_test.go b/exp/controllers/azuremachinepool_controller_unit_test.go index 68609f85564..054fae4a66f 100644 --- a/exp/controllers/azuremachinepool_controller_unit_test.go +++ b/exp/controllers/azuremachinepool_controller_unit_test.go @@ -33,12 +33,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/scalesets/mock_scalesets" - infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/internal/test/matchers" - "sigs.k8s.io/cluster-api-provider-azure/internal/test/mock_log" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/scalesets/mock_scalesets" + infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/internal/test/matchers" + "github.com/chlau-az/cluster-api-provider-azure/internal/test/mock_log" ) func Test_machinePoolToInfrastructureMapFunc(t *testing.T) { diff --git a/exp/controllers/azuremanagedcluster_controller.go b/exp/controllers/azuremanagedcluster_controller.go index 1c2b28c8feb..10364fb7010 100644 --- a/exp/controllers/azuremanagedcluster_controller.go +++ b/exp/controllers/azuremanagedcluster_controller.go @@ -37,8 +37,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" + infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/util/reconciler" ) // AzureManagedClusterReconciler reconciles a AzureManagedCluster object diff --git a/exp/controllers/azuremanagedmachinepool_controller.go b/exp/controllers/azuremanagedmachinepool_controller.go index 4291f97615d..791853dd9c3 100644 --- a/exp/controllers/azuremanagedmachinepool_controller.go +++ b/exp/controllers/azuremanagedmachinepool_controller.go @@ -37,10 +37,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" - infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/util/reconciler" ) // AzureManagedMachinePoolReconciler reconciles a AzureManagedMachinePool object diff --git a/exp/controllers/azuremanagedmachinepool_reconciler.go b/exp/controllers/azuremanagedmachinepool_reconciler.go index c933f1346f2..1d5bf0b400b 100644 --- a/exp/controllers/azuremanagedmachinepool_reconciler.go +++ b/exp/controllers/azuremanagedmachinepool_reconciler.go @@ -22,10 +22,10 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/pkg/errors" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/agentpools" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/scalesets" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/agentpools" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/scalesets" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/exp/controllers/azuremangedcontrolplane_controller.go b/exp/controllers/azuremangedcontrolplane_controller.go index e5979dc2e9c..1060fa22499 100644 --- a/exp/controllers/azuremangedcontrolplane_controller.go +++ b/exp/controllers/azuremangedcontrolplane_controller.go @@ -36,10 +36,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" - infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/util/reconciler" ) // AzureManagedControlPlaneReconciler reconciles a AzureManagedControlPlane object diff --git a/exp/controllers/azuremangedcontrolplane_reconciler.go b/exp/controllers/azuremangedcontrolplane_reconciler.go index dc5b3664db5..467ab4a3c11 100644 --- a/exp/controllers/azuremangedcontrolplane_reconciler.go +++ b/exp/controllers/azuremangedcontrolplane_reconciler.go @@ -25,10 +25,10 @@ import ( "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" - "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/managedclusters" - infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "github.com/chlau-az/cluster-api-provider-azure/cloud/services/managedclusters" + infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/cluster-api/util/secret" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/exp/controllers/helpers.go b/exp/controllers/helpers.go index 3244b38a43b..810b38b3135 100644 --- a/exp/controllers/helpers.go +++ b/exp/controllers/helpers.go @@ -33,10 +33,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/controllers" - infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/controllers" + infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/util/reconciler" ) // AzureClusterToAzureMachinePoolsMapper creates a mapping handler to transform AzureClusters into AzureMachinePools. The transform diff --git a/exp/controllers/helpers_test.go b/exp/controllers/helpers_test.go index 475c8094b06..73057ba7e54 100644 --- a/exp/controllers/helpers_test.go +++ b/exp/controllers/helpers_test.go @@ -31,9 +31,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/internal/test/mock_log" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/internal/test/mock_log" ) func TestAzureClusterToAzureMachinePoolsMapper(t *testing.T) { diff --git a/exp/controllers/suite_test.go b/exp/controllers/suite_test.go index 30641a6f7b6..0d3e1dd8cb8 100644 --- a/exp/controllers/suite_test.go +++ b/exp/controllers/suite_test.go @@ -33,8 +33,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" // +kubebuilder:scaffold:imports ) diff --git a/go.mod b/go.mod index 59d4a2afbe5..2c82d2b66e2 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,11 @@ -module sigs.k8s.io/cluster-api-provider-azure +module github.com/chlau-az/cluster-api-provider-azure go 1.13 require ( github.com/Azure/azure-sdk-for-go v43.2.0+incompatible github.com/Azure/go-autorest/autorest v0.10.2 + github.com/Azure/go-autorest/autorest/adal v0.8.2 github.com/Azure/go-autorest/autorest/azure/auth v0.4.2 github.com/Azure/go-autorest/autorest/to v0.3.0 github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect diff --git a/gosdk.log b/gosdk.log new file mode 100644 index 00000000000..e69de29bb2d diff --git a/hack/boilerplate/test/BUILD b/hack/boilerplate/test/BUILD index b3644d620e9..fc79a1a726f 100644 --- a/hack/boilerplate/test/BUILD +++ b/hack/boilerplate/test/BUILD @@ -11,7 +11,7 @@ go_library( "fail.go", "pass.go", ], - importpath = "sigs.k8s.io/cluster-api-provider-azure/hack/boilerplate/test", + importpath = "github.com/chlau-az/cluster-api-provider-azure/hack/boilerplate/test", ) filegroup( diff --git a/hack/version.sh b/hack/version.sh index a5e57c419d6..86ee81ca856 100755 --- a/hack/version.sh +++ b/hack/version.sh @@ -77,7 +77,7 @@ version::ldflags() { local key=${1} local val=${2} ldflags+=( - "-X 'sigs.k8s.io/cluster-api-provider-azure/version.${key}=${val}'" + "-X 'github.com/chlau-az/cluster-api-provider-azure/version.${key}=${val}'" ) } diff --git a/internal/test/env/env.go b/internal/test/env/env.go index 3587fd109f2..776d8086414 100644 --- a/internal/test/env/env.go +++ b/internal/test/env/env.go @@ -43,10 +43,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/manager" - infrav1alpha2 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha2" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/internal/test/record" + infrav1alpha2 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha2" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/internal/test/record" ) var ( diff --git a/main.go b/main.go index e3b4b911c54..8bd545213ea 100644 --- a/main.go +++ b/main.go @@ -35,13 +35,13 @@ import ( capifeature "sigs.k8s.io/cluster-api/feature" - infrav1alpha2 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha2" - infrav1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-azure/controllers" - infrav1alpha3exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" - infrav1controllersexp "sigs.k8s.io/cluster-api-provider-azure/exp/controllers" - "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" - version "sigs.k8s.io/cluster-api-provider-azure/version" + infrav1alpha2 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha2" + infrav1alpha3 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + "github.com/chlau-az/cluster-api-provider-azure/controllers" + infrav1alpha3exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" + infrav1controllersexp "github.com/chlau-az/cluster-api-provider-azure/exp/controllers" + "github.com/chlau-az/cluster-api-provider-azure/util/reconciler" + version "github.com/chlau-az/cluster-api-provider-azure/version" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" clusterv1exp "sigs.k8s.io/cluster-api/exp/api/v1alpha3" @@ -50,7 +50,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/healthz" - "sigs.k8s.io/cluster-api-provider-azure/feature" + "github.com/chlau-az/cluster-api-provider-azure/feature" ) var ( diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 6109bb516eb..ce7db7b2f1d 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -32,7 +32,7 @@ import ( "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/runtime" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/bootstrap" "sigs.k8s.io/cluster-api/test/framework/clusterctl" diff --git a/util/reconciler/defaults_test.go b/util/reconciler/defaults_test.go index 70834a2a1cf..3dfa8b64de8 100644 --- a/util/reconciler/defaults_test.go +++ b/util/reconciler/defaults_test.go @@ -22,7 +22,7 @@ import ( "github.com/onsi/gomega" - "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" + "github.com/chlau-az/cluster-api-provider-azure/util/reconciler" ) func TestDefaultedTimeout(t *testing.T) { From 0b587af9db03ae8310cfab240ceacf3d744d23c5 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Tue, 7 Jul 2020 22:26:20 -0700 Subject: [PATCH 06/47] Fixed runtime errors for azure cluster, network --- cloud/scope/clients.go | 10 ++-- cloud/scope/cluster.go | 6 +-- cloud/services/groups/client.go | 2 - .../internalloadbalancers.go | 52 +++++++++++-------- cloud/services/publicips/publicips.go | 4 +- .../publicloadbalancers.go | 7 ++- .../virtualmachines/virtualmachines.go | 6 +-- config/default/cloud_environment.yaml | 3 +- config/default/manager_cloud_env_patch.yaml | 7 ++- go.mod | 1 + go.sum | 14 +++++ templates/cluster-template.yaml | 2 + 12 files changed, 73 insertions(+), 41 deletions(-) diff --git a/cloud/scope/clients.go b/cloud/scope/clients.go index fce33db7029..3bbe7ff0857 100644 --- a/cloud/scope/clients.go +++ b/cloud/scope/clients.go @@ -58,23 +58,27 @@ func (c *AzureClients) setCredentials(subscriptionID string) error { c.ClientID = os.Getenv("AZURE_CLIENT_ID") c.ClientSecret = os.Getenv("AZURE_CLIENT_SECRET") c.TenantID = os.Getenv("AZURE_TENANT_ID") + log.Println("HERE client id: ", c.ClientID) + log.Println("HERE client secret: ", c.ClientSecret) + log.Println("HERE tenant id: ", c.TenantID) + log.Println("HERE subscription id: ", subID) settings, err := auth.GetSettingsFromEnvironment() if err != nil { return err } // To do: get arm endpoint in helper method - log.Println("HERE changing environment") armEndpoint := os.Getenv("AZURE_ARM_ENDPOINT") - log.Println("HERE armEndpoint: ", armEndpoint) settings.Environment, _ = azure.EnvironmentFromURL(armEndpoint) c.ResourceManagerEndpoint = settings.Environment.ResourceManagerEndpoint - log.Println("HERE resource manager endpoint: ", c.ResourceManagerEndpoint) c.ResourceManagerVMDNSSuffix = GetAzureDNSZoneForEnvironment(settings.Environment.Name) settings.Values[auth.SubscriptionID] = subscriptionID // c.Authorizer, err = settings.GetAuthorizer() c.Authorizer, err = c.getAuthorizerForResource(settings.Environment) + + log.Println("HERE armEndpoint: ", armEndpoint) + log.Println("HERE resource manager endpoint: ", c.ResourceManagerEndpoint) log.Println("HERE c.Authorizer: ", c.Authorizer, "err: ", err) return err } diff --git a/cloud/scope/cluster.go b/cloud/scope/cluster.go index a89d42cfb3c..ae29330e4b0 100644 --- a/cloud/scope/cluster.go +++ b/cloud/scope/cluster.go @@ -19,14 +19,13 @@ package scope import ( "context" "fmt" - "log" "github.com/Azure/go-autorest/autorest" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" "github.com/go-logr/logr" "github.com/pkg/errors" "k8s.io/klog/klogr" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/controller-runtime/pkg/client" @@ -93,7 +92,6 @@ func (s *ClusterScope) SubscriptionID() string { // BaseURI returns the Azure ResourceManagerEndpoint. func (s *ClusterScope) BaseURI() string { - log.Println("HERE BaseURI s.ResourceManagerEndpoint: ", s.ResourceManagerEndpoint) return s.ResourceManagerEndpoint } diff --git a/cloud/services/groups/client.go b/cloud/services/groups/client.go index 4bbdefc1f8b..c1bffb00d88 100644 --- a/cloud/services/groups/client.go +++ b/cloud/services/groups/client.go @@ -18,7 +18,6 @@ package groups import ( "context" - "log" "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/resources/mgmt/resources" "github.com/Azure/go-autorest/autorest" @@ -47,7 +46,6 @@ func NewClient(auth azure.Authorizer) *AzureClient { // newGroupsClient creates a new groups client from subscription ID. func newGroupsClient(subscriptionID string, baseURI string, authorizer autorest.Authorizer) resources.GroupsClient { - log.Println("HERE creating new group client with baseuri") groupsClient := resources.NewGroupsClientWithBaseURI(baseURI, subscriptionID) groupsClient.Authorizer = authorizer groupsClient.AddToUserAgent(azure.UserAgent()) diff --git a/cloud/services/internalloadbalancers/internalloadbalancers.go b/cloud/services/internalloadbalancers/internalloadbalancers.go index e090ae30bd4..c2db31fb190 100644 --- a/cloud/services/internalloadbalancers/internalloadbalancers.go +++ b/cloud/services/internalloadbalancers/internalloadbalancers.go @@ -19,13 +19,14 @@ package internalloadbalancers import ( "context" "fmt" + "log" "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest/to" - "github.com/pkg/errors" - "k8s.io/klog" infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + "github.com/pkg/errors" + "k8s.io/klog" ) // Spec specification for internal load balancer @@ -44,7 +45,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { return errors.New("invalid internal load balancer specification") } klog.V(2).Infof("creating internal load balancer %s", internalLBSpec.Name) - probeName := "HTTPSProbe" + // probeName := "HTTPSProbe" frontEndIPConfigName := "controlplane-internal-lbFrontEnd" backEndAddressPoolName := "controlplane-internal-backEndPool" idPrefix := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers", s.Scope.SubscriptionID(), s.Scope.ResourceGroup()) @@ -59,10 +60,14 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { } } else if azure.ResourceNotFound(err) { klog.V(2).Infof("internalLB %s not found in RG %s", internalLBSpec.Name, s.Scope.ResourceGroup()) - privateIP, err = s.getAvailablePrivateIP(ctx, s.Scope.Vnet().ResourceGroup, internalLBSpec.VnetName, internalLBSpec.SubnetCidr, internalLBSpec.IPAddress) - if err != nil { - return err - } + log.Println("HERE getting resource internal lb not found") + privateIP = "10.0.0.100" + /* + privateIP, err = s.getAvailablePrivateIP(ctx, s.Scope.Vnet().ResourceGroup, internalLBSpec.VnetName, internalLBSpec.SubnetCidr, internalLBSpec.IPAddress) + if err != nil { + return err + } + */ klog.V(2).Infof("setting internal load balancer IP to %s", privateIP) } else { return errors.Wrap(err, "failed to look for existing internal LB") @@ -81,7 +86,8 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { s.Scope.ResourceGroup(), lbName, network.LoadBalancer{ - Sku: &network.LoadBalancerSku{Name: network.LoadBalancerSkuNameStandard}, + Sku: &network.LoadBalancerSku{Name: network.LoadBalancerSkuNameBasic}, + // Sku: &network.LoadBalancerSku{Name: network.LoadBalancerSkuNameStandard}, Location: to.StringPtr(s.Scope.Location()), LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{ FrontendIPConfigurations: &[]network.FrontendIPConfiguration{ @@ -99,18 +105,20 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { Name: &backEndAddressPoolName, }, }, - Probes: &[]network.Probe{ - { - Name: &probeName, - ProbePropertiesFormat: &network.ProbePropertiesFormat{ - Protocol: "Https", - RequestPath: to.StringPtr("/healthz"), - Port: to.Int32Ptr(s.Scope.APIServerPort()), - IntervalInSeconds: to.Int32Ptr(15), - NumberOfProbes: to.Int32Ptr(4), + /* + Probes: &[]network.Probe{ + { + Name: &probeName, + ProbePropertiesFormat: &network.ProbePropertiesFormat{ + Protocol: "Http", + RequestPath: to.StringPtr("/healthz"), + Port: to.Int32Ptr(s.Scope.APIServerPort()), + IntervalInSeconds: to.Int32Ptr(15), + NumberOfProbes: to.Int32Ptr(4), + }, }, }, - }, + */ LoadBalancingRules: &[]network.LoadBalancingRule{ { Name: to.StringPtr("LBRuleHTTPS"), @@ -127,9 +135,11 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { BackendAddressPool: &network.SubResource{ ID: to.StringPtr(fmt.Sprintf("/%s/%s/backendAddressPools/%s", idPrefix, lbName, backEndAddressPoolName)), }, - Probe: &network.SubResource{ - ID: to.StringPtr(fmt.Sprintf("/%s/%s/probes/%s", idPrefix, lbName, probeName)), - }, + /* + Probe: &network.SubResource{ + ID: to.StringPtr(fmt.Sprintf("/%s/%s/probes/%s", idPrefix, lbName, probeName)), + }, + */ }, }, }, diff --git a/cloud/services/publicips/publicips.go b/cloud/services/publicips/publicips.go index 9c9f83d2704..11a027f072a 100644 --- a/cloud/services/publicips/publicips.go +++ b/cloud/services/publicips/publicips.go @@ -22,9 +22,9 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest/to" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" "github.com/pkg/errors" "k8s.io/klog" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" ) // Reconcile gets/creates/updates a public ip. @@ -37,7 +37,7 @@ func (s *Service) Reconcile(ctx context.Context) error { s.Scope.ResourceGroup(), ip.Name, network.PublicIPAddress{ - Sku: &network.PublicIPAddressSku{Name: network.PublicIPAddressSkuNameStandard}, + Sku: &network.PublicIPAddressSku{Name: network.PublicIPAddressSkuNameBasic}, Name: to.StringPtr(ip.Name), Location: to.StringPtr(s.Scope.Location()), PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ diff --git a/cloud/services/publicloadbalancers/publicloadbalancers.go b/cloud/services/publicloadbalancers/publicloadbalancers.go index bce585c08ff..0117f6475fc 100644 --- a/cloud/services/publicloadbalancers/publicloadbalancers.go +++ b/cloud/services/publicloadbalancers/publicloadbalancers.go @@ -23,11 +23,11 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest/to" - "github.com/pkg/errors" - "k8s.io/klog" infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" azure "github.com/chlau-az/cluster-api-provider-azure/cloud" "github.com/chlau-az/cluster-api-provider-azure/cloud/converters" + "github.com/pkg/errors" + "k8s.io/klog" ) // Spec specification for public load balancer @@ -61,10 +61,9 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { return errors.Wrap(err, "failed to look for existing public IP") } klog.V(2).Infof("successfully got public ip %s", publicLBSpec.PublicIPName) - log.Println("HI HERE RECONCILING") lb := network.LoadBalancer{ - Sku: &network.LoadBalancerSku{Name: network.LoadBalancerSkuNameStandard}, + Sku: &network.LoadBalancerSku{Name: network.LoadBalancerSkuNameBasic}, Location: to.StringPtr(s.Scope.Location()), Tags: converters.TagsToMap(infrav1.Build(infrav1.BuildParams{ ClusterName: s.Scope.ClusterName(), diff --git a/cloud/services/virtualmachines/virtualmachines.go b/cloud/services/virtualmachines/virtualmachines.go index 28a8bb3fb61..c6c80c865de 100644 --- a/cloud/services/virtualmachines/virtualmachines.go +++ b/cloud/services/virtualmachines/virtualmachines.go @@ -26,13 +26,13 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/authorization/mgmt/authorization" "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest/to" + infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + "github.com/chlau-az/cluster-api-provider-azure/cloud/converters" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/klog" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" - "github.com/chlau-az/cluster-api-provider-azure/cloud/converters" ) const azureBuiltInContributorID = "b24988ac-6180-42a0-ab88-20f7382dd24c" diff --git a/config/default/cloud_environment.yaml b/config/default/cloud_environment.yaml index f5db8af0b29..8d77e843a86 100644 --- a/config/default/cloud_environment.yaml +++ b/config/default/cloud_environment.yaml @@ -4,4 +4,5 @@ metadata: name: manager-bootstrap-cloud-env namespace: system data: - cloud-env: ${AZURE_ENVIRONMENT} \ No newline at end of file + cloud-env: ${AZURE_ENVIRONMENT} + cloud-arm-endpoint: ${AZURE_ARM_ENDPOINT} \ No newline at end of file diff --git a/config/default/manager_cloud_env_patch.yaml b/config/default/manager_cloud_env_patch.yaml index ee16064b6fa..12edb4e79bc 100644 --- a/config/default/manager_cloud_env_patch.yaml +++ b/config/default/manager_cloud_env_patch.yaml @@ -15,4 +15,9 @@ spec: name: manager-bootstrap-cloud-env key: cloud-env - name: AZURE_ARM_ENDPOINT - value: "https://management.ppe2.stackpoc.com" \ No newline at end of file + value: "https://management.ppe2.stackpoc.com" + + # valueFrom: + # configMapKeyRef: + # name: manager-bootstrap-cloud-env + # key: cloud-arm-endpoint \ No newline at end of file diff --git a/go.mod b/go.mod index 2c82d2b66e2..ea8b7eb811c 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( k8s.io/klog v1.0.0 k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab sigs.k8s.io/cluster-api v0.3.7-alpha.0 + sigs.k8s.io/cluster-api-provider-azure v0.4.5 sigs.k8s.io/controller-runtime v0.5.6 ) diff --git a/go.sum b/go.sum index 0f75b4db0c0..5d5e06db9e3 100644 --- a/go.sum +++ b/go.sum @@ -389,6 +389,7 @@ github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= @@ -410,6 +411,7 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.5.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -516,6 +518,7 @@ golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -589,6 +592,7 @@ golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -626,6 +630,7 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72 h1:bw9doJza/SFBEweII/rHQh338oozWyiFsBRHtrflcws= @@ -644,6 +649,7 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -725,6 +731,8 @@ k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= +k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c h1:/KUFqjjqAcY4Us6luF5RDNZ16KJtb49HfR3ZHB9qYXM= +k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab h1:I3f2hcBrepGRXI1z4sukzAb8w1R4eqbsHrAsx06LGYM= @@ -736,14 +744,20 @@ modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/cluster-api v0.3.6/go.mod h1:joh0d0Xu2VGQa3knsf2ZIHyOLX7puUp8LvJjyneeCb8= sigs.k8s.io/cluster-api v0.3.7-alpha.0 h1:sWeItWZdGoUxE+QwSPde9PEHzzTqz4ycPAbMqdJh7Kg= sigs.k8s.io/cluster-api v0.3.7-alpha.0/go.mod h1:gMLtsBVxRQOGHlj8uVG6qmlae8zfTiP8pVoGlX/vofs= +sigs.k8s.io/cluster-api-provider-azure v0.4.5 h1:K6EfLZ6rgvd+XijJuzCUyim58HUzZo8VQPeLp5gbsVk= +sigs.k8s.io/cluster-api-provider-azure v0.4.5/go.mod h1:tqQmT5BQfakjPq12GB/3OLC8EzPeOfKPnwh3VQbZTnc= +sigs.k8s.io/controller-runtime v0.5.2/go.mod h1:JZUwSMVbxDupo0lTJSSFP5pimEyxGynROImSsqIOx1A= +sigs.k8s.io/controller-runtime v0.5.5/go.mod h1:JZUwSMVbxDupo0lTJSSFP5pimEyxGynROImSsqIOx1A= sigs.k8s.io/controller-runtime v0.5.6 h1:84kdvjIfX6N/ExK5g99dnuuB5SeUsC798q6BfSzM7O0= sigs.k8s.io/controller-runtime v0.5.6/go.mod h1:JZUwSMVbxDupo0lTJSSFP5pimEyxGynROImSsqIOx1A= sigs.k8s.io/kind v0.7.1-0.20200303021537-981bd80d3802 h1:L6/8hETA7jvdx3xBcbDifrIN2xaYHE7tA58n+Kdp2Zw= sigs.k8s.io/kind v0.7.1-0.20200303021537-981bd80d3802/go.mod h1:HIZ3PWUezpklcjkqpFbnYOqaqsAE1JeCTEwkgvPLXjk= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index 0e9cff846bc..a6495e56d3c 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -112,6 +112,8 @@ metadata: spec: template: spec: + # image: + # id: https://capi1593538393.blob.core.windows.net/system/Microsoft.Compute/Images/cluster-api-vhds/capi-1593538415-osDisk.c4fa6c28-d876-443c-8415-b321dae95b44.vhd?st=2020-06-29T00%3A00Z&se=2021-06-30T00%3A00Z&sp=rl&sv=2018-11-09&sr=c&sig=y4S1vjKbexw1iMsyr0oK9SksIiLmBIpUD48j9n%2BoIaY%3D location: ${AZURE_LOCATION} osDisk: diskSizeGB: 128 From 185f181633513e6b73aaf02206f29670d1bf1647 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Wed, 8 Jul 2020 11:29:04 -0700 Subject: [PATCH 07/47] Set image to be raw ubuntu --- .../internalloadbalancers/internalloadbalancers.go | 2 -- cloud/services/publicloadbalancers/publicloadbalancers.go | 3 --- templates/cluster-template.yaml | 8 ++++++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/cloud/services/internalloadbalancers/internalloadbalancers.go b/cloud/services/internalloadbalancers/internalloadbalancers.go index c2db31fb190..125ebe259be 100644 --- a/cloud/services/internalloadbalancers/internalloadbalancers.go +++ b/cloud/services/internalloadbalancers/internalloadbalancers.go @@ -19,7 +19,6 @@ package internalloadbalancers import ( "context" "fmt" - "log" "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest/to" @@ -60,7 +59,6 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { } } else if azure.ResourceNotFound(err) { klog.V(2).Infof("internalLB %s not found in RG %s", internalLBSpec.Name, s.Scope.ResourceGroup()) - log.Println("HERE getting resource internal lb not found") privateIP = "10.0.0.100" /* privateIP, err = s.getAvailablePrivateIP(ctx, s.Scope.Vnet().ResourceGroup, internalLBSpec.VnetName, internalLBSpec.SubnetCidr, internalLBSpec.IPAddress) diff --git a/cloud/services/publicloadbalancers/publicloadbalancers.go b/cloud/services/publicloadbalancers/publicloadbalancers.go index 0117f6475fc..c36211396d0 100644 --- a/cloud/services/publicloadbalancers/publicloadbalancers.go +++ b/cloud/services/publicloadbalancers/publicloadbalancers.go @@ -19,7 +19,6 @@ package publicloadbalancers import ( "context" "fmt" - "log" "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest/to" @@ -147,12 +146,10 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { err = s.Client.CreateOrUpdate(ctx, s.Scope.ResourceGroup(), lbName, lb) if err != nil { - log.Println("cannot create public load balancer") return errors.Wrap(err, "cannot create public load balancer") } klog.V(2).Infof("successfully created public load balancer %s", lbName) - log.Println("sucessfully created public load balancer") return nil } diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index a6495e56d3c..682b1d48e51 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -112,8 +112,12 @@ metadata: spec: template: spec: - # image: - # id: https://capi1593538393.blob.core.windows.net/system/Microsoft.Compute/Images/cluster-api-vhds/capi-1593538415-osDisk.c4fa6c28-d876-443c-8415-b321dae95b44.vhd?st=2020-06-29T00%3A00Z&se=2021-06-30T00%3A00Z&sp=rl&sv=2018-11-09&sr=c&sig=y4S1vjKbexw1iMsyr0oK9SksIiLmBIpUD48j9n%2BoIaY%3D + image: + marketplace: + publisher: Canonical + offer: UbuntuServer + sku: 16.04-LTS + version: latest location: ${AZURE_LOCATION} osDisk: diskSizeGB: 128 From 353e0cc9742ba1720de8b882c4fd527b5a68daaf Mon Sep 17 00:00:00 2001 From: chlau-az Date: Fri, 10 Jul 2020 13:50:49 -0700 Subject: [PATCH 08/47] Implemented ADFS working environment --- .dockerignore | 1 + Dockerfile | 8 +++- Makefile | 2 +- Tiltfile | 3 +- cloud/scope/clients.go | 49 ++++++++++++++++++--- config/default/manager_cloud_env_patch.yaml | 2 +- config/manager/manager_image_patch.yaml | 2 +- config/manager/manager_pull_policy.yaml | 2 +- tilt-provider.json | 2 +- 9 files changed, 56 insertions(+), 15 deletions(-) diff --git a/.dockerignore b/.dockerignore index 6a029f49b77..142c69af9d3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,3 +13,4 @@ !/main.go !/go.mod !/go.sum +!/azs.crt \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 7a7ca05d911..36a98109915 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ # limitations under the License. # Build the manager binary -FROM golang:1.13.8 as builder +FROM golang:1.14 as builder WORKDIR /workspace # Run this with docker build --build_arg $(go env GOPROXY) to override the goproxy @@ -22,7 +22,7 @@ ENV GOPROXY=$goproxy # Copy the Go Modules manifests COPY go.mod go.mod -COPY go.sum go.sum +# COPY go.sum go.sum # Cache deps before building and copying source so that we don't need to re-download as much # and so that source changes don't invalidate our downloaded layer RUN go mod download @@ -36,9 +36,13 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \ go build -a -ldflags '-extldflags "-static"' \ -o manager . +COPY azs.crt /usr/local/share/ca-certificates/ +RUN update-ca-certificates + # Copy the controller-manager into a thin image FROM gcr.io/distroless/static:latest WORKDIR / +COPY --from=builder /etc/ssl/certs /etc/ssl/certs COPY --from=builder /workspace/manager . USER nobody ENTRYPOINT ["/manager"] diff --git a/Makefile b/Makefile index 8a8e24dbbf9..909dc78fb1d 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ REGISTRY ?= gcr.io/$(shell gcloud config get-value project) STAGING_REGISTRY := gcr.io/k8s-staging-cluster-api-azure PROD_REGISTRY := us.gcr.io/k8s-artifacts-prod/cluster-api-azure IMAGE_NAME ?= cluster-api-azure-controller -CONTROLLER_IMG ?= $(REGISTRY)/$(IMAGE_NAME) +CONTROLLER_IMG ?= $(REGISTRY)$(IMAGE_NAME) TAG ?= dev ARCH ?= amd64 ALL_ARCH = amd64 arm arm64 ppc64le s390x diff --git a/Tiltfile b/Tiltfile index c9523fbf5ef..c20b0e03ca1 100644 --- a/Tiltfile +++ b/Tiltfile @@ -180,7 +180,8 @@ def capz(): # Set up an image build for the provider. The live update configuration syncs the output from the local_resource # build into the container. docker_build( - ref = "gcr.io/k8s-staging-cluster-api-azure/cluster-api-azure-controller", + ref = "gcr.io/cluster-api-azure-controller-amd64:dev", + # ref = "gcr.io/k8s-staging-cluster-api-azure/cluster-api-azure-controller", context = "./.tiltbuild/", dockerfile_contents = dockerfile_contents, target = "tilt", diff --git a/cloud/scope/clients.go b/cloud/scope/clients.go index 3bbe7ff0857..390176b0455 100644 --- a/cloud/scope/clients.go +++ b/cloud/scope/clients.go @@ -50,35 +50,70 @@ type AzureClients struct { } func (c *AzureClients) setCredentials(subscriptionID string) error { + // log.Println("HI PRINTING DIRECTORY") + // // DIR + // files, er := ioutil.ReadDir("/etc/ssl/certs") + // if er != nil { + // log.Fatal(er) + // } + // for _, f := range files { + // log.Println(f.Name()) + // } + // log.Println("HI finished printing directory") + + // // CURL + // resp, er := http.Get("https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/metadata/endpoints?api-version=2015-01-01") + // if er != nil { + // log.Printf("HI ERROR: %s", er) + // } + // defer resp.Body.Close() + // body, er := ioutil.ReadAll(resp.Body) + // log.Println(string(body)) + + // log.Println("HI finished curling") + subID, err := getSubscriptionID(subscriptionID) if err != nil { return err } c.SubscriptionID = subID + + c.ClientID = "huey" + c.ClientSecret = "dewey" + c.TenantID = "louie" + log.Println("HERE 0client id: ", c.ClientID) + log.Println("HERE 0client secret: ", c.ClientSecret) + log.Println("HERE 0tenant id: ", c.TenantID) + log.Println("HERE 0subscription id: ", c.SubscriptionID) + c.ClientID = os.Getenv("AZURE_CLIENT_ID") c.ClientSecret = os.Getenv("AZURE_CLIENT_SECRET") c.TenantID = os.Getenv("AZURE_TENANT_ID") log.Println("HERE client id: ", c.ClientID) log.Println("HERE client secret: ", c.ClientSecret) log.Println("HERE tenant id: ", c.TenantID) - log.Println("HERE subscription id: ", subID) + log.Println("HERE subscription id: ", c.SubscriptionID) settings, err := auth.GetSettingsFromEnvironment() if err != nil { - return err + log.Println("HERE couldn't find environment") + // return err } // To do: get arm endpoint in helper method armEndpoint := os.Getenv("AZURE_ARM_ENDPOINT") - settings.Environment, _ = azure.EnvironmentFromURL(armEndpoint) + log.Println("HERE armEndpoint: ", armEndpoint) + settings.Environment, err = azure.EnvironmentFromURL(armEndpoint) + if err != nil { + log.Println("HERE error getting environment from armEndpoint: ", armEndpoint) + return err + } + log.Println("HERE resource manager endpoint: ", c.ResourceManagerEndpoint) c.ResourceManagerEndpoint = settings.Environment.ResourceManagerEndpoint c.ResourceManagerVMDNSSuffix = GetAzureDNSZoneForEnvironment(settings.Environment.Name) settings.Values[auth.SubscriptionID] = subscriptionID // c.Authorizer, err = settings.GetAuthorizer() c.Authorizer, err = c.getAuthorizerForResource(settings.Environment) - - log.Println("HERE armEndpoint: ", armEndpoint) - log.Println("HERE resource manager endpoint: ", c.ResourceManagerEndpoint) log.Println("HERE c.Authorizer: ", c.Authorizer, "err: ", err) return err } @@ -122,7 +157,7 @@ func (c *AzureClients) getAuthorizerForResource(env azure.Environment) (autorest log.Println("HERE TokenAudience: ", env.TokenAudience) log.Println("HERE ActiveDirectoryEndpoint: ", env.ActiveDirectoryEndpoint) oauthConfig, err = adal.NewOAuthConfig( - env.ActiveDirectoryEndpoint, c.TenantID) + env.ActiveDirectoryEndpoint, "adfs") if err != nil { return nil, err diff --git a/config/default/manager_cloud_env_patch.yaml b/config/default/manager_cloud_env_patch.yaml index 12edb4e79bc..e98d5e787fb 100644 --- a/config/default/manager_cloud_env_patch.yaml +++ b/config/default/manager_cloud_env_patch.yaml @@ -15,7 +15,7 @@ spec: name: manager-bootstrap-cloud-env key: cloud-env - name: AZURE_ARM_ENDPOINT - value: "https://management.ppe2.stackpoc.com" + value: "https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com" # valueFrom: # configMapKeyRef: diff --git a/config/manager/manager_image_patch.yaml b/config/manager/manager_image_patch.yaml index 0876a1db40d..e7e66801dfb 100644 --- a/config/manager/manager_image_patch.yaml +++ b/config/manager/manager_image_patch.yaml @@ -8,5 +8,5 @@ spec: spec: containers: # Change the value of image field below to your controller image URL - - image: gcr.io/k8s-staging-cluster-api-azure/cluster-api-azure-controller:latest + - image: gcr.io/cluster-api-azure-controller-amd64:dev name: manager diff --git a/config/manager/manager_pull_policy.yaml b/config/manager/manager_pull_policy.yaml index 74a0879c604..cd7ae12c01e 100644 --- a/config/manager/manager_pull_policy.yaml +++ b/config/manager/manager_pull_policy.yaml @@ -8,4 +8,4 @@ spec: spec: containers: - name: manager - imagePullPolicy: Always + imagePullPolicy: IfNotPresent diff --git a/tilt-provider.json b/tilt-provider.json index 4684dc5d6f1..61a06d90133 100644 --- a/tilt-provider.json +++ b/tilt-provider.json @@ -1,7 +1,7 @@ { "name": "azure", "config": { - "image": "gcr.io/k8s-staging-cluster-api-azure/cluster-api-azure-controller", + "image": "gcr.io/cluster-api-azure-controller-amd64:dev", "live_reload_deps": [ "main.go", "go.mod", From 472f6342f56ac460c90d3ce46fa0fb46bb068af6 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Fri, 10 Jul 2020 16:21:33 -0700 Subject: [PATCH 09/47] Referenced platform image for VM --- cloud/services/virtualmachines/virtualmachines.go | 6 ++++++ templates/cluster-template.yaml | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cloud/services/virtualmachines/virtualmachines.go b/cloud/services/virtualmachines/virtualmachines.go index c6c80c865de..147852ce49e 100644 --- a/cloud/services/virtualmachines/virtualmachines.go +++ b/cloud/services/virtualmachines/virtualmachines.go @@ -21,6 +21,7 @@ import ( "crypto/rand" "encoding/base64" "fmt" + "log" "strings" "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/authorization/mgmt/authorization" @@ -198,6 +199,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { */ } + log.Println("HI CREATING VM") err = s.Client.CreateOrUpdate( ctx, s.Scope.ResourceGroup(), @@ -351,6 +353,10 @@ func generateStorageProfile(vmSpec Spec) (*compute.StorageProfile, error) { }, } + log.Println("HI GENERATING STORAGE PROFILE") + log.Println("HI publisher: ", vmSpec.Image.Marketplace.Publisher) + log.Println("HI offer: ", vmSpec.Image.Marketplace.Offer) + log.Println("HI sku: ", vmSpec.Image.Marketplace.SKU) imageRef, err := converters.ImageToSDK(vmSpec.Image) if err != nil { return nil, err diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index 682b1d48e51..a70ead49816 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -114,9 +114,9 @@ spec: spec: image: marketplace: - publisher: Canonical - offer: UbuntuServer - sku: 16.04-LTS + publisher: AzureStack + offer: Test + sku: capz-test version: latest location: ${AZURE_LOCATION} osDisk: From b3ab4715895070388a5805d1212a975818c62c61 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Wed, 15 Jul 2020 16:08:47 -0700 Subject: [PATCH 10/47] Resolved authentication issue for kubelet and controller manager, control plane --- cloud/scope/clients.go | 3 +- templates/cluster-template.yaml | 64 +++++++++++++++++++++++++++++---- 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/cloud/scope/clients.go b/cloud/scope/clients.go index 390176b0455..a91421940d1 100644 --- a/cloud/scope/clients.go +++ b/cloud/scope/clients.go @@ -110,7 +110,8 @@ func (c *AzureClients) setCredentials(subscriptionID string) error { log.Println("HERE resource manager endpoint: ", c.ResourceManagerEndpoint) c.ResourceManagerEndpoint = settings.Environment.ResourceManagerEndpoint - c.ResourceManagerVMDNSSuffix = GetAzureDNSZoneForEnvironment(settings.Environment.Name) + c.ResourceManagerVMDNSSuffix = "cloudapp.ext-n31r1203.masd.stbtest.microsoft.com" + // c.ResourceManagerVMDNSSuffix = GetAzureDNSZoneForEnvironment(settings.Environment.Name) settings.Values[auth.SubscriptionID] = subscriptionID // c.Authorizer, err = settings.GetAuthorizer() c.Authorizer, err = c.getAuthorizerForResource(settings.Environment) diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index a70ead49816..9e2227d9ea1 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -41,6 +41,18 @@ spec: kind: AzureMachineTemplate name: ${CLUSTER_NAME}-control-plane kubeadmConfigSpec: + preKubeadmCommands: + - echo "hello world again!" + - "cat /etc/kubernetes/azurestackcloud.json" + - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + - sudo update-ca-certificates + - sudo echo "Environment='AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json'" >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + - sudo systemctl daemon-reload + - sudo systemctl restart kubelet + - echo "bye world!" + postKubeadmCommands: + - "sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml" + - echo "bye bye!" clusterConfiguration: apiServer: extraArgs: @@ -63,11 +75,15 @@ spec: mountPath: /etc/kubernetes/azure.json name: cloud-config readOnly: true + - hostPath: /etc/kubernetes/azurestackcloud.json + mountPath: /etc/kubernetes/azurestackcloud.json + name: cloud-config-stack + readOnly: true files: - content: | { "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "${AZURE_TENANT_ID}", + "tenantId": "adfs", "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", "aadClientId": "${AZURE_CLIENT_ID}", "aadClientSecret": "${AZURE_CLIENT_SECRET}", @@ -80,14 +96,43 @@ spec: "subnetName": "${CLUSTER_NAME}-node-subnet", "routeTableName": "${CLUSTER_NAME}-node-routetable", "userAssignedID": "${CLUSTER_NAME}", - "loadBalancerSku": "standard", + "loadBalancerSku": "Basic", "maximumLoadBalancerRuleCount": 250, "useManagedIdentityExtension": false, - "useInstanceMetadata": true + "useInstanceMetadata": false, + "excludeMasterFromStandardLB": false } owner: root:root path: /etc/kubernetes/azure.json permissions: "0644" + - content: | + { + "name":"${AZURE_ENVIRONMENT}", + "managementPortalURL":"https://portal.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "publishSettingsURL":"", + "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/98bbb015-dcf2-467c-a1cb-951df61a0948", + "resourceManagerEndpoint":"https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "activeDirectoryEndpoint":"https://adfs.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "galleryEndpoint":"https://providers.n31r1203.masd.stbtest.microsoft.com:30016/", + "keyVaultEndpoint":"", + "graphEndpoint":"https://graph.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "serviceBusEndpoint":"", + "batchManagementEndpoint":"", + "storageEndpointSuffix":"redmond.ext-n31r1203.masd.stbtest.microsoft.com", + "sqlDatabaseDNSSuffix":"", + "trafficManagerDNSSuffix":"", + "keyVaultDNSSuffix":"vault.redmond.ext-n31r1203.masd.stbtest.microsoft.com", + "serviceBusEndpointSuffix":"", + "serviceManagementVMDNSSuffix":"", + "resourceManagerVMDNSSuffix":"cloudapp.ext-n31r1203.masd.stbtest.microsoft.com", + "containerRegistryDNSSuffix":"", + "cosmosDBDNSSuffix":"", + "tokenAudience":"", + "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} + } + owner: root:root + path: /etc/kubernetes/azurestackcloud.json + permissions: "0644" initConfiguration: nodeRegistration: kubeletExtraArgs: @@ -159,6 +204,12 @@ metadata: spec: template: spec: + image: + marketplace: + publisher: AzureStack + offer: Test + sku: capz-test + version: latest location: ${AZURE_LOCATION} osDisk: diskSizeGB: 30 @@ -180,7 +231,7 @@ spec: - content: | { "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "${AZURE_TENANT_ID}", + "tenantId": "adfs", "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", "aadClientId": "${AZURE_CLIENT_ID}", "aadClientSecret": "${AZURE_CLIENT_SECRET}", @@ -192,10 +243,11 @@ spec: "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", "subnetName": "${CLUSTER_NAME}-node-subnet", "routeTableName": "${CLUSTER_NAME}-node-routetable", - "loadBalancerSku": "standard", + "loadBalancerSku": "Basic", "maximumLoadBalancerRuleCount": 250, "useManagedIdentityExtension": false, - "useInstanceMetadata": true + "useInstanceMetadata": false, + "excludeMasterFromStandardLB": false } owner: root:root path: /etc/kubernetes/azure.json From 0e6d78646f0db8d92c46443487e4409ab476a45b Mon Sep 17 00:00:00 2001 From: chlau-az Date: Fri, 17 Jul 2020 11:38:10 -0700 Subject: [PATCH 11/47] Put back internal load balancer tcp probe --- .../internalloadbalancers.go | 32 ++++++++----------- .../virtualmachines/virtualmachines.go | 4 ++- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/cloud/services/internalloadbalancers/internalloadbalancers.go b/cloud/services/internalloadbalancers/internalloadbalancers.go index 125ebe259be..8f13ad4c857 100644 --- a/cloud/services/internalloadbalancers/internalloadbalancers.go +++ b/cloud/services/internalloadbalancers/internalloadbalancers.go @@ -44,7 +44,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { return errors.New("invalid internal load balancer specification") } klog.V(2).Infof("creating internal load balancer %s", internalLBSpec.Name) - // probeName := "HTTPSProbe" + probeName := "tcpHTTPSProbe" frontEndIPConfigName := "controlplane-internal-lbFrontEnd" backEndAddressPoolName := "controlplane-internal-backEndPool" idPrefix := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers", s.Scope.SubscriptionID(), s.Scope.ResourceGroup()) @@ -103,20 +103,18 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { Name: &backEndAddressPoolName, }, }, - /* - Probes: &[]network.Probe{ - { - Name: &probeName, - ProbePropertiesFormat: &network.ProbePropertiesFormat{ - Protocol: "Http", - RequestPath: to.StringPtr("/healthz"), - Port: to.Int32Ptr(s.Scope.APIServerPort()), - IntervalInSeconds: to.Int32Ptr(15), - NumberOfProbes: to.Int32Ptr(4), - }, + Probes: &[]network.Probe{ + { + Name: &probeName, + ProbePropertiesFormat: &network.ProbePropertiesFormat{ + Protocol: "Tcp", + // RequestPath: to.StringPtr("/healthz"), + Port: to.Int32Ptr(s.Scope.APIServerPort()), + IntervalInSeconds: to.Int32Ptr(15), + NumberOfProbes: to.Int32Ptr(4), }, }, - */ + }, LoadBalancingRules: &[]network.LoadBalancingRule{ { Name: to.StringPtr("LBRuleHTTPS"), @@ -133,11 +131,9 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { BackendAddressPool: &network.SubResource{ ID: to.StringPtr(fmt.Sprintf("/%s/%s/backendAddressPools/%s", idPrefix, lbName, backEndAddressPoolName)), }, - /* - Probe: &network.SubResource{ - ID: to.StringPtr(fmt.Sprintf("/%s/%s/probes/%s", idPrefix, lbName, probeName)), - }, - */ + Probe: &network.SubResource{ + ID: to.StringPtr(fmt.Sprintf("/%s/%s/probes/%s", idPrefix, lbName, probeName)), + }, }, }, }, diff --git a/cloud/services/virtualmachines/virtualmachines.go b/cloud/services/virtualmachines/virtualmachines.go index 147852ce49e..dddf87a3e45 100644 --- a/cloud/services/virtualmachines/virtualmachines.go +++ b/cloud/services/virtualmachines/virtualmachines.go @@ -199,15 +199,17 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { */ } - log.Println("HI CREATING VM") + log.Println("HI CREATING VM: ", vmSpec.Name) err = s.Client.CreateOrUpdate( ctx, s.Scope.ResourceGroup(), vmSpec.Name, virtualMachine) if err != nil { + log.Println("HI CREATING VM ERROR: ", vmSpec.Name) return errors.Wrapf(err, "cannot create VM") } + log.Println("HI CREATING VM FINISHED: ", vmSpec.Name) if vmSpec.Identity == infrav1.VMIdentitySystemAssigned { err = s.createRoleAssignmentForIdentity(ctx, vmSpec.Name) From 47de0b098adc9537cc23bd8c19075bf7dc76e2a2 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Fri, 17 Jul 2020 13:08:55 -0700 Subject: [PATCH 12/47] Removed hardcode of ResourceManagerVMDNSSuffix --- cloud/scope/clients.go | 39 ++++++++++-------------------- cloud/scope/clients_test.go | 2 +- cloud/scope/cluster.go | 2 +- cloud/scope/managedcontrolplane.go | 5 ++-- 4 files changed, 18 insertions(+), 30 deletions(-) diff --git a/cloud/scope/clients.go b/cloud/scope/clients.go index a91421940d1..35aea642ad4 100644 --- a/cloud/scope/clients.go +++ b/cloud/scope/clients.go @@ -17,8 +17,10 @@ limitations under the License. package scope import ( + "fmt" "log" "os" + "strings" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/adal" @@ -49,29 +51,7 @@ type AzureClients struct { Authorizer autorest.Authorizer } -func (c *AzureClients) setCredentials(subscriptionID string) error { - // log.Println("HI PRINTING DIRECTORY") - // // DIR - // files, er := ioutil.ReadDir("/etc/ssl/certs") - // if er != nil { - // log.Fatal(er) - // } - // for _, f := range files { - // log.Println(f.Name()) - // } - // log.Println("HI finished printing directory") - - // // CURL - // resp, er := http.Get("https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/metadata/endpoints?api-version=2015-01-01") - // if er != nil { - // log.Printf("HI ERROR: %s", er) - // } - // defer resp.Body.Close() - // body, er := ioutil.ReadAll(resp.Body) - // log.Println(string(body)) - - // log.Println("HI finished curling") - +func (c *AzureClients) setCredentials(subscriptionID, location string) error { subID, err := getSubscriptionID(subscriptionID) if err != nil { return err @@ -110,10 +90,10 @@ func (c *AzureClients) setCredentials(subscriptionID string) error { log.Println("HERE resource manager endpoint: ", c.ResourceManagerEndpoint) c.ResourceManagerEndpoint = settings.Environment.ResourceManagerEndpoint - c.ResourceManagerVMDNSSuffix = "cloudapp.ext-n31r1203.masd.stbtest.microsoft.com" - // c.ResourceManagerVMDNSSuffix = GetAzureDNSZoneForEnvironment(settings.Environment.Name) + azsFQDNSuffix := getAzureStackFQDNSuffix(armEndpoint, location) + c.ResourceManagerVMDNSSuffix = fmt.Sprintf("cloudapp.%s", azsFQDNSuffix) + log.Println("HERE c.ResourceManagerVMDNSSuffix: ", c.ResourceManagerVMDNSSuffix) settings.Values[auth.SubscriptionID] = subscriptionID - // c.Authorizer, err = settings.GetAuthorizer() c.Authorizer, err = c.getAuthorizerForResource(settings.Environment) log.Println("HERE c.Authorizer: ", c.Authorizer, "err: ", err) return err @@ -148,6 +128,13 @@ func GetAzureDNSZoneForEnvironment(environmentName string) string { } } +func getAzureStackFQDNSuffix(portalURL, location string) string { + azsFQDNSuffix := strings.Replace(portalURL, fmt.Sprintf("https://management.%s.", location), "", -1) + azsFQDNSuffix = strings.TrimSuffix(azsFQDNSuffix, "/") + + return azsFQDNSuffix +} + // getAuthorizerForResource gets an OAuthTokenAuthorizer for Azure Resource Manager func (c *AzureClients) getAuthorizerForResource(env azure.Environment) (autorest.Authorizer, error) { var a autorest.Authorizer diff --git a/cloud/scope/clients_test.go b/cloud/scope/clients_test.go index b2413ee259b..621d8dcf995 100644 --- a/cloud/scope/clients_test.go +++ b/cloud/scope/clients_test.go @@ -69,7 +69,7 @@ func TestGettingEnvironment(t *testing.T) { t.Run(name, func(t *testing.T) { os.Setenv("AZURE_ENVIRONMENT", test.azureEnv) c := AzureClients{} - err := c.setCredentials("1234") + err := c.setCredentials("1234", "test_location") if test.expectedError { g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).To(ContainSubstring(test.expectedErrorMessage)) diff --git a/cloud/scope/cluster.go b/cloud/scope/cluster.go index ae29330e4b0..c907ade294e 100644 --- a/cloud/scope/cluster.go +++ b/cloud/scope/cluster.go @@ -55,7 +55,7 @@ func NewClusterScope(params ClusterScopeParams) (*ClusterScope, error) { params.Logger = klogr.New() } - err := params.AzureClients.setCredentials(params.AzureCluster.Spec.SubscriptionID) + err := params.AzureClients.setCredentials(params.AzureCluster.Spec.SubscriptionID, params.AzureCluster.Spec.Location) if err != nil { return nil, errors.Wrap(err, "failed to create Azure session") } diff --git a/cloud/scope/managedcontrolplane.go b/cloud/scope/managedcontrolplane.go index 1437063a25b..042366bc348 100644 --- a/cloud/scope/managedcontrolplane.go +++ b/cloud/scope/managedcontrolplane.go @@ -18,12 +18,13 @@ package scope import ( "context" + "github.com/Azure/go-autorest/autorest" + infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" "github.com/go-logr/logr" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/klog/klogr" - infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" expv1 "sigs.k8s.io/cluster-api/exp/api/v1alpha3" @@ -58,7 +59,7 @@ func NewManagedControlPlaneScope(params ManagedControlPlaneScopeParams) (*Manage params.Logger = klogr.New() } - if err := params.AzureClients.setCredentials(params.ControlPlane.Spec.SubscriptionID); err != nil { + if err := params.AzureClients.setCredentials(params.ControlPlane.Spec.SubscriptionID, params.ControlPlane.Spec.Location); err != nil { return nil, errors.Wrap(err, "failed to create Azure session") } From 94bf4aab8286d36a69219720239c28dbcc675ee2 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Sun, 19 Jul 2020 22:11:32 -0700 Subject: [PATCH 13/47] Removed hard code of arm endpoint --- config/default/manager_cloud_env_patch.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/config/default/manager_cloud_env_patch.yaml b/config/default/manager_cloud_env_patch.yaml index e98d5e787fb..d50c5c71a2c 100644 --- a/config/default/manager_cloud_env_patch.yaml +++ b/config/default/manager_cloud_env_patch.yaml @@ -15,9 +15,7 @@ spec: name: manager-bootstrap-cloud-env key: cloud-env - name: AZURE_ARM_ENDPOINT - value: "https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com" - - # valueFrom: - # configMapKeyRef: - # name: manager-bootstrap-cloud-env - # key: cloud-arm-endpoint \ No newline at end of file + valueFrom: + configMapKeyRef: + name: manager-bootstrap-cloud-env + key: cloud-arm-endpoint \ No newline at end of file From 64846b53c6322e25e1ff462da82e420262135ace Mon Sep 17 00:00:00 2001 From: chlau-az Date: Mon, 20 Jul 2020 11:48:52 -0700 Subject: [PATCH 14/47] Changed back file paths --- api/v1alpha2/azurecluster_conversion.go | 2 +- api/v1alpha2/azuremachine_conversion.go | 2 +- .../azuremachinetemplate_conversion.go | 2 +- api/v1alpha2/conversion_test.go | 2 +- api/v1alpha2/doc.go | 2 +- api/v1alpha2/zz_generated.conversion.go | 2 +- cloud/converters/image.go | 2 +- cloud/converters/tags.go | 2 +- cloud/converters/vm.go | 2 +- cloud/converters/vmss.go | 4 ++-- cloud/converters/vmss_test.go | 4 ++-- cloud/defaults.go | 4 ++-- cloud/interfaces.go | 2 +- cloud/mocks/service_mock.go | 2 +- cloud/scope/cluster.go | 4 ++-- cloud/scope/machine.go | 4 ++-- cloud/scope/machinepool.go | 6 ++--- cloud/scope/managedcontrolplane.go | 2 +- cloud/services/agentpools/agentpools.go | 2 +- cloud/services/agentpools/agentpools_test.go | 2 +- cloud/services/agentpools/client.go | 2 +- cloud/services/agentpools/service.go | 2 +- .../availabilityzones_test.go | 6 ++--- cloud/services/availabilityzones/client.go | 2 +- cloud/services/availabilityzones/service.go | 2 +- cloud/services/disks/client.go | 2 +- cloud/services/disks/disks.go | 2 +- cloud/services/disks/disks_test.go | 6 ++--- cloud/services/disks/service.go | 2 +- cloud/services/groups/client.go | 2 +- cloud/services/groups/groups.go | 6 ++--- cloud/services/groups/groups_test.go | 8 +++---- cloud/services/groups/service.go | 2 +- cloud/services/inboundnatrules/client.go | 2 +- .../services/internalloadbalancers/client.go | 2 +- .../internalloadbalancers.go | 4 ++-- .../internalloadbalancers_test.go | 10 ++++---- .../services/internalloadbalancers/service.go | 6 ++--- cloud/services/managedclusters/client.go | 2 +- .../managedclusters/managedclusters.go | 2 +- .../managedclusters/managedclusters_test.go | 2 +- cloud/services/managedclusters/service.go | 2 +- cloud/services/networkinterfaces/client.go | 2 +- .../networkinterfaces/networkinterfaces.go | 4 ++-- .../networkinterfaces_test.go | 20 ++++++++-------- cloud/services/networkinterfaces/service.go | 14 +++++------ cloud/services/publicips/client.go | 2 +- .../mock_publicips/publicips_mock.go | 4 ++-- cloud/services/publicips/publicips.go | 2 +- cloud/services/publicips/publicips_test.go | 4 ++-- cloud/services/publicips/service.go | 2 +- cloud/services/publicloadbalancers/client.go | 2 +- .../publicloadbalancers.go | 6 ++--- .../publicloadbalancers_test.go | 10 ++++---- cloud/services/publicloadbalancers/service.go | 4 ++-- cloud/services/resourceskus/client.go | 2 +- cloud/services/roleassignments/client.go | 2 +- cloud/services/routetables/client.go | 2 +- cloud/services/routetables/routetables.go | 2 +- .../services/routetables/routetables_test.go | 6 ++--- cloud/services/routetables/service.go | 2 +- cloud/services/scalesets/client.go | 2 +- cloud/services/scalesets/service.go | 6 ++--- cloud/services/scalesets/vmss.go | 8 +++---- cloud/services/scalesets/vmss_test.go | 16 ++++++------- cloud/services/securitygroups/client.go | 2 +- .../services/securitygroups/securitygroups.go | 2 +- .../securitygroups/securitygroups_test.go | 6 ++--- cloud/services/securitygroups/service.go | 2 +- cloud/services/subnets/client.go | 2 +- cloud/services/subnets/service.go | 6 ++--- cloud/services/subnets/subnets.go | 4 ++-- cloud/services/subnets/subnets_test.go | 10 ++++---- .../virtualmachineextensions/client.go | 2 +- .../virtualmachineextensions/service.go | 2 +- cloud/services/virtualmachines/client.go | 2 +- cloud/services/virtualmachines/service.go | 8 +++---- .../virtualmachines/virtualmachines.go | 6 ++--- .../virtualmachines/virtualmachines_test.go | 12 +++++----- cloud/services/virtualnetworks/client.go | 2 +- cloud/services/virtualnetworks/service.go | 2 +- .../virtualnetworks/virtualnetworks.go | 6 ++--- .../virtualnetworks/virtualnetworks_test.go | 6 ++--- config/default/manager_credentials_patch.yaml | 2 +- controllers/azurecluster_controller.go | 6 ++--- controllers/azurecluster_controller_test.go | 8 +++---- controllers/azurecluster_reconciler.go | 24 +++++++++---------- controllers/azuremachine_annotations.go | 2 +- controllers/azuremachine_controller.go | 6 ++--- controllers/azuremachine_controller_test.go | 6 ++--- controllers/azuremachine_reconciler.go | 16 ++++++------- controllers/azuremachine_reconciler_test.go | 6 ++--- controllers/azuremachine_tags.go | 4 ++-- controllers/helpers.go | 4 ++-- controllers/helpers_test.go | 4 ++-- controllers/suite_test.go | 2 +- docs/development.md | 2 +- exp/PROJECT | 2 +- exp/api/v1alpha3/azuremachinepool_test.go | 4 ++-- exp/api/v1alpha3/azuremachinepool_types.go | 2 +- exp/api/v1alpha3/azuremachinepool_webhook.go | 2 +- exp/api/v1alpha3/types.go | 2 +- exp/api/v1alpha3/zz_generated.deepcopy.go | 2 +- .../azuremachinepool_controller.go | 14 +++++------ .../azuremachinepool_controller_test.go | 2 +- .../azuremachinepool_controller_unit_test.go | 12 +++++----- .../azuremanagedcluster_controller.go | 4 ++-- .../azuremanagedmachinepool_controller.go | 8 +++---- .../azuremanagedmachinepool_reconciler.go | 8 +++---- .../azuremangedcontrolplane_controller.go | 8 +++---- .../azuremangedcontrolplane_reconciler.go | 8 +++---- exp/controllers/helpers.go | 8 +++---- exp/controllers/helpers_test.go | 6 ++--- exp/controllers/suite_test.go | 4 ++-- go.mod | 4 ++-- hack/boilerplate/test/BUILD | 2 +- hack/version.sh | 2 +- internal/test/env/env.go | 8 +++---- main.go | 16 ++++++------- test/e2e/e2e_suite_test.go | 2 +- util/reconciler/defaults_test.go | 2 +- 121 files changed, 280 insertions(+), 280 deletions(-) diff --git a/api/v1alpha2/azurecluster_conversion.go b/api/v1alpha2/azurecluster_conversion.go index e5e27655b18..f3c583240e1 100644 --- a/api/v1alpha2/azurecluster_conversion.go +++ b/api/v1alpha2/azurecluster_conversion.go @@ -18,7 +18,7 @@ package v1alpha2 import ( apiconversion "k8s.io/apimachinery/pkg/conversion" - infrav1alpha3 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + infrav1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" utilconversion "sigs.k8s.io/cluster-api/util/conversion" "sigs.k8s.io/controller-runtime/pkg/conversion" ) diff --git a/api/v1alpha2/azuremachine_conversion.go b/api/v1alpha2/azuremachine_conversion.go index 191cb3adc67..fb7e39db297 100644 --- a/api/v1alpha2/azuremachine_conversion.go +++ b/api/v1alpha2/azuremachine_conversion.go @@ -18,7 +18,7 @@ package v1alpha2 import ( apiconversion "k8s.io/apimachinery/pkg/conversion" - infrav1alpha3 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + infrav1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" utilconversion "sigs.k8s.io/cluster-api/util/conversion" "sigs.k8s.io/controller-runtime/pkg/conversion" ) diff --git a/api/v1alpha2/azuremachinetemplate_conversion.go b/api/v1alpha2/azuremachinetemplate_conversion.go index c95b050c54a..5887c26eea7 100644 --- a/api/v1alpha2/azuremachinetemplate_conversion.go +++ b/api/v1alpha2/azuremachinetemplate_conversion.go @@ -17,7 +17,7 @@ limitations under the License. package v1alpha2 import ( - infrav1alpha3 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + infrav1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" utilconversion "sigs.k8s.io/cluster-api/util/conversion" "sigs.k8s.io/controller-runtime/pkg/conversion" ) diff --git a/api/v1alpha2/conversion_test.go b/api/v1alpha2/conversion_test.go index 1efbb5adaef..1d6e0d721ed 100644 --- a/api/v1alpha2/conversion_test.go +++ b/api/v1alpha2/conversion_test.go @@ -24,7 +24,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" - v1alpha3 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + v1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" utilconversion "sigs.k8s.io/cluster-api/util/conversion" ) diff --git a/api/v1alpha2/doc.go b/api/v1alpha2/doc.go index c613f85957c..3147930676a 100644 --- a/api/v1alpha2/doc.go +++ b/api/v1alpha2/doc.go @@ -16,4 +16,4 @@ limitations under the License. package v1alpha2 -// +k8s:conversion-gen=github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3 +// +k8s:conversion-gen=sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3 diff --git a/api/v1alpha2/zz_generated.conversion.go b/api/v1alpha2/zz_generated.conversion.go index f5fa211a379..218e1569a06 100644 --- a/api/v1alpha2/zz_generated.conversion.go +++ b/api/v1alpha2/zz_generated.conversion.go @@ -23,7 +23,7 @@ package v1alpha2 import ( unsafe "unsafe" - v1alpha3 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + v1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" v1 "k8s.io/api/core/v1" conversion "k8s.io/apimachinery/pkg/conversion" runtime "k8s.io/apimachinery/pkg/runtime" diff --git a/cloud/converters/image.go b/cloud/converters/image.go index bcc8a298015..10c7994e7f1 100644 --- a/cloud/converters/image.go +++ b/cloud/converters/image.go @@ -22,7 +22,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/pkg/errors" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" ) // ImageToSDK converts a CAPZ Image (as RawExtension) to a Azure SDK Image Reference. diff --git a/cloud/converters/tags.go b/cloud/converters/tags.go index 442c0491f36..a62cc4b0448 100644 --- a/cloud/converters/tags.go +++ b/cloud/converters/tags.go @@ -18,7 +18,7 @@ package converters import ( "github.com/Azure/go-autorest/autorest/to" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" ) // MapToTags converts a map[string]*string into a infrav1.Tags. diff --git a/cloud/converters/vm.go b/cloud/converters/vm.go index 1a1d9c96b23..ee28ed98391 100644 --- a/cloud/converters/vm.go +++ b/cloud/converters/vm.go @@ -19,7 +19,7 @@ package converters import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest/to" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" ) // SDKToVM converts an Azure SDK VirtualMachine to the CAPZ VM type. diff --git a/cloud/converters/vmss.go b/cloud/converters/vmss.go index 5ea59c93a41..7611b0d8428 100644 --- a/cloud/converters/vmss.go +++ b/cloud/converters/vmss.go @@ -20,8 +20,8 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest/to" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" ) // SDKToVMSS converts an Azure SDK VirtualMachineScaleSet to the AzureMachinePool type. diff --git a/cloud/converters/vmss_test.go b/cloud/converters/vmss_test.go index 49850a7d0b4..be509ef98df 100644 --- a/cloud/converters/vmss_test.go +++ b/cloud/converters/vmss_test.go @@ -24,8 +24,8 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/onsi/gomega" - "github.com/chlau-az/cluster-api-provider-azure/cloud/converters" - infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/cloud/converters" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" ) func Test_SDKToVMSS(t *testing.T) { diff --git a/cloud/defaults.go b/cloud/defaults.go index 281d3cce1a3..fc654519df7 100644 --- a/cloud/defaults.go +++ b/cloud/defaults.go @@ -21,8 +21,8 @@ import ( "github.com/blang/semver" "github.com/pkg/errors" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/version" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/version" ) const ( diff --git a/cloud/interfaces.go b/cloud/interfaces.go index fbe099fbb3f..d6be0728377 100644 --- a/cloud/interfaces.go +++ b/cloud/interfaces.go @@ -19,7 +19,7 @@ package azure import ( "context" "github.com/Azure/go-autorest/autorest" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" ) // Service is a generic interface used by components offering a type of service. diff --git a/cloud/mocks/service_mock.go b/cloud/mocks/service_mock.go index 5ee10358619..ef9a4110bd9 100644 --- a/cloud/mocks/service_mock.go +++ b/cloud/mocks/service_mock.go @@ -15,7 +15,7 @@ limitations under the License. */ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/chlau-az/cluster-api-provider-azure/cloud (interfaces: Service,GetterService) +// Source: sigs.k8s.io/cluster-api-provider-azure/cloud (interfaces: Service,GetterService) // Package mocks is a generated GoMock package. package mocks diff --git a/cloud/scope/cluster.go b/cloud/scope/cluster.go index c907ade294e..8ae176fd9b5 100644 --- a/cloud/scope/cluster.go +++ b/cloud/scope/cluster.go @@ -21,8 +21,8 @@ import ( "fmt" "github.com/Azure/go-autorest/autorest" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" "github.com/go-logr/logr" "github.com/pkg/errors" "k8s.io/klog/klogr" diff --git a/cloud/scope/machine.go b/cloud/scope/machine.go index 76cb28af473..a846b879c07 100644 --- a/cloud/scope/machine.go +++ b/cloud/scope/machine.go @@ -26,8 +26,8 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/klog/klogr" "k8s.io/utils/pointer" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/cluster-api/controllers/noderefutil" capierrors "sigs.k8s.io/cluster-api/errors" diff --git a/cloud/scope/machinepool.go b/cloud/scope/machinepool.go index a80a8946f41..9798f861f06 100644 --- a/cloud/scope/machinepool.go +++ b/cloud/scope/machinepool.go @@ -26,14 +26,14 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/klog/klogr" "k8s.io/utils/pointer" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" "sigs.k8s.io/cluster-api/controllers/noderefutil" capierrors "sigs.k8s.io/cluster-api/errors" capiv1exp "sigs.k8s.io/cluster-api/exp/api/v1alpha3" "sigs.k8s.io/cluster-api/util/patch" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/cloud/scope/managedcontrolplane.go b/cloud/scope/managedcontrolplane.go index 042366bc348..484d5548adc 100644 --- a/cloud/scope/managedcontrolplane.go +++ b/cloud/scope/managedcontrolplane.go @@ -20,7 +20,7 @@ import ( "context" "github.com/Azure/go-autorest/autorest" - infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" "github.com/go-logr/logr" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" diff --git a/cloud/services/agentpools/agentpools.go b/cloud/services/agentpools/agentpools.go index 29692f8ed97..4b3187dec00 100644 --- a/cloud/services/agentpools/agentpools.go +++ b/cloud/services/agentpools/agentpools.go @@ -23,7 +23,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/pkg/errors" "k8s.io/klog" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Spec contains properties to create a agent pool. diff --git a/cloud/services/agentpools/agentpools_test.go b/cloud/services/agentpools/agentpools_test.go index 47a7a0935e1..001541f1702 100644 --- a/cloud/services/agentpools/agentpools_test.go +++ b/cloud/services/agentpools/agentpools_test.go @@ -27,7 +27,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/golang/mock/gomock" . "github.com/onsi/gomega" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/agentpools/mock_agentpools" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/agentpools/mock_agentpools" ) const ( diff --git a/cloud/services/agentpools/client.go b/cloud/services/agentpools/client.go index 0293d29e997..f680ebe0511 100644 --- a/cloud/services/agentpools/client.go +++ b/cloud/services/agentpools/client.go @@ -22,7 +22,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2020-02-01/containerservice" "github.com/Azure/go-autorest/autorest" "github.com/pkg/errors" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/agentpools/service.go b/cloud/services/agentpools/service.go index 5448560ff6d..9e3ef623b4c 100644 --- a/cloud/services/agentpools/service.go +++ b/cloud/services/agentpools/service.go @@ -17,7 +17,7 @@ limitations under the License. package agentpools import ( - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Service provides operations on azure resources diff --git a/cloud/services/availabilityzones/availabilityzones_test.go b/cloud/services/availabilityzones/availabilityzones_test.go index 602165609b0..d46bb487cce 100644 --- a/cloud/services/availabilityzones/availabilityzones_test.go +++ b/cloud/services/availabilityzones/availabilityzones_test.go @@ -22,7 +22,7 @@ import ( "testing" . "github.com/onsi/gomega" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/availabilityzones/mock_availabilityzones" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/availabilityzones/mock_availabilityzones" "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest" @@ -32,8 +32,8 @@ import ( network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) diff --git a/cloud/services/availabilityzones/client.go b/cloud/services/availabilityzones/client.go index 4f391e156a3..4c502f29330 100644 --- a/cloud/services/availabilityzones/client.go +++ b/cloud/services/availabilityzones/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/availabilityzones/service.go b/cloud/services/availabilityzones/service.go index c119e1bb2e0..5a3aa546f81 100644 --- a/cloud/services/availabilityzones/service.go +++ b/cloud/services/availabilityzones/service.go @@ -17,7 +17,7 @@ limitations under the License. package availabilityzones import ( - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" ) // Service provides operations on azure resources diff --git a/cloud/services/disks/client.go b/cloud/services/disks/client.go index d32ca128425..eb0664e68c3 100644 --- a/cloud/services/disks/client.go +++ b/cloud/services/disks/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/disks/disks.go b/cloud/services/disks/disks.go index 9f51db77183..8240ef6870f 100644 --- a/cloud/services/disks/disks.go +++ b/cloud/services/disks/disks.go @@ -21,7 +21,7 @@ import ( "github.com/pkg/errors" "k8s.io/klog" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Spec specification for disk diff --git a/cloud/services/disks/disks_test.go b/cloud/services/disks/disks_test.go index 95f84377863..0d144436b52 100644 --- a/cloud/services/disks/disks_test.go +++ b/cloud/services/disks/disks_test.go @@ -22,7 +22,7 @@ import ( "testing" . "github.com/onsi/gomega" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/disks/mock_disks" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/disks/mock_disks" "github.com/Azure/go-autorest/autorest" "github.com/golang/mock/gomock" @@ -30,8 +30,8 @@ import ( network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) diff --git a/cloud/services/disks/service.go b/cloud/services/disks/service.go index e9643293f64..e3114864528 100644 --- a/cloud/services/disks/service.go +++ b/cloud/services/disks/service.go @@ -17,7 +17,7 @@ limitations under the License. package disks import ( - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" ) // Service provides operations on azure resources diff --git a/cloud/services/groups/client.go b/cloud/services/groups/client.go index c1bffb00d88..ded75d2dafd 100644 --- a/cloud/services/groups/client.go +++ b/cloud/services/groups/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/resources/mgmt/resources" "github.com/Azure/go-autorest/autorest" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/groups/groups.go b/cloud/services/groups/groups.go index 978b8ab7092..e1041d70438 100644 --- a/cloud/services/groups/groups.go +++ b/cloud/services/groups/groups.go @@ -23,9 +23,9 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" - "github.com/chlau-az/cluster-api-provider-azure/cloud/converters" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + "sigs.k8s.io/cluster-api-provider-azure/cloud/converters" ) // Reconcile gets/creates/updates a resource group. diff --git a/cloud/services/groups/groups_test.go b/cloud/services/groups/groups_test.go index 6e69d70d23c..aa766e005a1 100644 --- a/cloud/services/groups/groups_test.go +++ b/cloud/services/groups/groups_test.go @@ -22,7 +22,7 @@ import ( "testing" . "github.com/onsi/gomega" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/groups/mock_groups" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/groups/mock_groups" "github.com/Azure/go-autorest/autorest" "github.com/golang/mock/gomock" @@ -30,9 +30,9 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/resources/mgmt/resources" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/cloud/converters" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/cloud/converters" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) diff --git a/cloud/services/groups/service.go b/cloud/services/groups/service.go index a8233fbd564..b57544dd445 100644 --- a/cloud/services/groups/service.go +++ b/cloud/services/groups/service.go @@ -17,7 +17,7 @@ limitations under the License. package groups import ( - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" ) // Service provides operations on azure resources diff --git a/cloud/services/inboundnatrules/client.go b/cloud/services/inboundnatrules/client.go index 9af1a46a6d9..4cc64d8107b 100644 --- a/cloud/services/inboundnatrules/client.go +++ b/cloud/services/inboundnatrules/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/internalloadbalancers/client.go b/cloud/services/internalloadbalancers/client.go index 2949d66599d..c07dd0d582d 100644 --- a/cloud/services/internalloadbalancers/client.go +++ b/cloud/services/internalloadbalancers/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/internalloadbalancers/internalloadbalancers.go b/cloud/services/internalloadbalancers/internalloadbalancers.go index 8f13ad4c857..ffadd128855 100644 --- a/cloud/services/internalloadbalancers/internalloadbalancers.go +++ b/cloud/services/internalloadbalancers/internalloadbalancers.go @@ -22,8 +22,8 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest/to" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" "github.com/pkg/errors" "k8s.io/klog" ) diff --git a/cloud/services/internalloadbalancers/internalloadbalancers_test.go b/cloud/services/internalloadbalancers/internalloadbalancers_test.go index b30c71c9935..fa5598802ab 100644 --- a/cloud/services/internalloadbalancers/internalloadbalancers_test.go +++ b/cloud/services/internalloadbalancers/internalloadbalancers_test.go @@ -22,9 +22,9 @@ import ( "testing" . "github.com/onsi/gomega" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/internalloadbalancers/mock_internalloadbalancers" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/subnets/mock_subnets" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/virtualnetworks/mock_virtualnetworks" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/internalloadbalancers/mock_internalloadbalancers" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/subnets/mock_subnets" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/virtualnetworks/mock_virtualnetworks" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" @@ -33,8 +33,8 @@ import ( network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) diff --git a/cloud/services/internalloadbalancers/service.go b/cloud/services/internalloadbalancers/service.go index db06b4a6555..7c85d4bc3f4 100644 --- a/cloud/services/internalloadbalancers/service.go +++ b/cloud/services/internalloadbalancers/service.go @@ -17,9 +17,9 @@ limitations under the License. package internalloadbalancers import ( - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/subnets" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/virtualnetworks" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/subnets" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/virtualnetworks" ) // Service provides operations on azure resources diff --git a/cloud/services/managedclusters/client.go b/cloud/services/managedclusters/client.go index b5e2bcda77b..93648055614 100644 --- a/cloud/services/managedclusters/client.go +++ b/cloud/services/managedclusters/client.go @@ -22,7 +22,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2020-02-01/containerservice" "github.com/Azure/go-autorest/autorest" "github.com/pkg/errors" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/managedclusters/managedclusters.go b/cloud/services/managedclusters/managedclusters.go index ebd2edb3088..ce17b130536 100644 --- a/cloud/services/managedclusters/managedclusters.go +++ b/cloud/services/managedclusters/managedclusters.go @@ -25,7 +25,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2020-02-01/containerservice" "github.com/pkg/errors" "k8s.io/klog" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) var ( diff --git a/cloud/services/managedclusters/managedclusters_test.go b/cloud/services/managedclusters/managedclusters_test.go index e130d348b49..5ea90817141 100644 --- a/cloud/services/managedclusters/managedclusters_test.go +++ b/cloud/services/managedclusters/managedclusters_test.go @@ -25,7 +25,7 @@ import ( "github.com/Azure/go-autorest/autorest" "github.com/golang/mock/gomock" . "github.com/onsi/gomega" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/managedclusters/mock_managedclusters" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/managedclusters/mock_managedclusters" ) func TestReconcile(t *testing.T) { diff --git a/cloud/services/managedclusters/service.go b/cloud/services/managedclusters/service.go index 13300870981..dfc744d6f1a 100644 --- a/cloud/services/managedclusters/service.go +++ b/cloud/services/managedclusters/service.go @@ -17,7 +17,7 @@ limitations under the License. package managedclusters import ( - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Service provides operations on azure resources diff --git a/cloud/services/networkinterfaces/client.go b/cloud/services/networkinterfaces/client.go index b05095fa1f9..77b83bb27e3 100644 --- a/cloud/services/networkinterfaces/client.go +++ b/cloud/services/networkinterfaces/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/networkinterfaces/networkinterfaces.go b/cloud/services/networkinterfaces/networkinterfaces.go index ccf1678d8bc..b328f3cabc5 100644 --- a/cloud/services/networkinterfaces/networkinterfaces.go +++ b/cloud/services/networkinterfaces/networkinterfaces.go @@ -24,8 +24,8 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Spec specification for routetable diff --git a/cloud/services/networkinterfaces/networkinterfaces_test.go b/cloud/services/networkinterfaces/networkinterfaces_test.go index 5cb585497d9..bb2dd0f88f3 100644 --- a/cloud/services/networkinterfaces/networkinterfaces_test.go +++ b/cloud/services/networkinterfaces/networkinterfaces_test.go @@ -22,14 +22,14 @@ import ( "testing" . "github.com/onsi/gomega" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/inboundnatrules/mock_inboundnatrules" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/internalloadbalancers/mock_internalloadbalancers" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/networkinterfaces/mock_networkinterfaces" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicips/mock_publicips" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicloadbalancers/mock_publicloadbalancers" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/resourceskus/mock_resourceskus" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/subnets/mock_subnets" - "github.com/chlau-az/cluster-api-provider-azure/internal/test/matchers" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/inboundnatrules/mock_inboundnatrules" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/internalloadbalancers/mock_internalloadbalancers" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/networkinterfaces/mock_networkinterfaces" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicips/mock_publicips" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicloadbalancers/mock_publicloadbalancers" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/resourceskus/mock_resourceskus" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/subnets/mock_subnets" + "sigs.k8s.io/cluster-api-provider-azure/internal/test/matchers" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" @@ -39,8 +39,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/pointer" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) diff --git a/cloud/services/networkinterfaces/service.go b/cloud/services/networkinterfaces/service.go index 2ed2b6c00a5..7f443abac25 100644 --- a/cloud/services/networkinterfaces/service.go +++ b/cloud/services/networkinterfaces/service.go @@ -17,13 +17,13 @@ limitations under the License. package networkinterfaces import ( - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/inboundnatrules" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/internalloadbalancers" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicips" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicloadbalancers" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/resourceskus" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/subnets" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/inboundnatrules" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/internalloadbalancers" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicips" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicloadbalancers" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/resourceskus" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/subnets" ) // Service provides operations on azure resources diff --git a/cloud/services/publicips/client.go b/cloud/services/publicips/client.go index 41b3aa79f53..aa92ccffef1 100644 --- a/cloud/services/publicips/client.go +++ b/cloud/services/publicips/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/publicips/mock_publicips/publicips_mock.go b/cloud/services/publicips/mock_publicips/publicips_mock.go index bdb49340ee9..948a1b4e768 100644 --- a/cloud/services/publicips/mock_publicips/publicips_mock.go +++ b/cloud/services/publicips/mock_publicips/publicips_mock.go @@ -24,8 +24,8 @@ import ( autorest "github.com/Azure/go-autorest/autorest" gomock "github.com/golang/mock/gomock" reflect "reflect" - v1alpha3 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + v1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // MockPublicIPScope is a mock of PublicIPScope interface. diff --git a/cloud/services/publicips/publicips.go b/cloud/services/publicips/publicips.go index 11a027f072a..0fc58d4901c 100644 --- a/cloud/services/publicips/publicips.go +++ b/cloud/services/publicips/publicips.go @@ -22,7 +22,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest/to" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" "github.com/pkg/errors" "k8s.io/klog" ) diff --git a/cloud/services/publicips/publicips_test.go b/cloud/services/publicips/publicips_test.go index efcb7d917dc..d11237a27c7 100644 --- a/cloud/services/publicips/publicips_test.go +++ b/cloud/services/publicips/publicips_test.go @@ -21,10 +21,10 @@ import ( "net/http" "testing" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" . "github.com/onsi/gomega" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicips/mock_publicips" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicips/mock_publicips" "github.com/Azure/go-autorest/autorest" "github.com/golang/mock/gomock" diff --git a/cloud/services/publicips/service.go b/cloud/services/publicips/service.go index d52f5369a99..73898c8bdf8 100644 --- a/cloud/services/publicips/service.go +++ b/cloud/services/publicips/service.go @@ -17,7 +17,7 @@ limitations under the License. package publicips import ( - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // PublicIPScope defines the scope interface for a public IP service. diff --git a/cloud/services/publicloadbalancers/client.go b/cloud/services/publicloadbalancers/client.go index a7dd31bcb61..7405d932441 100644 --- a/cloud/services/publicloadbalancers/client.go +++ b/cloud/services/publicloadbalancers/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/publicloadbalancers/publicloadbalancers.go b/cloud/services/publicloadbalancers/publicloadbalancers.go index c36211396d0..d900361013c 100644 --- a/cloud/services/publicloadbalancers/publicloadbalancers.go +++ b/cloud/services/publicloadbalancers/publicloadbalancers.go @@ -22,9 +22,9 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest/to" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" - "github.com/chlau-az/cluster-api-provider-azure/cloud/converters" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + "sigs.k8s.io/cluster-api-provider-azure/cloud/converters" "github.com/pkg/errors" "k8s.io/klog" ) diff --git a/cloud/services/publicloadbalancers/publicloadbalancers_test.go b/cloud/services/publicloadbalancers/publicloadbalancers_test.go index 966c3241b61..9bb40ca9b86 100644 --- a/cloud/services/publicloadbalancers/publicloadbalancers_test.go +++ b/cloud/services/publicloadbalancers/publicloadbalancers_test.go @@ -21,9 +21,9 @@ import ( "net/http" "testing" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicips/mock_publicips" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicloadbalancers/mock_publicloadbalancers" - "github.com/chlau-az/cluster-api-provider-azure/internal/test/matchers" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicips/mock_publicips" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicloadbalancers/mock_publicloadbalancers" + "sigs.k8s.io/cluster-api-provider-azure/internal/test/matchers" . "github.com/onsi/gomega" "github.com/Azure/go-autorest/autorest" @@ -31,8 +31,8 @@ import ( "github.com/golang/mock/gomock" network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" diff --git a/cloud/services/publicloadbalancers/service.go b/cloud/services/publicloadbalancers/service.go index ea8283675f5..ee5b22b5cac 100644 --- a/cloud/services/publicloadbalancers/service.go +++ b/cloud/services/publicloadbalancers/service.go @@ -17,8 +17,8 @@ limitations under the License. package publicloadbalancers import ( - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicips" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicips" ) // Service provides operations on azure resources diff --git a/cloud/services/resourceskus/client.go b/cloud/services/resourceskus/client.go index bcbfb030dff..f5d5f7e2a8c 100644 --- a/cloud/services/resourceskus/client.go +++ b/cloud/services/resourceskus/client.go @@ -24,7 +24,7 @@ import ( "github.com/Azure/go-autorest/autorest" "github.com/pkg/errors" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/roleassignments/client.go b/cloud/services/roleassignments/client.go index aed3c78b522..09a61d0f21b 100644 --- a/cloud/services/roleassignments/client.go +++ b/cloud/services/roleassignments/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/authorization/mgmt/authorization" "github.com/Azure/go-autorest/autorest" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/routetables/client.go b/cloud/services/routetables/client.go index d85e4e033c9..3f1c9e49576 100644 --- a/cloud/services/routetables/client.go +++ b/cloud/services/routetables/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/routetables/routetables.go b/cloud/services/routetables/routetables.go index 652ec865139..6b97e0d4b4a 100644 --- a/cloud/services/routetables/routetables.go +++ b/cloud/services/routetables/routetables.go @@ -22,7 +22,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Spec specification for route table. diff --git a/cloud/services/routetables/routetables_test.go b/cloud/services/routetables/routetables_test.go index 4c3b7ab7843..dab3ed6a7b4 100644 --- a/cloud/services/routetables/routetables_test.go +++ b/cloud/services/routetables/routetables_test.go @@ -23,7 +23,7 @@ import ( "testing" . "github.com/onsi/gomega" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/routetables/mock_routetables" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/routetables/mock_routetables" "github.com/Azure/go-autorest/autorest" "github.com/golang/mock/gomock" @@ -31,8 +31,8 @@ import ( network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) diff --git a/cloud/services/routetables/service.go b/cloud/services/routetables/service.go index 94ee500b16b..14b8d91a18c 100644 --- a/cloud/services/routetables/service.go +++ b/cloud/services/routetables/service.go @@ -17,7 +17,7 @@ limitations under the License. package routetables import ( - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" ) // Service provides operations on azure resources diff --git a/cloud/services/scalesets/client.go b/cloud/services/scalesets/client.go index b914c6f02ec..edda4e5ca7f 100644 --- a/cloud/services/scalesets/client.go +++ b/cloud/services/scalesets/client.go @@ -24,7 +24,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/scalesets/service.go b/cloud/services/scalesets/service.go index 1d09828885c..992e659fb4a 100644 --- a/cloud/services/scalesets/service.go +++ b/cloud/services/scalesets/service.go @@ -16,9 +16,9 @@ limitations under the License. package scalesets import ( - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicloadbalancers" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/resourceskus" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicloadbalancers" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/resourceskus" ) // Service provides operations on azure resources diff --git a/cloud/services/scalesets/vmss.go b/cloud/services/scalesets/vmss.go index ec0684a27ae..eaa5250740b 100644 --- a/cloud/services/scalesets/vmss.go +++ b/cloud/services/scalesets/vmss.go @@ -20,16 +20,16 @@ import ( "context" "fmt" - infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" - "github.com/chlau-az/cluster-api-provider-azure/cloud/converters" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + "sigs.k8s.io/cluster-api-provider-azure/cloud/converters" ) // Spec contains properties to create a managed cluster. diff --git a/cloud/services/scalesets/vmss_test.go b/cloud/services/scalesets/vmss_test.go index 94748b4f42f..17307702fbc 100644 --- a/cloud/services/scalesets/vmss_test.go +++ b/cloud/services/scalesets/vmss_test.go @@ -35,14 +35,14 @@ import ( clusterv1exp "sigs.k8s.io/cluster-api/exp/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicloadbalancers/mock_publicloadbalancers" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/resourceskus/mock_resourceskus" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/scalesets/mock_scalesets" - infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/internal/test/matchers" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicloadbalancers/mock_publicloadbalancers" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/resourceskus/mock_resourceskus" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/scalesets/mock_scalesets" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/internal/test/matchers" ) func init() { diff --git a/cloud/services/securitygroups/client.go b/cloud/services/securitygroups/client.go index 52bdcacb96c..5b9b84d47c0 100644 --- a/cloud/services/securitygroups/client.go +++ b/cloud/services/securitygroups/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/securitygroups/securitygroups.go b/cloud/services/securitygroups/securitygroups.go index 53a1aedb62e..5af8aae53d0 100644 --- a/cloud/services/securitygroups/securitygroups.go +++ b/cloud/services/securitygroups/securitygroups.go @@ -25,7 +25,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) const ( diff --git a/cloud/services/securitygroups/securitygroups_test.go b/cloud/services/securitygroups/securitygroups_test.go index 24f66ecc8c9..b8972089f28 100644 --- a/cloud/services/securitygroups/securitygroups_test.go +++ b/cloud/services/securitygroups/securitygroups_test.go @@ -22,7 +22,7 @@ import ( "testing" . "github.com/onsi/gomega" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/securitygroups/mock_securitygroups" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/securitygroups/mock_securitygroups" "github.com/Azure/go-autorest/autorest" "github.com/golang/mock/gomock" @@ -30,8 +30,8 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) diff --git a/cloud/services/securitygroups/service.go b/cloud/services/securitygroups/service.go index 827dde02c5b..6a4285f8465 100644 --- a/cloud/services/securitygroups/service.go +++ b/cloud/services/securitygroups/service.go @@ -17,7 +17,7 @@ limitations under the License. package securitygroups import ( - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" ) // Service provides operations on azure resources diff --git a/cloud/services/subnets/client.go b/cloud/services/subnets/client.go index 74bb4802b81..57cfae82e87 100644 --- a/cloud/services/subnets/client.go +++ b/cloud/services/subnets/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/subnets/service.go b/cloud/services/subnets/service.go index b0f830efcd0..229aae45b47 100644 --- a/cloud/services/subnets/service.go +++ b/cloud/services/subnets/service.go @@ -17,9 +17,9 @@ limitations under the License. package subnets import ( - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/routetables" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/securitygroups" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/routetables" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/securitygroups" ) // Service provides operations on azure resources diff --git a/cloud/services/subnets/subnets.go b/cloud/services/subnets/subnets.go index dd950a8deef..223dde154e2 100644 --- a/cloud/services/subnets/subnets.go +++ b/cloud/services/subnets/subnets.go @@ -25,8 +25,8 @@ import ( "github.com/pkg/errors" "k8s.io/klog" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Spec input specification for Get/CreateOrUpdate/Delete calls diff --git a/cloud/services/subnets/subnets_test.go b/cloud/services/subnets/subnets_test.go index dbd0578d213..0fa9572aa3d 100644 --- a/cloud/services/subnets/subnets_test.go +++ b/cloud/services/subnets/subnets_test.go @@ -22,9 +22,9 @@ import ( "testing" . "github.com/onsi/gomega" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/routetables/mock_routetables" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/securitygroups/mock_securitygroups" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/subnets/mock_subnets" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/routetables/mock_routetables" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/securitygroups/mock_securitygroups" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/subnets/mock_subnets" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" @@ -33,8 +33,8 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) diff --git a/cloud/services/virtualmachineextensions/client.go b/cloud/services/virtualmachineextensions/client.go index 34d33e9481c..356328798fc 100644 --- a/cloud/services/virtualmachineextensions/client.go +++ b/cloud/services/virtualmachineextensions/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/virtualmachineextensions/service.go b/cloud/services/virtualmachineextensions/service.go index 9a8d81613d5..cc438f7b0b6 100644 --- a/cloud/services/virtualmachineextensions/service.go +++ b/cloud/services/virtualmachineextensions/service.go @@ -17,7 +17,7 @@ limitations under the License. package virtualmachineextensions import ( - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" ) // Service provides operations on azure resources diff --git a/cloud/services/virtualmachines/client.go b/cloud/services/virtualmachines/client.go index f5cc229b90f..fdc70fd436d 100644 --- a/cloud/services/virtualmachines/client.go +++ b/cloud/services/virtualmachines/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/virtualmachines/service.go b/cloud/services/virtualmachines/service.go index 248ccb209d6..0cbd3fef03d 100644 --- a/cloud/services/virtualmachines/service.go +++ b/cloud/services/virtualmachines/service.go @@ -17,10 +17,10 @@ limitations under the License. package virtualmachines import ( - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/networkinterfaces" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicips" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/roleassignments" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/networkinterfaces" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicips" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/roleassignments" ) // Service provides operations on azure resources diff --git a/cloud/services/virtualmachines/virtualmachines.go b/cloud/services/virtualmachines/virtualmachines.go index dddf87a3e45..b1c745c9d8e 100644 --- a/cloud/services/virtualmachines/virtualmachines.go +++ b/cloud/services/virtualmachines/virtualmachines.go @@ -27,9 +27,9 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/authorization/mgmt/authorization" "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest/to" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" - "github.com/chlau-az/cluster-api-provider-azure/cloud/converters" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + "sigs.k8s.io/cluster-api-provider-azure/cloud/converters" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/uuid" diff --git a/cloud/services/virtualmachines/virtualmachines_test.go b/cloud/services/virtualmachines/virtualmachines_test.go index e38a3035ca6..0f1f424a000 100644 --- a/cloud/services/virtualmachines/virtualmachines_test.go +++ b/cloud/services/virtualmachines/virtualmachines_test.go @@ -22,10 +22,10 @@ import ( "testing" . "github.com/onsi/gomega" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/networkinterfaces/mock_networkinterfaces" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicips/mock_publicips" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/roleassignments/mock_roleassignments" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/virtualmachines/mock_virtualmachines" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/networkinterfaces/mock_networkinterfaces" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicips/mock_publicips" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/roleassignments/mock_roleassignments" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/virtualmachines/mock_virtualmachines" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" @@ -36,8 +36,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) diff --git a/cloud/services/virtualnetworks/client.go b/cloud/services/virtualnetworks/client.go index f7299fa70c8..0d79c926df6 100644 --- a/cloud/services/virtualnetworks/client.go +++ b/cloud/services/virtualnetworks/client.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" "github.com/Azure/go-autorest/autorest" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Client wraps go-sdk diff --git a/cloud/services/virtualnetworks/service.go b/cloud/services/virtualnetworks/service.go index ca5e4381ac6..abddc3b7304 100644 --- a/cloud/services/virtualnetworks/service.go +++ b/cloud/services/virtualnetworks/service.go @@ -17,7 +17,7 @@ limitations under the License. package virtualnetworks import ( - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" ) // Service provides operations on azure resources diff --git a/cloud/services/virtualnetworks/virtualnetworks.go b/cloud/services/virtualnetworks/virtualnetworks.go index dbb07157f6d..43a116201ce 100644 --- a/cloud/services/virtualnetworks/virtualnetworks.go +++ b/cloud/services/virtualnetworks/virtualnetworks.go @@ -23,9 +23,9 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" - "github.com/chlau-az/cluster-api-provider-azure/cloud/converters" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + "sigs.k8s.io/cluster-api-provider-azure/cloud/converters" ) // Spec input specification for Get/CreateOrUpdate/Delete calls diff --git a/cloud/services/virtualnetworks/virtualnetworks_test.go b/cloud/services/virtualnetworks/virtualnetworks_test.go index 36d0d5707e6..25e6181b71f 100644 --- a/cloud/services/virtualnetworks/virtualnetworks_test.go +++ b/cloud/services/virtualnetworks/virtualnetworks_test.go @@ -24,7 +24,7 @@ import ( "testing" . "github.com/onsi/gomega" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/virtualnetworks/mock_virtualnetworks" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/virtualnetworks/mock_virtualnetworks" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" @@ -33,8 +33,8 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) diff --git a/config/default/manager_credentials_patch.yaml b/config/default/manager_credentials_patch.yaml index e7d9c80d8df..2feffd0a33e 100644 --- a/config/default/manager_credentials_patch.yaml +++ b/config/default/manager_credentials_patch.yaml @@ -32,4 +32,4 @@ spec: - name: AZURE_GO_SDK_LOG_LEVEL value: DEBUG - name: AZURE_GO_SDK_LOG_FILE - value: /home/chlau/go/src/github.com/chlau-az/cluster-api-provider-azure/gosdk.log + value: /home/chlau/go/src/sigs.k8s.io/cluster-api-provider-azure/gosdk.log diff --git a/controllers/azurecluster_controller.go b/controllers/azurecluster_controller.go index 807def5dbaa..980ddb88390 100644 --- a/controllers/azurecluster_controller.go +++ b/controllers/azurecluster_controller.go @@ -36,9 +36,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" - "github.com/chlau-az/cluster-api-provider-azure/util/reconciler" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" ) // AzureClusterReconciler reconciles a AzureCluster object diff --git a/controllers/azurecluster_controller_test.go b/controllers/azurecluster_controller_test.go index ec505f6be83..529c339b69c 100644 --- a/controllers/azurecluster_controller_test.go +++ b/controllers/azurecluster_controller_test.go @@ -29,10 +29,10 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/internal/test" - "github.com/chlau-az/cluster-api-provider-azure/internal/test/mock_log" - "github.com/chlau-az/cluster-api-provider-azure/internal/test/record" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/internal/test" + "sigs.k8s.io/cluster-api-provider-azure/internal/test/mock_log" + "sigs.k8s.io/cluster-api-provider-azure/internal/test/record" ) var _ = Describe("AzureClusterReconciler", func() { diff --git a/controllers/azurecluster_reconciler.go b/controllers/azurecluster_reconciler.go index 45c4fd2aa03..68378e04c33 100644 --- a/controllers/azurecluster_reconciler.go +++ b/controllers/azurecluster_reconciler.go @@ -26,18 +26,18 @@ import ( clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/availabilityzones" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/groups" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/internalloadbalancers" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicips" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicloadbalancers" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/routetables" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/securitygroups" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/subnets" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/virtualnetworks" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/availabilityzones" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/groups" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/internalloadbalancers" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicips" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicloadbalancers" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/routetables" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/securitygroups" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/subnets" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/virtualnetworks" ) // azureClusterReconciler is the reconciler called by the AzureCluster controller diff --git a/controllers/azuremachine_annotations.go b/controllers/azuremachine_annotations.go index 9ca8f91b9d6..e70192830f3 100644 --- a/controllers/azuremachine_annotations.go +++ b/controllers/azuremachine_annotations.go @@ -19,7 +19,7 @@ package controllers import ( "encoding/json" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" ) // updateMachineAnnotationJSON updates the `annotation` on `machine` with diff --git a/controllers/azuremachine_controller.go b/controllers/azuremachine_controller.go index b9503eeef3f..90e1742cef9 100644 --- a/controllers/azuremachine_controller.go +++ b/controllers/azuremachine_controller.go @@ -39,9 +39,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" - "github.com/chlau-az/cluster-api-provider-azure/util/reconciler" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" ) // AzureMachineReconciler reconciles a AzureMachine object diff --git a/controllers/azuremachine_controller_test.go b/controllers/azuremachine_controller_test.go index 5587fec7a73..97d59601100 100644 --- a/controllers/azuremachine_controller_test.go +++ b/controllers/azuremachine_controller_test.go @@ -28,9 +28,9 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/internal/test" - "github.com/chlau-az/cluster-api-provider-azure/internal/test/record" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/internal/test" + "sigs.k8s.io/cluster-api-provider-azure/internal/test/record" ) var _ = Describe("AzureMachineReconciler", func() { diff --git a/controllers/azuremachine_reconciler.go b/controllers/azuremachine_reconciler.go index ed61f97933e..7a1c8bd9230 100644 --- a/controllers/azuremachine_reconciler.go +++ b/controllers/azuremachine_reconciler.go @@ -19,19 +19,19 @@ package controllers import ( "context" "encoding/base64" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/publicips" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicips" "time" "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/availabilityzones" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/disks" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/networkinterfaces" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/virtualmachines" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/availabilityzones" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/disks" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/networkinterfaces" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/virtualmachines" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/cluster-api/util" ) diff --git a/controllers/azuremachine_reconciler_test.go b/controllers/azuremachine_reconciler_test.go index 074dcfcdb20..a1cc4bd38b1 100644 --- a/controllers/azuremachine_reconciler_test.go +++ b/controllers/azuremachine_reconciler_test.go @@ -17,14 +17,14 @@ limitations under the License. package controllers import ( - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" "testing" . "github.com/onsi/gomega" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/log" ) diff --git a/controllers/azuremachine_tags.go b/controllers/azuremachine_tags.go index 99353abeb14..10ca8577484 100644 --- a/controllers/azuremachine_tags.go +++ b/controllers/azuremachine_tags.go @@ -21,8 +21,8 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/virtualmachines" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/virtualmachines" ) const ( diff --git a/controllers/helpers.go b/controllers/helpers.go index b9d3689affc..6d06636ad75 100644 --- a/controllers/helpers.go +++ b/controllers/helpers.go @@ -31,8 +31,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/handler" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/util/reconciler" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" ) // AzureClusterToAzureMachinesMapper creates a mapping handler to transform AzureClusters into AzureMachines. The transform diff --git a/controllers/helpers_test.go b/controllers/helpers_test.go index 7c0c49be11b..4139a170fc8 100644 --- a/controllers/helpers_test.go +++ b/controllers/helpers_test.go @@ -29,8 +29,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/handler" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/internal/test/mock_log" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/internal/test/mock_log" ) func TestAzureClusterToAzureMachinesMapper(t *testing.T) { diff --git a/controllers/suite_test.go b/controllers/suite_test.go index bec13255a3e..c2a56266e13 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/envtest/printer" - "github.com/chlau-az/cluster-api-provider-azure/internal/test/env" + "sigs.k8s.io/cluster-api-provider-azure/internal/test/env" // +kubebuilder:scaffold:imports ) diff --git a/docs/development.md b/docs/development.md index b7dbaa697fd..97fbd86f436 100644 --- a/docs/development.md +++ b/docs/development.md @@ -338,7 +338,7 @@ You can optionally set the following variables: | Variable | Description | Default | |----------------------------|---------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------| -| `E2E_CONF_FILE` | The path of the [E2E configuration file](https://cluster-api.sigs.k8s.io/developer/e2e.html#defining-an-e2e-config-file). | `${GOPATH}/src/github.com/chlau-az/cluster-api-provider-azure/test/e2e/config/azure-dev.yaml` | +| `E2E_CONF_FILE` | The path of the [E2E configuration file](https://cluster-api.sigs.k8s.io/developer/e2e.html#defining-an-e2e-config-file). | `${GOPATH}/src/sigs.k8s.io/cluster-api-provider-azure/test/e2e/config/azure-dev.yaml` | | `SKIP_CLEANUP` | Set to `true` if you do not want the bootstrap and workload clusters to be cleaned up after running E2E tests. | `false` | | `SKIP_CREATE_MGMT_CLUSTER` | Skip management cluster creation. | `false` | diff --git a/exp/PROJECT b/exp/PROJECT index c1d2793e927..1e99aa6fcd8 100644 --- a/exp/PROJECT +++ b/exp/PROJECT @@ -1,5 +1,5 @@ domain: exp.infrastructure.x-k8s.io -repo: github.com/chlau-az/cluster-api-provider-azure/exp +repo: sigs.k8s.io/cluster-api-provider-azure/exp version: "2" resources: - group: exp.infrastructure diff --git a/exp/api/v1alpha3/azuremachinepool_test.go b/exp/api/v1alpha3/azuremachinepool_test.go index 0badbfe4395..92c6afb82c0 100644 --- a/exp/api/v1alpha3/azuremachinepool_test.go +++ b/exp/api/v1alpha3/azuremachinepool_test.go @@ -21,8 +21,8 @@ import ( "github.com/onsi/gomega" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" ) func TestAzureMachinePool_Validate(t *testing.T) { diff --git a/exp/api/v1alpha3/azuremachinepool_types.go b/exp/api/v1alpha3/azuremachinepool_types.go index ca574721fdf..ca49e5f492b 100644 --- a/exp/api/v1alpha3/azuremachinepool_types.go +++ b/exp/api/v1alpha3/azuremachinepool_types.go @@ -20,7 +20,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/cluster-api/errors" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" ) type ( diff --git a/exp/api/v1alpha3/azuremachinepool_webhook.go b/exp/api/v1alpha3/azuremachinepool_webhook.go index bb1d0485a52..218a321f788 100644 --- a/exp/api/v1alpha3/azuremachinepool_webhook.go +++ b/exp/api/v1alpha3/azuremachinepool_webhook.go @@ -24,7 +24,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" "sigs.k8s.io/controller-runtime/pkg/webhook" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" ) // log is for logging in this package. diff --git a/exp/api/v1alpha3/types.go b/exp/api/v1alpha3/types.go index 69f70d9c277..67fff704148 100644 --- a/exp/api/v1alpha3/types.go +++ b/exp/api/v1alpha3/types.go @@ -17,7 +17,7 @@ limitations under the License. package v1alpha3 import ( - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" ) type ( diff --git a/exp/api/v1alpha3/zz_generated.deepcopy.go b/exp/api/v1alpha3/zz_generated.deepcopy.go index 9a1e8ff80fa..11a9ce1c844 100644 --- a/exp/api/v1alpha3/zz_generated.deepcopy.go +++ b/exp/api/v1alpha3/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ package v1alpha3 import ( "k8s.io/apimachinery/pkg/runtime" - apiv1alpha3 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + apiv1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" "sigs.k8s.io/cluster-api/errors" ) diff --git a/exp/controllers/azuremachinepool_controller.go b/exp/controllers/azuremachinepool_controller.go index 545ddc90a64..05e069932a3 100644 --- a/exp/controllers/azuremachinepool_controller.go +++ b/exp/controllers/azuremachinepool_controller.go @@ -45,13 +45,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/scalesets" - "github.com/chlau-az/cluster-api-provider-azure/controllers" - infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/util/reconciler" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/scalesets" + "sigs.k8s.io/cluster-api-provider-azure/controllers" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" ) type ( diff --git a/exp/controllers/azuremachinepool_controller_test.go b/exp/controllers/azuremachinepool_controller_test.go index 8a8d0eb7562..8a8cc776971 100644 --- a/exp/controllers/azuremachinepool_controller_test.go +++ b/exp/controllers/azuremachinepool_controller_test.go @@ -25,7 +25,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" - infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" ) var _ = Describe("AzureMachinePoolReconciler", func() { diff --git a/exp/controllers/azuremachinepool_controller_unit_test.go b/exp/controllers/azuremachinepool_controller_unit_test.go index 054fae4a66f..68609f85564 100644 --- a/exp/controllers/azuremachinepool_controller_unit_test.go +++ b/exp/controllers/azuremachinepool_controller_unit_test.go @@ -33,12 +33,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/scalesets/mock_scalesets" - infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/internal/test/matchers" - "github.com/chlau-az/cluster-api-provider-azure/internal/test/mock_log" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/scalesets/mock_scalesets" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/internal/test/matchers" + "sigs.k8s.io/cluster-api-provider-azure/internal/test/mock_log" ) func Test_machinePoolToInfrastructureMapFunc(t *testing.T) { diff --git a/exp/controllers/azuremanagedcluster_controller.go b/exp/controllers/azuremanagedcluster_controller.go index 10364fb7010..1c2b28c8feb 100644 --- a/exp/controllers/azuremanagedcluster_controller.go +++ b/exp/controllers/azuremanagedcluster_controller.go @@ -37,8 +37,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/util/reconciler" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" ) // AzureManagedClusterReconciler reconciles a AzureManagedCluster object diff --git a/exp/controllers/azuremanagedmachinepool_controller.go b/exp/controllers/azuremanagedmachinepool_controller.go index 791853dd9c3..4291f97615d 100644 --- a/exp/controllers/azuremanagedmachinepool_controller.go +++ b/exp/controllers/azuremanagedmachinepool_controller.go @@ -37,10 +37,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" - infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/util/reconciler" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" ) // AzureManagedMachinePoolReconciler reconciles a AzureManagedMachinePool object diff --git a/exp/controllers/azuremanagedmachinepool_reconciler.go b/exp/controllers/azuremanagedmachinepool_reconciler.go index 1d5bf0b400b..c933f1346f2 100644 --- a/exp/controllers/azuremanagedmachinepool_reconciler.go +++ b/exp/controllers/azuremanagedmachinepool_reconciler.go @@ -22,10 +22,10 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/pkg/errors" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/agentpools" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/scalesets" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/agentpools" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/scalesets" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/exp/controllers/azuremangedcontrolplane_controller.go b/exp/controllers/azuremangedcontrolplane_controller.go index 1060fa22499..e5979dc2e9c 100644 --- a/exp/controllers/azuremangedcontrolplane_controller.go +++ b/exp/controllers/azuremangedcontrolplane_controller.go @@ -36,10 +36,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" - infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/util/reconciler" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" ) // AzureManagedControlPlaneReconciler reconciles a AzureManagedControlPlane object diff --git a/exp/controllers/azuremangedcontrolplane_reconciler.go b/exp/controllers/azuremangedcontrolplane_reconciler.go index 467ab4a3c11..dc5b3664db5 100644 --- a/exp/controllers/azuremangedcontrolplane_reconciler.go +++ b/exp/controllers/azuremangedcontrolplane_reconciler.go @@ -25,10 +25,10 @@ import ( "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - azure "github.com/chlau-az/cluster-api-provider-azure/cloud" - "github.com/chlau-az/cluster-api-provider-azure/cloud/scope" - "github.com/chlau-az/cluster-api-provider-azure/cloud/services/managedclusters" - infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/managedclusters" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/cluster-api/util/secret" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/exp/controllers/helpers.go b/exp/controllers/helpers.go index 810b38b3135..3244b38a43b 100644 --- a/exp/controllers/helpers.go +++ b/exp/controllers/helpers.go @@ -33,10 +33,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/controllers" - infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/util/reconciler" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/controllers" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" ) // AzureClusterToAzureMachinePoolsMapper creates a mapping handler to transform AzureClusters into AzureMachinePools. The transform diff --git a/exp/controllers/helpers_test.go b/exp/controllers/helpers_test.go index 73057ba7e54..475c8094b06 100644 --- a/exp/controllers/helpers_test.go +++ b/exp/controllers/helpers_test.go @@ -31,9 +31,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/internal/test/mock_log" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/internal/test/mock_log" ) func TestAzureClusterToAzureMachinePoolsMapper(t *testing.T) { diff --git a/exp/controllers/suite_test.go b/exp/controllers/suite_test.go index 0d3e1dd8cb8..30641a6f7b6 100644 --- a/exp/controllers/suite_test.go +++ b/exp/controllers/suite_test.go @@ -33,8 +33,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" // +kubebuilder:scaffold:imports ) diff --git a/go.mod b/go.mod index ea8b7eb811c..1e5ed5ba4ee 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/chlau-az/cluster-api-provider-azure +module sigs.k8s.io/cluster-api-provider-azure go 1.13 @@ -26,9 +26,9 @@ require ( k8s.io/client-go v0.17.2 k8s.io/component-base v0.17.2 k8s.io/klog v1.0.0 + k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c // indirect k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab sigs.k8s.io/cluster-api v0.3.7-alpha.0 - sigs.k8s.io/cluster-api-provider-azure v0.4.5 sigs.k8s.io/controller-runtime v0.5.6 ) diff --git a/hack/boilerplate/test/BUILD b/hack/boilerplate/test/BUILD index fc79a1a726f..b3644d620e9 100644 --- a/hack/boilerplate/test/BUILD +++ b/hack/boilerplate/test/BUILD @@ -11,7 +11,7 @@ go_library( "fail.go", "pass.go", ], - importpath = "github.com/chlau-az/cluster-api-provider-azure/hack/boilerplate/test", + importpath = "sigs.k8s.io/cluster-api-provider-azure/hack/boilerplate/test", ) filegroup( diff --git a/hack/version.sh b/hack/version.sh index 86ee81ca856..a5e57c419d6 100755 --- a/hack/version.sh +++ b/hack/version.sh @@ -77,7 +77,7 @@ version::ldflags() { local key=${1} local val=${2} ldflags+=( - "-X 'github.com/chlau-az/cluster-api-provider-azure/version.${key}=${val}'" + "-X 'sigs.k8s.io/cluster-api-provider-azure/version.${key}=${val}'" ) } diff --git a/internal/test/env/env.go b/internal/test/env/env.go index 776d8086414..3587fd109f2 100644 --- a/internal/test/env/env.go +++ b/internal/test/env/env.go @@ -43,10 +43,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/manager" - infrav1alpha2 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha2" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - infrav1exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/internal/test/record" + infrav1alpha2 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha2" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/internal/test/record" ) var ( diff --git a/main.go b/main.go index 8bd545213ea..e3b4b911c54 100644 --- a/main.go +++ b/main.go @@ -35,13 +35,13 @@ import ( capifeature "sigs.k8s.io/cluster-api/feature" - infrav1alpha2 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha2" - infrav1alpha3 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" - "github.com/chlau-az/cluster-api-provider-azure/controllers" - infrav1alpha3exp "github.com/chlau-az/cluster-api-provider-azure/exp/api/v1alpha3" - infrav1controllersexp "github.com/chlau-az/cluster-api-provider-azure/exp/controllers" - "github.com/chlau-az/cluster-api-provider-azure/util/reconciler" - version "github.com/chlau-az/cluster-api-provider-azure/version" + infrav1alpha2 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha2" + infrav1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/controllers" + infrav1alpha3exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" + infrav1controllersexp "sigs.k8s.io/cluster-api-provider-azure/exp/controllers" + "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" + version "sigs.k8s.io/cluster-api-provider-azure/version" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" clusterv1exp "sigs.k8s.io/cluster-api/exp/api/v1alpha3" @@ -50,7 +50,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/healthz" - "github.com/chlau-az/cluster-api-provider-azure/feature" + "sigs.k8s.io/cluster-api-provider-azure/feature" ) var ( diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index ce7db7b2f1d..6109bb516eb 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -32,7 +32,7 @@ import ( "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/runtime" - infrav1 "github.com/chlau-az/cluster-api-provider-azure/api/v1alpha3" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/bootstrap" "sigs.k8s.io/cluster-api/test/framework/clusterctl" diff --git a/util/reconciler/defaults_test.go b/util/reconciler/defaults_test.go index 3dfa8b64de8..70834a2a1cf 100644 --- a/util/reconciler/defaults_test.go +++ b/util/reconciler/defaults_test.go @@ -22,7 +22,7 @@ import ( "github.com/onsi/gomega" - "github.com/chlau-az/cluster-api-provider-azure/util/reconciler" + "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" ) func TestDefaultedTimeout(t *testing.T) { From 3ac64e3d16ae985f5acbce85efe61b44e64ebc00 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Mon, 20 Jul 2020 13:47:29 -0700 Subject: [PATCH 15/47] Removed location from setCredentials, added azure stack to DNSZoneForEnvironment --- cloud/scope/clients.go | 68 ++++++++++++------------------ cloud/scope/clients_test.go | 2 +- cloud/scope/cluster.go | 6 +-- cloud/scope/managedcontrolplane.go | 4 +- 4 files changed, 33 insertions(+), 47 deletions(-) diff --git a/cloud/scope/clients.go b/cloud/scope/clients.go index 35aea642ad4..85998b4f0ba 100644 --- a/cloud/scope/clients.go +++ b/cloud/scope/clients.go @@ -25,7 +25,6 @@ import ( "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/adal" "github.com/Azure/go-autorest/autorest/azure" - "github.com/Azure/go-autorest/autorest/azure/auth" "github.com/pkg/errors" ) @@ -38,64 +37,39 @@ const ( PublicCloud = "AzurePublicCloud" // USGovernmentCloud is the cloud environment for the US Government USGovernmentCloud = "AzureUSGovernmentCloud" + // AzureStackCloud + AzureStackCloud = "AzureStackCloud" ) // AzureClients contains all the Azure clients used by the scopes. type AzureClients struct { SubscriptionID string - ClientID string - ClientSecret string - TenantID string ResourceManagerEndpoint string ResourceManagerVMDNSSuffix string Authorizer autorest.Authorizer } -func (c *AzureClients) setCredentials(subscriptionID, location string) error { +func (c *AzureClients) setCredentials(subscriptionID string) error { subID, err := getSubscriptionID(subscriptionID) if err != nil { return err } c.SubscriptionID = subID - c.ClientID = "huey" - c.ClientSecret = "dewey" - c.TenantID = "louie" - log.Println("HERE 0client id: ", c.ClientID) - log.Println("HERE 0client secret: ", c.ClientSecret) - log.Println("HERE 0tenant id: ", c.TenantID) - log.Println("HERE 0subscription id: ", c.SubscriptionID) - - c.ClientID = os.Getenv("AZURE_CLIENT_ID") - c.ClientSecret = os.Getenv("AZURE_CLIENT_SECRET") - c.TenantID = os.Getenv("AZURE_TENANT_ID") - log.Println("HERE client id: ", c.ClientID) - log.Println("HERE client secret: ", c.ClientSecret) - log.Println("HERE tenant id: ", c.TenantID) - log.Println("HERE subscription id: ", c.SubscriptionID) - settings, err := auth.GetSettingsFromEnvironment() - if err != nil { - log.Println("HERE couldn't find environment") - // return err - } - - // To do: get arm endpoint in helper method armEndpoint := os.Getenv("AZURE_ARM_ENDPOINT") log.Println("HERE armEndpoint: ", armEndpoint) - settings.Environment, err = azure.EnvironmentFromURL(armEndpoint) + env, err := azure.EnvironmentFromURL(armEndpoint) if err != nil { log.Println("HERE error getting environment from armEndpoint: ", armEndpoint) return err } - log.Println("HERE resource manager endpoint: ", c.ResourceManagerEndpoint) - c.ResourceManagerEndpoint = settings.Environment.ResourceManagerEndpoint - azsFQDNSuffix := getAzureStackFQDNSuffix(armEndpoint, location) - c.ResourceManagerVMDNSSuffix = fmt.Sprintf("cloudapp.%s", azsFQDNSuffix) + c.ResourceManagerEndpoint = env.ResourceManagerEndpoint + log.Println("HERE c.ResourceManagerEndpoint: ", c.ResourceManagerEndpoint) + c.ResourceManagerVMDNSSuffix = GetAzureDNSZoneForEnvironment("AzureStackCloud") log.Println("HERE c.ResourceManagerVMDNSSuffix: ", c.ResourceManagerVMDNSSuffix) - settings.Values[auth.SubscriptionID] = subscriptionID - c.Authorizer, err = c.getAuthorizerForResource(settings.Environment) - log.Println("HERE c.Authorizer: ", c.Authorizer, "err: ", err) + c.Authorizer, err = getAuthorizerForResource(env) + log.Println("HERE c.Authorizer: ", c.Authorizer) return err } @@ -123,24 +97,35 @@ func GetAzureDNSZoneForEnvironment(environmentName string) string { return "cloudapp.azure.com" case USGovernmentCloud: return "cloudapp.usgovcloudapi.net" + case AzureStackCloud: + armEndpoint := os.Getenv("AZURE_ARM_ENDPOINT") + azsFQDNSuffix := getAzureStackFQDNSuffix(armEndpoint) + return fmt.Sprintf("cloudapp.%s", azsFQDNSuffix) default: return "cloudapp.azure.com" } } -func getAzureStackFQDNSuffix(portalURL, location string) string { - azsFQDNSuffix := strings.Replace(portalURL, fmt.Sprintf("https://management.%s.", location), "", -1) +func getAzureStackFQDNSuffix(portalURL string) string { + azsFQDNSuffix := strings.Replace(portalURL, "https://management.", "", -1) + azsFQDNSuffix = strings.Join(strings.Split(azsFQDNSuffix, ".")[1:], ".") //remove location prefix azsFQDNSuffix = strings.TrimSuffix(azsFQDNSuffix, "/") - return azsFQDNSuffix } // getAuthorizerForResource gets an OAuthTokenAuthorizer for Azure Resource Manager -func (c *AzureClients) getAuthorizerForResource(env azure.Environment) (autorest.Authorizer, error) { +func getAuthorizerForResource(env azure.Environment) (autorest.Authorizer, error) { var a autorest.Authorizer var err error var oauthConfig *adal.OAuthConfig + clientID := os.Getenv("AZURE_CLIENT_ID") + clientSecret := os.Getenv("AZURE_CLIENT_SECRET") + tenantID := os.Getenv("AZURE_TENANT_ID") + log.Println("HERE client id: ", clientID) + log.Println("HERE client secret: ", clientSecret) + log.Println("HERE tenant id: ", tenantID) + tokenAudience := env.TokenAudience log.Println("HERE TokenAudience: ", env.TokenAudience) log.Println("HERE ActiveDirectoryEndpoint: ", env.ActiveDirectoryEndpoint) @@ -150,10 +135,11 @@ func (c *AzureClients) getAuthorizerForResource(env azure.Environment) (autorest if err != nil { return nil, err } + token, err := adal.NewServicePrincipalToken( *oauthConfig, - c.ClientID, - c.ClientSecret, + clientID, + clientSecret, tokenAudience) log.Println("HERE generated token") diff --git a/cloud/scope/clients_test.go b/cloud/scope/clients_test.go index 621d8dcf995..b2413ee259b 100644 --- a/cloud/scope/clients_test.go +++ b/cloud/scope/clients_test.go @@ -69,7 +69,7 @@ func TestGettingEnvironment(t *testing.T) { t.Run(name, func(t *testing.T) { os.Setenv("AZURE_ENVIRONMENT", test.azureEnv) c := AzureClients{} - err := c.setCredentials("1234", "test_location") + err := c.setCredentials("1234") if test.expectedError { g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).To(ContainSubstring(test.expectedErrorMessage)) diff --git a/cloud/scope/cluster.go b/cloud/scope/cluster.go index 8ae176fd9b5..629e2c75acd 100644 --- a/cloud/scope/cluster.go +++ b/cloud/scope/cluster.go @@ -21,11 +21,11 @@ import ( "fmt" "github.com/Azure/go-autorest/autorest" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" "github.com/go-logr/logr" "github.com/pkg/errors" "k8s.io/klog/klogr" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/controller-runtime/pkg/client" @@ -55,7 +55,7 @@ func NewClusterScope(params ClusterScopeParams) (*ClusterScope, error) { params.Logger = klogr.New() } - err := params.AzureClients.setCredentials(params.AzureCluster.Spec.SubscriptionID, params.AzureCluster.Spec.Location) + err := params.AzureClients.setCredentials(params.AzureCluster.Spec.SubscriptionID) if err != nil { return nil, errors.Wrap(err, "failed to create Azure session") } diff --git a/cloud/scope/managedcontrolplane.go b/cloud/scope/managedcontrolplane.go index 484d5548adc..b2741c5f899 100644 --- a/cloud/scope/managedcontrolplane.go +++ b/cloud/scope/managedcontrolplane.go @@ -20,11 +20,11 @@ import ( "context" "github.com/Azure/go-autorest/autorest" - infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" "github.com/go-logr/logr" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/klog/klogr" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" expv1 "sigs.k8s.io/cluster-api/exp/api/v1alpha3" @@ -59,7 +59,7 @@ func NewManagedControlPlaneScope(params ManagedControlPlaneScopeParams) (*Manage params.Logger = klogr.New() } - if err := params.AzureClients.setCredentials(params.ControlPlane.Spec.SubscriptionID, params.ControlPlane.Spec.Location); err != nil { + if err := params.AzureClients.setCredentials(params.ControlPlane.Spec.SubscriptionID); err != nil { return nil, errors.Wrap(err, "failed to create Azure session") } From b7f3efce546e816b4aa1ee50afeb015f614afaff Mon Sep 17 00:00:00 2001 From: chlau-az Date: Wed, 22 Jul 2020 16:58:30 -0700 Subject: [PATCH 16/47] Added flavor azure-stack for clusterctl --- Makefile | 2 +- templates/cluster-template-azure-stack.yaml | 286 +++++++++++++++++ templates/cluster-template.yaml | 70 +---- .../flavors/azure-stack/cluster-template.yaml | 292 ++++++++++++++++++ .../flavors/azure-stack/kustomization.yaml | 5 + 5 files changed, 590 insertions(+), 65 deletions(-) create mode 100644 templates/cluster-template-azure-stack.yaml create mode 100644 templates/flavors/azure-stack/cluster-template.yaml create mode 100644 templates/flavors/azure-stack/kustomization.yaml diff --git a/Makefile b/Makefile index 909dc78fb1d..9ec8017130b 100644 --- a/Makefile +++ b/Makefile @@ -123,7 +123,7 @@ LDFLAGS := $(shell hack/version.sh) FEATURE_GATE_MACHINE_POOL ?= false FEATURE_GATES_JSON_PATCH := [{"op": "add", "path": "/spec/template/spec/containers/1/args/-", "value": "--feature-gates=MachinePool=$(FEATURE_GATE_MACHINE_POOL)"}] -CLUSTER_TEMPLATE ?= cluster-template.yaml +CLUSTER_TEMPLATE ?= cluster-template-azure-stack.yaml MANAGED_CLUSTER_TEMPLATE ?= cluster-template-aks.yaml ## -------------------------------------- diff --git a/templates/cluster-template-azure-stack.yaml b/templates/cluster-template-azure-stack.yaml new file mode 100644 index 00000000000..e1a67e1544a --- /dev/null +++ b/templates/cluster-template-azure-stack.yaml @@ -0,0 +1,286 @@ +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + location: ${AZURE_LOCATION} + networkSpec: + vnet: + name: ${AZURE_VNET_NAME} + resourceGroup: ${AZURE_RESOURCE_GROUP} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + cluster-name: ${CLUSTER_NAME} + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + - hostPath: /etc/kubernetes/azurestackcloud.json + mountPath: /etc/kubernetes/azurestackcloud.json + name: cloud-config-stack + readOnly: true + files: + - content: "{\n \"cloud\": \"${AZURE_ENVIRONMENT}\",\n \"tenantId\": \"adfs\",\n + \ \"subscriptionId\": \"${AZURE_SUBSCRIPTION_ID}\",\n \"aadClientId\": \"${AZURE_CLIENT_ID}\",\n + \ \"aadClientSecret\": \"${AZURE_CLIENT_SECRET}\",\n \"resourceGroup\": \"${AZURE_RESOURCE_GROUP}\",\n + \ \"securityGroupName\": \"${CLUSTER_NAME}-node-nsg\",\n \"location\": \"${AZURE_LOCATION}\",\n + \ \"vmType\": \"vmss\",\n \"vnetName\": \"${CLUSTER_NAME}-vnet\",\n \"vnetResourceGroup\": + \"${AZURE_RESOURCE_GROUP}\",\n \"subnetName\": \"${CLUSTER_NAME}-node-subnet\",\n + \ \"routeTableName\": \"${CLUSTER_NAME}-node-routetable\",\n \"userAssignedID\": + \"${CLUSTER_NAME}\",\n \"loadBalancerSku\": \"Basic\",\n \"maximumLoadBalancerRuleCount\": + 250,\n \"useManagedIdentityExtension\": false,\n \"useInstanceMetadata\": + false, \n \"excludeMasterFromStandardLB\": false\n}\n" + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + { + "name":"${AZURE_ENVIRONMENT}", + "managementPortalURL":"https://portal.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "publishSettingsURL":"", + "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/98bbb015-dcf2-467c-a1cb-951df61a0948", + "resourceManagerEndpoint":"https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "activeDirectoryEndpoint":"https://adfs.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "galleryEndpoint":"https://providers.n31r1203.masd.stbtest.microsoft.com:30016/", + "keyVaultEndpoint":"", + "graphEndpoint":"https://graph.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "serviceBusEndpoint":"", + "batchManagementEndpoint":"", + "storageEndpointSuffix":"redmond.ext-n31r1203.masd.stbtest.microsoft.com", + "sqlDatabaseDNSSuffix":"", + "trafficManagerDNSSuffix":"", + "keyVaultDNSSuffix":"vault.redmond.ext-n31r1203.masd.stbtest.microsoft.com", + "serviceBusEndpointSuffix":"", + "serviceManagementVMDNSSuffix":"", + "resourceManagerVMDNSSuffix":"cloudapp.ext-n31r1203.masd.stbtest.microsoft.com", + "containerRegistryDNSSuffix":"", + "cosmosDBDNSSuffix":"", + "tokenAudience":"", + "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} + } + owner: root:root + path: /etc/kubernetes/azurestackcloud.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - 'sudo sed -i ''s/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\n env:\n - + name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json/g'' + /etc/kubernetes/manifests/kube-controller-manager.yaml' + preKubeadmCommands: + - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + - sudo update-ca-certificates + - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' + \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + - sudo systemctl daemon-reload + - sudo systemctl restart kubelet + useExperimentalRetryJoin: true + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + image: + marketplace: + offer: Test + publisher: AzureStack + sku: capz-test + version: latest + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + image: + marketplace: + offer: Test + publisher: AzureStack + sku: capz-test + version: latest + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 30 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - content: | + { + "cloud": "${AZURE_ENVIRONMENT}", + "tenantId": "adfs", + "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", + "aadClientId": "${AZURE_CLIENT_ID}", + "aadClientSecret": "${AZURE_CLIENT_SECRET}", + "resourceGroup": "${AZURE_RESOURCE_GROUP}", + "securityGroupName": "${CLUSTER_NAME}-node-nsg", + "location": "${AZURE_LOCATION}", + "vmType": "vmss", + "vnetName": "${CLUSTER_NAME}-vnet", + "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", + "subnetName": "${CLUSTER_NAME}-node-subnet", + "routeTableName": "${CLUSTER_NAME}-node-routetable", + "loadBalancerSku": "Basic", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": false, + "excludeMasterFromStandardLB": false + } + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + { + "name":"${AZURE_ENVIRONMENT}", + "managementPortalURL":"https://portal.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "publishSettingsURL":"", + "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/98bbb015-dcf2-467c-a1cb-951df61a0948", + "resourceManagerEndpoint":"https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "activeDirectoryEndpoint":"https://adfs.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "galleryEndpoint":"https://providers.n31r1203.masd.stbtest.microsoft.com:30016/", + "keyVaultEndpoint":"", + "graphEndpoint":"https://graph.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "serviceBusEndpoint":"", + "batchManagementEndpoint":"", + "storageEndpointSuffix":"redmond.ext-n31r1203.masd.stbtest.microsoft.com", + "sqlDatabaseDNSSuffix":"", + "trafficManagerDNSSuffix":"", + "keyVaultDNSSuffix":"vault.redmond.ext-n31r1203.masd.stbtest.microsoft.com", + "serviceBusEndpointSuffix":"", + "serviceManagementVMDNSSuffix":"", + "resourceManagerVMDNSSuffix":"cloudapp.ext-n31r1203.masd.stbtest.microsoft.com", + "containerRegistryDNSSuffix":"", + "cosmosDBDNSSuffix":"", + "tokenAudience":"", + "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} + } + owner: root:root + path: /etc/kubernetes/azurestackcloud.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - 'sudo sed -i ''s/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\n env:\n - + name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json/g'' + /etc/kubernetes/manifests/kube-controller-manager.yaml' + preKubeadmCommands: + - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + - sudo update-ca-certificates + - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' + \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + - sudo systemctl daemon-reload + - sudo systemctl restart kubelet diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index 9e2227d9ea1..0e9cff846bc 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -41,18 +41,6 @@ spec: kind: AzureMachineTemplate name: ${CLUSTER_NAME}-control-plane kubeadmConfigSpec: - preKubeadmCommands: - - echo "hello world again!" - - "cat /etc/kubernetes/azurestackcloud.json" - - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt - - sudo update-ca-certificates - - sudo echo "Environment='AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json'" >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf - - sudo systemctl daemon-reload - - sudo systemctl restart kubelet - - echo "bye world!" - postKubeadmCommands: - - "sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml" - - echo "bye bye!" clusterConfiguration: apiServer: extraArgs: @@ -75,15 +63,11 @@ spec: mountPath: /etc/kubernetes/azure.json name: cloud-config readOnly: true - - hostPath: /etc/kubernetes/azurestackcloud.json - mountPath: /etc/kubernetes/azurestackcloud.json - name: cloud-config-stack - readOnly: true files: - content: | { "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "adfs", + "tenantId": "${AZURE_TENANT_ID}", "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", "aadClientId": "${AZURE_CLIENT_ID}", "aadClientSecret": "${AZURE_CLIENT_SECRET}", @@ -96,43 +80,14 @@ spec: "subnetName": "${CLUSTER_NAME}-node-subnet", "routeTableName": "${CLUSTER_NAME}-node-routetable", "userAssignedID": "${CLUSTER_NAME}", - "loadBalancerSku": "Basic", + "loadBalancerSku": "standard", "maximumLoadBalancerRuleCount": 250, "useManagedIdentityExtension": false, - "useInstanceMetadata": false, - "excludeMasterFromStandardLB": false + "useInstanceMetadata": true } owner: root:root path: /etc/kubernetes/azure.json permissions: "0644" - - content: | - { - "name":"${AZURE_ENVIRONMENT}", - "managementPortalURL":"https://portal.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "publishSettingsURL":"", - "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/98bbb015-dcf2-467c-a1cb-951df61a0948", - "resourceManagerEndpoint":"https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "activeDirectoryEndpoint":"https://adfs.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "galleryEndpoint":"https://providers.n31r1203.masd.stbtest.microsoft.com:30016/", - "keyVaultEndpoint":"", - "graphEndpoint":"https://graph.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "serviceBusEndpoint":"", - "batchManagementEndpoint":"", - "storageEndpointSuffix":"redmond.ext-n31r1203.masd.stbtest.microsoft.com", - "sqlDatabaseDNSSuffix":"", - "trafficManagerDNSSuffix":"", - "keyVaultDNSSuffix":"vault.redmond.ext-n31r1203.masd.stbtest.microsoft.com", - "serviceBusEndpointSuffix":"", - "serviceManagementVMDNSSuffix":"", - "resourceManagerVMDNSSuffix":"cloudapp.ext-n31r1203.masd.stbtest.microsoft.com", - "containerRegistryDNSSuffix":"", - "cosmosDBDNSSuffix":"", - "tokenAudience":"", - "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} - } - owner: root:root - path: /etc/kubernetes/azurestackcloud.json - permissions: "0644" initConfiguration: nodeRegistration: kubeletExtraArgs: @@ -157,12 +112,6 @@ metadata: spec: template: spec: - image: - marketplace: - publisher: AzureStack - offer: Test - sku: capz-test - version: latest location: ${AZURE_LOCATION} osDisk: diskSizeGB: 128 @@ -204,12 +153,6 @@ metadata: spec: template: spec: - image: - marketplace: - publisher: AzureStack - offer: Test - sku: capz-test - version: latest location: ${AZURE_LOCATION} osDisk: diskSizeGB: 30 @@ -231,7 +174,7 @@ spec: - content: | { "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "adfs", + "tenantId": "${AZURE_TENANT_ID}", "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", "aadClientId": "${AZURE_CLIENT_ID}", "aadClientSecret": "${AZURE_CLIENT_SECRET}", @@ -243,11 +186,10 @@ spec: "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", "subnetName": "${CLUSTER_NAME}-node-subnet", "routeTableName": "${CLUSTER_NAME}-node-routetable", - "loadBalancerSku": "Basic", + "loadBalancerSku": "standard", "maximumLoadBalancerRuleCount": 250, "useManagedIdentityExtension": false, - "useInstanceMetadata": false, - "excludeMasterFromStandardLB": false + "useInstanceMetadata": true } owner: root:root path: /etc/kubernetes/azure.json diff --git a/templates/flavors/azure-stack/cluster-template.yaml b/templates/flavors/azure-stack/cluster-template.yaml new file mode 100644 index 00000000000..55d5f65a76c --- /dev/null +++ b/templates/flavors/azure-stack/cluster-template.yaml @@ -0,0 +1,292 @@ +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + location: ${AZURE_LOCATION} + networkSpec: + vnet: + name: ${AZURE_VNET_NAME} + resourceGroup: ${AZURE_RESOURCE_GROUP} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + kubeadmConfigSpec: + preKubeadmCommands: + - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + - sudo update-ca-certificates + - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + - sudo systemctl daemon-reload + - sudo systemctl restart kubelet + postKubeadmCommands: + - "sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml" + clusterConfiguration: + apiServer: + extraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + cluster-name: ${CLUSTER_NAME} + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + - hostPath: /etc/kubernetes/azurestackcloud.json + mountPath: /etc/kubernetes/azurestackcloud.json + name: cloud-config-stack + readOnly: true + files: + - content: | + { + "cloud": "${AZURE_ENVIRONMENT}", + "tenantId": "adfs", + "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", + "aadClientId": "${AZURE_CLIENT_ID}", + "aadClientSecret": "${AZURE_CLIENT_SECRET}", + "resourceGroup": "${AZURE_RESOURCE_GROUP}", + "securityGroupName": "${CLUSTER_NAME}-node-nsg", + "location": "${AZURE_LOCATION}", + "vmType": "vmss", + "vnetName": "${CLUSTER_NAME}-vnet", + "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", + "subnetName": "${CLUSTER_NAME}-node-subnet", + "routeTableName": "${CLUSTER_NAME}-node-routetable", + "userAssignedID": "${CLUSTER_NAME}", + "loadBalancerSku": "Basic", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": false, + "excludeMasterFromStandardLB": false + } + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + { + "name":"${AZURE_ENVIRONMENT}", + "managementPortalURL":"https://portal.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "publishSettingsURL":"", + "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/98bbb015-dcf2-467c-a1cb-951df61a0948", + "resourceManagerEndpoint":"https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "activeDirectoryEndpoint":"https://adfs.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "galleryEndpoint":"https://providers.n31r1203.masd.stbtest.microsoft.com:30016/", + "keyVaultEndpoint":"", + "graphEndpoint":"https://graph.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "serviceBusEndpoint":"", + "batchManagementEndpoint":"", + "storageEndpointSuffix":"redmond.ext-n31r1203.masd.stbtest.microsoft.com", + "sqlDatabaseDNSSuffix":"", + "trafficManagerDNSSuffix":"", + "keyVaultDNSSuffix":"vault.redmond.ext-n31r1203.masd.stbtest.microsoft.com", + "serviceBusEndpointSuffix":"", + "serviceManagementVMDNSSuffix":"", + "resourceManagerVMDNSSuffix":"cloudapp.ext-n31r1203.masd.stbtest.microsoft.com", + "containerRegistryDNSSuffix":"", + "cosmosDBDNSSuffix":"", + "tokenAudience":"", + "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} + } + owner: root:root + path: /etc/kubernetes/azurestackcloud.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + useExperimentalRetryJoin: true + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + image: + marketplace: + publisher: AzureStack + offer: Test + sku: capz-test + version: latest + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + image: + marketplace: + publisher: AzureStack + offer: Test + sku: capz-test + version: latest + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 30 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + preKubeadmCommands: + - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + - sudo update-ca-certificates + - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + - sudo systemctl daemon-reload + - sudo systemctl restart kubelet + postKubeadmCommands: + - "sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml" + files: + - content: | + { + "cloud": "${AZURE_ENVIRONMENT}", + "tenantId": "adfs", + "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", + "aadClientId": "${AZURE_CLIENT_ID}", + "aadClientSecret": "${AZURE_CLIENT_SECRET}", + "resourceGroup": "${AZURE_RESOURCE_GROUP}", + "securityGroupName": "${CLUSTER_NAME}-node-nsg", + "location": "${AZURE_LOCATION}", + "vmType": "vmss", + "vnetName": "${CLUSTER_NAME}-vnet", + "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", + "subnetName": "${CLUSTER_NAME}-node-subnet", + "routeTableName": "${CLUSTER_NAME}-node-routetable", + "loadBalancerSku": "Basic", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": false, + "excludeMasterFromStandardLB": false + } + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + { + "name":"${AZURE_ENVIRONMENT}", + "managementPortalURL":"https://portal.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "publishSettingsURL":"", + "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/98bbb015-dcf2-467c-a1cb-951df61a0948", + "resourceManagerEndpoint":"https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "activeDirectoryEndpoint":"https://adfs.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "galleryEndpoint":"https://providers.n31r1203.masd.stbtest.microsoft.com:30016/", + "keyVaultEndpoint":"", + "graphEndpoint":"https://graph.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "serviceBusEndpoint":"", + "batchManagementEndpoint":"", + "storageEndpointSuffix":"redmond.ext-n31r1203.masd.stbtest.microsoft.com", + "sqlDatabaseDNSSuffix":"", + "trafficManagerDNSSuffix":"", + "keyVaultDNSSuffix":"vault.redmond.ext-n31r1203.masd.stbtest.microsoft.com", + "serviceBusEndpointSuffix":"", + "serviceManagementVMDNSSuffix":"", + "resourceManagerVMDNSSuffix":"cloudapp.ext-n31r1203.masd.stbtest.microsoft.com", + "containerRegistryDNSSuffix":"", + "cosmosDBDNSSuffix":"", + "tokenAudience":"", + "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} + } + owner: root:root + path: /etc/kubernetes/azurestackcloud.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' diff --git a/templates/flavors/azure-stack/kustomization.yaml b/templates/flavors/azure-stack/kustomization.yaml new file mode 100644 index 00000000000..16a0a22fe60 --- /dev/null +++ b/templates/flavors/azure-stack/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: +- cluster-template.yaml From 9bf82d5cf5e3aa364ab623aefde4cad5f67e0f71 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Thu, 23 Jul 2020 16:53:29 -0700 Subject: [PATCH 17/47] Added support for Azure Stack AAD --- .dockerignore | 2 +- Dockerfile | 8 +- Makefile | 4 +- cloud/scope/clients.go | 11 +- config/default/cloud_environment.yaml | 3 +- config/default/manager_cloud_env_patch.yaml | 7 +- .../cluster-template-azure-stack-aad.yaml | 284 ++++++++++++++++++ 7 files changed, 309 insertions(+), 10 deletions(-) create mode 100644 templates/cluster-template-azure-stack-aad.yaml diff --git a/.dockerignore b/.dockerignore index 142c69af9d3..29b318896df 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,4 +13,4 @@ !/main.go !/go.mod !/go.sum -!/azs.crt \ No newline at end of file +!/.vscode/** \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 36a98109915..4f01278dda6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ # limitations under the License. # Build the manager binary -FROM golang:1.14 as builder +FROM golang:1.13.8 as builder WORKDIR /workspace # Run this with docker build --build_arg $(go env GOPROXY) to override the goproxy @@ -22,7 +22,7 @@ ENV GOPROXY=$goproxy # Copy the Go Modules manifests COPY go.mod go.mod -# COPY go.sum go.sum +COPY go.sum go.sum # Cache deps before building and copying source so that we don't need to re-download as much # and so that source changes don't invalidate our downloaded layer RUN go mod download @@ -36,7 +36,9 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \ go build -a -ldflags '-extldflags "-static"' \ -o manager . -COPY azs.crt /usr/local/share/ca-certificates/ + +COPY .vscode/azs.crt /usr/local/share/ca-certificates/ +COPY .vscode/azs02.crt /usr/local/share/ca-certificates/ RUN update-ca-certificates # Copy the controller-manager into a thin image diff --git a/Makefile b/Makefile index 9ec8017130b..8a8e24dbbf9 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ REGISTRY ?= gcr.io/$(shell gcloud config get-value project) STAGING_REGISTRY := gcr.io/k8s-staging-cluster-api-azure PROD_REGISTRY := us.gcr.io/k8s-artifacts-prod/cluster-api-azure IMAGE_NAME ?= cluster-api-azure-controller -CONTROLLER_IMG ?= $(REGISTRY)$(IMAGE_NAME) +CONTROLLER_IMG ?= $(REGISTRY)/$(IMAGE_NAME) TAG ?= dev ARCH ?= amd64 ALL_ARCH = amd64 arm arm64 ppc64le s390x @@ -123,7 +123,7 @@ LDFLAGS := $(shell hack/version.sh) FEATURE_GATE_MACHINE_POOL ?= false FEATURE_GATES_JSON_PATCH := [{"op": "add", "path": "/spec/template/spec/containers/1/args/-", "value": "--feature-gates=MachinePool=$(FEATURE_GATE_MACHINE_POOL)"}] -CLUSTER_TEMPLATE ?= cluster-template-azure-stack.yaml +CLUSTER_TEMPLATE ?= cluster-template.yaml MANAGED_CLUSTER_TEMPLATE ?= cluster-template-aks.yaml ## -------------------------------------- diff --git a/cloud/scope/clients.go b/cloud/scope/clients.go index 85998b4f0ba..b1de6bd915c 100644 --- a/cloud/scope/clients.go +++ b/cloud/scope/clients.go @@ -129,9 +129,16 @@ func getAuthorizerForResource(env azure.Environment) (autorest.Authorizer, error tokenAudience := env.TokenAudience log.Println("HERE TokenAudience: ", env.TokenAudience) log.Println("HERE ActiveDirectoryEndpoint: ", env.ActiveDirectoryEndpoint) - oauthConfig, err = adal.NewOAuthConfig( - env.ActiveDirectoryEndpoint, "adfs") + identitySystem := os.Getenv("IDENTITY_SYSTEM") + log.Println("HERE identity system: ", identitySystem) + if identitySystem == "adfs" { + oauthConfig, err = adal.NewOAuthConfig( + env.ActiveDirectoryEndpoint, "adfs") + } else { + oauthConfig, err = adal.NewOAuthConfig( + env.ActiveDirectoryEndpoint, tenantID) + } if err != nil { return nil, err } diff --git a/config/default/cloud_environment.yaml b/config/default/cloud_environment.yaml index 8d77e843a86..61eb121c8f9 100644 --- a/config/default/cloud_environment.yaml +++ b/config/default/cloud_environment.yaml @@ -5,4 +5,5 @@ metadata: namespace: system data: cloud-env: ${AZURE_ENVIRONMENT} - cloud-arm-endpoint: ${AZURE_ARM_ENDPOINT} \ No newline at end of file + cloud-arm-endpoint: ${AZURE_ARM_ENDPOINT} + identity-system: ${IDENTITY_SYSTEM} \ No newline at end of file diff --git a/config/default/manager_cloud_env_patch.yaml b/config/default/manager_cloud_env_patch.yaml index d50c5c71a2c..fcf41c3ff2e 100644 --- a/config/default/manager_cloud_env_patch.yaml +++ b/config/default/manager_cloud_env_patch.yaml @@ -18,4 +18,9 @@ spec: valueFrom: configMapKeyRef: name: manager-bootstrap-cloud-env - key: cloud-arm-endpoint \ No newline at end of file + key: cloud-arm-endpoint + - name: IDENTITY_SYSTEM + valueFrom: + configMapKeyRef: + name: manager-bootstrap-cloud-env + key: identity-system \ No newline at end of file diff --git a/templates/cluster-template-azure-stack-aad.yaml b/templates/cluster-template-azure-stack-aad.yaml new file mode 100644 index 00000000000..09bbfd3dd8b --- /dev/null +++ b/templates/cluster-template-azure-stack-aad.yaml @@ -0,0 +1,284 @@ +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + location: ${AZURE_LOCATION} + networkSpec: + vnet: + name: ${AZURE_VNET_NAME} + resourceGroup: ${AZURE_RESOURCE_GROUP} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + cluster-name: ${CLUSTER_NAME} + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + - hostPath: /etc/kubernetes/azurestackcloud.json + mountPath: /etc/kubernetes/azurestackcloud.json + name: cloud-config-stack + readOnly: true + files: + - content: "{\n \"cloud\": \"${AZURE_ENVIRONMENT}\",\n \"tenantId\": \"${AZURE_TENANT_ID}\",\n + \ \"subscriptionId\": \"${AZURE_SUBSCRIPTION_ID}\",\n \"aadClientId\": \"${AZURE_CLIENT_ID}\",\n + \ \"aadClientSecret\": \"${AZURE_CLIENT_SECRET}\",\n \"resourceGroup\": \"${AZURE_RESOURCE_GROUP}\",\n + \ \"securityGroupName\": \"${CLUSTER_NAME}-node-nsg\",\n \"location\": \"${AZURE_LOCATION}\",\n + \ \"vmType\": \"vmss\",\n \"vnetName\": \"${CLUSTER_NAME}-vnet\",\n \"vnetResourceGroup\": + \"${AZURE_RESOURCE_GROUP}\",\n \"subnetName\": \"${CLUSTER_NAME}-node-subnet\",\n + \ \"routeTableName\": \"${CLUSTER_NAME}-node-routetable\",\n \"userAssignedID\": + \"${CLUSTER_NAME}\",\n \"loadBalancerSku\": \"Basic\",\n \"maximumLoadBalancerRuleCount\": + 250,\n \"useManagedIdentityExtension\": false,\n \"useInstanceMetadata\": + false, \n \"excludeMasterFromStandardLB\": false\n}\n" + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + { + "name":"${AZURE_ENVIRONMENT}", + "managementPortalURL":"https://portal.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", + "publishSettingsURL":"", + "serviceManagementEndpoint":"https://management.azurestackci02.onmicrosoft.com/6bbece7e-df47-402c-bf11-101f74fdf525", + "resourceManagerEndpoint":"https://management.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", + "activeDirectoryEndpoint":"https://login.microsoftonline.com/", + "galleryEndpoint":"https://providers.n25r1306.masd.stbtest.microsoft.com:30016/", + "keyVaultEndpoint":"","graphEndpoint":"https://graph.windows.net/", + "serviceBusEndpoint":"", + "batchManagementEndpoint":"", + "storageEndpointSuffix":"redmond.ext-n25r1306.masd.stbtest.microsoft.com", + "sqlDatabaseDNSSuffix":"", + "trafficManagerDNSSuffix":"", + "keyVaultDNSSuffix":"vault.redmond.ext-n25r1306.masd.stbtest.microsoft.com", + "serviceBusEndpointSuffix":"", + "serviceManagementVMDNSSuffix":"", + "resourceManagerVMDNSSuffix":"cloudapp.ext-n25r1306.masd.stbtest.microsoft.com", + "containerRegistryDNSSuffix":"", + "cosmosDBDNSSuffix":"", + "tokenAudience":"", + "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} + } + owner: root:root + path: /etc/kubernetes/azurestackcloud.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - 'sudo sed -i ''s/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\n env:\n - + name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json/g'' + /etc/kubernetes/manifests/kube-controller-manager.yaml' + preKubeadmCommands: + - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + - sudo update-ca-certificates + - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' + \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + - sudo systemctl daemon-reload + - sudo systemctl restart kubelet + useExperimentalRetryJoin: true + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + image: + marketplace: + offer: Test + publisher: AzureStack + sku: capz-test + version: latest + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + image: + marketplace: + offer: Test + publisher: AzureStack + sku: capz-test + version: latest + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 30 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - content: | + { + "cloud": "${AZURE_ENVIRONMENT}", + "tenantId": "${AZURE_TENANT_ID}", + "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", + "aadClientId": "${AZURE_CLIENT_ID}", + "aadClientSecret": "${AZURE_CLIENT_SECRET}", + "resourceGroup": "${AZURE_RESOURCE_GROUP}", + "securityGroupName": "${CLUSTER_NAME}-node-nsg", + "location": "${AZURE_LOCATION}", + "vmType": "vmss", + "vnetName": "${CLUSTER_NAME}-vnet", + "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", + "subnetName": "${CLUSTER_NAME}-node-subnet", + "routeTableName": "${CLUSTER_NAME}-node-routetable", + "loadBalancerSku": "Basic", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": false, + "excludeMasterFromStandardLB": false + } + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + { + "name":"${AZURE_ENVIRONMENT}", + "managementPortalURL":"https://portal.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", + "publishSettingsURL":"", + "serviceManagementEndpoint":"https://management.azurestackci02.onmicrosoft.com/6bbece7e-df47-402c-bf11-101f74fdf525", + "resourceManagerEndpoint":"https://management.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", + "activeDirectoryEndpoint":"https://login.microsoftonline.com/", + "galleryEndpoint":"https://providers.n25r1306.masd.stbtest.microsoft.com:30016/", + "keyVaultEndpoint":"","graphEndpoint":"https://graph.windows.net/", + "serviceBusEndpoint":"", + "batchManagementEndpoint":"", + "storageEndpointSuffix":"redmond.ext-n25r1306.masd.stbtest.microsoft.com", + "sqlDatabaseDNSSuffix":"", + "trafficManagerDNSSuffix":"", + "keyVaultDNSSuffix":"vault.redmond.ext-n25r1306.masd.stbtest.microsoft.com", + "serviceBusEndpointSuffix":"", + "serviceManagementVMDNSSuffix":"", + "resourceManagerVMDNSSuffix":"cloudapp.ext-n25r1306.masd.stbtest.microsoft.com", + "containerRegistryDNSSuffix":"", + "cosmosDBDNSSuffix":"", + "tokenAudience":"", + "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} + } + owner: root:root + path: /etc/kubernetes/azurestackcloud.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - 'sudo sed -i ''s/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\n env:\n - + name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json/g'' + /etc/kubernetes/manifests/kube-controller-manager.yaml' + preKubeadmCommands: + - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + - sudo update-ca-certificates + - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' + \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + - sudo systemctl daemon-reload + - sudo systemctl restart kubelet From 5ca0944e4232b96f0d79013a83ef2801f98d0bd1 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Fri, 24 Jul 2020 10:18:31 -0700 Subject: [PATCH 18/47] Fixed new loadbalancer service --- cloud/services/loadbalancers/loadbalancers.go | 14 +++++------ .../loadbalancers/loadbalancers_test.go | 24 ++++++++----------- .../mock_virtualnetworks/client_mock.go | 5 ++-- 3 files changed, 19 insertions(+), 24 deletions(-) diff --git a/cloud/services/loadbalancers/loadbalancers.go b/cloud/services/loadbalancers/loadbalancers.go index 63228d656fa..dd2afabbb03 100644 --- a/cloud/services/loadbalancers/loadbalancers.go +++ b/cloud/services/loadbalancers/loadbalancers.go @@ -108,12 +108,10 @@ func (s *Service) Reconcile(ctx context.Context) error { Name: &backEndAddressPoolName, }, }, - OutboundRules: &[]network.OutboundRule{ + OutboundNatRules: &[]network.OutboundNatRule{ { Name: to.StringPtr("OutboundNATAllProtocols"), - OutboundRulePropertiesFormat: &network.OutboundRulePropertiesFormat{ - Protocol: network.LoadBalancerOutboundRuleProtocolAll, - IdleTimeoutInMinutes: to.Int32Ptr(4), + OutboundNatRulePropertiesFormat: &network.OutboundNatRulePropertiesFormat{ FrontendIPConfigurations: &[]network.SubResource{ { ID: to.StringPtr(fmt.Sprintf("/%s/%s/frontendIPConfigurations/%s", idPrefix, lbSpec.Name, frontEndIPConfigName)), @@ -129,12 +127,12 @@ func (s *Service) Reconcile(ctx context.Context) error { } if lbSpec.Role == infrav1.APIServerRole || lbSpec.Role == infrav1.InternalRole { - probeName := "HTTPSProbe" + probeName := "tcpHTTPSProbe" lb.LoadBalancerPropertiesFormat.Probes = &[]network.Probe{ { Name: to.StringPtr(probeName), ProbePropertiesFormat: &network.ProbePropertiesFormat{ - Protocol: network.ProbeProtocolHTTPS, + Protocol: network.ProbeProtocolTCP, RequestPath: to.StringPtr("/healthz"), Port: to.Int32Ptr(lbSpec.APIServerPort), IntervalInSeconds: to.Int32Ptr(15), @@ -150,7 +148,7 @@ func (s *Service) Reconcile(ctx context.Context) error { BackendPort: to.Int32Ptr(lbSpec.APIServerPort), IdleTimeoutInMinutes: to.Int32Ptr(4), EnableFloatingIP: to.BoolPtr(false), - LoadDistribution: network.LoadDistributionDefault, + LoadDistribution: "Default", FrontendIPConfiguration: &network.SubResource{ ID: to.StringPtr(fmt.Sprintf("/%s/%s/frontendIPConfigurations/%s", idPrefix, lbSpec.Name, frontEndIPConfigName)), }, @@ -168,7 +166,7 @@ func (s *Service) Reconcile(ctx context.Context) error { // For more information on Standard LB outbound connections see https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-connections. lbRule.LoadBalancingRulePropertiesFormat.DisableOutboundSnat = to.BoolPtr(true) } else if lbSpec.Role == infrav1.InternalRole { - lb.LoadBalancerPropertiesFormat.OutboundRules = nil + lb.LoadBalancerPropertiesFormat.OutboundNatRules = nil } lb.LoadBalancerPropertiesFormat.LoadBalancingRules = &[]network.LoadBalancingRule{lbRule} } diff --git a/cloud/services/loadbalancers/loadbalancers_test.go b/cloud/services/loadbalancers/loadbalancers_test.go index 2f954e73fb1..41c33a497b9 100644 --- a/cloud/services/loadbalancers/loadbalancers_test.go +++ b/cloud/services/loadbalancers/loadbalancers_test.go @@ -157,7 +157,7 @@ func TestReconcileLoadBalancer(t *testing.T) { BackendPort: to.Int32Ptr(6443), IdleTimeoutInMinutes: to.Int32Ptr(4), EnableFloatingIP: to.BoolPtr(false), - LoadDistribution: network.LoadDistributionDefault, + LoadDistribution: "Default", FrontendIPConfiguration: &network.SubResource{ ID: to.StringPtr("//subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-publiclb/frontendIPConfigurations/my-publiclb-frontEnd"), }, @@ -172,9 +172,9 @@ func TestReconcileLoadBalancer(t *testing.T) { }, Probes: &[]network.Probe{ { - Name: to.StringPtr("HTTPSProbe"), + Name: to.StringPtr("tcpHTTPSProbe"), ProbePropertiesFormat: &network.ProbePropertiesFormat{ - Protocol: network.ProbeProtocolHTTPS, + Protocol: network.ProbeProtocolTCP, Port: to.Int32Ptr(6443), RequestPath: to.StringPtr("/healthz"), IntervalInSeconds: to.Int32Ptr(15), @@ -182,18 +182,16 @@ func TestReconcileLoadBalancer(t *testing.T) { }, }, }, - OutboundRules: &[]network.OutboundRule{ + OutboundNatRules: &[]network.OutboundNatRule{ { Name: to.StringPtr("OutboundNATAllProtocols"), - OutboundRulePropertiesFormat: &network.OutboundRulePropertiesFormat{ + OutboundNatRulePropertiesFormat: &network.OutboundNatRulePropertiesFormat{ FrontendIPConfigurations: &[]network.SubResource{ {ID: to.StringPtr("//subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-publiclb/frontendIPConfigurations/my-publiclb-frontEnd")}, }, BackendAddressPool: &network.SubResource{ ID: to.StringPtr("//subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-publiclb/backendAddressPools/my-publiclb-backendPool"), }, - Protocol: network.LoadBalancerOutboundRuleProtocolAll, - IdleTimeoutInMinutes: to.Int32Ptr(4), }, }, }, @@ -243,18 +241,16 @@ func TestReconcileLoadBalancer(t *testing.T) { Name: to.StringPtr("cluster-name-outboundBackendPool"), }, }, - OutboundRules: &[]network.OutboundRule{ + OutboundNatRules: &[]network.OutboundNatRule{ { Name: to.StringPtr("OutboundNATAllProtocols"), - OutboundRulePropertiesFormat: &network.OutboundRulePropertiesFormat{ + OutboundNatRulePropertiesFormat: &network.OutboundNatRulePropertiesFormat{ FrontendIPConfigurations: &[]network.SubResource{ {ID: to.StringPtr("//subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/cluster-name/frontendIPConfigurations/cluster-name-frontEnd")}, }, BackendAddressPool: &network.SubResource{ ID: to.StringPtr("//subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/cluster-name/backendAddressPools/cluster-name-outboundBackendPool"), }, - Protocol: network.LoadBalancerOutboundRuleProtocolAll, - IdleTimeoutInMinutes: to.Int32Ptr(4), }, }, }, @@ -375,9 +371,9 @@ func TestReconcileLoadBalancer(t *testing.T) { }, Probes: &[]network.Probe{ { - Name: to.StringPtr("HTTPSProbe"), + Name: to.StringPtr("tcpHTTPSProbe"), ProbePropertiesFormat: &network.ProbePropertiesFormat{ - Protocol: network.ProbeProtocolHTTPS, + Protocol: network.ProbeProtocolTCP, RequestPath: to.StringPtr("/healthz"), Port: to.Int32Ptr(100), IntervalInSeconds: to.Int32Ptr(15), @@ -399,7 +395,7 @@ func TestReconcileLoadBalancer(t *testing.T) { BackendPort: to.Int32Ptr(100), IdleTimeoutInMinutes: to.Int32Ptr(4), EnableFloatingIP: to.BoolPtr(false), - LoadDistribution: network.LoadDistributionDefault, + LoadDistribution: "Default", FrontendIPConfiguration: &network.SubResource{ ID: to.StringPtr("//subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-lb/frontendIPConfigurations/my-lb-frontEnd"), }, diff --git a/cloud/services/virtualnetworks/mock_virtualnetworks/client_mock.go b/cloud/services/virtualnetworks/mock_virtualnetworks/client_mock.go index cd86079bc4e..acd179f1d24 100644 --- a/cloud/services/virtualnetworks/mock_virtualnetworks/client_mock.go +++ b/cloud/services/virtualnetworks/mock_virtualnetworks/client_mock.go @@ -22,9 +22,10 @@ package mock_virtualnetworks import ( context "context" - network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" - gomock "github.com/golang/mock/gomock" reflect "reflect" + + network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" + gomock "github.com/golang/mock/gomock" ) // MockClient is a mock of Client interface. From 64c6317c5a6413661d0319da05799cec6bcfebaf Mon Sep 17 00:00:00 2001 From: chlau-az Date: Fri, 24 Jul 2020 10:28:19 -0700 Subject: [PATCH 19/47] Changed API versions for new services added --- api/v1alpha3/azuremachine_validation.go | 5 ++--- api/v1alpha3/azuremachine_validation_test.go | 5 ++--- cloud/services/groups/mock_groups/client_mock.go | 5 +++-- cloud/services/resourceskus/cache_test.go | 2 +- cloud/services/resourceskus/sku.go | 2 +- cloud/services/roleassignments/roleassignments_test.go | 7 ++++--- cloud/services/routetables/mock_routetables/client_mock.go | 5 +++-- cloud/services/subnets/mock_subnets/client_mock.go | 5 +++-- 8 files changed, 19 insertions(+), 17 deletions(-) diff --git a/api/v1alpha3/azuremachine_validation.go b/api/v1alpha3/azuremachine_validation.go index 2cf81dd86ab..338b91cd0b7 100644 --- a/api/v1alpha3/azuremachine_validation.go +++ b/api/v1alpha3/azuremachine_validation.go @@ -20,7 +20,6 @@ import ( "encoding/base64" "fmt" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute" "golang.org/x/crypto/ssh" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -110,7 +109,7 @@ func ValidateOSDisk(osDisk OSDisk, fieldPath *field.Path) field.ErrorList { allErrs = append(allErrs, err) } - if osDisk.DiffDiskSettings != nil && osDisk.DiffDiskSettings.Option == string(compute.Local) && osDisk.ManagedDisk.StorageAccountType != "Standard_LRS" { + if osDisk.DiffDiskSettings != nil && osDisk.DiffDiskSettings.Option == "Local" && osDisk.ManagedDisk.StorageAccountType != "Standard_LRS" { allErrs = append(allErrs, field.Invalid( fieldPath.Child("managedDisks").Child("storageAccountType"), osDisk.ManagedDisk.StorageAccountType, @@ -134,7 +133,7 @@ func ValidateManagedDisk(old, new ManagedDisk, fieldPath *field.Path) field.Erro func validateDiffDiskSetings(d *DiffDiskSettings, fldPath *field.Path) *field.Error { if d != nil { - if d.Option != string(compute.Local) { + if d.Option != "Local" { return field.Invalid( fldPath.Child("option"), d, diff --git a/api/v1alpha3/azuremachine_validation_test.go b/api/v1alpha3/azuremachine_validation_test.go index d30e2e740aa..7fd489ba4a5 100644 --- a/api/v1alpha3/azuremachine_validation_test.go +++ b/api/v1alpha3/azuremachine_validation_test.go @@ -23,7 +23,6 @@ import ( "fmt" "testing" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute" "github.com/Azure/go-autorest/autorest/to" . "github.com/onsi/gomega" @@ -91,7 +90,7 @@ func TestAzureMachine_ValidateOSDisk(t *testing.T) { DiskSizeGB: 30, OSType: "blah", DiffDiskSettings: &DiffDiskSettings{ - Option: string(compute.Local), + Option: "Local", }, ManagedDisk: ManagedDisk{ StorageAccountType: "Standard_LRS", @@ -161,7 +160,7 @@ func generateNegativeTestCases() []osDiskTestInput { StorageAccountType: "Premium_LRS", }, DiffDiskSettings: &DiffDiskSettings{ - Option: string(compute.Local), + Option: "Local", }, }, } diff --git a/cloud/services/groups/mock_groups/client_mock.go b/cloud/services/groups/mock_groups/client_mock.go index a36a3873e56..29b500af8a9 100644 --- a/cloud/services/groups/mock_groups/client_mock.go +++ b/cloud/services/groups/mock_groups/client_mock.go @@ -22,9 +22,10 @@ package mock_groups import ( context "context" - resources "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-05-01/resources" - gomock "github.com/golang/mock/gomock" reflect "reflect" + + resources "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/resources/mgmt/resources" + gomock "github.com/golang/mock/gomock" ) // MockClient is a mock of Client interface. diff --git a/cloud/services/resourceskus/cache_test.go b/cloud/services/resourceskus/cache_test.go index 324ac9c5161..9dc9732e445 100644 --- a/cloud/services/resourceskus/cache_test.go +++ b/cloud/services/resourceskus/cache_test.go @@ -20,7 +20,7 @@ import ( "context" "testing" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest/to" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" diff --git a/cloud/services/resourceskus/sku.go b/cloud/services/resourceskus/sku.go index 9b22b49ad12..1f603f3a0cd 100644 --- a/cloud/services/resourceskus/sku.go +++ b/cloud/services/resourceskus/sku.go @@ -20,7 +20,7 @@ import ( "strconv" "strings" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/pkg/errors" ) diff --git a/cloud/services/roleassignments/roleassignments_test.go b/cloud/services/roleassignments/roleassignments_test.go index da0065b8eb7..d139cc2ea51 100644 --- a/cloud/services/roleassignments/roleassignments_test.go +++ b/cloud/services/roleassignments/roleassignments_test.go @@ -18,16 +18,17 @@ package roleassignments import ( "context" + "net/http" + "testing" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/authorization/mgmt/authorization" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute" + "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" "github.com/golang/mock/gomock" . "github.com/onsi/gomega" "k8s.io/klog/klogr" - "net/http" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" - "testing" "sigs.k8s.io/cluster-api-provider-azure/cloud/services/roleassignments/mock_roleassignments" "sigs.k8s.io/cluster-api-provider-azure/cloud/services/virtualmachines/mock_virtualmachines" diff --git a/cloud/services/routetables/mock_routetables/client_mock.go b/cloud/services/routetables/mock_routetables/client_mock.go index 8fc01540d8b..981043680ec 100644 --- a/cloud/services/routetables/mock_routetables/client_mock.go +++ b/cloud/services/routetables/mock_routetables/client_mock.go @@ -22,9 +22,10 @@ package mock_routetables import ( context "context" - network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" - gomock "github.com/golang/mock/gomock" reflect "reflect" + + network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" + gomock "github.com/golang/mock/gomock" ) // MockClient is a mock of Client interface. diff --git a/cloud/services/subnets/mock_subnets/client_mock.go b/cloud/services/subnets/mock_subnets/client_mock.go index e155d84ed56..c88d0170b19 100644 --- a/cloud/services/subnets/mock_subnets/client_mock.go +++ b/cloud/services/subnets/mock_subnets/client_mock.go @@ -22,9 +22,10 @@ package mock_subnets import ( context "context" - network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" - gomock "github.com/golang/mock/gomock" reflect "reflect" + + network "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/network/mgmt/network" + gomock "github.com/golang/mock/gomock" ) // MockClient is a mock of Client interface. From 20b7a0584ade0857fe61cc0b10378b53cd637162 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Fri, 24 Jul 2020 10:52:28 -0700 Subject: [PATCH 20/47] Fixed scalesets service --- cloud/services/scalesets/vmss.go | 8 ++++---- cloud/services/scalesets/vmss_test.go | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cloud/services/scalesets/vmss.go b/cloud/services/scalesets/vmss.go index da9c35a3ffe..1d6194917f4 100644 --- a/cloud/services/scalesets/vmss.go +++ b/cloud/services/scalesets/vmss.go @@ -126,7 +126,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { }, VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ UpgradePolicy: &compute.UpgradePolicy{ - Mode: compute.UpgradeModeManual, + Mode: "Manual", }, VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ OsProfile: &compute.VirtualMachineScaleSetOSProfile{ @@ -241,9 +241,9 @@ func (s *Service) generateStorageProfile(ctx context.Context, vmssSpec Spec, sku return nil, fmt.Errorf("vm size %s does not support ephemeral os. select a different vm size or disable ephemeral os", vmssSpec.Sku) } - storageProfile.OsDisk.DiffDiskSettings = &compute.DiffDiskSettings{ - Option: compute.DiffDiskOptions(vmssSpec.OSDisk.DiffDiskSettings.Option), - } + // storageProfile.OsDisk.DiffDiskSettings = &compute.DiffDiskSettings{ + // Option: compute.DiffDiskOptions(vmssSpec.OSDisk.DiffDiskSettings.Option), + // } } dataDisks := []compute.VirtualMachineScaleSetDataDisk{} diff --git a/cloud/services/scalesets/vmss_test.go b/cloud/services/scalesets/vmss_test.go index 05837df754f..880442e3af2 100644 --- a/cloud/services/scalesets/vmss_test.go +++ b/cloud/services/scalesets/vmss_test.go @@ -343,7 +343,7 @@ func TestService_Reconcile(t *testing.T) { }, VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ UpgradePolicy: &compute.UpgradePolicy{ - Mode: compute.UpgradeModeManual, + Mode: "Manual", }, VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ OsProfile: &compute.VirtualMachineScaleSetOSProfile{ @@ -478,7 +478,7 @@ func TestService_Reconcile(t *testing.T) { }, VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ UpgradePolicy: &compute.UpgradePolicy{ - Mode: compute.UpgradeModeManual, + Mode: "Manual", }, VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ OsProfile: &compute.VirtualMachineScaleSetOSProfile{ @@ -614,7 +614,7 @@ func TestService_Reconcile(t *testing.T) { }, VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ UpgradePolicy: &compute.UpgradePolicy{ - Mode: compute.UpgradeModeManual, + Mode: "Manual", }, VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ OsProfile: &compute.VirtualMachineScaleSetOSProfile{ @@ -677,7 +677,7 @@ func TestService_Reconcile(t *testing.T) { }, VirtualMachineScaleSetUpdateProperties: &compute.VirtualMachineScaleSetUpdateProperties{ UpgradePolicy: &compute.UpgradePolicy{ - Mode: compute.UpgradeModeManual, + Mode: "Manual", }, VirtualMachineProfile: &compute.VirtualMachineScaleSetUpdateVMProfile{ OsProfile: &compute.VirtualMachineScaleSetUpdateOSProfile{ @@ -849,7 +849,7 @@ func TestGetVMSSUpdateFromVMSS(t *testing.T) { }, VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ UpgradePolicy: &compute.UpgradePolicy{ - Mode: compute.UpgradeModeManual, + Mode: "Manual", }, VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ OsProfile: &compute.VirtualMachineScaleSetOSProfile{ @@ -881,7 +881,7 @@ func TestGetVMSSUpdateFromVMSS(t *testing.T) { }, VirtualMachineScaleSetUpdateProperties: &compute.VirtualMachineScaleSetUpdateProperties{ UpgradePolicy: &compute.UpgradePolicy{ - Mode: compute.UpgradeModeManual, + Mode: "Manual", }, VirtualMachineProfile: &compute.VirtualMachineScaleSetUpdateVMProfile{ OsProfile: &compute.VirtualMachineScaleSetUpdateOSProfile{ From 7655c1bc04bde0a622971591841cde225da50bb2 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Fri, 24 Jul 2020 10:55:22 -0700 Subject: [PATCH 21/47] Fixed virtualmachines service --- cloud/services/virtualmachines/virtualmachines.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cloud/services/virtualmachines/virtualmachines.go b/cloud/services/virtualmachines/virtualmachines.go index 85913eb2ce6..cb4d3d8e8e1 100644 --- a/cloud/services/virtualmachines/virtualmachines.go +++ b/cloud/services/virtualmachines/virtualmachines.go @@ -24,9 +24,6 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest/to" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" - azure "sigs.k8s.io/cluster-api-provider-azure/cloud" - "sigs.k8s.io/cluster-api-provider-azure/cloud/converters" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" @@ -336,9 +333,9 @@ func (s *Service) generateStorageProfile(ctx context.Context, vmSpec Spec) (*com return nil, fmt.Errorf("vm size %s does not support ephemeral os. select a different vm size or disable ephemeral os", vmSpec.Size) } - storageProfile.OsDisk.DiffDiskSettings = &compute.DiffDiskSettings{ - Option: compute.DiffDiskOptions(vmSpec.OSDisk.DiffDiskSettings.Option), - } + // storageProfile.OsDisk.DiffDiskSettings = &compute.DiffDiskSettings{ + // Option: compute.DiffDiskOptions(vmSpec.OSDisk.DiffDiskSettings.Option), + // } } dataDisks := []compute.DataDisk{} @@ -380,3 +377,4 @@ func getSpotVMOptions(spotVMOptions *infrav1.SpotVMOptions) (compute.VirtualMach } return compute.Spot, compute.Deallocate, billingProfile, nil } +*/ From 3daf6bf4f1ded8393458c8b0c6eeb14169083341 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Fri, 24 Jul 2020 14:11:27 -0700 Subject: [PATCH 22/47] Fixed loadbalancer service, removed call to getAvailablePrivateIP --- cloud/services/loadbalancers/loadbalancers.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cloud/services/loadbalancers/loadbalancers.go b/cloud/services/loadbalancers/loadbalancers.go index dd2afabbb03..0683c3331dc 100644 --- a/cloud/services/loadbalancers/loadbalancers.go +++ b/cloud/services/loadbalancers/loadbalancers.go @@ -53,10 +53,13 @@ func (s *Service) Reconcile(ctx context.Context) error { } } else if azure.ResourceNotFound(err) { s.Scope.V(2).Info("internalLB not found in RG", "internal lb", lbSpec.Name, "resource group", s.Scope.ResourceGroup()) - privateIP, err = s.getAvailablePrivateIP(ctx, s.Scope.Vnet().ResourceGroup, s.Scope.Vnet().Name, lbSpec.SubnetCidr, lbSpec.PrivateIPAddress) - if err != nil { - return err - } + privateIP = "10.0.0.100" + /* + privateIP, err = s.getAvailablePrivateIP(ctx, s.Scope.Vnet().ResourceGroup, s.Scope.Vnet().Name, lbSpec.SubnetCidr, lbSpec.PrivateIPAddress) + if err != nil { + return err + } + */ s.Scope.V(2).Info("setting internal load balancer IP", "private ip", privateIP) } else { return errors.Wrap(err, "failed to look for existing internal LB") @@ -88,7 +91,7 @@ func (s *Service) Reconcile(ctx context.Context) error { } lb := network.LoadBalancer{ - Sku: &network.LoadBalancerSku{Name: network.LoadBalancerSkuNameStandard}, + Sku: &network.LoadBalancerSku{Name: network.LoadBalancerSkuNameBasic}, Location: to.StringPtr(s.Scope.Location()), Tags: converters.TagsToMap(infrav1.Build(infrav1.BuildParams{ ClusterName: s.Scope.ClusterName(), @@ -133,7 +136,6 @@ func (s *Service) Reconcile(ctx context.Context) error { Name: to.StringPtr(probeName), ProbePropertiesFormat: &network.ProbePropertiesFormat{ Protocol: network.ProbeProtocolTCP, - RequestPath: to.StringPtr("/healthz"), Port: to.Int32Ptr(lbSpec.APIServerPort), IntervalInSeconds: to.Int32Ptr(15), NumberOfProbes: to.Int32Ptr(4), From e9f4fb00f1d96215da144abbcf5572d7eea3b55c Mon Sep 17 00:00:00 2001 From: chlau-az Date: Fri, 24 Jul 2020 14:35:54 -0700 Subject: [PATCH 23/47] Removed accelerated networking check to avoid resource sku cache --- .../networkinterfaces/networkinterfaces.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/cloud/services/networkinterfaces/networkinterfaces.go b/cloud/services/networkinterfaces/networkinterfaces.go index a9ca247b172..9a315189223 100644 --- a/cloud/services/networkinterfaces/networkinterfaces.go +++ b/cloud/services/networkinterfaces/networkinterfaces.go @@ -25,7 +25,6 @@ import ( "github.com/pkg/errors" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/resourceskus" ) // Reconcile gets/creates/updates a network interface. @@ -88,16 +87,16 @@ func (s *Service) Reconcile(ctx context.Context) error { nicConfig.PublicIPAddress = &publicIP } - if nicSpec.AcceleratedNetworking == nil { - // set accelerated networking to the capability of the VMSize - sku, err := s.ResourceSKUCache.Get(ctx, nicSpec.VMSize, resourceskus.VirtualMachines) - if err != nil { - return errors.Wrapf(err, "failed to get find vm sku %s in compute api", nicSpec.VMSize) - } + // if nicSpec.AcceleratedNetworking == nil { + // // set accelerated networking to the capability of the VMSize + // sku, err := s.ResourceSKUCache.Get(ctx, nicSpec.VMSize, resourceskus.VirtualMachines) + // if err != nil { + // return errors.Wrapf(err, "failed to get find vm sku %s in compute api", nicSpec.VMSize) + // } - accelNet := sku.HasCapability(resourceskus.AcceleratedNetworking) - nicSpec.AcceleratedNetworking = &accelNet - } + // accelNet := sku.HasCapability(resourceskus.AcceleratedNetworking) + // nicSpec.AcceleratedNetworking = &accelNet + // } err = s.Client.CreateOrUpdate(ctx, s.Scope.ResourceGroup(), From 33f5efe63e3b2fcad9d9116b0474a02aeeca9e10 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Fri, 24 Jul 2020 17:23:47 -0700 Subject: [PATCH 24/47] Created documentation for Azure Stack clusterctl and environment --- docs/topics/azure-stack-clusterctl.md | 67 +++++++++++++++++++++ docs/topics/azure-stack.md | 85 +++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 docs/topics/azure-stack-clusterctl.md create mode 100644 docs/topics/azure-stack.md diff --git a/docs/topics/azure-stack-clusterctl.md b/docs/topics/azure-stack-clusterctl.md new file mode 100644 index 00000000000..5c76dbedb1a --- /dev/null +++ b/docs/topics/azure-stack-clusterctl.md @@ -0,0 +1,67 @@ +# Azure Stack clusterctl + +## Development Workflow + +### Setup +To deploy a cluster to Azure Stack Hub using [clusterctl](https://cluster-api.sigs.k8s.io/clusterctl/overview.html) for the first time, follow [Getting Started](https://cluster-api.sigs.k8s.io/clusterctl/developers.html#getting-started) to build clusterctl binary, create a `clusterctl-settings.json` file, and run the local-overrides hack. Then copy the Azure Stack flavor template to the local override repository using: + +```bash +cp repo/cluster-api-provider-azure/templates/cluster-template-azure-stack.yaml ~/.cluster-api/overrides/infrastructure-azure/v0.4.0/ +``` + +### Create kind cluster + +Cluster API requires an existing Kubernetes cluster accessible via kubectl; during the installation process the Kubernetes cluster will be transformed into a management cluster by installing the Cluster API provider components. +```bash +kind create cluster +kind load docker-image gcr.io/cluster-api-azure-controller-amd64:dev +``` + +### Initalize management cluster +Transform the Kubernetes cluster into a management cluster by using `clusterctl init`. The command accepts as input a list of providers to install. +```bash +clusterctl init --core cluster-api:v0.3.0 --bootstrap kubeadm:v0.3.0 --control-plane kubeadm:v0.3.0 --infrastructure azure:v0.4.0 +``` + +### Create workload cluster +Once the management cluster is ready, you can create your workload cluster. The `clusterctl config cluster` command returns a YAML template for creating a workload cluster. + +Set the required [Azure Stack environment variables](./azure-stack.md). + +Generate the cluster configuration, either by flavor or directly from file. + +To generate by flavor: +```bash +clusterctl config cluster capz-cluster2 --kubernetes-version v1.17.8 --control-plane-machine-count 1 --worker-machine-count 1 --flavor azure-stack > my-cluster.yaml +``` +To generate directly from file: +```bash +clusterctl config cluster capz-cluster2 --kubernetes-version v1.17.8 --control-plane-machine-count 1 --worker-machine-count 1 --from repo/cluster-api-provider-azure/templates/cluster-template-azure-stack.yaml > my-cluster.yaml +``` +This creates a YAML file named `my-cluster.yaml` with a predefined list of Cluster API objects; Cluster, Machines, Machine Deployments, etc. + +When ready, run the following command to apply the cluster manifest, creating a workload cluster on Azure Stack Hub: +```bash +kubectl apply -f my-cluster.yaml +``` + +### Accessing the workload cluster +The cluster will now start provisioning. You can check status with: +```bash +kubectl get cluster --all-namespaces +``` +After the first control plane node is up and running, we can retrieve the worload cluster Kubeconfig: +```bash +kubectl get secrets capz-cluster2-kubeconfig -o json | jq -r .data.value | base64 --decode > ./kubeconfig +``` + +### Deploy a CNI solution +Calico is used here as an example. +```bash +kubectl --kubeconfig=./kubeconfig apply -f https://raw.githubusercontent.com/kubernetes-sigs/cluster-api-provider-azure/master/templates/addons/calico.yaml +``` + +After a short while, our nodes should be running and in `Ready` state, let's check the status using `kubectl get nodes` +```bash +kubectl --kubeconfig=./kubeconfig get nodes +``` \ No newline at end of file diff --git a/docs/topics/azure-stack.md b/docs/topics/azure-stack.md new file mode 100644 index 00000000000..7b867a9ca4d --- /dev/null +++ b/docs/topics/azure-stack.md @@ -0,0 +1,85 @@ +# Azure Stack + +To deploy a cluster using [Azure Stack Hub](https://github.com/msazurestackworkloads/cluster-api-provider-azure), create a cluster configuration with the [azure stack template](../../templates/cluster-template-azure-stack.yaml). + + +## Set environment variables + +### Azure cloud settings +```bash +export AZURE_ARM_ENDPOINT="https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com" +export AZURE_LOCATION="redmond" +export AZURE_ENVIRONMENT=AzureStackCloud +export AZURE_ENVIRONMENT_FILEPATH=”/etc/kubernetes/azurestackcloud.json” +export IDENTITY_SYSTEM=adfs +``` +Azure Stack offers both Azure Active Directory and ADFS authentication. Environment variable `IDENTITY_SYSTEM` can be either `azure_ad` or `adfs`. + +### Azure Service Principal +```bash +export AZURE_TENANT_ID="" +export AZURE_CLIENT_ID="" +export AZURE_CLIENT_SECRET="" +export AZURE_SUBSCRIPTION_ID="" + +export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZURE_SUBSCRIPTION_ID" | base64 | tr -d '\n')" +export AZURE_TENANT_ID_B64="$(echo -n "$AZURE_TENANT_ID" | base64 | tr -d '\n')" +export AZURE_CLIENT_ID_B64="$(echo -n "$AZURE_CLIENT_ID" | base64 | tr -d '\n')" +export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZURE_CLIENT_SECRET" | base64 | tr -d '\n')" +``` + +### Cluster settings +```bash +export CLUSTER_NAME="capz-cluster" +export AZURE_RESOURCE_GROUP=${CLUSTER_NAME} +export AZURE_VNET_NAME=${CLUSTER_NAME}-vnet +``` +### Machine settings +```bash +export CONTROL_PLANE_MACHINE_COUNT=1 +export AZURE_CONTROL_PLANE_MACHINE_TYPE="Standard_DS2_v2" +export AZURE_NODE_MACHINE_TYPE="Standard_DS2_v2" +export WORKER_MACHINE_COUNT=2 +export KUBERNETES_VERSION="v1.17.8" +``` + +### Generate SSH key +If you want to provide your own key, skip this step and set AZURE_SSH_PUBLIC_KEY to your existing file. +```bash +SSH_KEY_FILE=.sshkey +rm -f "${SSH_KEY_FILE}" 2>/dev/null +ssh-keygen -t rsa -b 2048 -f "${SSH_KEY_FILE}" -N '' 1>/dev/null +echo "Machine SSH key generated in ${SSH_KEY_FILE}" +export AZURE_SSH_PUBLIC_KEY=$(cat "${SSH_KEY_FILE}.pub" | base64 | tr -d '\r\n') +``` + +## Build docker image after changes to codebase +```bash +export REGISTRY="gcr.io" +export PULL_POLICY=IfNotPresent +make docker-build +``` + +## Create management cluster +```bash +make create-management-cluster +``` + +## Create workload cluster +```bash +export CLUSTER_TEMPLATE=cluster-template-azure-stack.yaml +make create-workload-cluster +``` + +## Debug +### Retrieve CAPZ logs +```bash +kubectl logs deployment/capz-controller-manager -n capz-system --all-containers=true > stack.log +``` + +### ssh into virtual machine +```bash +ssh -i repo/cluster-api-provider-azure/.sshkey capi@ +cat /var/log/cloud-init-output.log > init.log +sudo journalctl -u kubelet -l --no-pager > kubelet.log +``` \ No newline at end of file From 97fa9a64dd8afe8ac19b9e939626bdcc840b8f68 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Fri, 24 Jul 2020 17:27:21 -0700 Subject: [PATCH 25/47] Fixed documentation typo --- docs/topics/azure-stack-clusterctl.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/azure-stack-clusterctl.md b/docs/topics/azure-stack-clusterctl.md index 5c76dbedb1a..10077e4c49a 100644 --- a/docs/topics/azure-stack-clusterctl.md +++ b/docs/topics/azure-stack-clusterctl.md @@ -50,7 +50,7 @@ The cluster will now start provisioning. You can check status with: ```bash kubectl get cluster --all-namespaces ``` -After the first control plane node is up and running, we can retrieve the worload cluster Kubeconfig: +After the first control plane node is up and running, we can retrieve the workload cluster Kubeconfig: ```bash kubectl get secrets capz-cluster2-kubeconfig -o json | jq -r .data.value | base64 --decode > ./kubeconfig ``` From ba8714c485166cfc15cad35d163e5951ad38fa18 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Mon, 27 Jul 2020 15:06:37 -0700 Subject: [PATCH 26/47] Moved pre/post kubeadm commands to script --- .../cluster-template-azure-stack-aad.yaml | 58 +++- ...=> cluster-template-azure-stack-adfs.yaml} | 54 ++- .../azure-stack-aad/cluster-template.yaml | 318 ++++++++++++++++++ .../kustomization.yaml | 0 .../cluster-template.yaml | 48 ++- .../azure-stack-adfs/kustomization.yaml | 5 + 6 files changed, 435 insertions(+), 48 deletions(-) rename templates/{cluster-template-azure-stack.yaml => cluster-template-azure-stack-adfs.yaml} (83%) create mode 100644 templates/flavors/azure-stack-aad/cluster-template.yaml rename templates/flavors/{azure-stack => azure-stack-aad}/kustomization.yaml (100%) rename templates/flavors/{azure-stack => azure-stack-adfs}/cluster-template.yaml (85%) create mode 100644 templates/flavors/azure-stack-adfs/kustomization.yaml diff --git a/templates/cluster-template-azure-stack-aad.yaml b/templates/cluster-template-azure-stack-aad.yaml index 09bbfd3dd8b..0bdd6359dd7 100644 --- a/templates/cluster-template-azure-stack-aad.yaml +++ b/templates/cluster-template-azure-stack-aad.yaml @@ -67,7 +67,25 @@ spec: mountPath: /etc/kubernetes/azurestackcloud.json name: cloud-config-stack readOnly: true + - hostPath: /etc/kubernetes/prekubeadmcommands.sh + mountPath: /etc/kubernetes/prekubeadmcommands.sh + name: cloud-config-stack + readOnly: true files: + - content: | + sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + sudo update-ca-certificates + sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + sudo systemctl daemon-reload + sudo systemctl restart kubelet + owner: root:root + path: /etc/kubernetes/prekubeadmcommands.sh + permissions: "0644" + - content: | + "sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml" + owner: root:root + path: /etc/kubernetes/postkubeadmcommands.sh + permissions: "0644" - content: "{\n \"cloud\": \"${AZURE_ENVIRONMENT}\",\n \"tenantId\": \"${AZURE_TENANT_ID}\",\n \ \"subscriptionId\": \"${AZURE_SUBSCRIPTION_ID}\",\n \"aadClientId\": \"${AZURE_CLIENT_ID}\",\n \ \"aadClientSecret\": \"${AZURE_CLIENT_SECRET}\",\n \"resourceGroup\": \"${AZURE_RESOURCE_GROUP}\",\n @@ -121,16 +139,11 @@ spec: cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - - 'sudo sed -i ''s/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\n env:\n - - name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json/g'' - /etc/kubernetes/manifests/kube-controller-manager.yaml' + - hi loading post script + - sudo bash /etc/kubernetes/postkubeadmcommands.sh preKubeadmCommands: - - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt - - sudo update-ca-certificates - - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' - \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf - - sudo systemctl daemon-reload - - sudo systemctl restart kubelet + - hi loading pre script + - sudo bash /etc/kubernetes/prekubeadmcommands.sh useExperimentalRetryJoin: true replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} @@ -214,6 +227,20 @@ spec: template: spec: files: + - content: | + sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + sudo update-ca-certificates + sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + sudo systemctl daemon-reload + sudo systemctl restart kubelet + owner: root:root + path: /etc/kubernetes/prekubeadmcommands.sh + permissions: "0644" + - content: | + "sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml" + owner: root:root + path: /etc/kubernetes/postkubeadmcommands.sh + permissions: "0644" - content: | { "cloud": "${AZURE_ENVIRONMENT}", @@ -272,13 +299,8 @@ spec: cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - - 'sudo sed -i ''s/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\n env:\n - - name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json/g'' - /etc/kubernetes/manifests/kube-controller-manager.yaml' + - hi loading post script + - sudo bash /etc/kubernetes/postkubeadmcommands.sh preKubeadmCommands: - - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt - - sudo update-ca-certificates - - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' - \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf - - sudo systemctl daemon-reload - - sudo systemctl restart kubelet + - hi loading pre script + - sudo bash /etc/kubernetes/prekubeadmcommands.sh diff --git a/templates/cluster-template-azure-stack.yaml b/templates/cluster-template-azure-stack-adfs.yaml similarity index 83% rename from templates/cluster-template-azure-stack.yaml rename to templates/cluster-template-azure-stack-adfs.yaml index e1a67e1544a..1eb178d8d97 100644 --- a/templates/cluster-template-azure-stack.yaml +++ b/templates/cluster-template-azure-stack-adfs.yaml @@ -68,6 +68,20 @@ spec: name: cloud-config-stack readOnly: true files: + - content: | + sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + sudo update-ca-certificates + sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + sudo systemctl daemon-reload + sudo systemctl restart kubelet + owner: root:root + path: /etc/kubernetes/prekubeadmcommands.sh + permissions: "0644" + - content: | + "sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml" + owner: root:root + path: /etc/kubernetes/postkubeadmcommands.sh + permissions: "0644" - content: "{\n \"cloud\": \"${AZURE_ENVIRONMENT}\",\n \"tenantId\": \"adfs\",\n \ \"subscriptionId\": \"${AZURE_SUBSCRIPTION_ID}\",\n \"aadClientId\": \"${AZURE_CLIENT_ID}\",\n \ \"aadClientSecret\": \"${AZURE_CLIENT_SECRET}\",\n \"resourceGroup\": \"${AZURE_RESOURCE_GROUP}\",\n @@ -122,16 +136,11 @@ spec: cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - - 'sudo sed -i ''s/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\n env:\n - - name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json/g'' - /etc/kubernetes/manifests/kube-controller-manager.yaml' + - hi loading post script + - sudo bash /etc/kubernetes/postkubeadmcommands.sh preKubeadmCommands: - - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt - - sudo update-ca-certificates - - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' - \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf - - sudo systemctl daemon-reload - - sudo systemctl restart kubelet + - hi loading pre script + - sudo bash /etc/kubernetes/prekubeadmcommands.sh useExperimentalRetryJoin: true replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} @@ -215,6 +224,20 @@ spec: template: spec: files: + - content: | + sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + sudo update-ca-certificates + sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + sudo systemctl daemon-reload + sudo systemctl restart kubelet + owner: root:root + path: /etc/kubernetes/prekubeadmcommands.sh + permissions: "0644" + - content: | + "sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml" + owner: root:root + path: /etc/kubernetes/postkubeadmcommands.sh + permissions: "0644" - content: | { "cloud": "${AZURE_ENVIRONMENT}", @@ -274,13 +297,8 @@ spec: cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - - 'sudo sed -i ''s/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\n env:\n - - name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json/g'' - /etc/kubernetes/manifests/kube-controller-manager.yaml' + - hi loading post script + - sudo bash /etc/kubernetes/postkubeadmcommands.sh preKubeadmCommands: - - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt - - sudo update-ca-certificates - - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' - \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf - - sudo systemctl daemon-reload - - sudo systemctl restart kubelet + - hi loading pre script + - sudo bash /etc/kubernetes/prekubeadmcommands.sh diff --git a/templates/flavors/azure-stack-aad/cluster-template.yaml b/templates/flavors/azure-stack-aad/cluster-template.yaml new file mode 100644 index 00000000000..0edfdb41c06 --- /dev/null +++ b/templates/flavors/azure-stack-aad/cluster-template.yaml @@ -0,0 +1,318 @@ +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + location: ${AZURE_LOCATION} + networkSpec: + vnet: + name: ${AZURE_VNET_NAME} + resourceGroup: ${AZURE_RESOURCE_GROUP} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + kubeadmConfigSpec: + preKubeadmCommands: + - "hi loading pre script" + - sudo bash /etc/kubernetes/prekubeadmcommands.sh + postKubeadmCommands: + - "hi loading post script" + - sudo bash /etc/kubernetes/postkubeadmcommands.sh + clusterConfiguration: + apiServer: + extraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + cluster-name: ${CLUSTER_NAME} + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + - hostPath: /etc/kubernetes/azurestackcloud.json + mountPath: /etc/kubernetes/azurestackcloud.json + name: cloud-config-stack + readOnly: true + - hostPath: /etc/kubernetes/prekubeadmcommands.sh + mountPath: /etc/kubernetes/prekubeadmcommands.sh + name: cloud-config-stack + readOnly: true + files: + - content: | + sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + sudo update-ca-certificates + sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + sudo systemctl daemon-reload + sudo systemctl restart kubelet + owner: root:root + path: /etc/kubernetes/prekubeadmcommands.sh + permissions: "0644" + - content: | + "sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml" + owner: root:root + path: /etc/kubernetes/postkubeadmcommands.sh + permissions: "0644" + - content: | + { + "cloud": "${AZURE_ENVIRONMENT}", + "tenantId": "${AZURE_TENANT_ID}", + "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", + "aadClientId": "${AZURE_CLIENT_ID}", + "aadClientSecret": "${AZURE_CLIENT_SECRET}", + "resourceGroup": "${AZURE_RESOURCE_GROUP}", + "securityGroupName": "${CLUSTER_NAME}-node-nsg", + "location": "${AZURE_LOCATION}", + "vmType": "vmss", + "vnetName": "${CLUSTER_NAME}-vnet", + "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", + "subnetName": "${CLUSTER_NAME}-node-subnet", + "routeTableName": "${CLUSTER_NAME}-node-routetable", + "userAssignedID": "${CLUSTER_NAME}", + "loadBalancerSku": "Basic", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": false, + "excludeMasterFromStandardLB": false + } + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + { + "name":"${AZURE_ENVIRONMENT}", + "managementPortalURL":"https://portal.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", + "publishSettingsURL":"", + "serviceManagementEndpoint":"https://management.azurestackci02.onmicrosoft.com/6bbece7e-df47-402c-bf11-101f74fdf525", + "resourceManagerEndpoint":"https://management.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", + "activeDirectoryEndpoint":"https://login.microsoftonline.com/", + "galleryEndpoint":"https://providers.n25r1306.masd.stbtest.microsoft.com:30016/", + "keyVaultEndpoint":"","graphEndpoint":"https://graph.windows.net/", + "serviceBusEndpoint":"", + "batchManagementEndpoint":"", + "storageEndpointSuffix":"redmond.ext-n25r1306.masd.stbtest.microsoft.com", + "sqlDatabaseDNSSuffix":"", + "trafficManagerDNSSuffix":"", + "keyVaultDNSSuffix":"vault.redmond.ext-n25r1306.masd.stbtest.microsoft.com", + "serviceBusEndpointSuffix":"", + "serviceManagementVMDNSSuffix":"", + "resourceManagerVMDNSSuffix":"cloudapp.ext-n25r1306.masd.stbtest.microsoft.com", + "containerRegistryDNSSuffix":"", + "cosmosDBDNSSuffix":"", + "tokenAudience":"", + "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} + } + owner: root:root + path: /etc/kubernetes/azurestackcloud.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + useExperimentalRetryJoin: true + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + image: + marketplace: + publisher: AzureStack + offer: Test + sku: capz-test + version: latest + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + image: + marketplace: + publisher: AzureStack + offer: Test + sku: capz-test + version: latest + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 30 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + preKubeadmCommands: + - "hi loading pre script" + - sudo bash /etc/kubernetes/prekubeadmcommands.sh + postKubeadmCommands: + - "hi loading post script" + - sudo bash /etc/kubernetes/postkubeadmcommands.sh + files: + - content: | + sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + sudo update-ca-certificates + sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + sudo systemctl daemon-reload + sudo systemctl restart kubelet + owner: root:root + path: /etc/kubernetes/prekubeadmcommands.sh + permissions: "0644" + - content: | + "sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml" + owner: root:root + path: /etc/kubernetes/postkubeadmcommands.sh + permissions: "0644" + - content: | + { + "cloud": "${AZURE_ENVIRONMENT}", + "tenantId": "${AZURE_TENANT_ID}", + "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", + "aadClientId": "${AZURE_CLIENT_ID}", + "aadClientSecret": "${AZURE_CLIENT_SECRET}", + "resourceGroup": "${AZURE_RESOURCE_GROUP}", + "securityGroupName": "${CLUSTER_NAME}-node-nsg", + "location": "${AZURE_LOCATION}", + "vmType": "vmss", + "vnetName": "${CLUSTER_NAME}-vnet", + "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", + "subnetName": "${CLUSTER_NAME}-node-subnet", + "routeTableName": "${CLUSTER_NAME}-node-routetable", + "loadBalancerSku": "Basic", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": false, + "excludeMasterFromStandardLB": false + } + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + { + "name":"${AZURE_ENVIRONMENT}", + "managementPortalURL":"https://portal.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", + "publishSettingsURL":"", + "serviceManagementEndpoint":"https://management.azurestackci02.onmicrosoft.com/6bbece7e-df47-402c-bf11-101f74fdf525", + "resourceManagerEndpoint":"https://management.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", + "activeDirectoryEndpoint":"https://login.microsoftonline.com/", + "galleryEndpoint":"https://providers.n25r1306.masd.stbtest.microsoft.com:30016/", + "keyVaultEndpoint":"","graphEndpoint":"https://graph.windows.net/", + "serviceBusEndpoint":"", + "batchManagementEndpoint":"", + "storageEndpointSuffix":"redmond.ext-n25r1306.masd.stbtest.microsoft.com", + "sqlDatabaseDNSSuffix":"", + "trafficManagerDNSSuffix":"", + "keyVaultDNSSuffix":"vault.redmond.ext-n25r1306.masd.stbtest.microsoft.com", + "serviceBusEndpointSuffix":"", + "serviceManagementVMDNSSuffix":"", + "resourceManagerVMDNSSuffix":"cloudapp.ext-n25r1306.masd.stbtest.microsoft.com", + "containerRegistryDNSSuffix":"", + "cosmosDBDNSSuffix":"", + "tokenAudience":"", + "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} + } + owner: root:root + path: /etc/kubernetes/azurestackcloud.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' diff --git a/templates/flavors/azure-stack/kustomization.yaml b/templates/flavors/azure-stack-aad/kustomization.yaml similarity index 100% rename from templates/flavors/azure-stack/kustomization.yaml rename to templates/flavors/azure-stack-aad/kustomization.yaml diff --git a/templates/flavors/azure-stack/cluster-template.yaml b/templates/flavors/azure-stack-adfs/cluster-template.yaml similarity index 85% rename from templates/flavors/azure-stack/cluster-template.yaml rename to templates/flavors/azure-stack-adfs/cluster-template.yaml index 55d5f65a76c..9455ae219ed 100644 --- a/templates/flavors/azure-stack/cluster-template.yaml +++ b/templates/flavors/azure-stack-adfs/cluster-template.yaml @@ -42,13 +42,11 @@ spec: name: ${CLUSTER_NAME}-control-plane kubeadmConfigSpec: preKubeadmCommands: - - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt - - sudo update-ca-certificates - - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf - - sudo systemctl daemon-reload - - sudo systemctl restart kubelet + - "hi loading pre script" + - sudo bash /etc/kubernetes/prekubeadmcommands.sh postKubeadmCommands: - - "sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml" + - "hi loading post script" + - sudo bash /etc/kubernetes/postkubeadmcommands.sh clusterConfiguration: apiServer: extraArgs: @@ -76,6 +74,20 @@ spec: name: cloud-config-stack readOnly: true files: + - content: | + sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + sudo update-ca-certificates + sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + sudo systemctl daemon-reload + sudo systemctl restart kubelet + owner: root:root + path: /etc/kubernetes/prekubeadmcommands.sh + permissions: "0644" + - content: | + "sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml" + owner: root:root + path: /etc/kubernetes/postkubeadmcommands.sh + permissions: "0644" - content: | { "cloud": "${AZURE_ENVIRONMENT}", @@ -224,14 +236,26 @@ spec: template: spec: preKubeadmCommands: - - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt - - sudo update-ca-certificates - - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf - - sudo systemctl daemon-reload - - sudo systemctl restart kubelet + - "hi loading pre script" + - sudo bash /etc/kubernetes/prekubeadmcommands.sh postKubeadmCommands: - - "sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml" + - "hi loading post script" + - sudo bash /etc/kubernetes/postkubeadmcommands.sh files: + - content: | + sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + sudo update-ca-certificates + sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + sudo systemctl daemon-reload + sudo systemctl restart kubelet + owner: root:root + path: /etc/kubernetes/prekubeadmcommands.sh + permissions: "0644" + - content: | + "sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml" + owner: root:root + path: /etc/kubernetes/postkubeadmcommands.sh + permissions: "0644" - content: | { "cloud": "${AZURE_ENVIRONMENT}", diff --git a/templates/flavors/azure-stack-adfs/kustomization.yaml b/templates/flavors/azure-stack-adfs/kustomization.yaml new file mode 100644 index 00000000000..16a0a22fe60 --- /dev/null +++ b/templates/flavors/azure-stack-adfs/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: +- cluster-template.yaml From 086df1c332bc9a473e73761d80587a99136029e3 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Mon, 27 Jul 2020 16:22:05 -0700 Subject: [PATCH 27/47] Reverted tilt files --- Tiltfile | 3 +-- config/default/manager_credentials_patch.yaml | 6 +----- tilt-provider.json | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/Tiltfile b/Tiltfile index 6a6cbb48562..151364d19a4 100644 --- a/Tiltfile +++ b/Tiltfile @@ -187,8 +187,7 @@ def capz(): # Set up an image build for the provider. The live update configuration syncs the output from the local_resource # build into the container. docker_build( - ref = "gcr.io/cluster-api-azure-controller-amd64:dev", - # ref = "gcr.io/k8s-staging-cluster-api-azure/cluster-api-azure-controller", + ref = "gcr.io/k8s-staging-cluster-api-azure/cluster-api-azure-controller", context = "./.tiltbuild/", dockerfile_contents = dockerfile_contents, target = "tilt", diff --git a/config/default/manager_credentials_patch.yaml b/config/default/manager_credentials_patch.yaml index 2feffd0a33e..f5f17828634 100644 --- a/config/default/manager_credentials_patch.yaml +++ b/config/default/manager_credentials_patch.yaml @@ -28,8 +28,4 @@ spec: valueFrom: secretKeyRef: name: manager-bootstrap-credentials - key: client-secret - - name: AZURE_GO_SDK_LOG_LEVEL - value: DEBUG - - name: AZURE_GO_SDK_LOG_FILE - value: /home/chlau/go/src/sigs.k8s.io/cluster-api-provider-azure/gosdk.log + key: client-secret \ No newline at end of file diff --git a/tilt-provider.json b/tilt-provider.json index 61a06d90133..4684dc5d6f1 100644 --- a/tilt-provider.json +++ b/tilt-provider.json @@ -1,7 +1,7 @@ { "name": "azure", "config": { - "image": "gcr.io/cluster-api-azure-controller-amd64:dev", + "image": "gcr.io/k8s-staging-cluster-api-azure/cluster-api-azure-controller", "live_reload_deps": [ "main.go", "go.mod", From f4b54e9c813d357c7780718d0f8089be929c9aaf Mon Sep 17 00:00:00 2001 From: chlau-az Date: Mon, 27 Jul 2020 16:55:13 -0700 Subject: [PATCH 28/47] Removed log debug statements --- cloud/scope/clients.go | 14 -------------- cloud/services/virtualmachines/virtualmachines.go | 6 +++--- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/cloud/scope/clients.go b/cloud/scope/clients.go index b1de6bd915c..bf43cece70a 100644 --- a/cloud/scope/clients.go +++ b/cloud/scope/clients.go @@ -18,7 +18,6 @@ package scope import ( "fmt" - "log" "os" "strings" @@ -57,19 +56,14 @@ func (c *AzureClients) setCredentials(subscriptionID string) error { c.SubscriptionID = subID armEndpoint := os.Getenv("AZURE_ARM_ENDPOINT") - log.Println("HERE armEndpoint: ", armEndpoint) env, err := azure.EnvironmentFromURL(armEndpoint) if err != nil { - log.Println("HERE error getting environment from armEndpoint: ", armEndpoint) return err } c.ResourceManagerEndpoint = env.ResourceManagerEndpoint - log.Println("HERE c.ResourceManagerEndpoint: ", c.ResourceManagerEndpoint) c.ResourceManagerVMDNSSuffix = GetAzureDNSZoneForEnvironment("AzureStackCloud") - log.Println("HERE c.ResourceManagerVMDNSSuffix: ", c.ResourceManagerVMDNSSuffix) c.Authorizer, err = getAuthorizerForResource(env) - log.Println("HERE c.Authorizer: ", c.Authorizer) return err } @@ -122,16 +116,10 @@ func getAuthorizerForResource(env azure.Environment) (autorest.Authorizer, error clientID := os.Getenv("AZURE_CLIENT_ID") clientSecret := os.Getenv("AZURE_CLIENT_SECRET") tenantID := os.Getenv("AZURE_TENANT_ID") - log.Println("HERE client id: ", clientID) - log.Println("HERE client secret: ", clientSecret) - log.Println("HERE tenant id: ", tenantID) tokenAudience := env.TokenAudience - log.Println("HERE TokenAudience: ", env.TokenAudience) - log.Println("HERE ActiveDirectoryEndpoint: ", env.ActiveDirectoryEndpoint) identitySystem := os.Getenv("IDENTITY_SYSTEM") - log.Println("HERE identity system: ", identitySystem) if identitySystem == "adfs" { oauthConfig, err = adal.NewOAuthConfig( env.ActiveDirectoryEndpoint, "adfs") @@ -149,8 +137,6 @@ func getAuthorizerForResource(env azure.Environment) (autorest.Authorizer, error clientSecret, tokenAudience) - log.Println("HERE generated token") a = autorest.NewBearerAuthorizer(token) - log.Println("HERE generated authorizer") return a, err } diff --git a/cloud/services/virtualmachines/virtualmachines.go b/cloud/services/virtualmachines/virtualmachines.go index cb4d3d8e8e1..d60f25cc5c6 100644 --- a/cloud/services/virtualmachines/virtualmachines.go +++ b/cloud/services/virtualmachines/virtualmachines.go @@ -197,17 +197,17 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { */ } - log.Println("HI CREATING VM: ", vmSpec.Name) + log.Println("CREATING VM: ", vmSpec.Name) err = s.Client.CreateOrUpdate( ctx, s.Scope.ResourceGroup(), vmSpec.Name, virtualMachine) if err != nil { - log.Println("HI CREATING VM ERROR: ", vmSpec.Name) + log.Println("CREATING VM ERROR: ", vmSpec.Name) return errors.Wrapf(err, "cannot create VM") } - log.Println("HI CREATING VM FINISHED: ", vmSpec.Name) + log.Println("CREATING VM FINISHED: ", vmSpec.Name) s.Scope.V(2).Info("successfully created VM", "vm", vmSpec.Name) return nil From 17f2d67cda0698068415903575e76e4bead682ef Mon Sep 17 00:00:00 2001 From: chlau-az Date: Tue, 28 Jul 2020 09:16:38 -0700 Subject: [PATCH 29/47] Corrected Azure Stack Hub disk storage account types --- api/v1alpha3/azuremachine_validation.go | 2 +- config/default/manager_credentials_patch.yaml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/api/v1alpha3/azuremachine_validation.go b/api/v1alpha3/azuremachine_validation.go index 338b91cd0b7..5aa3cfef567 100644 --- a/api/v1alpha3/azuremachine_validation.go +++ b/api/v1alpha3/azuremachine_validation.go @@ -183,7 +183,7 @@ func validateStorageAccountType(storageAccountType string, fieldPath *field.Path return allErrs } - possibleDiskStorageAccountTypesValues := []string{"Premium_LRS", "Standard_LRS", "StandardSSD_LRS", "UltraSSD_LRS"} + possibleDiskStorageAccountTypesValues := []string{"Premium_LRS", "Standard_LRS"} for _, possibleStorageAccountType := range possibleDiskStorageAccountTypesValues { if string(possibleStorageAccountType) == storageAccountType { return allErrs diff --git a/config/default/manager_credentials_patch.yaml b/config/default/manager_credentials_patch.yaml index f5f17828634..3048f8f9a3f 100644 --- a/config/default/manager_credentials_patch.yaml +++ b/config/default/manager_credentials_patch.yaml @@ -28,4 +28,6 @@ spec: valueFrom: secretKeyRef: name: manager-bootstrap-credentials - key: client-secret \ No newline at end of file + key: client-secret + - name: AZURE_GO_SDK_LOG_LEVEL + value: DEBUG \ No newline at end of file From a4cc6a10df3f26a36b8516326258f9d9bf56f4f2 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Thu, 30 Jul 2020 16:50:44 -0700 Subject: [PATCH 30/47] Debugging kube controller manager --- .../cluster-template-azure-stack-aad.yaml | 50 +++++++++++++++++-- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/templates/cluster-template-azure-stack-aad.yaml b/templates/cluster-template-azure-stack-aad.yaml index 0bdd6359dd7..8c0a3e6a0df 100644 --- a/templates/cluster-template-azure-stack-aad.yaml +++ b/templates/cluster-template-azure-stack-aad.yaml @@ -69,8 +69,32 @@ spec: readOnly: true - hostPath: /etc/kubernetes/prekubeadmcommands.sh mountPath: /etc/kubernetes/prekubeadmcommands.sh - name: cloud-config-stack + name: prekubeadmcommands + readOnly: true + - hostPath: /etc/kubernetes/postkubeadmcommands.sh + mountPath: /etc/kubernetes/postkubeadmcommands.sh + name: postkubeadmcommands readOnly: true + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt files: - content: | sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt @@ -139,11 +163,21 @@ spec: cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - - hi loading post script + - echo 'hi loading post script' + - echo 'This is kube-controller-manager.yaml' + - sudo cat /etc/kubernetes/manifests/kube-controller-manager.yaml + - echo 'This is azurestackcloud.json' + - sudo cat /etc/kubernetes/azurestackcloud.json + - echo 'Now running post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh + - echo 'hi finished loading post script' preKubeadmCommands: - - hi loading pre script + - echo 'hi loading pre script' - sudo bash /etc/kubernetes/prekubeadmcommands.sh + - echo 'hi finished loading pre script' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk useExperimentalRetryJoin: true replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} @@ -162,6 +196,10 @@ spec: publisher: AzureStack sku: capz-test version: latest + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk location: ${AZURE_LOCATION} osDisk: diskSizeGB: 128 @@ -299,8 +337,10 @@ spec: cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - - hi loading post script + - echo 'hi loading post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh + - echo 'hi finished loading post script' preKubeadmCommands: - - hi loading pre script + - echo 'hi loading pre script' - sudo bash /etc/kubernetes/prekubeadmcommands.sh + - echo 'hi finished loading pre script' From 4fad98549e7e0a1cd818000228d7cd404a2ab45a Mon Sep 17 00:00:00 2001 From: chlau-az Date: Thu, 30 Jul 2020 16:56:31 -0700 Subject: [PATCH 31/47] Still debugging kube controller manager --- templates/cluster-template-azure-stack-aad.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/templates/cluster-template-azure-stack-aad.yaml b/templates/cluster-template-azure-stack-aad.yaml index 8c0a3e6a0df..552dfa5b1df 100644 --- a/templates/cluster-template-azure-stack-aad.yaml +++ b/templates/cluster-template-azure-stack-aad.yaml @@ -164,11 +164,6 @@ spec: name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - echo 'hi loading post script' - - echo 'This is kube-controller-manager.yaml' - - sudo cat /etc/kubernetes/manifests/kube-controller-manager.yaml - - echo 'This is azurestackcloud.json' - - sudo cat /etc/kubernetes/azurestackcloud.json - - echo 'Now running post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh - echo 'hi finished loading post script' preKubeadmCommands: From 0ab9aaf8aa5edb361587bb28c7007f035e6616e7 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Thu, 30 Jul 2020 18:26:34 -0700 Subject: [PATCH 32/47] Fixed kube controller manager error, in template --- templates/cluster-template-azure-stack-aad.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/templates/cluster-template-azure-stack-aad.yaml b/templates/cluster-template-azure-stack-aad.yaml index 552dfa5b1df..11081c5f2b4 100644 --- a/templates/cluster-template-azure-stack-aad.yaml +++ b/templates/cluster-template-azure-stack-aad.yaml @@ -106,7 +106,7 @@ spec: path: /etc/kubernetes/prekubeadmcommands.sh permissions: "0644" - content: | - "sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml" + sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml owner: root:root path: /etc/kubernetes/postkubeadmcommands.sh permissions: "0644" @@ -164,6 +164,11 @@ spec: name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - echo 'hi loading post script' + - echo 'This is kube-controller-manager.yaml' + - sudo cat /etc/kubernetes/manifests/kube-controller-manager.yaml + - echo 'This is azurestackcloud.json' + - sudo cat /etc/kubernetes/azurestackcloud.json + - echo 'Now running post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh - echo 'hi finished loading post script' preKubeadmCommands: @@ -270,7 +275,7 @@ spec: path: /etc/kubernetes/prekubeadmcommands.sh permissions: "0644" - content: | - "sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml" + sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml owner: root:root path: /etc/kubernetes/postkubeadmcommands.sh permissions: "0644" From f77596de84d1864e712a69441357afe338546916 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Fri, 31 Jul 2020 12:48:54 -0700 Subject: [PATCH 33/47] Created machinepool template --- cloud/services/scalesets/vmss.go | 6 +- .../azuremachinepool_controller.go | 2 + ...cluster-template-azure-stack-aad-copy.yaml | 348 ++++++++++++++++++ .../cluster-template-azure-stack-aad.yaml | 5 - 4 files changed, 354 insertions(+), 7 deletions(-) create mode 100644 templates/cluster-template-azure-stack-aad-copy.yaml diff --git a/cloud/services/scalesets/vmss.go b/cloud/services/scalesets/vmss.go index 1d6194917f4..fa7c6b3ab13 100644 --- a/cloud/services/scalesets/vmss.go +++ b/cloud/services/scalesets/vmss.go @@ -19,6 +19,7 @@ package scalesets import ( "context" "fmt" + "log" "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest/to" @@ -70,6 +71,7 @@ func (s *Service) Get(ctx context.Context, vmssSpec *Spec) (*infrav1exp.VMSS, er } func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { + log.Println("calliou reconcile vmss") vmssSpec, ok := spec.(*Spec) if !ok { return errors.New("invalid VMSS specification") @@ -151,8 +153,8 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { { Name: to.StringPtr(vmssSpec.Name + "-netconfig"), VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ - Primary: to.BoolPtr(true), - EnableIPForwarding: to.BoolPtr(true), + Primary: to.BoolPtr(true), + // EnableIPForwarding: to.BoolPtr(true), IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{ { Name: to.StringPtr(vmssSpec.Name + "-ipconfig"), diff --git a/exp/controllers/azuremachinepool_controller.go b/exp/controllers/azuremachinepool_controller.go index 23516a39b3e..ad4c27964e0 100644 --- a/exp/controllers/azuremachinepool_controller.go +++ b/exp/controllers/azuremachinepool_controller.go @@ -21,6 +21,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "log" "time" "github.com/Azure/go-autorest/autorest/to" @@ -227,6 +228,7 @@ func (r *AzureMachinePoolReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, } func (r *AzureMachinePoolReconciler) reconcileNormal(ctx context.Context, machinePoolScope *scope.MachinePoolScope, clusterScope *scope.ClusterScope) (_ reconcile.Result, reterr error) { + log.Println("calliou reconcile machine pool") machinePoolScope.Info("Reconciling AzureMachinePool") // If the AzureMachine is in an error state, return early. if machinePoolScope.AzureMachinePool.Status.FailureReason != nil || machinePoolScope.AzureMachinePool.Status.FailureMessage != nil { diff --git a/templates/cluster-template-azure-stack-aad-copy.yaml b/templates/cluster-template-azure-stack-aad-copy.yaml new file mode 100644 index 00000000000..85ec3c22117 --- /dev/null +++ b/templates/cluster-template-azure-stack-aad-copy.yaml @@ -0,0 +1,348 @@ +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + location: ${AZURE_LOCATION} + networkSpec: + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + cluster-name: ${CLUSTER_NAME} + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + - hostPath: /etc/kubernetes/azurestackcloud.json + mountPath: /etc/kubernetes/azurestackcloud.json + name: cloud-config-stack + readOnly: true + - hostPath: /etc/kubernetes/prekubeadmcommands.sh + mountPath: /etc/kubernetes/prekubeadmcommands.sh + name: prekubeadmcommands + readOnly: true + - hostPath: /etc/kubernetes/postkubeadmcommands.sh + mountPath: /etc/kubernetes/postkubeadmcommands.sh + name: postkubeadmcommands + readOnly: true + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - content: | + sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + sudo update-ca-certificates + sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + sudo systemctl daemon-reload + sudo systemctl restart kubelet + owner: root:root + path: /etc/kubernetes/prekubeadmcommands.sh + permissions: "0644" + - content: | + sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml + owner: root:root + path: /etc/kubernetes/postkubeadmcommands.sh + permissions: "0644" + - content: | + { + "cloud": "${AZURE_ENVIRONMENT}", + "tenantId": "${AZURE_TENANT_ID}", + "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", + "aadClientId": "${AZURE_CLIENT_ID}", + "aadClientSecret": "${AZURE_CLIENT_SECRET}", + "resourceGroup": "${AZURE_RESOURCE_GROUP}", + "securityGroupName": "${CLUSTER_NAME}-node-nsg", + "location": "${AZURE_LOCATION}", + "vmType": "vmss", + "vnetName": "${CLUSTER_NAME}-vnet", + "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", + "subnetName": "${CLUSTER_NAME}-node-subnet", + "routeTableName": "${CLUSTER_NAME}-node-routetable", + "userAssignedID": "${CLUSTER_NAME}", + "loadBalancerSku": "basic", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": false, + "excludeMasterFromStandardLB": false + } + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + { + "name":"${AZURE_ENVIRONMENT}", + "managementPortalURL":"https://portal.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", + "publishSettingsURL":"", + "serviceManagementEndpoint":"https://management.azurestackci02.onmicrosoft.com/6bbece7e-df47-402c-bf11-101f74fdf525", + "resourceManagerEndpoint":"https://management.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", + "activeDirectoryEndpoint":"https://login.microsoftonline.com/", + "galleryEndpoint":"https://providers.n25r1306.masd.stbtest.microsoft.com:30016/", + "keyVaultEndpoint":"","graphEndpoint":"https://graph.windows.net/", + "serviceBusEndpoint":"", + "batchManagementEndpoint":"", + "storageEndpointSuffix":"redmond.ext-n25r1306.masd.stbtest.microsoft.com", + "sqlDatabaseDNSSuffix":"", + "trafficManagerDNSSuffix":"", + "keyVaultDNSSuffix":"vault.redmond.ext-n25r1306.masd.stbtest.microsoft.com", + "serviceBusEndpointSuffix":"", + "serviceManagementVMDNSSuffix":"", + "resourceManagerVMDNSSuffix":"cloudapp.ext-n25r1306.masd.stbtest.microsoft.com", + "containerRegistryDNSSuffix":"", + "cosmosDBDNSSuffix":"", + "tokenAudience":"", + "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} + } + owner: root:root + path: /etc/kubernetes/azurestackcloud.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - echo 'hi loading post script' + - sudo bash /etc/kubernetes/postkubeadmcommands.sh + - echo 'hi finished loading post script' + preKubeadmCommands: + - echo 'hi loading pre script' + - sudo bash /etc/kubernetes/prekubeadmcommands.sh + - echo 'hi finished loading pre script' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + useExperimentalRetryJoin: true + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + image: + marketplace: + offer: Test + publisher: AzureStack + sku: capz-test-1804 + version: latest + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: exp.cluster.x-k8s.io/v1alpha3 +kind: MachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 + kind: KubeadmConfig + name: ${CLUSTER_NAME}-mp-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachinePool + name: ${CLUSTER_NAME}-mp-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 +spec: + location: ${AZURE_LOCATION} + template: + image: + marketplace: + offer: Test + publisher: AzureStack + sku: capz-test-1804 + version: latest + osDisk: + diskSizeGB: 30 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 +kind: KubeadmConfig +metadata: + name: ${CLUSTER_NAME}-mp-0 +spec: + files: + - content: | + sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + sudo update-ca-certificates + sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + sudo systemctl daemon-reload + sudo systemctl restart kubelet + owner: root:root + path: /etc/kubernetes/prekubeadmcommands.sh + permissions: "0644" + - content: | + sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml + owner: root:root + path: /etc/kubernetes/postkubeadmcommands.sh + permissions: "0644" + - content: | + { + "cloud": "${AZURE_ENVIRONMENT}", + "tenantId": "${AZURE_TENANT_ID}", + "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", + "aadClientId": "${AZURE_CLIENT_ID}", + "aadClientSecret": "${AZURE_CLIENT_SECRET}", + "resourceGroup": "${AZURE_RESOURCE_GROUP}", + "securityGroupName": "${CLUSTER_NAME}-node-nsg", + "location": "${AZURE_LOCATION}", + "vmType": "vmss", + "vnetName": "${CLUSTER_NAME}-vnet", + "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", + "subnetName": "${CLUSTER_NAME}-node-subnet", + "routeTableName": "${CLUSTER_NAME}-node-routetable", + "userAssignedID": "${CLUSTER_NAME}", + "loadBalancerSku": "basic", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": false, + "excludeMasterFromStandardLB": false + } + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + { + "name":"${AZURE_ENVIRONMENT}", + "managementPortalURL":"https://portal.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", + "publishSettingsURL":"", + "serviceManagementEndpoint":"https://management.azurestackci02.onmicrosoft.com/6bbece7e-df47-402c-bf11-101f74fdf525", + "resourceManagerEndpoint":"https://management.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", + "activeDirectoryEndpoint":"https://login.microsoftonline.com/", + "galleryEndpoint":"https://providers.n25r1306.masd.stbtest.microsoft.com:30016/", + "keyVaultEndpoint":"","graphEndpoint":"https://graph.windows.net/", + "serviceBusEndpoint":"", + "batchManagementEndpoint":"", + "storageEndpointSuffix":"redmond.ext-n25r1306.masd.stbtest.microsoft.com", + "sqlDatabaseDNSSuffix":"", + "trafficManagerDNSSuffix":"", + "keyVaultDNSSuffix":"vault.redmond.ext-n25r1306.masd.stbtest.microsoft.com", + "serviceBusEndpointSuffix":"", + "serviceManagementVMDNSSuffix":"", + "resourceManagerVMDNSSuffix":"cloudapp.ext-n25r1306.masd.stbtest.microsoft.com", + "containerRegistryDNSSuffix":"", + "cosmosDBDNSSuffix":"", + "tokenAudience":"", + "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} + } + owner: root:root + path: /etc/kubernetes/azurestackcloud.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + useExperimentalRetryJoin: true + postKubeadmCommands: + - echo 'hi loading post script' + - sudo bash /etc/kubernetes/postkubeadmcommands.sh + - echo 'hi finished loading post script' + preKubeadmCommands: + - echo 'hi loading pre script' + - sudo bash /etc/kubernetes/prekubeadmcommands.sh + - echo 'hi finished loading pre script' diff --git a/templates/cluster-template-azure-stack-aad.yaml b/templates/cluster-template-azure-stack-aad.yaml index 11081c5f2b4..8589f181ced 100644 --- a/templates/cluster-template-azure-stack-aad.yaml +++ b/templates/cluster-template-azure-stack-aad.yaml @@ -164,11 +164,6 @@ spec: name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - echo 'hi loading post script' - - echo 'This is kube-controller-manager.yaml' - - sudo cat /etc/kubernetes/manifests/kube-controller-manager.yaml - - echo 'This is azurestackcloud.json' - - sudo cat /etc/kubernetes/azurestackcloud.json - - echo 'Now running post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh - echo 'hi finished loading post script' preKubeadmCommands: From 41bf25d0142b12fc8b8d3f416d6008520fba12ad Mon Sep 17 00:00:00 2001 From: chlau-az Date: Fri, 31 Jul 2020 15:18:35 -0700 Subject: [PATCH 34/47] Fixed template typo --- templates/cluster-template-azure-stack-aad-copy.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/cluster-template-azure-stack-aad-copy.yaml b/templates/cluster-template-azure-stack-aad-copy.yaml index 85ec3c22117..64ac702e9c6 100644 --- a/templates/cluster-template-azure-stack-aad-copy.yaml +++ b/templates/cluster-template-azure-stack-aad-copy.yaml @@ -242,6 +242,7 @@ apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 kind: AzureMachinePool metadata: name: ${CLUSTER_NAME}-mp-0 + namespace: default spec: location: ${AZURE_LOCATION} template: @@ -263,6 +264,7 @@ apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 kind: KubeadmConfig metadata: name: ${CLUSTER_NAME}-mp-0 + namespace: default spec: files: - content: | From 9ab84200f8db6e973483a0080d38ff8aab2e8e84 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Fri, 31 Jul 2020 17:32:30 -0700 Subject: [PATCH 35/47] Fixed machine pool vmss not being created --- Makefile | 10 +++++ cloud/services/resourceskus/cache.go | 1 - cloud/services/scalesets/vmss.go | 43 +++++++++---------- cloud/services/scalesets/vmss_test.go | 6 +-- .../azuremachinepool_controller.go | 2 +- 5 files changed, 35 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index 246b93fccfa..e58d455569d 100644 --- a/Makefile +++ b/Makefile @@ -118,6 +118,10 @@ SKIP_CREATE_MGMT_CLUSTER ?= false # Build time versioning details. LDFLAGS := $(shell hack/version.sh) +# Allow overriding the feature gates +EXP_MACHINE_POOL ?= true +FEATURE_GATES_JSON_PATCH := [{"op": "add", "path": "/spec/template/spec/containers/1/args/-", "value": "--feature-gates=MachinePool=$(EXP_MACHINE_POOL)"}] + CLUSTER_TEMPLATE ?= cluster-template.yaml MANAGED_CLUSTER_TEMPLATE ?= cluster-template-aks.yaml @@ -390,6 +394,12 @@ create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) kind load docker-image $(CONTROLLER_IMG)-$(ARCH):$(TAG) --name=capz $(KUSTOMIZE) build config | $(ENVSUBST) | kubectl apply -f - + # Patch controllers with feature gates flag + kubectl patch deployment capi-controller-manager -n capi-system --type=json -p='$(FEATURE_GATES_JSON_PATCH)' + kubectl patch deployment capi-kubeadm-bootstrap-controller-manager -n capi-kubeadm-bootstrap-system --type=json -p='$(FEATURE_GATES_JSON_PATCH)' + kubectl patch deployment capz-controller-manager -n capz-system --type=json -p='$(FEATURE_GATES_JSON_PATCH)' + kubectl patch deployment capi-controller-manager -n capi-webhook-system --type=json -p='$(FEATURE_GATES_JSON_PATCH)' + # Wait for CAPI deployments kubectl wait --for=condition=Available --timeout=5m -n capi-system deployment -l cluster.x-k8s.io/provider=cluster-api kubectl wait --for=condition=Available --timeout=5m -n capi-kubeadm-bootstrap-system deployment -l cluster.x-k8s.io/provider=bootstrap-kubeadm diff --git a/cloud/services/resourceskus/cache.go b/cloud/services/resourceskus/cache.go index 1e58e98155f..21a6f34e868 100644 --- a/cloud/services/resourceskus/cache.go +++ b/cloud/services/resourceskus/cache.go @@ -75,7 +75,6 @@ func (c *Cache) refresh(ctx context.Context, location string) error { } c.data = data - return nil } diff --git a/cloud/services/scalesets/vmss.go b/cloud/services/scalesets/vmss.go index fa7c6b3ab13..22ccdf28fb8 100644 --- a/cloud/services/scalesets/vmss.go +++ b/cloud/services/scalesets/vmss.go @@ -25,7 +25,6 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/resourceskus" infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" @@ -71,7 +70,7 @@ func (s *Service) Get(ctx context.Context, vmssSpec *Spec) (*infrav1exp.VMSS, er } func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { - log.Println("calliou reconcile vmss") + log.Println("HERE reconcile vmss") vmssSpec, ok := spec.(*Spec) if !ok { return errors.New("invalid VMSS specification") @@ -84,18 +83,18 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { } vmssSpec.AdditionalTags[infrav1.ClusterAzureCloudProviderTagKey(vmssSpec.MachinePoolName)] = string(infrav1.ResourceLifecycleOwned) - sku, err := s.ResourceSKUCache.Get(ctx, vmssSpec.Sku, resourceskus.VirtualMachines) - if err != nil { - return errors.Wrapf(err, "failed to get find vm sku %s in compute api", vmssSpec.Sku) - } + // sku, err := s.ResourceSKUCache.Get(ctx, vmssSpec.Sku, resourceskus.VirtualMachines) + // if err != nil { + // return errors.Wrapf(err, "failed to get find vm sku %s in compute api", vmssSpec.Sku) + // } - if vmssSpec.AcceleratedNetworking == nil { - // set accelerated networking to the capability of the VMSize - accelNet := sku.HasCapability(resourceskus.AcceleratedNetworking) - vmssSpec.AcceleratedNetworking = &accelNet - } + // if vmssSpec.AcceleratedNetworking == nil { + // // set accelerated networking to the capability of the VMSize + // accelNet := sku.HasCapability(resourceskus.AcceleratedNetworking) + // vmssSpec.AcceleratedNetworking = &accelNet + // } - storageProfile, err := s.generateStorageProfile(ctx, *vmssSpec, sku) + storageProfile, err := s.generateStorageProfile(ctx, *vmssSpec) if err != nil { return err } @@ -225,7 +224,7 @@ func (s *Service) Delete(ctx context.Context, spec interface{}) error { } // generateStorageProfile generates a pointer to a compute.VirtualMachineScaleSetStorageProfile which can utilized for VM creation. -func (s *Service) generateStorageProfile(ctx context.Context, vmssSpec Spec, sku resourceskus.SKU) (*compute.VirtualMachineScaleSetStorageProfile, error) { +func (s *Service) generateStorageProfile(ctx context.Context, vmssSpec Spec) (*compute.VirtualMachineScaleSetStorageProfile, error) { storageProfile := &compute.VirtualMachineScaleSetStorageProfile{ OsDisk: &compute.VirtualMachineScaleSetOSDisk{ OsType: compute.OperatingSystemTypes(vmssSpec.OSDisk.OSType), @@ -237,16 +236,16 @@ func (s *Service) generateStorageProfile(ctx context.Context, vmssSpec Spec, sku }, } - // enable ephemeral OS - if vmssSpec.OSDisk.DiffDiskSettings != nil { - if !sku.HasCapability(resourceskus.EphemeralOSDisk) { - return nil, fmt.Errorf("vm size %s does not support ephemeral os. select a different vm size or disable ephemeral os", vmssSpec.Sku) - } + // // enable ephemeral OS + // if vmssSpec.OSDisk.DiffDiskSettings != nil { + // if !sku.HasCapability(resourceskus.EphemeralOSDisk) { + // return nil, fmt.Errorf("vm size %s does not support ephemeral os. select a different vm size or disable ephemeral os", vmssSpec.Sku) + // } - // storageProfile.OsDisk.DiffDiskSettings = &compute.DiffDiskSettings{ - // Option: compute.DiffDiskOptions(vmssSpec.OSDisk.DiffDiskSettings.Option), - // } - } + // // storageProfile.OsDisk.DiffDiskSettings = &compute.DiffDiskSettings{ + // // Option: compute.DiffDiskOptions(vmssSpec.OSDisk.DiffDiskSettings.Option), + // // } + // } dataDisks := []compute.VirtualMachineScaleSetDataDisk{} for _, disk := range vmssSpec.DataDisks { diff --git a/cloud/services/scalesets/vmss_test.go b/cloud/services/scalesets/vmss_test.go index 880442e3af2..1ad51c0cd80 100644 --- a/cloud/services/scalesets/vmss_test.go +++ b/cloud/services/scalesets/vmss_test.go @@ -325,7 +325,7 @@ func TestService_Reconcile(t *testing.T) { lbMock := mock_loadbalancers.NewMockClient(mockCtrl) svc.LoadBalancersClient = lbMock - storageProfile, err := svc.generateStorageProfile(ctx, *spec, resourceskus.SKU(skus[0])) + storageProfile, err := svc.generateStorageProfile(ctx, *spec) g.Expect(err).ToNot(gomega.HaveOccurred()) vmss := compute.VirtualMachineScaleSet{ @@ -460,7 +460,7 @@ func TestService_Reconcile(t *testing.T) { lbMock := mock_loadbalancers.NewMockClient(mockCtrl) svc.LoadBalancersClient = lbMock - storageProfile, err := svc.generateStorageProfile(ctx, *spec, resourceskus.SKU(skus[0])) + storageProfile, err := svc.generateStorageProfile(ctx, *spec) g.Expect(err).ToNot(gomega.HaveOccurred()) vmss := compute.VirtualMachineScaleSet{ @@ -596,7 +596,7 @@ func TestService_Reconcile(t *testing.T) { lbMock := mock_loadbalancers.NewMockClient(mockCtrl) svc.LoadBalancersClient = lbMock - storageProfile, err := svc.generateStorageProfile(ctx, *spec, resourceskus.SKU(skus[0])) + storageProfile, err := svc.generateStorageProfile(ctx, *spec) g.Expect(err).ToNot(gomega.HaveOccurred()) vmss := compute.VirtualMachineScaleSet{ diff --git a/exp/controllers/azuremachinepool_controller.go b/exp/controllers/azuremachinepool_controller.go index ad4c27964e0..16f3653bcd0 100644 --- a/exp/controllers/azuremachinepool_controller.go +++ b/exp/controllers/azuremachinepool_controller.go @@ -228,7 +228,7 @@ func (r *AzureMachinePoolReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, } func (r *AzureMachinePoolReconciler) reconcileNormal(ctx context.Context, machinePoolScope *scope.MachinePoolScope, clusterScope *scope.ClusterScope) (_ reconcile.Result, reterr error) { - log.Println("calliou reconcile machine pool") + log.Println("HERE reconcile machine pool") machinePoolScope.Info("Reconciling AzureMachinePool") // If the AzureMachine is in an error state, return early. if machinePoolScope.AzureMachinePool.Status.FailureReason != nil || machinePoolScope.AzureMachinePool.Status.FailureMessage != nil { From ecda6bc0913aadc67f64498d50ad61b9d041308f Mon Sep 17 00:00:00 2001 From: chlau-az Date: Mon, 3 Aug 2020 11:21:27 -0700 Subject: [PATCH 36/47] Fixed flavors and templates for aad and adfs, using machinepool --- ...cluster-template-azure-stack-aad-copy.yaml | 350 ------------------ .../cluster-template-azure-stack-aad.yaml | 273 +++++++------- .../cluster-template-azure-stack-adfs.yaml | 302 ++++++++------- .../azure-stack-aad/cluster-template.yaml | 301 ++++++++------- .../azure-stack-adfs/cluster-template.yaml | 305 ++++++++------- 5 files changed, 646 insertions(+), 885 deletions(-) delete mode 100644 templates/cluster-template-azure-stack-aad-copy.yaml diff --git a/templates/cluster-template-azure-stack-aad-copy.yaml b/templates/cluster-template-azure-stack-aad-copy.yaml deleted file mode 100644 index 64ac702e9c6..00000000000 --- a/templates/cluster-template-azure-stack-aad-copy.yaml +++ /dev/null @@ -1,350 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1alpha3 -kind: Cluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - location: ${AZURE_LOCATION} - networkSpec: - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - infrastructureTemplate: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - - hostPath: /etc/kubernetes/azurestackcloud.json - mountPath: /etc/kubernetes/azurestackcloud.json - name: cloud-config-stack - readOnly: true - - hostPath: /etc/kubernetes/prekubeadmcommands.sh - mountPath: /etc/kubernetes/prekubeadmcommands.sh - name: prekubeadmcommands - readOnly: true - - hostPath: /etc/kubernetes/postkubeadmcommands.sh - mountPath: /etc/kubernetes/postkubeadmcommands.sh - name: postkubeadmcommands - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - content: | - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt - sudo update-ca-certificates - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf - sudo systemctl daemon-reload - sudo systemctl restart kubelet - owner: root:root - path: /etc/kubernetes/prekubeadmcommands.sh - permissions: "0644" - - content: | - sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml - owner: root:root - path: /etc/kubernetes/postkubeadmcommands.sh - permissions: "0644" - - content: | - { - "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "${AZURE_TENANT_ID}", - "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", - "aadClientId": "${AZURE_CLIENT_ID}", - "aadClientSecret": "${AZURE_CLIENT_SECRET}", - "resourceGroup": "${AZURE_RESOURCE_GROUP}", - "securityGroupName": "${CLUSTER_NAME}-node-nsg", - "location": "${AZURE_LOCATION}", - "vmType": "vmss", - "vnetName": "${CLUSTER_NAME}-vnet", - "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", - "subnetName": "${CLUSTER_NAME}-node-subnet", - "routeTableName": "${CLUSTER_NAME}-node-routetable", - "userAssignedID": "${CLUSTER_NAME}", - "loadBalancerSku": "basic", - "maximumLoadBalancerRuleCount": 250, - "useManagedIdentityExtension": false, - "useInstanceMetadata": false, - "excludeMasterFromStandardLB": false - } - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - { - "name":"${AZURE_ENVIRONMENT}", - "managementPortalURL":"https://portal.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", - "publishSettingsURL":"", - "serviceManagementEndpoint":"https://management.azurestackci02.onmicrosoft.com/6bbece7e-df47-402c-bf11-101f74fdf525", - "resourceManagerEndpoint":"https://management.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", - "activeDirectoryEndpoint":"https://login.microsoftonline.com/", - "galleryEndpoint":"https://providers.n25r1306.masd.stbtest.microsoft.com:30016/", - "keyVaultEndpoint":"","graphEndpoint":"https://graph.windows.net/", - "serviceBusEndpoint":"", - "batchManagementEndpoint":"", - "storageEndpointSuffix":"redmond.ext-n25r1306.masd.stbtest.microsoft.com", - "sqlDatabaseDNSSuffix":"", - "trafficManagerDNSSuffix":"", - "keyVaultDNSSuffix":"vault.redmond.ext-n25r1306.masd.stbtest.microsoft.com", - "serviceBusEndpointSuffix":"", - "serviceManagementVMDNSSuffix":"", - "resourceManagerVMDNSSuffix":"cloudapp.ext-n25r1306.masd.stbtest.microsoft.com", - "containerRegistryDNSSuffix":"", - "cosmosDBDNSSuffix":"", - "tokenAudience":"", - "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} - } - owner: root:root - path: /etc/kubernetes/azurestackcloud.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - echo 'hi loading post script' - - sudo bash /etc/kubernetes/postkubeadmcommands.sh - - echo 'hi finished loading post script' - preKubeadmCommands: - - echo 'hi loading pre script' - - sudo bash /etc/kubernetes/prekubeadmcommands.sh - - echo 'hi finished loading pre script' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - useExperimentalRetryJoin: true - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - image: - marketplace: - offer: Test - publisher: AzureStack - sku: capz-test-1804 - version: latest - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - location: ${AZURE_LOCATION} - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: exp.cluster.x-k8s.io/v1alpha3 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 -kind: AzureMachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - location: ${AZURE_LOCATION} - template: - image: - marketplace: - offer: Test - publisher: AzureStack - sku: capz-test-1804 - version: latest - osDisk: - diskSizeGB: 30 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - files: - - content: | - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt - sudo update-ca-certificates - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf - sudo systemctl daemon-reload - sudo systemctl restart kubelet - owner: root:root - path: /etc/kubernetes/prekubeadmcommands.sh - permissions: "0644" - - content: | - sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml - owner: root:root - path: /etc/kubernetes/postkubeadmcommands.sh - permissions: "0644" - - content: | - { - "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "${AZURE_TENANT_ID}", - "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", - "aadClientId": "${AZURE_CLIENT_ID}", - "aadClientSecret": "${AZURE_CLIENT_SECRET}", - "resourceGroup": "${AZURE_RESOURCE_GROUP}", - "securityGroupName": "${CLUSTER_NAME}-node-nsg", - "location": "${AZURE_LOCATION}", - "vmType": "vmss", - "vnetName": "${CLUSTER_NAME}-vnet", - "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", - "subnetName": "${CLUSTER_NAME}-node-subnet", - "routeTableName": "${CLUSTER_NAME}-node-routetable", - "userAssignedID": "${CLUSTER_NAME}", - "loadBalancerSku": "basic", - "maximumLoadBalancerRuleCount": 250, - "useManagedIdentityExtension": false, - "useInstanceMetadata": false, - "excludeMasterFromStandardLB": false - } - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - { - "name":"${AZURE_ENVIRONMENT}", - "managementPortalURL":"https://portal.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", - "publishSettingsURL":"", - "serviceManagementEndpoint":"https://management.azurestackci02.onmicrosoft.com/6bbece7e-df47-402c-bf11-101f74fdf525", - "resourceManagerEndpoint":"https://management.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", - "activeDirectoryEndpoint":"https://login.microsoftonline.com/", - "galleryEndpoint":"https://providers.n25r1306.masd.stbtest.microsoft.com:30016/", - "keyVaultEndpoint":"","graphEndpoint":"https://graph.windows.net/", - "serviceBusEndpoint":"", - "batchManagementEndpoint":"", - "storageEndpointSuffix":"redmond.ext-n25r1306.masd.stbtest.microsoft.com", - "sqlDatabaseDNSSuffix":"", - "trafficManagerDNSSuffix":"", - "keyVaultDNSSuffix":"vault.redmond.ext-n25r1306.masd.stbtest.microsoft.com", - "serviceBusEndpointSuffix":"", - "serviceManagementVMDNSSuffix":"", - "resourceManagerVMDNSSuffix":"cloudapp.ext-n25r1306.masd.stbtest.microsoft.com", - "containerRegistryDNSSuffix":"", - "cosmosDBDNSSuffix":"", - "tokenAudience":"", - "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} - } - owner: root:root - path: /etc/kubernetes/azurestackcloud.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - useExperimentalRetryJoin: true - postKubeadmCommands: - - echo 'hi loading post script' - - sudo bash /etc/kubernetes/postkubeadmcommands.sh - - echo 'hi finished loading post script' - preKubeadmCommands: - - echo 'hi loading pre script' - - sudo bash /etc/kubernetes/prekubeadmcommands.sh - - echo 'hi finished loading pre script' diff --git a/templates/cluster-template-azure-stack-aad.yaml b/templates/cluster-template-azure-stack-aad.yaml index 8589f181ced..480d2fb9e6b 100644 --- a/templates/cluster-template-azure-stack-aad.yaml +++ b/templates/cluster-template-azure-stack-aad.yaml @@ -26,8 +26,8 @@ spec: location: ${AZURE_LOCATION} networkSpec: vnet: - name: ${AZURE_VNET_NAME} - resourceGroup: ${AZURE_RESOURCE_GROUP} + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} --- apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 @@ -110,16 +110,28 @@ spec: owner: root:root path: /etc/kubernetes/postkubeadmcommands.sh permissions: "0644" - - content: "{\n \"cloud\": \"${AZURE_ENVIRONMENT}\",\n \"tenantId\": \"${AZURE_TENANT_ID}\",\n - \ \"subscriptionId\": \"${AZURE_SUBSCRIPTION_ID}\",\n \"aadClientId\": \"${AZURE_CLIENT_ID}\",\n - \ \"aadClientSecret\": \"${AZURE_CLIENT_SECRET}\",\n \"resourceGroup\": \"${AZURE_RESOURCE_GROUP}\",\n - \ \"securityGroupName\": \"${CLUSTER_NAME}-node-nsg\",\n \"location\": \"${AZURE_LOCATION}\",\n - \ \"vmType\": \"vmss\",\n \"vnetName\": \"${CLUSTER_NAME}-vnet\",\n \"vnetResourceGroup\": - \"${AZURE_RESOURCE_GROUP}\",\n \"subnetName\": \"${CLUSTER_NAME}-node-subnet\",\n - \ \"routeTableName\": \"${CLUSTER_NAME}-node-routetable\",\n \"userAssignedID\": - \"${CLUSTER_NAME}\",\n \"loadBalancerSku\": \"Basic\",\n \"maximumLoadBalancerRuleCount\": - 250,\n \"useManagedIdentityExtension\": false,\n \"useInstanceMetadata\": - false, \n \"excludeMasterFromStandardLB\": false\n}\n" + - content: | + { + "cloud": "${AZURE_ENVIRONMENT}", + "tenantId": "${AZURE_TENANT_ID}", + "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", + "aadClientId": "${AZURE_CLIENT_ID}", + "aadClientSecret": "${AZURE_CLIENT_SECRET}", + "resourceGroup": "${AZURE_RESOURCE_GROUP}", + "securityGroupName": "${CLUSTER_NAME}-node-nsg", + "location": "${AZURE_LOCATION}", + "vmType": "vmss", + "vnetName": "${CLUSTER_NAME}-vnet", + "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", + "subnetName": "${CLUSTER_NAME}-node-subnet", + "routeTableName": "${CLUSTER_NAME}-node-routetable", + "userAssignedID": "${CLUSTER_NAME}", + "loadBalancerSku": "basic", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": false, + "excludeMasterFromStandardLB": false + } owner: root:root path: /etc/kubernetes/azure.json permissions: "0644" @@ -162,6 +174,9 @@ spec: cloud-config: /etc/kubernetes/azure.json cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk postKubeadmCommands: - echo 'hi loading post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh @@ -170,9 +185,6 @@ spec: - echo 'hi loading pre script' - sudo bash /etc/kubernetes/prekubeadmcommands.sh - echo 'hi finished loading pre script' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk useExperimentalRetryJoin: true replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} @@ -185,16 +197,16 @@ metadata: spec: template: spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk image: marketplace: offer: Test publisher: AzureStack - sku: capz-test + sku: capz-test-1804 version: latest - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk location: ${AZURE_LOCATION} osDisk: diskSizeGB: 128 @@ -204,138 +216,135 @@ spec: sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} --- -apiVersion: cluster.x-k8s.io/v1alpha3 -kind: MachineDeployment +apiVersion: exp.cluster.x-k8s.io/v1alpha3 +kind: MachinePool metadata: - name: ${CLUSTER_NAME}-md-0 + name: ${CLUSTER_NAME}-mp-0 namespace: default spec: clusterName: ${CLUSTER_NAME} replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: null template: spec: bootstrap: configRef: apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-0 + kind: KubeadmConfig + name: ${CLUSTER_NAME}-mp-0 clusterName: ${CLUSTER_NAME} infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-0 + apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachinePool + name: ${CLUSTER_NAME}-mp-0 version: ${KUBERNETES_VERSION} --- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 -kind: AzureMachineTemplate +apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachinePool metadata: - name: ${CLUSTER_NAME}-md-0 + name: ${CLUSTER_NAME}-mp-0 namespace: default spec: + location: ${AZURE_LOCATION} template: - spec: - image: - marketplace: - offer: Test - publisher: AzureStack - sku: capz-test - version: latest - location: ${AZURE_LOCATION} - osDisk: - diskSizeGB: 30 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} - vmSize: ${AZURE_NODE_MACHINE_TYPE} + image: + marketplace: + offer: Test + publisher: AzureStack + sku: capz-test-1804 + version: latest + osDisk: + diskSizeGB: 30 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} + vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 -kind: KubeadmConfigTemplate +kind: KubeadmConfig metadata: - name: ${CLUSTER_NAME}-md-0 + name: ${CLUSTER_NAME}-mp-0 namespace: default spec: - template: - spec: - files: - - content: | - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt - sudo update-ca-certificates - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf - sudo systemctl daemon-reload - sudo systemctl restart kubelet - owner: root:root - path: /etc/kubernetes/prekubeadmcommands.sh - permissions: "0644" - - content: | - sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml - owner: root:root - path: /etc/kubernetes/postkubeadmcommands.sh - permissions: "0644" - - content: | - { - "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "${AZURE_TENANT_ID}", - "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", - "aadClientId": "${AZURE_CLIENT_ID}", - "aadClientSecret": "${AZURE_CLIENT_SECRET}", - "resourceGroup": "${AZURE_RESOURCE_GROUP}", - "securityGroupName": "${CLUSTER_NAME}-node-nsg", - "location": "${AZURE_LOCATION}", - "vmType": "vmss", - "vnetName": "${CLUSTER_NAME}-vnet", - "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", - "subnetName": "${CLUSTER_NAME}-node-subnet", - "routeTableName": "${CLUSTER_NAME}-node-routetable", - "loadBalancerSku": "Basic", - "maximumLoadBalancerRuleCount": 250, - "useManagedIdentityExtension": false, - "useInstanceMetadata": false, - "excludeMasterFromStandardLB": false - } - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - { - "name":"${AZURE_ENVIRONMENT}", - "managementPortalURL":"https://portal.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", - "publishSettingsURL":"", - "serviceManagementEndpoint":"https://management.azurestackci02.onmicrosoft.com/6bbece7e-df47-402c-bf11-101f74fdf525", - "resourceManagerEndpoint":"https://management.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", - "activeDirectoryEndpoint":"https://login.microsoftonline.com/", - "galleryEndpoint":"https://providers.n25r1306.masd.stbtest.microsoft.com:30016/", - "keyVaultEndpoint":"","graphEndpoint":"https://graph.windows.net/", - "serviceBusEndpoint":"", - "batchManagementEndpoint":"", - "storageEndpointSuffix":"redmond.ext-n25r1306.masd.stbtest.microsoft.com", - "sqlDatabaseDNSSuffix":"", - "trafficManagerDNSSuffix":"", - "keyVaultDNSSuffix":"vault.redmond.ext-n25r1306.masd.stbtest.microsoft.com", - "serviceBusEndpointSuffix":"", - "serviceManagementVMDNSSuffix":"", - "resourceManagerVMDNSSuffix":"cloudapp.ext-n25r1306.masd.stbtest.microsoft.com", - "containerRegistryDNSSuffix":"", - "cosmosDBDNSSuffix":"", - "tokenAudience":"", - "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} - } - owner: root:root - path: /etc/kubernetes/azurestackcloud.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - echo 'hi loading post script' - - sudo bash /etc/kubernetes/postkubeadmcommands.sh - - echo 'hi finished loading post script' - preKubeadmCommands: - - echo 'hi loading pre script' - - sudo bash /etc/kubernetes/prekubeadmcommands.sh - - echo 'hi finished loading pre script' + files: + - content: | + sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + sudo update-ca-certificates + sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + sudo systemctl daemon-reload + sudo systemctl restart kubelet + owner: root:root + path: /etc/kubernetes/prekubeadmcommands.sh + permissions: "0644" + - content: | + sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml + owner: root:root + path: /etc/kubernetes/postkubeadmcommands.sh + permissions: "0644" + - content: | + { + "cloud": "${AZURE_ENVIRONMENT}", + "tenantId": "${AZURE_TENANT_ID}", + "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", + "aadClientId": "${AZURE_CLIENT_ID}", + "aadClientSecret": "${AZURE_CLIENT_SECRET}", + "resourceGroup": "${AZURE_RESOURCE_GROUP}", + "securityGroupName": "${CLUSTER_NAME}-node-nsg", + "location": "${AZURE_LOCATION}", + "vmType": "vmss", + "vnetName": "${CLUSTER_NAME}-vnet", + "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", + "subnetName": "${CLUSTER_NAME}-node-subnet", + "routeTableName": "${CLUSTER_NAME}-node-routetable", + "userAssignedID": "${CLUSTER_NAME}", + "loadBalancerSku": "basic", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": false, + "excludeMasterFromStandardLB": false + } + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + { + "name":"${AZURE_ENVIRONMENT}", + "managementPortalURL":"https://portal.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", + "publishSettingsURL":"", + "serviceManagementEndpoint":"https://management.azurestackci02.onmicrosoft.com/6bbece7e-df47-402c-bf11-101f74fdf525", + "resourceManagerEndpoint":"https://management.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", + "activeDirectoryEndpoint":"https://login.microsoftonline.com/", + "galleryEndpoint":"https://providers.n25r1306.masd.stbtest.microsoft.com:30016/", + "keyVaultEndpoint":"","graphEndpoint":"https://graph.windows.net/", + "serviceBusEndpoint":"", + "batchManagementEndpoint":"", + "storageEndpointSuffix":"redmond.ext-n25r1306.masd.stbtest.microsoft.com", + "sqlDatabaseDNSSuffix":"", + "trafficManagerDNSSuffix":"", + "keyVaultDNSSuffix":"vault.redmond.ext-n25r1306.masd.stbtest.microsoft.com", + "serviceBusEndpointSuffix":"", + "serviceManagementVMDNSSuffix":"", + "resourceManagerVMDNSSuffix":"cloudapp.ext-n25r1306.masd.stbtest.microsoft.com", + "containerRegistryDNSSuffix":"", + "cosmosDBDNSSuffix":"", + "tokenAudience":"", + "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} + } + owner: root:root + path: /etc/kubernetes/azurestackcloud.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - echo 'hi loading post script' + - sudo bash /etc/kubernetes/postkubeadmcommands.sh + - echo 'hi finished loading post script' + preKubeadmCommands: + - echo 'hi loading pre script' + - sudo bash /etc/kubernetes/prekubeadmcommands.sh + - echo 'hi finished loading pre script' + useExperimentalRetryJoin: true diff --git a/templates/cluster-template-azure-stack-adfs.yaml b/templates/cluster-template-azure-stack-adfs.yaml index 1eb178d8d97..343ccd2e3fc 100644 --- a/templates/cluster-template-azure-stack-adfs.yaml +++ b/templates/cluster-template-azure-stack-adfs.yaml @@ -26,8 +26,8 @@ spec: location: ${AZURE_LOCATION} networkSpec: vnet: - name: ${AZURE_VNET_NAME} - resourceGroup: ${AZURE_RESOURCE_GROUP} + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} --- apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 @@ -67,6 +67,34 @@ spec: mountPath: /etc/kubernetes/azurestackcloud.json name: cloud-config-stack readOnly: true + - hostPath: /etc/kubernetes/prekubeadmcommands.sh + mountPath: /etc/kubernetes/prekubeadmcommands.sh + name: prekubeadmcommands + readOnly: true + - hostPath: /etc/kubernetes/postkubeadmcommands.sh + mountPath: /etc/kubernetes/postkubeadmcommands.sh + name: postkubeadmcommands + readOnly: true + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt files: - content: | sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt @@ -78,20 +106,32 @@ spec: path: /etc/kubernetes/prekubeadmcommands.sh permissions: "0644" - content: | - "sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml" + sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml owner: root:root path: /etc/kubernetes/postkubeadmcommands.sh permissions: "0644" - - content: "{\n \"cloud\": \"${AZURE_ENVIRONMENT}\",\n \"tenantId\": \"adfs\",\n - \ \"subscriptionId\": \"${AZURE_SUBSCRIPTION_ID}\",\n \"aadClientId\": \"${AZURE_CLIENT_ID}\",\n - \ \"aadClientSecret\": \"${AZURE_CLIENT_SECRET}\",\n \"resourceGroup\": \"${AZURE_RESOURCE_GROUP}\",\n - \ \"securityGroupName\": \"${CLUSTER_NAME}-node-nsg\",\n \"location\": \"${AZURE_LOCATION}\",\n - \ \"vmType\": \"vmss\",\n \"vnetName\": \"${CLUSTER_NAME}-vnet\",\n \"vnetResourceGroup\": - \"${AZURE_RESOURCE_GROUP}\",\n \"subnetName\": \"${CLUSTER_NAME}-node-subnet\",\n - \ \"routeTableName\": \"${CLUSTER_NAME}-node-routetable\",\n \"userAssignedID\": - \"${CLUSTER_NAME}\",\n \"loadBalancerSku\": \"Basic\",\n \"maximumLoadBalancerRuleCount\": - 250,\n \"useManagedIdentityExtension\": false,\n \"useInstanceMetadata\": - false, \n \"excludeMasterFromStandardLB\": false\n}\n" + - content: | + { + "cloud": "${AZURE_ENVIRONMENT}", + "tenantId": "adfs", + "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", + "aadClientId": "${AZURE_CLIENT_ID}", + "aadClientSecret": "${AZURE_CLIENT_SECRET}", + "resourceGroup": "${AZURE_RESOURCE_GROUP}", + "securityGroupName": "${CLUSTER_NAME}-node-nsg", + "location": "${AZURE_LOCATION}", + "vmType": "vmss", + "vnetName": "${CLUSTER_NAME}-vnet", + "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", + "subnetName": "${CLUSTER_NAME}-node-subnet", + "routeTableName": "${CLUSTER_NAME}-node-routetable", + "userAssignedID": "${CLUSTER_NAME}", + "loadBalancerSku": "basic", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": false, + "excludeMasterFromStandardLB": false + } owner: root:root path: /etc/kubernetes/azure.json permissions: "0644" @@ -135,12 +175,17 @@ spec: cloud-config: /etc/kubernetes/azure.json cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk postKubeadmCommands: - - hi loading post script + - echo 'hi loading post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh + - echo 'hi finished loading post script' preKubeadmCommands: - - hi loading pre script + - echo 'hi loading pre script' - sudo bash /etc/kubernetes/prekubeadmcommands.sh + - echo 'hi finished loading pre script' useExperimentalRetryJoin: true replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} @@ -153,11 +198,15 @@ metadata: spec: template: spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk image: marketplace: offer: Test publisher: AzureStack - sku: capz-test + sku: capz-test-1804 version: latest location: ${AZURE_LOCATION} osDisk: @@ -168,137 +217,136 @@ spec: sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} --- -apiVersion: cluster.x-k8s.io/v1alpha3 -kind: MachineDeployment +apiVersion: exp.cluster.x-k8s.io/v1alpha3 +kind: MachinePool metadata: - name: ${CLUSTER_NAME}-md-0 + name: ${CLUSTER_NAME}-mp-0 namespace: default spec: clusterName: ${CLUSTER_NAME} replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: null template: spec: bootstrap: configRef: apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-0 + kind: KubeadmConfig + name: ${CLUSTER_NAME}-mp-0 clusterName: ${CLUSTER_NAME} infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-0 + apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachinePool + name: ${CLUSTER_NAME}-mp-0 version: ${KUBERNETES_VERSION} --- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 -kind: AzureMachineTemplate +apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachinePool metadata: - name: ${CLUSTER_NAME}-md-0 + name: ${CLUSTER_NAME}-mp-0 namespace: default spec: + location: ${AZURE_LOCATION} template: - spec: - image: - marketplace: - offer: Test - publisher: AzureStack - sku: capz-test - version: latest - location: ${AZURE_LOCATION} - osDisk: - diskSizeGB: 30 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} - vmSize: ${AZURE_NODE_MACHINE_TYPE} + image: + marketplace: + offer: Test + publisher: AzureStack + sku: capz-test-1804 + version: latest + osDisk: + diskSizeGB: 30 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} + vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 -kind: KubeadmConfigTemplate +kind: KubeadmConfig metadata: - name: ${CLUSTER_NAME}-md-0 + name: ${CLUSTER_NAME}-mp-0 namespace: default spec: - template: - spec: - files: - - content: | - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt - sudo update-ca-certificates - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf - sudo systemctl daemon-reload - sudo systemctl restart kubelet - owner: root:root - path: /etc/kubernetes/prekubeadmcommands.sh - permissions: "0644" - - content: | - "sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml" - owner: root:root - path: /etc/kubernetes/postkubeadmcommands.sh - permissions: "0644" - - content: | - { - "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "adfs", - "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", - "aadClientId": "${AZURE_CLIENT_ID}", - "aadClientSecret": "${AZURE_CLIENT_SECRET}", - "resourceGroup": "${AZURE_RESOURCE_GROUP}", - "securityGroupName": "${CLUSTER_NAME}-node-nsg", - "location": "${AZURE_LOCATION}", - "vmType": "vmss", - "vnetName": "${CLUSTER_NAME}-vnet", - "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", - "subnetName": "${CLUSTER_NAME}-node-subnet", - "routeTableName": "${CLUSTER_NAME}-node-routetable", - "loadBalancerSku": "Basic", - "maximumLoadBalancerRuleCount": 250, - "useManagedIdentityExtension": false, - "useInstanceMetadata": false, - "excludeMasterFromStandardLB": false - } - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - { - "name":"${AZURE_ENVIRONMENT}", - "managementPortalURL":"https://portal.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "publishSettingsURL":"", - "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/98bbb015-dcf2-467c-a1cb-951df61a0948", - "resourceManagerEndpoint":"https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "activeDirectoryEndpoint":"https://adfs.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "galleryEndpoint":"https://providers.n31r1203.masd.stbtest.microsoft.com:30016/", - "keyVaultEndpoint":"", - "graphEndpoint":"https://graph.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "serviceBusEndpoint":"", - "batchManagementEndpoint":"", - "storageEndpointSuffix":"redmond.ext-n31r1203.masd.stbtest.microsoft.com", - "sqlDatabaseDNSSuffix":"", - "trafficManagerDNSSuffix":"", - "keyVaultDNSSuffix":"vault.redmond.ext-n31r1203.masd.stbtest.microsoft.com", - "serviceBusEndpointSuffix":"", - "serviceManagementVMDNSSuffix":"", - "resourceManagerVMDNSSuffix":"cloudapp.ext-n31r1203.masd.stbtest.microsoft.com", - "containerRegistryDNSSuffix":"", - "cosmosDBDNSSuffix":"", - "tokenAudience":"", - "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} - } - owner: root:root - path: /etc/kubernetes/azurestackcloud.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - hi loading post script - - sudo bash /etc/kubernetes/postkubeadmcommands.sh - preKubeadmCommands: - - hi loading pre script - - sudo bash /etc/kubernetes/prekubeadmcommands.sh + files: + - content: | + sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + sudo update-ca-certificates + sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + sudo systemctl daemon-reload + sudo systemctl restart kubelet + owner: root:root + path: /etc/kubernetes/prekubeadmcommands.sh + permissions: "0644" + - content: | + sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml + owner: root:root + path: /etc/kubernetes/postkubeadmcommands.sh + permissions: "0644" + - content: | + { + "cloud": "${AZURE_ENVIRONMENT}", + "tenantId": "adfs", + "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", + "aadClientId": "${AZURE_CLIENT_ID}", + "aadClientSecret": "${AZURE_CLIENT_SECRET}", + "resourceGroup": "${AZURE_RESOURCE_GROUP}", + "securityGroupName": "${CLUSTER_NAME}-node-nsg", + "location": "${AZURE_LOCATION}", + "vmType": "vmss", + "vnetName": "${CLUSTER_NAME}-vnet", + "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", + "subnetName": "${CLUSTER_NAME}-node-subnet", + "routeTableName": "${CLUSTER_NAME}-node-routetable", + "userAssignedID": "${CLUSTER_NAME}", + "loadBalancerSku": "basic", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": false, + "excludeMasterFromStandardLB": false + } + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + { + "name":"${AZURE_ENVIRONMENT}", + "managementPortalURL":"https://portal.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "publishSettingsURL":"", + "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/98bbb015-dcf2-467c-a1cb-951df61a0948", + "resourceManagerEndpoint":"https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "activeDirectoryEndpoint":"https://adfs.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "galleryEndpoint":"https://providers.n31r1203.masd.stbtest.microsoft.com:30016/", + "keyVaultEndpoint":"", + "graphEndpoint":"https://graph.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "serviceBusEndpoint":"", + "batchManagementEndpoint":"", + "storageEndpointSuffix":"redmond.ext-n31r1203.masd.stbtest.microsoft.com", + "sqlDatabaseDNSSuffix":"", + "trafficManagerDNSSuffix":"", + "keyVaultDNSSuffix":"vault.redmond.ext-n31r1203.masd.stbtest.microsoft.com", + "serviceBusEndpointSuffix":"", + "serviceManagementVMDNSSuffix":"", + "resourceManagerVMDNSSuffix":"cloudapp.ext-n31r1203.masd.stbtest.microsoft.com", + "containerRegistryDNSSuffix":"", + "cosmosDBDNSSuffix":"", + "tokenAudience":"", + "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} + } + owner: root:root + path: /etc/kubernetes/azurestackcloud.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - echo 'hi loading post script' + - sudo bash /etc/kubernetes/postkubeadmcommands.sh + - echo 'hi finished loading post script' + preKubeadmCommands: + - echo 'hi loading pre script' + - sudo bash /etc/kubernetes/prekubeadmcommands.sh + - echo 'hi finished loading pre script' + useExperimentalRetryJoin: true diff --git a/templates/flavors/azure-stack-aad/cluster-template.yaml b/templates/flavors/azure-stack-aad/cluster-template.yaml index 0edfdb41c06..fc3d31e38af 100644 --- a/templates/flavors/azure-stack-aad/cluster-template.yaml +++ b/templates/flavors/azure-stack-aad/cluster-template.yaml @@ -2,7 +2,6 @@ apiVersion: cluster.x-k8s.io/v1alpha3 kind: Cluster metadata: name: ${CLUSTER_NAME} - namespace: default spec: clusterNetwork: pods: @@ -21,32 +20,24 @@ apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 kind: AzureCluster metadata: name: ${CLUSTER_NAME} - namespace: default spec: location: ${AZURE_LOCATION} networkSpec: vnet: - name: ${AZURE_VNET_NAME} - resourceGroup: ${AZURE_RESOURCE_GROUP} + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} --- apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 kind: KubeadmControlPlane metadata: name: ${CLUSTER_NAME}-control-plane - namespace: default spec: infrastructureTemplate: apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 kind: AzureMachineTemplate name: ${CLUSTER_NAME}-control-plane kubeadmConfigSpec: - preKubeadmCommands: - - "hi loading pre script" - - sudo bash /etc/kubernetes/prekubeadmcommands.sh - postKubeadmCommands: - - "hi loading post script" - - sudo bash /etc/kubernetes/postkubeadmcommands.sh clusterConfiguration: apiServer: extraArgs: @@ -75,20 +66,44 @@ spec: readOnly: true - hostPath: /etc/kubernetes/prekubeadmcommands.sh mountPath: /etc/kubernetes/prekubeadmcommands.sh - name: cloud-config-stack + name: prekubeadmcommands + readOnly: true + - hostPath: /etc/kubernetes/postkubeadmcommands.sh + mountPath: /etc/kubernetes/postkubeadmcommands.sh + name: postkubeadmcommands readOnly: true + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt files: - content: | - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt - sudo update-ca-certificates - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf - sudo systemctl daemon-reload - sudo systemctl restart kubelet + sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + sudo update-ca-certificates + sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + sudo systemctl daemon-reload + sudo systemctl restart kubelet owner: root:root path: /etc/kubernetes/prekubeadmcommands.sh permissions: "0644" - content: | - "sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml" + sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml owner: root:root path: /etc/kubernetes/postkubeadmcommands.sh permissions: "0644" @@ -108,10 +123,10 @@ spec: "subnetName": "${CLUSTER_NAME}-node-subnet", "routeTableName": "${CLUSTER_NAME}-node-routetable", "userAssignedID": "${CLUSTER_NAME}", - "loadBalancerSku": "Basic", + "loadBalancerSku": "basic", "maximumLoadBalancerRuleCount": 250, "useManagedIdentityExtension": false, - "useInstanceMetadata": false, + "useInstanceMetadata": false, "excludeMasterFromStandardLB": false } owner: root:root @@ -156,6 +171,17 @@ spec: cloud-config: /etc/kubernetes/azure.json cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - echo 'hi loading post script' + - sudo bash /etc/kubernetes/postkubeadmcommands.sh + - echo 'hi finished loading post script' + preKubeadmCommands: + - echo 'hi loading pre script' + - sudo bash /etc/kubernetes/prekubeadmcommands.sh + - echo 'hi finished loading pre script' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk useExperimentalRetryJoin: true replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} @@ -164,16 +190,19 @@ apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 kind: AzureMachineTemplate metadata: name: ${CLUSTER_NAME}-control-plane - namespace: default spec: template: spec: - image: - marketplace: - publisher: AzureStack + image: + marketplace: offer: Test - sku: capz-test + publisher: AzureStack + sku: capz-test-1804 version: latest + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk location: ${AZURE_LOCATION} osDisk: diskSizeGB: 128 @@ -183,136 +212,132 @@ spec: sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} --- -apiVersion: cluster.x-k8s.io/v1alpha3 -kind: MachineDeployment +apiVersion: exp.cluster.x-k8s.io/v1alpha3 +kind: MachinePool metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default + name: ${CLUSTER_NAME}-mp-0 spec: clusterName: ${CLUSTER_NAME} replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: null template: spec: bootstrap: configRef: apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-0 + kind: KubeadmConfig + name: ${CLUSTER_NAME}-mp-0 clusterName: ${CLUSTER_NAME} infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-0 + apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachinePool + name: ${CLUSTER_NAME}-mp-0 version: ${KUBERNETES_VERSION} --- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 -kind: AzureMachineTemplate +apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachinePool metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default + name: ${CLUSTER_NAME}-mp-0 spec: + location: ${AZURE_LOCATION} template: - spec: - image: - marketplace: - publisher: AzureStack - offer: Test - sku: capz-test - version: latest - location: ${AZURE_LOCATION} - osDisk: - diskSizeGB: 30 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} - vmSize: ${AZURE_NODE_MACHINE_TYPE} + image: + marketplace: + offer: Test + publisher: AzureStack + sku: capz-test-1804 + version: latest + osDisk: + diskSizeGB: 30 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} + vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 -kind: KubeadmConfigTemplate +kind: KubeadmConfig metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default + name: ${CLUSTER_NAME}-mp-0 spec: - template: - spec: - preKubeadmCommands: - - "hi loading pre script" - - sudo bash /etc/kubernetes/prekubeadmcommands.sh - postKubeadmCommands: - - "hi loading post script" - - sudo bash /etc/kubernetes/postkubeadmcommands.sh - files: - - content: | - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt - sudo update-ca-certificates - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf - sudo systemctl daemon-reload - sudo systemctl restart kubelet - owner: root:root - path: /etc/kubernetes/prekubeadmcommands.sh - permissions: "0644" - - content: | - "sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml" - owner: root:root - path: /etc/kubernetes/postkubeadmcommands.sh - permissions: "0644" - - content: | - { - "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "${AZURE_TENANT_ID}", - "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", - "aadClientId": "${AZURE_CLIENT_ID}", - "aadClientSecret": "${AZURE_CLIENT_SECRET}", - "resourceGroup": "${AZURE_RESOURCE_GROUP}", - "securityGroupName": "${CLUSTER_NAME}-node-nsg", - "location": "${AZURE_LOCATION}", - "vmType": "vmss", - "vnetName": "${CLUSTER_NAME}-vnet", - "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", - "subnetName": "${CLUSTER_NAME}-node-subnet", - "routeTableName": "${CLUSTER_NAME}-node-routetable", - "loadBalancerSku": "Basic", - "maximumLoadBalancerRuleCount": 250, - "useManagedIdentityExtension": false, - "useInstanceMetadata": false, - "excludeMasterFromStandardLB": false - } - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - { - "name":"${AZURE_ENVIRONMENT}", - "managementPortalURL":"https://portal.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", - "publishSettingsURL":"", - "serviceManagementEndpoint":"https://management.azurestackci02.onmicrosoft.com/6bbece7e-df47-402c-bf11-101f74fdf525", - "resourceManagerEndpoint":"https://management.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", - "activeDirectoryEndpoint":"https://login.microsoftonline.com/", - "galleryEndpoint":"https://providers.n25r1306.masd.stbtest.microsoft.com:30016/", - "keyVaultEndpoint":"","graphEndpoint":"https://graph.windows.net/", - "serviceBusEndpoint":"", - "batchManagementEndpoint":"", - "storageEndpointSuffix":"redmond.ext-n25r1306.masd.stbtest.microsoft.com", - "sqlDatabaseDNSSuffix":"", - "trafficManagerDNSSuffix":"", - "keyVaultDNSSuffix":"vault.redmond.ext-n25r1306.masd.stbtest.microsoft.com", - "serviceBusEndpointSuffix":"", - "serviceManagementVMDNSSuffix":"", - "resourceManagerVMDNSSuffix":"cloudapp.ext-n25r1306.masd.stbtest.microsoft.com", - "containerRegistryDNSSuffix":"", - "cosmosDBDNSSuffix":"", - "tokenAudience":"", - "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} - } - owner: root:root - path: /etc/kubernetes/azurestackcloud.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' + files: + - content: | + sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + sudo update-ca-certificates + sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + sudo systemctl daemon-reload + sudo systemctl restart kubelet + owner: root:root + path: /etc/kubernetes/prekubeadmcommands.sh + permissions: "0644" + - content: | + sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml + owner: root:root + path: /etc/kubernetes/postkubeadmcommands.sh + permissions: "0644" + - content: | + { + "cloud": "${AZURE_ENVIRONMENT}", + "tenantId": "${AZURE_TENANT_ID}", + "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", + "aadClientId": "${AZURE_CLIENT_ID}", + "aadClientSecret": "${AZURE_CLIENT_SECRET}", + "resourceGroup": "${AZURE_RESOURCE_GROUP}", + "securityGroupName": "${CLUSTER_NAME}-node-nsg", + "location": "${AZURE_LOCATION}", + "vmType": "vmss", + "vnetName": "${CLUSTER_NAME}-vnet", + "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", + "subnetName": "${CLUSTER_NAME}-node-subnet", + "routeTableName": "${CLUSTER_NAME}-node-routetable", + "userAssignedID": "${CLUSTER_NAME}", + "loadBalancerSku": "basic", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": false, + "excludeMasterFromStandardLB": false + } + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + { + "name":"${AZURE_ENVIRONMENT}", + "managementPortalURL":"https://portal.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", + "publishSettingsURL":"", + "serviceManagementEndpoint":"https://management.azurestackci02.onmicrosoft.com/6bbece7e-df47-402c-bf11-101f74fdf525", + "resourceManagerEndpoint":"https://management.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", + "activeDirectoryEndpoint":"https://login.microsoftonline.com/", + "galleryEndpoint":"https://providers.n25r1306.masd.stbtest.microsoft.com:30016/", + "keyVaultEndpoint":"","graphEndpoint":"https://graph.windows.net/", + "serviceBusEndpoint":"", + "batchManagementEndpoint":"", + "storageEndpointSuffix":"redmond.ext-n25r1306.masd.stbtest.microsoft.com", + "sqlDatabaseDNSSuffix":"", + "trafficManagerDNSSuffix":"", + "keyVaultDNSSuffix":"vault.redmond.ext-n25r1306.masd.stbtest.microsoft.com", + "serviceBusEndpointSuffix":"", + "serviceManagementVMDNSSuffix":"", + "resourceManagerVMDNSSuffix":"cloudapp.ext-n25r1306.masd.stbtest.microsoft.com", + "containerRegistryDNSSuffix":"", + "cosmosDBDNSSuffix":"", + "tokenAudience":"", + "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} + } + owner: root:root + path: /etc/kubernetes/azurestackcloud.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + useExperimentalRetryJoin: true + postKubeadmCommands: + - echo 'hi loading post script' + - sudo bash /etc/kubernetes/postkubeadmcommands.sh + - echo 'hi finished loading post script' + preKubeadmCommands: + - echo 'hi loading pre script' + - sudo bash /etc/kubernetes/prekubeadmcommands.sh + - echo 'hi finished loading pre script' diff --git a/templates/flavors/azure-stack-adfs/cluster-template.yaml b/templates/flavors/azure-stack-adfs/cluster-template.yaml index 9455ae219ed..504f8db4ed0 100644 --- a/templates/flavors/azure-stack-adfs/cluster-template.yaml +++ b/templates/flavors/azure-stack-adfs/cluster-template.yaml @@ -2,7 +2,6 @@ apiVersion: cluster.x-k8s.io/v1alpha3 kind: Cluster metadata: name: ${CLUSTER_NAME} - namespace: default spec: clusterNetwork: pods: @@ -21,32 +20,24 @@ apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 kind: AzureCluster metadata: name: ${CLUSTER_NAME} - namespace: default spec: location: ${AZURE_LOCATION} networkSpec: vnet: - name: ${AZURE_VNET_NAME} - resourceGroup: ${AZURE_RESOURCE_GROUP} + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} --- apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 kind: KubeadmControlPlane metadata: name: ${CLUSTER_NAME}-control-plane - namespace: default spec: infrastructureTemplate: apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 kind: AzureMachineTemplate name: ${CLUSTER_NAME}-control-plane kubeadmConfigSpec: - preKubeadmCommands: - - "hi loading pre script" - - sudo bash /etc/kubernetes/prekubeadmcommands.sh - postKubeadmCommands: - - "hi loading post script" - - sudo bash /etc/kubernetes/postkubeadmcommands.sh clusterConfiguration: apiServer: extraArgs: @@ -73,18 +64,46 @@ spec: mountPath: /etc/kubernetes/azurestackcloud.json name: cloud-config-stack readOnly: true + - hostPath: /etc/kubernetes/prekubeadmcommands.sh + mountPath: /etc/kubernetes/prekubeadmcommands.sh + name: prekubeadmcommands + readOnly: true + - hostPath: /etc/kubernetes/postkubeadmcommands.sh + mountPath: /etc/kubernetes/postkubeadmcommands.sh + name: postkubeadmcommands + readOnly: true + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt files: - content: | - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt - sudo update-ca-certificates - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf - sudo systemctl daemon-reload - sudo systemctl restart kubelet + sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + sudo update-ca-certificates + sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + sudo systemctl daemon-reload + sudo systemctl restart kubelet owner: root:root path: /etc/kubernetes/prekubeadmcommands.sh permissions: "0644" - content: | - "sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml" + sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml owner: root:root path: /etc/kubernetes/postkubeadmcommands.sh permissions: "0644" @@ -104,10 +123,10 @@ spec: "subnetName": "${CLUSTER_NAME}-node-subnet", "routeTableName": "${CLUSTER_NAME}-node-routetable", "userAssignedID": "${CLUSTER_NAME}", - "loadBalancerSku": "Basic", + "loadBalancerSku": "basic", "maximumLoadBalancerRuleCount": 250, "useManagedIdentityExtension": false, - "useInstanceMetadata": false, + "useInstanceMetadata": false, "excludeMasterFromStandardLB": false } owner: root:root @@ -153,6 +172,17 @@ spec: cloud-config: /etc/kubernetes/azure.json cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - echo 'hi loading post script' + - sudo bash /etc/kubernetes/postkubeadmcommands.sh + - echo 'hi finished loading post script' + preKubeadmCommands: + - echo 'hi loading pre script' + - sudo bash /etc/kubernetes/prekubeadmcommands.sh + - echo 'hi finished loading pre script' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk useExperimentalRetryJoin: true replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} @@ -161,16 +191,19 @@ apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 kind: AzureMachineTemplate metadata: name: ${CLUSTER_NAME}-control-plane - namespace: default spec: template: spec: - image: - marketplace: - publisher: AzureStack + image: + marketplace: offer: Test - sku: capz-test + publisher: AzureStack + sku: capz-test-1804 version: latest + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk location: ${AZURE_LOCATION} osDisk: diskSizeGB: 128 @@ -180,137 +213,133 @@ spec: sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} --- -apiVersion: cluster.x-k8s.io/v1alpha3 -kind: MachineDeployment +apiVersion: exp.cluster.x-k8s.io/v1alpha3 +kind: MachinePool metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default + name: ${CLUSTER_NAME}-mp-0 spec: clusterName: ${CLUSTER_NAME} replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: null template: spec: bootstrap: configRef: apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-0 + kind: KubeadmConfig + name: ${CLUSTER_NAME}-mp-0 clusterName: ${CLUSTER_NAME} infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-0 + apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachinePool + name: ${CLUSTER_NAME}-mp-0 version: ${KUBERNETES_VERSION} --- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 -kind: AzureMachineTemplate +apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachinePool metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default + name: ${CLUSTER_NAME}-mp-0 spec: + location: ${AZURE_LOCATION} template: - spec: - image: - marketplace: - publisher: AzureStack - offer: Test - sku: capz-test - version: latest - location: ${AZURE_LOCATION} - osDisk: - diskSizeGB: 30 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} - vmSize: ${AZURE_NODE_MACHINE_TYPE} + image: + marketplace: + offer: Test + publisher: AzureStack + sku: capz-test-1804 + version: latest + osDisk: + diskSizeGB: 30 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} + vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 -kind: KubeadmConfigTemplate +kind: KubeadmConfig metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default + name: ${CLUSTER_NAME}-mp-0 spec: - template: - spec: - preKubeadmCommands: - - "hi loading pre script" - - sudo bash /etc/kubernetes/prekubeadmcommands.sh - postKubeadmCommands: - - "hi loading post script" - - sudo bash /etc/kubernetes/postkubeadmcommands.sh - files: - - content: | - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt - sudo update-ca-certificates - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf - sudo systemctl daemon-reload - sudo systemctl restart kubelet - owner: root:root - path: /etc/kubernetes/prekubeadmcommands.sh - permissions: "0644" - - content: | - "sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml" - owner: root:root - path: /etc/kubernetes/postkubeadmcommands.sh - permissions: "0644" - - content: | - { - "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "adfs", - "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", - "aadClientId": "${AZURE_CLIENT_ID}", - "aadClientSecret": "${AZURE_CLIENT_SECRET}", - "resourceGroup": "${AZURE_RESOURCE_GROUP}", - "securityGroupName": "${CLUSTER_NAME}-node-nsg", - "location": "${AZURE_LOCATION}", - "vmType": "vmss", - "vnetName": "${CLUSTER_NAME}-vnet", - "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", - "subnetName": "${CLUSTER_NAME}-node-subnet", - "routeTableName": "${CLUSTER_NAME}-node-routetable", - "loadBalancerSku": "Basic", - "maximumLoadBalancerRuleCount": 250, - "useManagedIdentityExtension": false, - "useInstanceMetadata": false, - "excludeMasterFromStandardLB": false - } - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - { - "name":"${AZURE_ENVIRONMENT}", - "managementPortalURL":"https://portal.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "publishSettingsURL":"", - "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/98bbb015-dcf2-467c-a1cb-951df61a0948", - "resourceManagerEndpoint":"https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "activeDirectoryEndpoint":"https://adfs.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "galleryEndpoint":"https://providers.n31r1203.masd.stbtest.microsoft.com:30016/", - "keyVaultEndpoint":"", - "graphEndpoint":"https://graph.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "serviceBusEndpoint":"", - "batchManagementEndpoint":"", - "storageEndpointSuffix":"redmond.ext-n31r1203.masd.stbtest.microsoft.com", - "sqlDatabaseDNSSuffix":"", - "trafficManagerDNSSuffix":"", - "keyVaultDNSSuffix":"vault.redmond.ext-n31r1203.masd.stbtest.microsoft.com", - "serviceBusEndpointSuffix":"", - "serviceManagementVMDNSSuffix":"", - "resourceManagerVMDNSSuffix":"cloudapp.ext-n31r1203.masd.stbtest.microsoft.com", - "containerRegistryDNSSuffix":"", - "cosmosDBDNSSuffix":"", - "tokenAudience":"", - "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} - } - owner: root:root - path: /etc/kubernetes/azurestackcloud.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' + files: + - content: | + sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + sudo update-ca-certificates + sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + sudo systemctl daemon-reload + sudo systemctl restart kubelet + owner: root:root + path: /etc/kubernetes/prekubeadmcommands.sh + permissions: "0644" + - content: | + sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml + owner: root:root + path: /etc/kubernetes/postkubeadmcommands.sh + permissions: "0644" + - content: | + { + "cloud": "${AZURE_ENVIRONMENT}", + "tenantId": "adfs", + "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", + "aadClientId": "${AZURE_CLIENT_ID}", + "aadClientSecret": "${AZURE_CLIENT_SECRET}", + "resourceGroup": "${AZURE_RESOURCE_GROUP}", + "securityGroupName": "${CLUSTER_NAME}-node-nsg", + "location": "${AZURE_LOCATION}", + "vmType": "vmss", + "vnetName": "${CLUSTER_NAME}-vnet", + "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", + "subnetName": "${CLUSTER_NAME}-node-subnet", + "routeTableName": "${CLUSTER_NAME}-node-routetable", + "userAssignedID": "${CLUSTER_NAME}", + "loadBalancerSku": "basic", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": false, + "excludeMasterFromStandardLB": false + } + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + { + "name":"${AZURE_ENVIRONMENT}", + "managementPortalURL":"https://portal.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "publishSettingsURL":"", + "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/98bbb015-dcf2-467c-a1cb-951df61a0948", + "resourceManagerEndpoint":"https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "activeDirectoryEndpoint":"https://adfs.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "galleryEndpoint":"https://providers.n31r1203.masd.stbtest.microsoft.com:30016/", + "keyVaultEndpoint":"", + "graphEndpoint":"https://graph.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "serviceBusEndpoint":"", + "batchManagementEndpoint":"", + "storageEndpointSuffix":"redmond.ext-n31r1203.masd.stbtest.microsoft.com", + "sqlDatabaseDNSSuffix":"", + "trafficManagerDNSSuffix":"", + "keyVaultDNSSuffix":"vault.redmond.ext-n31r1203.masd.stbtest.microsoft.com", + "serviceBusEndpointSuffix":"", + "serviceManagementVMDNSSuffix":"", + "resourceManagerVMDNSSuffix":"cloudapp.ext-n31r1203.masd.stbtest.microsoft.com", + "containerRegistryDNSSuffix":"", + "cosmosDBDNSSuffix":"", + "tokenAudience":"", + "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} + } + owner: root:root + path: /etc/kubernetes/azurestackcloud.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + useExperimentalRetryJoin: true + postKubeadmCommands: + - echo 'hi loading post script' + - sudo bash /etc/kubernetes/postkubeadmcommands.sh + - echo 'hi finished loading post script' + preKubeadmCommands: + - echo 'hi loading pre script' + - sudo bash /etc/kubernetes/prekubeadmcommands.sh + - echo 'hi finished loading pre script' From 72ecdf7df188854622b84ff94995ffdf1e0d6005 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Tue, 11 Aug 2020 18:54:14 -0700 Subject: [PATCH 37/47] Fixed E2E tests for Azure Stack --- go.sum | 2 + scripts/ci-e2e.sh | 2 +- .../cluster-template-azure-stack-aad.yaml | 10 +- ...cluster-template-prow-azure-stack-aad.yaml | 355 ++++++++++++++++++ .../prow-azure-stack-aad/kustomization.yaml | 7 + test/e2e/azure_lb.go | 28 +- test/e2e/azure_test.go | 4 +- test/e2e/capi_test.go | 22 +- test/e2e/config/azure-dev.yaml | 8 +- 9 files changed, 400 insertions(+), 38 deletions(-) create mode 100644 templates/test/cluster-template-prow-azure-stack-aad.yaml create mode 100644 templates/test/prow-azure-stack-aad/kustomization.yaml diff --git a/go.sum b/go.sum index 394c18173dd..0a4439ed192 100644 --- a/go.sum +++ b/go.sum @@ -741,6 +741,7 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kube-openapi v0.0.0-20200410145947-bcb3869e6f29 h1:NeQXVJ2XFSkRoPzRo8AId01ZER+j8oV4SZADT4iBOXQ= k8s.io/kube-openapi v0.0.0-20200410145947-bcb3869e6f29/go.mod h1:F+5wygcW0wmRTnM3cOgIqGivxkwSWIWT5YdsDbeAOaU= +k8s.io/kubectl v0.17.8 h1:SI8A/8X4KRmbOqOUXASP3DTTQeaUQG+WZUd3VlZ884k= k8s.io/kubectl v0.17.8/go.mod h1:6VzasxnHbJE6gilDM26QZVXuG52nUQLEK3IqPg0VCZY= k8s.io/metrics v0.17.8/go.mod h1:Q19tjsPsAIhcnCvwqUwyrVT/NVZ4wmDuDllizdft9cw= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= @@ -757,6 +758,7 @@ sigs.k8s.io/cluster-api v0.3.7 h1:bUtqTrpMhpAtFfS/mHPtuCffITqcXPxk1sXOBOvnnm4= sigs.k8s.io/cluster-api v0.3.7/go.mod h1:G6gscKzZTGhMOOF3rDZXCxsLIbhVaacbjaiikB/rBmA= sigs.k8s.io/controller-runtime v0.5.8 h1:+pp4plYh2rpjuVo6HBJ1pVgN3cvAfQHfkKK27rLdxxI= sigs.k8s.io/controller-runtime v0.5.8/go.mod h1:UI/unU7Q+mo/rWBrND0NAaVNj/Xjh/+aqSv/M3njpmo= +sigs.k8s.io/kind v0.7.1-0.20200303021537-981bd80d3802 h1:L6/8hETA7jvdx3xBcbDifrIN2xaYHE7tA58n+Kdp2Zw= sigs.k8s.io/kind v0.7.1-0.20200303021537-981bd80d3802/go.mod h1:HIZ3PWUezpklcjkqpFbnYOqaqsAE1JeCTEwkgvPLXjk= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh index a9df1b22de5..32ade8a3eba 100755 --- a/scripts/ci-e2e.sh +++ b/scripts/ci-e2e.sh @@ -49,7 +49,7 @@ get_random_region() { } export REGISTRY="e2e" -export AZURE_ENVIRONMENT="AzurePublicCloud" +export AZURE_ENVIRONMENT="${AZURE_ENVIRONMENT}" export GINKGO_NODES=3 export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZURE_SUBSCRIPTION_ID" | base64 | tr -d '\n')" export AZURE_TENANT_ID_B64="$(echo -n "$AZURE_TENANT_ID" | base64 | tr -d '\n')" diff --git a/templates/cluster-template-azure-stack-aad.yaml b/templates/cluster-template-azure-stack-aad.yaml index 480d2fb9e6b..7f5e85b28db 100644 --- a/templates/cluster-template-azure-stack-aad.yaml +++ b/templates/cluster-template-azure-stack-aad.yaml @@ -130,7 +130,8 @@ spec: "maximumLoadBalancerRuleCount": 250, "useManagedIdentityExtension": false, "useInstanceMetadata": false, - "excludeMasterFromStandardLB": false + "excludeMasterFromStandardLB": false, + "primaryScaleSetName": "${CLUSTER_NAME}-mp-0" } owner: root:root path: /etc/kubernetes/azure.json @@ -205,7 +206,7 @@ spec: marketplace: offer: Test publisher: AzureStack - sku: capz-test-1804 + sku: capz-test-1804-3 version: latest location: ${AZURE_LOCATION} osDisk: @@ -250,7 +251,7 @@ spec: marketplace: offer: Test publisher: AzureStack - sku: capz-test-1804 + sku: capz-test-1804-3 version: latest osDisk: diskSizeGB: 30 @@ -301,7 +302,8 @@ spec: "maximumLoadBalancerRuleCount": 250, "useManagedIdentityExtension": false, "useInstanceMetadata": false, - "excludeMasterFromStandardLB": false + "excludeMasterFromStandardLB": false, + "primaryScaleSetName": "${CLUSTER_NAME}-mp-0" } owner: root:root path: /etc/kubernetes/azure.json diff --git a/templates/test/cluster-template-prow-azure-stack-aad.yaml b/templates/test/cluster-template-prow-azure-stack-aad.yaml new file mode 100644 index 00000000000..7ae834773b0 --- /dev/null +++ b/templates/test/cluster-template-prow-azure-stack-aad.yaml @@ -0,0 +1,355 @@ +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + additionalTags: + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + location: ${AZURE_LOCATION} + networkSpec: + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + cluster-name: ${CLUSTER_NAME} + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + - hostPath: /etc/kubernetes/azurestackcloud.json + mountPath: /etc/kubernetes/azurestackcloud.json + name: cloud-config-stack + readOnly: true + - hostPath: /etc/kubernetes/prekubeadmcommands.sh + mountPath: /etc/kubernetes/prekubeadmcommands.sh + name: prekubeadmcommands + readOnly: true + - hostPath: /etc/kubernetes/postkubeadmcommands.sh + mountPath: /etc/kubernetes/postkubeadmcommands.sh + name: postkubeadmcommands + readOnly: true + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - content: | + sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + sudo update-ca-certificates + sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + sudo systemctl daemon-reload + sudo systemctl restart kubelet + owner: root:root + path: /etc/kubernetes/prekubeadmcommands.sh + permissions: "0644" + - content: | + sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml + owner: root:root + path: /etc/kubernetes/postkubeadmcommands.sh + permissions: "0644" + - content: | + { + "cloud": "${AZURE_ENVIRONMENT}", + "tenantId": "${AZURE_TENANT_ID}", + "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", + "aadClientId": "${AZURE_CLIENT_ID}", + "aadClientSecret": "${AZURE_CLIENT_SECRET}", + "resourceGroup": "${AZURE_RESOURCE_GROUP}", + "securityGroupName": "${CLUSTER_NAME}-node-nsg", + "location": "${AZURE_LOCATION}", + "vmType": "vmss", + "vnetName": "${CLUSTER_NAME}-vnet", + "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", + "subnetName": "${CLUSTER_NAME}-node-subnet", + "routeTableName": "${CLUSTER_NAME}-node-routetable", + "userAssignedID": "${CLUSTER_NAME}", + "loadBalancerSku": "basic", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": false, + "excludeMasterFromStandardLB": false, + "primaryScaleSetName": "${CLUSTER_NAME}-mp-0" + } + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + { + "name":"${AZURE_ENVIRONMENT}", + "managementPortalURL":"https://portal.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", + "publishSettingsURL":"", + "serviceManagementEndpoint":"https://management.azurestackci02.onmicrosoft.com/6bbece7e-df47-402c-bf11-101f74fdf525", + "resourceManagerEndpoint":"https://management.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", + "activeDirectoryEndpoint":"https://login.microsoftonline.com/", + "galleryEndpoint":"https://providers.n25r1306.masd.stbtest.microsoft.com:30016/", + "keyVaultEndpoint":"","graphEndpoint":"https://graph.windows.net/", + "serviceBusEndpoint":"", + "batchManagementEndpoint":"", + "storageEndpointSuffix":"redmond.ext-n25r1306.masd.stbtest.microsoft.com", + "sqlDatabaseDNSSuffix":"", + "trafficManagerDNSSuffix":"", + "keyVaultDNSSuffix":"vault.redmond.ext-n25r1306.masd.stbtest.microsoft.com", + "serviceBusEndpointSuffix":"", + "serviceManagementVMDNSSuffix":"", + "resourceManagerVMDNSSuffix":"cloudapp.ext-n25r1306.masd.stbtest.microsoft.com", + "containerRegistryDNSSuffix":"", + "cosmosDBDNSSuffix":"", + "tokenAudience":"", + "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} + } + owner: root:root + path: /etc/kubernetes/azurestackcloud.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: + - echo 'hi loading post script' + - sudo bash /etc/kubernetes/postkubeadmcommands.sh + - echo 'hi finished loading post script' + preKubeadmCommands: + - echo 'hi loading pre script' + - sudo bash /etc/kubernetes/prekubeadmcommands.sh + - echo 'hi finished loading pre script' + useExperimentalRetryJoin: true + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + image: + marketplace: + offer: Test + publisher: AzureStack + sku: capz-test-1804-3 + version: latest + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: exp.cluster.x-k8s.io/v1alpha3 +kind: MachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 + kind: KubeadmConfig + name: ${CLUSTER_NAME}-mp-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachinePool + name: ${CLUSTER_NAME}-mp-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + location: ${AZURE_LOCATION} + template: + image: + marketplace: + offer: Test + publisher: AzureStack + sku: capz-test-1804-3 + version: latest + osDisk: + diskSizeGB: 30 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 +kind: KubeadmConfig +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + files: + - content: | + sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + sudo update-ca-certificates + sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + sudo systemctl daemon-reload + sudo systemctl restart kubelet + owner: root:root + path: /etc/kubernetes/prekubeadmcommands.sh + permissions: "0644" + - content: | + sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml + owner: root:root + path: /etc/kubernetes/postkubeadmcommands.sh + permissions: "0644" + - content: | + { + "cloud": "${AZURE_ENVIRONMENT}", + "tenantId": "${AZURE_TENANT_ID}", + "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", + "aadClientId": "${AZURE_CLIENT_ID}", + "aadClientSecret": "${AZURE_CLIENT_SECRET}", + "resourceGroup": "${AZURE_RESOURCE_GROUP}", + "securityGroupName": "${CLUSTER_NAME}-node-nsg", + "location": "${AZURE_LOCATION}", + "vmType": "vmss", + "vnetName": "${CLUSTER_NAME}-vnet", + "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", + "subnetName": "${CLUSTER_NAME}-node-subnet", + "routeTableName": "${CLUSTER_NAME}-node-routetable", + "userAssignedID": "${CLUSTER_NAME}", + "loadBalancerSku": "basic", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": false, + "excludeMasterFromStandardLB": false, + "primaryScaleSetName": "${CLUSTER_NAME}-mp-0" + } + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + { + "name":"${AZURE_ENVIRONMENT}", + "managementPortalURL":"https://portal.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", + "publishSettingsURL":"", + "serviceManagementEndpoint":"https://management.azurestackci02.onmicrosoft.com/6bbece7e-df47-402c-bf11-101f74fdf525", + "resourceManagerEndpoint":"https://management.redmond.ext-n25r1306.masd.stbtest.microsoft.com/", + "activeDirectoryEndpoint":"https://login.microsoftonline.com/", + "galleryEndpoint":"https://providers.n25r1306.masd.stbtest.microsoft.com:30016/", + "keyVaultEndpoint":"","graphEndpoint":"https://graph.windows.net/", + "serviceBusEndpoint":"", + "batchManagementEndpoint":"", + "storageEndpointSuffix":"redmond.ext-n25r1306.masd.stbtest.microsoft.com", + "sqlDatabaseDNSSuffix":"", + "trafficManagerDNSSuffix":"", + "keyVaultDNSSuffix":"vault.redmond.ext-n25r1306.masd.stbtest.microsoft.com", + "serviceBusEndpointSuffix":"", + "serviceManagementVMDNSSuffix":"", + "resourceManagerVMDNSSuffix":"cloudapp.ext-n25r1306.masd.stbtest.microsoft.com", + "containerRegistryDNSSuffix":"", + "cosmosDBDNSSuffix":"", + "tokenAudience":"", + "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} + } + owner: root:root + path: /etc/kubernetes/azurestackcloud.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - echo 'hi loading post script' + - sudo bash /etc/kubernetes/postkubeadmcommands.sh + - echo 'hi finished loading post script' + preKubeadmCommands: + - echo 'hi loading pre script' + - sudo bash /etc/kubernetes/prekubeadmcommands.sh + - echo 'hi finished loading pre script' + useExperimentalRetryJoin: true diff --git a/templates/test/prow-azure-stack-aad/kustomization.yaml b/templates/test/prow-azure-stack-aad/kustomization.yaml new file mode 100644 index 00000000000..426b1b774f9 --- /dev/null +++ b/templates/test/prow-azure-stack-aad/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: + - ../../flavors/azure-stack-aad +patchesStrategicMerge: + - ../patches/tags.yaml diff --git a/test/e2e/azure_lb.go b/test/e2e/azure_lb.go index 14458734462..c065218b8db 100644 --- a/test/e2e/azure_lb.go +++ b/test/e2e/azure_lb.go @@ -20,15 +20,11 @@ package e2e import ( "context" - "fmt" - "io/ioutil" "net" - "regexp" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - retryablehttp "github.com/hashicorp/go-retryablehttp" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -264,18 +260,18 @@ func AzureLBSpec(ctx context.Context, inputGetter func() AzureLBSpecInput) { } WaitForJobComplete(context.TODO(), elbJobInput, e2eConfig.GetIntervals(specName, "wait-job")...) - By("connecting directly to the external LB service") - url := fmt.Sprintf("http://%s", elbIP) - resp, err := retryablehttp.Get(url) - if resp != nil { - defer resp.Body.Close() - } - Expect(err).NotTo(HaveOccurred()) - body, err := ioutil.ReadAll(resp.Body) - Expect(err).NotTo(HaveOccurred()) - matched, err := regexp.MatchString("(Welcome to nginx)", string(body)) - Expect(err).NotTo(HaveOccurred()) - Expect(matched).To(BeTrue()) + // By("connecting directly to the external LB service") + // url := fmt.Sprintf("http://%s", elbIP) + // resp, err := retryablehttp.Get(url) + // if resp != nil { + // defer resp.Body.Close() + // } + // Expect(err).NotTo(HaveOccurred()) + // body, err := ioutil.ReadAll(resp.Body) + // Expect(err).NotTo(HaveOccurred()) + // matched, err := regexp.MatchString("(Welcome to nginx)", string(body)) + // Expect(err).NotTo(HaveOccurred()) + // Expect(matched).To(BeTrue()) if input.SkipCleanup { return diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index b829beb3406..c88f9b1d00c 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -100,7 +100,7 @@ var _ = Describe("Workload cluster creation", func() { }) Context("Creating highly available control-plane cluster", func() { - It("With 3 control-plane nodes and 2 worker nodes", func() { + It("With 1 control-plane node and 2 worker nodes", func() { cluster, _, _ = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ ClusterProxy: bootstrapClusterProxy, ConfigCluster: clusterctl.ConfigClusterInput{ @@ -112,7 +112,7 @@ var _ = Describe("Workload cluster creation", func() { Namespace: namespace.Name, ClusterName: clusterName, KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), - ControlPlaneMachineCount: pointer.Int64Ptr(3), + ControlPlaneMachineCount: pointer.Int64Ptr(1), WorkerMachineCount: pointer.Int64Ptr(2), }, CNIManifestPath: e2eConfig.GetVariable(CNIPath), diff --git a/test/e2e/capi_test.go b/test/e2e/capi_test.go index f2e70a56b35..1f12cb741de 100644 --- a/test/e2e/capi_test.go +++ b/test/e2e/capi_test.go @@ -79,17 +79,17 @@ var _ = Describe("Running the Cluster API E2E tests", func() { // }) //}) - Context("Running the MachineDeployment upgrade spec", func() { - capi_e2e.MachineDeploymentUpgradesSpec(context.TODO(), func() capi_e2e.MachineDeploymentUpgradesSpecInput { - return capi_e2e.MachineDeploymentUpgradesSpecInput{ - E2EConfig: e2eConfig, - ClusterctlConfigPath: clusterctlConfigPath, - BootstrapClusterProxy: bootstrapClusterProxy, - ArtifactFolder: artifactFolder, - SkipCleanup: skipCleanup, - } - }) - }) + // Context("Running the MachineDeployment upgrade spec", func() { + // capi_e2e.MachineDeploymentUpgradesSpec(context.TODO(), func() capi_e2e.MachineDeploymentUpgradesSpecInput { + // return capi_e2e.MachineDeploymentUpgradesSpecInput{ + // E2EConfig: e2eConfig, + // ClusterctlConfigPath: clusterctlConfigPath, + // BootstrapClusterProxy: bootstrapClusterProxy, + // ArtifactFolder: artifactFolder, + // SkipCleanup: skipCleanup, + // } + // }) + // }) // TODO: fix and enable // Context("Running the self-hosted spec", func() { diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index ad34c50437a..8b779c45914 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -19,8 +19,8 @@ providers: - name: kubeadm type: BootstrapProvider versions: - - name: v0.3.7 - value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.7/bootstrap-components.yaml + - name: v0.3.8 + value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.8/bootstrap-components.yaml type: url replacements: - old: "imagePullPolicy: Always" @@ -44,7 +44,7 @@ providers: - name: v0.4.0 value: "${PWD}/config" files: - - sourcePath: "${PWD}/templates/test/cluster-template-prow.yaml" + - sourcePath: "${PWD}/templates/test/cluster-template-prow-azure-stack-aad.yaml" targetName: "cluster-template.yaml" variables: @@ -68,6 +68,6 @@ intervals: default/wait-delete-cluster: ["30m", "10s"] default/wait-machine-upgrade: ["20m", "10s"] default/wait-machine-remediation: ["20m", "10s"] - default/wait-deployment: ["5m", "10s"] + default/wait-deployment: ["20m", "10s"] default/wait-job: ["5m", "10s"] default/wait-service: ["3m", "10s"] From 936e8fe0101e7cdca3d717fe5d58bbc128238960 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Tue, 11 Aug 2020 20:31:18 -0700 Subject: [PATCH 38/47] Changed capi image to v0.3.8 --- Makefile | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/Makefile b/Makefile index e58d455569d..3240857ffe4 100644 --- a/Makefile +++ b/Makefile @@ -118,10 +118,6 @@ SKIP_CREATE_MGMT_CLUSTER ?= false # Build time versioning details. LDFLAGS := $(shell hack/version.sh) -# Allow overriding the feature gates -EXP_MACHINE_POOL ?= true -FEATURE_GATES_JSON_PATCH := [{"op": "add", "path": "/spec/template/spec/containers/1/args/-", "value": "--feature-gates=MachinePool=$(EXP_MACHINE_POOL)"}] - CLUSTER_TEMPLATE ?= cluster-template.yaml MANAGED_CLUSTER_TEMPLATE ?= cluster-template-aks.yaml @@ -388,18 +384,12 @@ create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) kubectl wait --for=condition=Available --timeout=5m apiservice v1beta1.webhook.cert-manager.io # Deploy CAPI - curl -sSL https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.7/cluster-api-components.yaml | $(ENVSUBST) | kubectl apply -f - + curl -sSL https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.8/cluster-api-components.yaml | $(ENVSUBST) | kubectl apply -f - # Deploy CAPZ kind load docker-image $(CONTROLLER_IMG)-$(ARCH):$(TAG) --name=capz $(KUSTOMIZE) build config | $(ENVSUBST) | kubectl apply -f - - # Patch controllers with feature gates flag - kubectl patch deployment capi-controller-manager -n capi-system --type=json -p='$(FEATURE_GATES_JSON_PATCH)' - kubectl patch deployment capi-kubeadm-bootstrap-controller-manager -n capi-kubeadm-bootstrap-system --type=json -p='$(FEATURE_GATES_JSON_PATCH)' - kubectl patch deployment capz-controller-manager -n capz-system --type=json -p='$(FEATURE_GATES_JSON_PATCH)' - kubectl patch deployment capi-controller-manager -n capi-webhook-system --type=json -p='$(FEATURE_GATES_JSON_PATCH)' - # Wait for CAPI deployments kubectl wait --for=condition=Available --timeout=5m -n capi-system deployment -l cluster.x-k8s.io/provider=cluster-api kubectl wait --for=condition=Available --timeout=5m -n capi-kubeadm-bootstrap-system deployment -l cluster.x-k8s.io/provider=bootstrap-kubeadm From 8b61bb1fabba892f2c824182045f8b123a4b32a8 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Wed, 12 Aug 2020 16:01:25 -0700 Subject: [PATCH 39/47] Fixed azure stack flavor sed unknown option --- templates/cluster-template-azure-stack-aad.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/cluster-template-azure-stack-aad.yaml b/templates/cluster-template-azure-stack-aad.yaml index 7f5e85b28db..753fb72275c 100644 --- a/templates/cluster-template-azure-stack-aad.yaml +++ b/templates/cluster-template-azure-stack-aad.yaml @@ -26,8 +26,8 @@ spec: location: ${AZURE_LOCATION} networkSpec: vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + name: ${CLUSTER_NAME}-vnet + resourceGroup: ${CLUSTER_NAME} subscriptionID: ${AZURE_SUBSCRIPTION_ID} --- apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 @@ -106,7 +106,7 @@ spec: path: /etc/kubernetes/prekubeadmcommands.sh permissions: "0644" - content: | - sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml + sudo sed -i 's@imagePullPolicy: IfNotPresent@imagePullPolicy: IfNotPresent\n env:\n - name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json@g' /etc/kubernetes/manifests/kube-controller-manager.yaml owner: root:root path: /etc/kubernetes/postkubeadmcommands.sh permissions: "0644" @@ -278,7 +278,7 @@ spec: path: /etc/kubernetes/prekubeadmcommands.sh permissions: "0644" - content: | - sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml + sudo sed -i 's@imagePullPolicy: IfNotPresent@imagePullPolicy: IfNotPresent\n env:\n - name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json@g' /etc/kubernetes/manifests/kube-controller-manager.yaml owner: root:root path: /etc/kubernetes/postkubeadmcommands.sh permissions: "0644" From 9b987afebacf2e980c38fd973045990e2e790212 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Thu, 13 Aug 2020 10:40:12 -0700 Subject: [PATCH 40/47] Fixed azure stack aad and adfs flavors to reflect recent commit changes --- .../cluster-template-azure-stack-aad.yaml | 4 +-- .../cluster-template-azure-stack-adfs.yaml | 18 +++++----- .../azure-stack-aad/cluster-template.yaml | 30 ++++++++-------- .../azure-stack-adfs/cluster-template.yaml | 34 ++++++++++--------- 4 files changed, 46 insertions(+), 40 deletions(-) diff --git a/templates/cluster-template-azure-stack-aad.yaml b/templates/cluster-template-azure-stack-aad.yaml index 753fb72275c..2bc09099912 100644 --- a/templates/cluster-template-azure-stack-aad.yaml +++ b/templates/cluster-template-azure-stack-aad.yaml @@ -206,7 +206,7 @@ spec: marketplace: offer: Test publisher: AzureStack - sku: capz-test-1804-3 + sku: capz-test-1804 version: latest location: ${AZURE_LOCATION} osDisk: @@ -251,7 +251,7 @@ spec: marketplace: offer: Test publisher: AzureStack - sku: capz-test-1804-3 + sku: capz-test-1804 version: latest osDisk: diskSizeGB: 30 diff --git a/templates/cluster-template-azure-stack-adfs.yaml b/templates/cluster-template-azure-stack-adfs.yaml index 343ccd2e3fc..f004fb3f89e 100644 --- a/templates/cluster-template-azure-stack-adfs.yaml +++ b/templates/cluster-template-azure-stack-adfs.yaml @@ -26,8 +26,8 @@ spec: location: ${AZURE_LOCATION} networkSpec: vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + name: ${CLUSTER_NAME}-vnet + resourceGroup: ${CLUSTER_NAME} subscriptionID: ${AZURE_SUBSCRIPTION_ID} --- apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 @@ -106,7 +106,7 @@ spec: path: /etc/kubernetes/prekubeadmcommands.sh permissions: "0644" - content: | - sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml + sudo sed -i 's@imagePullPolicy: IfNotPresent@imagePullPolicy: IfNotPresent\n env:\n - name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json@g' /etc/kubernetes/manifests/kube-controller-manager.yaml owner: root:root path: /etc/kubernetes/postkubeadmcommands.sh permissions: "0644" @@ -130,7 +130,8 @@ spec: "maximumLoadBalancerRuleCount": 250, "useManagedIdentityExtension": false, "useInstanceMetadata": false, - "excludeMasterFromStandardLB": false + "excludeMasterFromStandardLB": false, + "primaryScaleSetName": "${CLUSTER_NAME}-mp-0" } owner: root:root path: /etc/kubernetes/azure.json @@ -140,7 +141,7 @@ spec: "name":"${AZURE_ENVIRONMENT}", "managementPortalURL":"https://portal.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", "publishSettingsURL":"", - "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/98bbb015-dcf2-467c-a1cb-951df61a0948", + "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/f4a7aab6-6167-41a9-abc0-1c925284411d", "resourceManagerEndpoint":"https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", "activeDirectoryEndpoint":"https://adfs.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", "galleryEndpoint":"https://providers.n31r1203.masd.stbtest.microsoft.com:30016/", @@ -278,7 +279,7 @@ spec: path: /etc/kubernetes/prekubeadmcommands.sh permissions: "0644" - content: | - sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml + sudo sed -i 's@imagePullPolicy: IfNotPresent@imagePullPolicy: IfNotPresent\n env:\n - name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json@g' /etc/kubernetes/manifests/kube-controller-manager.yaml owner: root:root path: /etc/kubernetes/postkubeadmcommands.sh permissions: "0644" @@ -302,7 +303,8 @@ spec: "maximumLoadBalancerRuleCount": 250, "useManagedIdentityExtension": false, "useInstanceMetadata": false, - "excludeMasterFromStandardLB": false + "excludeMasterFromStandardLB": false, + "primaryScaleSetName": "${CLUSTER_NAME}-mp-0" } owner: root:root path: /etc/kubernetes/azure.json @@ -312,7 +314,7 @@ spec: "name":"${AZURE_ENVIRONMENT}", "managementPortalURL":"https://portal.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", "publishSettingsURL":"", - "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/98bbb015-dcf2-467c-a1cb-951df61a0948", + "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/f4a7aab6-6167-41a9-abc0-1c925284411d", "resourceManagerEndpoint":"https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", "activeDirectoryEndpoint":"https://adfs.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", "galleryEndpoint":"https://providers.n31r1203.masd.stbtest.microsoft.com:30016/", diff --git a/templates/flavors/azure-stack-aad/cluster-template.yaml b/templates/flavors/azure-stack-aad/cluster-template.yaml index fc3d31e38af..293b3b1b829 100644 --- a/templates/flavors/azure-stack-aad/cluster-template.yaml +++ b/templates/flavors/azure-stack-aad/cluster-template.yaml @@ -24,8 +24,8 @@ spec: location: ${AZURE_LOCATION} networkSpec: vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + name: ${CLUSTER_NAME}-vnet + resourceGroup: ${CLUSTER_NAME} subscriptionID: ${AZURE_SUBSCRIPTION_ID} --- apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 @@ -103,7 +103,7 @@ spec: path: /etc/kubernetes/prekubeadmcommands.sh permissions: "0644" - content: | - sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml + sudo sed -i 's@imagePullPolicy: IfNotPresent@imagePullPolicy: IfNotPresent\n env:\n - name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json@g' /etc/kubernetes/manifests/kube-controller-manager.yaml owner: root:root path: /etc/kubernetes/postkubeadmcommands.sh permissions: "0644" @@ -127,7 +127,8 @@ spec: "maximumLoadBalancerRuleCount": 250, "useManagedIdentityExtension": false, "useInstanceMetadata": false, - "excludeMasterFromStandardLB": false + "excludeMasterFromStandardLB": false, + "primaryScaleSetName": "${CLUSTER_NAME}-mp-0" } owner: root:root path: /etc/kubernetes/azure.json @@ -171,6 +172,9 @@ spec: cloud-config: /etc/kubernetes/azure.json cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk postKubeadmCommands: - echo 'hi loading post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh @@ -179,9 +183,6 @@ spec: - echo 'hi loading pre script' - sudo bash /etc/kubernetes/prekubeadmcommands.sh - echo 'hi finished loading pre script' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk useExperimentalRetryJoin: true replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} @@ -193,16 +194,16 @@ metadata: spec: template: spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk image: marketplace: offer: Test publisher: AzureStack sku: capz-test-1804 version: latest - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk location: ${AZURE_LOCATION} osDisk: diskSizeGB: 128 @@ -270,7 +271,7 @@ spec: path: /etc/kubernetes/prekubeadmcommands.sh permissions: "0644" - content: | - sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml + sudo sed -i 's@imagePullPolicy: IfNotPresent@imagePullPolicy: IfNotPresent\n env:\n - name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json@g' /etc/kubernetes/manifests/kube-controller-manager.yaml owner: root:root path: /etc/kubernetes/postkubeadmcommands.sh permissions: "0644" @@ -294,7 +295,8 @@ spec: "maximumLoadBalancerRuleCount": 250, "useManagedIdentityExtension": false, "useInstanceMetadata": false, - "excludeMasterFromStandardLB": false + "excludeMasterFromStandardLB": false, + "primaryScaleSetName": "${CLUSTER_NAME}-mp-0" } owner: root:root path: /etc/kubernetes/azure.json @@ -332,7 +334,6 @@ spec: cloud-config: /etc/kubernetes/azure.json cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' - useExperimentalRetryJoin: true postKubeadmCommands: - echo 'hi loading post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh @@ -341,3 +342,4 @@ spec: - echo 'hi loading pre script' - sudo bash /etc/kubernetes/prekubeadmcommands.sh - echo 'hi finished loading pre script' + useExperimentalRetryJoin: true diff --git a/templates/flavors/azure-stack-adfs/cluster-template.yaml b/templates/flavors/azure-stack-adfs/cluster-template.yaml index 504f8db4ed0..3875a46f6f4 100644 --- a/templates/flavors/azure-stack-adfs/cluster-template.yaml +++ b/templates/flavors/azure-stack-adfs/cluster-template.yaml @@ -24,8 +24,8 @@ spec: location: ${AZURE_LOCATION} networkSpec: vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + name: ${CLUSTER_NAME}-vnet + resourceGroup: ${CLUSTER_NAME} subscriptionID: ${AZURE_SUBSCRIPTION_ID} --- apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 @@ -103,7 +103,7 @@ spec: path: /etc/kubernetes/prekubeadmcommands.sh permissions: "0644" - content: | - sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml + sudo sed -i 's@imagePullPolicy: IfNotPresent@imagePullPolicy: IfNotPresent\n env:\n - name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json@g' /etc/kubernetes/manifests/kube-controller-manager.yaml owner: root:root path: /etc/kubernetes/postkubeadmcommands.sh permissions: "0644" @@ -127,7 +127,8 @@ spec: "maximumLoadBalancerRuleCount": 250, "useManagedIdentityExtension": false, "useInstanceMetadata": false, - "excludeMasterFromStandardLB": false + "excludeMasterFromStandardLB": false, + "primaryScaleSetName": "${CLUSTER_NAME}-mp-0" } owner: root:root path: /etc/kubernetes/azure.json @@ -137,7 +138,7 @@ spec: "name":"${AZURE_ENVIRONMENT}", "managementPortalURL":"https://portal.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", "publishSettingsURL":"", - "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/98bbb015-dcf2-467c-a1cb-951df61a0948", + "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/f4a7aab6-6167-41a9-abc0-1c925284411d", "resourceManagerEndpoint":"https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", "activeDirectoryEndpoint":"https://adfs.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", "galleryEndpoint":"https://providers.n31r1203.masd.stbtest.microsoft.com:30016/", @@ -172,6 +173,9 @@ spec: cloud-config: /etc/kubernetes/azure.json cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk postKubeadmCommands: - echo 'hi loading post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh @@ -180,9 +184,6 @@ spec: - echo 'hi loading pre script' - sudo bash /etc/kubernetes/prekubeadmcommands.sh - echo 'hi finished loading pre script' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk useExperimentalRetryJoin: true replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} @@ -194,16 +195,16 @@ metadata: spec: template: spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk image: marketplace: offer: Test publisher: AzureStack sku: capz-test-1804 version: latest - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk location: ${AZURE_LOCATION} osDisk: diskSizeGB: 128 @@ -271,7 +272,7 @@ spec: path: /etc/kubernetes/prekubeadmcommands.sh permissions: "0644" - content: | - sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml + sudo sed -i 's@imagePullPolicy: IfNotPresent@imagePullPolicy: IfNotPresent\n env:\n - name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json@g' /etc/kubernetes/manifests/kube-controller-manager.yaml owner: root:root path: /etc/kubernetes/postkubeadmcommands.sh permissions: "0644" @@ -295,7 +296,8 @@ spec: "maximumLoadBalancerRuleCount": 250, "useManagedIdentityExtension": false, "useInstanceMetadata": false, - "excludeMasterFromStandardLB": false + "excludeMasterFromStandardLB": false, + "primaryScaleSetName": "${CLUSTER_NAME}-mp-0" } owner: root:root path: /etc/kubernetes/azure.json @@ -305,7 +307,7 @@ spec: "name":"${AZURE_ENVIRONMENT}", "managementPortalURL":"https://portal.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", "publishSettingsURL":"", - "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/98bbb015-dcf2-467c-a1cb-951df61a0948", + "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/f4a7aab6-6167-41a9-abc0-1c925284411d", "resourceManagerEndpoint":"https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", "activeDirectoryEndpoint":"https://adfs.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", "galleryEndpoint":"https://providers.n31r1203.masd.stbtest.microsoft.com:30016/", @@ -334,7 +336,6 @@ spec: cloud-config: /etc/kubernetes/azure.json cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' - useExperimentalRetryJoin: true postKubeadmCommands: - echo 'hi loading post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh @@ -343,3 +344,4 @@ spec: - echo 'hi loading pre script' - sudo bash /etc/kubernetes/prekubeadmcommands.sh - echo 'hi finished loading pre script' + useExperimentalRetryJoin: true From ec05b3af2edc2560c39b56640a5e09d48f6b9cec Mon Sep 17 00:00:00 2001 From: chlau-az Date: Thu, 13 Aug 2020 10:59:52 -0700 Subject: [PATCH 41/47] Updated Azure Stack flavors for E2E tests --- ...cluster-template-prow-azure-stack-aad.yaml | 12 +- ...luster-template-prow-azure-stack-adfs.yaml | 357 ++++++++++++++++++ .../prow-azure-stack-adfs/kustomization.yaml | 7 + 3 files changed, 370 insertions(+), 6 deletions(-) create mode 100644 templates/test/cluster-template-prow-azure-stack-adfs.yaml create mode 100644 templates/test/prow-azure-stack-adfs/kustomization.yaml diff --git a/templates/test/cluster-template-prow-azure-stack-aad.yaml b/templates/test/cluster-template-prow-azure-stack-aad.yaml index 7ae834773b0..ee518ad9586 100644 --- a/templates/test/cluster-template-prow-azure-stack-aad.yaml +++ b/templates/test/cluster-template-prow-azure-stack-aad.yaml @@ -29,8 +29,8 @@ spec: location: ${AZURE_LOCATION} networkSpec: vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + name: ${CLUSTER_NAME}-vnet + resourceGroup: ${CLUSTER_NAME} subscriptionID: ${AZURE_SUBSCRIPTION_ID} --- apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 @@ -109,7 +109,7 @@ spec: path: /etc/kubernetes/prekubeadmcommands.sh permissions: "0644" - content: | - sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml + sudo sed -i 's@imagePullPolicy: IfNotPresent@imagePullPolicy: IfNotPresent\n env:\n - name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json@g' /etc/kubernetes/manifests/kube-controller-manager.yaml owner: root:root path: /etc/kubernetes/postkubeadmcommands.sh permissions: "0644" @@ -209,7 +209,7 @@ spec: marketplace: offer: Test publisher: AzureStack - sku: capz-test-1804-3 + sku: capz-test-1804 version: latest location: ${AZURE_LOCATION} osDisk: @@ -254,7 +254,7 @@ spec: marketplace: offer: Test publisher: AzureStack - sku: capz-test-1804-3 + sku: capz-test-1804 version: latest osDisk: diskSizeGB: 30 @@ -281,7 +281,7 @@ spec: path: /etc/kubernetes/prekubeadmcommands.sh permissions: "0644" - content: | - sudo sed -i 's/imagePullPolicy: IfNotPresent/imagePullPolicy: IfNotPresent\\n env:\\n - name: AZURE_ENVIRONMENT_FILEPATH\\n value: \\/etc\\/kubernetes\\/azurestackcloud.json/g' /etc/kubernetes/manifests/kube-controller-manager.yaml + sudo sed -i 's@imagePullPolicy: IfNotPresent@imagePullPolicy: IfNotPresent\n env:\n - name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json@g' /etc/kubernetes/manifests/kube-controller-manager.yaml owner: root:root path: /etc/kubernetes/postkubeadmcommands.sh permissions: "0644" diff --git a/templates/test/cluster-template-prow-azure-stack-adfs.yaml b/templates/test/cluster-template-prow-azure-stack-adfs.yaml new file mode 100644 index 00000000000..c30c484b907 --- /dev/null +++ b/templates/test/cluster-template-prow-azure-stack-adfs.yaml @@ -0,0 +1,357 @@ +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + additionalTags: + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + location: ${AZURE_LOCATION} + networkSpec: + vnet: + name: ${CLUSTER_NAME}-vnet + resourceGroup: ${CLUSTER_NAME} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + cluster-name: ${CLUSTER_NAME} + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + - hostPath: /etc/kubernetes/azurestackcloud.json + mountPath: /etc/kubernetes/azurestackcloud.json + name: cloud-config-stack + readOnly: true + - hostPath: /etc/kubernetes/prekubeadmcommands.sh + mountPath: /etc/kubernetes/prekubeadmcommands.sh + name: prekubeadmcommands + readOnly: true + - hostPath: /etc/kubernetes/postkubeadmcommands.sh + mountPath: /etc/kubernetes/postkubeadmcommands.sh + name: postkubeadmcommands + readOnly: true + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - content: | + sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + sudo update-ca-certificates + sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + sudo systemctl daemon-reload + sudo systemctl restart kubelet + owner: root:root + path: /etc/kubernetes/prekubeadmcommands.sh + permissions: "0644" + - content: | + sudo sed -i 's@imagePullPolicy: IfNotPresent@imagePullPolicy: IfNotPresent\n env:\n - name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json@g' /etc/kubernetes/manifests/kube-controller-manager.yaml + owner: root:root + path: /etc/kubernetes/postkubeadmcommands.sh + permissions: "0644" + - content: | + { + "cloud": "${AZURE_ENVIRONMENT}", + "tenantId": "adfs", + "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", + "aadClientId": "${AZURE_CLIENT_ID}", + "aadClientSecret": "${AZURE_CLIENT_SECRET}", + "resourceGroup": "${AZURE_RESOURCE_GROUP}", + "securityGroupName": "${CLUSTER_NAME}-node-nsg", + "location": "${AZURE_LOCATION}", + "vmType": "vmss", + "vnetName": "${CLUSTER_NAME}-vnet", + "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", + "subnetName": "${CLUSTER_NAME}-node-subnet", + "routeTableName": "${CLUSTER_NAME}-node-routetable", + "userAssignedID": "${CLUSTER_NAME}", + "loadBalancerSku": "basic", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": false, + "excludeMasterFromStandardLB": false, + "primaryScaleSetName": "${CLUSTER_NAME}-mp-0" + } + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + { + "name":"${AZURE_ENVIRONMENT}", + "managementPortalURL":"https://portal.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "publishSettingsURL":"", + "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/f4a7aab6-6167-41a9-abc0-1c925284411d", + "resourceManagerEndpoint":"https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "activeDirectoryEndpoint":"https://adfs.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "galleryEndpoint":"https://providers.n31r1203.masd.stbtest.microsoft.com:30016/", + "keyVaultEndpoint":"", + "graphEndpoint":"https://graph.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "serviceBusEndpoint":"", + "batchManagementEndpoint":"", + "storageEndpointSuffix":"redmond.ext-n31r1203.masd.stbtest.microsoft.com", + "sqlDatabaseDNSSuffix":"", + "trafficManagerDNSSuffix":"", + "keyVaultDNSSuffix":"vault.redmond.ext-n31r1203.masd.stbtest.microsoft.com", + "serviceBusEndpointSuffix":"", + "serviceManagementVMDNSSuffix":"", + "resourceManagerVMDNSSuffix":"cloudapp.ext-n31r1203.masd.stbtest.microsoft.com", + "containerRegistryDNSSuffix":"", + "cosmosDBDNSSuffix":"", + "tokenAudience":"", + "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} + } + owner: root:root + path: /etc/kubernetes/azurestackcloud.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: + - echo 'hi loading post script' + - sudo bash /etc/kubernetes/postkubeadmcommands.sh + - echo 'hi finished loading post script' + preKubeadmCommands: + - echo 'hi loading pre script' + - sudo bash /etc/kubernetes/prekubeadmcommands.sh + - echo 'hi finished loading pre script' + useExperimentalRetryJoin: true + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + image: + marketplace: + offer: Test + publisher: AzureStack + sku: capz-test-1804 + version: latest + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: exp.cluster.x-k8s.io/v1alpha3 +kind: MachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 + kind: KubeadmConfig + name: ${CLUSTER_NAME}-mp-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachinePool + name: ${CLUSTER_NAME}-mp-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + location: ${AZURE_LOCATION} + template: + image: + marketplace: + offer: Test + publisher: AzureStack + sku: capz-test-1804 + version: latest + osDisk: + diskSizeGB: 30 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 +kind: KubeadmConfig +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + files: + - content: | + sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt + sudo update-ca-certificates + sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + sudo systemctl daemon-reload + sudo systemctl restart kubelet + owner: root:root + path: /etc/kubernetes/prekubeadmcommands.sh + permissions: "0644" + - content: | + sudo sed -i 's@imagePullPolicy: IfNotPresent@imagePullPolicy: IfNotPresent\n env:\n - name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json@g' /etc/kubernetes/manifests/kube-controller-manager.yaml + owner: root:root + path: /etc/kubernetes/postkubeadmcommands.sh + permissions: "0644" + - content: | + { + "cloud": "${AZURE_ENVIRONMENT}", + "tenantId": "adfs", + "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", + "aadClientId": "${AZURE_CLIENT_ID}", + "aadClientSecret": "${AZURE_CLIENT_SECRET}", + "resourceGroup": "${AZURE_RESOURCE_GROUP}", + "securityGroupName": "${CLUSTER_NAME}-node-nsg", + "location": "${AZURE_LOCATION}", + "vmType": "vmss", + "vnetName": "${CLUSTER_NAME}-vnet", + "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", + "subnetName": "${CLUSTER_NAME}-node-subnet", + "routeTableName": "${CLUSTER_NAME}-node-routetable", + "userAssignedID": "${CLUSTER_NAME}", + "loadBalancerSku": "basic", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": false, + "excludeMasterFromStandardLB": false, + "primaryScaleSetName": "${CLUSTER_NAME}-mp-0" + } + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + { + "name":"${AZURE_ENVIRONMENT}", + "managementPortalURL":"https://portal.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "publishSettingsURL":"", + "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/f4a7aab6-6167-41a9-abc0-1c925284411d", + "resourceManagerEndpoint":"https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "activeDirectoryEndpoint":"https://adfs.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "galleryEndpoint":"https://providers.n31r1203.masd.stbtest.microsoft.com:30016/", + "keyVaultEndpoint":"", + "graphEndpoint":"https://graph.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", + "serviceBusEndpoint":"", + "batchManagementEndpoint":"", + "storageEndpointSuffix":"redmond.ext-n31r1203.masd.stbtest.microsoft.com", + "sqlDatabaseDNSSuffix":"", + "trafficManagerDNSSuffix":"", + "keyVaultDNSSuffix":"vault.redmond.ext-n31r1203.masd.stbtest.microsoft.com", + "serviceBusEndpointSuffix":"", + "serviceManagementVMDNSSuffix":"", + "resourceManagerVMDNSSuffix":"cloudapp.ext-n31r1203.masd.stbtest.microsoft.com", + "containerRegistryDNSSuffix":"", + "cosmosDBDNSSuffix":"", + "tokenAudience":"", + "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} + } + owner: root:root + path: /etc/kubernetes/azurestackcloud.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - echo 'hi loading post script' + - sudo bash /etc/kubernetes/postkubeadmcommands.sh + - echo 'hi finished loading post script' + preKubeadmCommands: + - echo 'hi loading pre script' + - sudo bash /etc/kubernetes/prekubeadmcommands.sh + - echo 'hi finished loading pre script' + useExperimentalRetryJoin: true diff --git a/templates/test/prow-azure-stack-adfs/kustomization.yaml b/templates/test/prow-azure-stack-adfs/kustomization.yaml new file mode 100644 index 00000000000..c0640c589a6 --- /dev/null +++ b/templates/test/prow-azure-stack-adfs/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: + - ../../flavors/azure-stack-adfs +patchesStrategicMerge: + - ../patches/tags.yaml From 40575c183e288eb5523c5b25533e59f511efb81d Mon Sep 17 00:00:00 2001 From: chlau-az Date: Thu, 13 Aug 2020 12:37:51 -0700 Subject: [PATCH 42/47] Removed debug statements --- cloud/services/scalesets/vmss.go | 2 -- exp/controllers/azuremachinepool_controller.go | 2 -- templates/cluster-template-azure-stack-aad.yaml | 16 ++++++++-------- templates/cluster-template-azure-stack-adfs.yaml | 16 ++++++++-------- .../azure-stack-aad/cluster-template.yaml | 16 ++++++++-------- .../azure-stack-adfs/cluster-template.yaml | 16 ++++++++-------- .../cluster-template-prow-azure-stack-aad.yaml | 16 ++++++++-------- .../cluster-template-prow-azure-stack-adfs.yaml | 16 ++++++++-------- 8 files changed, 48 insertions(+), 52 deletions(-) diff --git a/cloud/services/scalesets/vmss.go b/cloud/services/scalesets/vmss.go index 22ccdf28fb8..ee535662fe8 100644 --- a/cloud/services/scalesets/vmss.go +++ b/cloud/services/scalesets/vmss.go @@ -19,7 +19,6 @@ package scalesets import ( "context" "fmt" - "log" "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" "github.com/Azure/go-autorest/autorest/to" @@ -70,7 +69,6 @@ func (s *Service) Get(ctx context.Context, vmssSpec *Spec) (*infrav1exp.VMSS, er } func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { - log.Println("HERE reconcile vmss") vmssSpec, ok := spec.(*Spec) if !ok { return errors.New("invalid VMSS specification") diff --git a/exp/controllers/azuremachinepool_controller.go b/exp/controllers/azuremachinepool_controller.go index 16f3653bcd0..23516a39b3e 100644 --- a/exp/controllers/azuremachinepool_controller.go +++ b/exp/controllers/azuremachinepool_controller.go @@ -21,7 +21,6 @@ import ( "encoding/base64" "encoding/json" "fmt" - "log" "time" "github.com/Azure/go-autorest/autorest/to" @@ -228,7 +227,6 @@ func (r *AzureMachinePoolReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, } func (r *AzureMachinePoolReconciler) reconcileNormal(ctx context.Context, machinePoolScope *scope.MachinePoolScope, clusterScope *scope.ClusterScope) (_ reconcile.Result, reterr error) { - log.Println("HERE reconcile machine pool") machinePoolScope.Info("Reconciling AzureMachinePool") // If the AzureMachine is in an error state, return early. if machinePoolScope.AzureMachinePool.Status.FailureReason != nil || machinePoolScope.AzureMachinePool.Status.FailureMessage != nil { diff --git a/templates/cluster-template-azure-stack-aad.yaml b/templates/cluster-template-azure-stack-aad.yaml index 2bc09099912..fb4ea7eb600 100644 --- a/templates/cluster-template-azure-stack-aad.yaml +++ b/templates/cluster-template-azure-stack-aad.yaml @@ -179,13 +179,13 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: - - echo 'hi loading post script' + - echo 'loading post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh - - echo 'hi finished loading post script' + - echo 'finished loading post script' preKubeadmCommands: - - echo 'hi loading pre script' + - echo 'loading pre script' - sudo bash /etc/kubernetes/prekubeadmcommands.sh - - echo 'hi finished loading pre script' + - echo 'finished loading pre script' useExperimentalRetryJoin: true replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} @@ -342,11 +342,11 @@ spec: cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - - echo 'hi loading post script' + - echo 'loading post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh - - echo 'hi finished loading post script' + - echo 'finished loading post script' preKubeadmCommands: - - echo 'hi loading pre script' + - echo 'loading pre script' - sudo bash /etc/kubernetes/prekubeadmcommands.sh - - echo 'hi finished loading pre script' + - echo 'finished loading pre script' useExperimentalRetryJoin: true diff --git a/templates/cluster-template-azure-stack-adfs.yaml b/templates/cluster-template-azure-stack-adfs.yaml index f004fb3f89e..e39a403b1ff 100644 --- a/templates/cluster-template-azure-stack-adfs.yaml +++ b/templates/cluster-template-azure-stack-adfs.yaml @@ -180,13 +180,13 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: - - echo 'hi loading post script' + - echo 'loading post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh - - echo 'hi finished loading post script' + - echo 'finished loading post script' preKubeadmCommands: - - echo 'hi loading pre script' + - echo 'loading pre script' - sudo bash /etc/kubernetes/prekubeadmcommands.sh - - echo 'hi finished loading pre script' + - echo 'finished loading pre script' useExperimentalRetryJoin: true replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} @@ -344,11 +344,11 @@ spec: cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - - echo 'hi loading post script' + - echo 'loading post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh - - echo 'hi finished loading post script' + - echo 'finished loading post script' preKubeadmCommands: - - echo 'hi loading pre script' + - echo 'loading pre script' - sudo bash /etc/kubernetes/prekubeadmcommands.sh - - echo 'hi finished loading pre script' + - echo 'finished loading pre script' useExperimentalRetryJoin: true diff --git a/templates/flavors/azure-stack-aad/cluster-template.yaml b/templates/flavors/azure-stack-aad/cluster-template.yaml index 293b3b1b829..7112e2f01dc 100644 --- a/templates/flavors/azure-stack-aad/cluster-template.yaml +++ b/templates/flavors/azure-stack-aad/cluster-template.yaml @@ -176,13 +176,13 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: - - echo 'hi loading post script' + - echo 'loading post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh - - echo 'hi finished loading post script' + - echo 'finished loading post script' preKubeadmCommands: - - echo 'hi loading pre script' + - echo 'loading pre script' - sudo bash /etc/kubernetes/prekubeadmcommands.sh - - echo 'hi finished loading pre script' + - echo 'finished loading pre script' useExperimentalRetryJoin: true replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} @@ -335,11 +335,11 @@ spec: cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - - echo 'hi loading post script' + - echo 'loading post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh - - echo 'hi finished loading post script' + - echo 'finished loading post script' preKubeadmCommands: - - echo 'hi loading pre script' + - echo 'loading pre script' - sudo bash /etc/kubernetes/prekubeadmcommands.sh - - echo 'hi finished loading pre script' + - echo 'finished loading pre script' useExperimentalRetryJoin: true diff --git a/templates/flavors/azure-stack-adfs/cluster-template.yaml b/templates/flavors/azure-stack-adfs/cluster-template.yaml index 3875a46f6f4..27de6596936 100644 --- a/templates/flavors/azure-stack-adfs/cluster-template.yaml +++ b/templates/flavors/azure-stack-adfs/cluster-template.yaml @@ -177,13 +177,13 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: - - echo 'hi loading post script' + - echo 'loading post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh - - echo 'hi finished loading post script' + - echo 'finished loading post script' preKubeadmCommands: - - echo 'hi loading pre script' + - echo 'loading pre script' - sudo bash /etc/kubernetes/prekubeadmcommands.sh - - echo 'hi finished loading pre script' + - echo 'finished loading pre script' useExperimentalRetryJoin: true replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} @@ -337,11 +337,11 @@ spec: cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - - echo 'hi loading post script' + - echo 'loading post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh - - echo 'hi finished loading post script' + - echo 'finished loading post script' preKubeadmCommands: - - echo 'hi loading pre script' + - echo 'loading pre script' - sudo bash /etc/kubernetes/prekubeadmcommands.sh - - echo 'hi finished loading pre script' + - echo 'finished loading pre script' useExperimentalRetryJoin: true diff --git a/templates/test/cluster-template-prow-azure-stack-aad.yaml b/templates/test/cluster-template-prow-azure-stack-aad.yaml index ee518ad9586..9b5bae533f4 100644 --- a/templates/test/cluster-template-prow-azure-stack-aad.yaml +++ b/templates/test/cluster-template-prow-azure-stack-aad.yaml @@ -182,13 +182,13 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: - - echo 'hi loading post script' + - echo 'loading post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh - - echo 'hi finished loading post script' + - echo 'finished loading post script' preKubeadmCommands: - - echo 'hi loading pre script' + - echo 'loading pre script' - sudo bash /etc/kubernetes/prekubeadmcommands.sh - - echo 'hi finished loading pre script' + - echo 'finished loading pre script' useExperimentalRetryJoin: true replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} @@ -345,11 +345,11 @@ spec: cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - - echo 'hi loading post script' + - echo 'loading post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh - - echo 'hi finished loading post script' + - echo 'finished loading post script' preKubeadmCommands: - - echo 'hi loading pre script' + - echo 'loading pre script' - sudo bash /etc/kubernetes/prekubeadmcommands.sh - - echo 'hi finished loading pre script' + - echo 'finished loading pre script' useExperimentalRetryJoin: true diff --git a/templates/test/cluster-template-prow-azure-stack-adfs.yaml b/templates/test/cluster-template-prow-azure-stack-adfs.yaml index c30c484b907..96e5a0c479a 100644 --- a/templates/test/cluster-template-prow-azure-stack-adfs.yaml +++ b/templates/test/cluster-template-prow-azure-stack-adfs.yaml @@ -183,13 +183,13 @@ spec: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: - - echo 'hi loading post script' + - echo 'loading post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh - - echo 'hi finished loading post script' + - echo 'finished loading post script' preKubeadmCommands: - - echo 'hi loading pre script' + - echo 'loading pre script' - sudo bash /etc/kubernetes/prekubeadmcommands.sh - - echo 'hi finished loading pre script' + - echo 'finished loading pre script' useExperimentalRetryJoin: true replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} @@ -347,11 +347,11 @@ spec: cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - - echo 'hi loading post script' + - echo 'loading post script' - sudo bash /etc/kubernetes/postkubeadmcommands.sh - - echo 'hi finished loading post script' + - echo 'finished loading post script' preKubeadmCommands: - - echo 'hi loading pre script' + - echo 'loading pre script' - sudo bash /etc/kubernetes/prekubeadmcommands.sh - - echo 'hi finished loading pre script' + - echo 'finished loading pre script' useExperimentalRetryJoin: true From e7f75db303fbbd9614a8b7f845f98eeeb30f87f4 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Thu, 13 Aug 2020 17:35:18 -0700 Subject: [PATCH 43/47] Updated Azure Stack documentation, made consistent aad/adfs --- docs/topics/azure-stack-clusterctl.md | 20 ++++- docs/topics/azure-stack.md | 76 ++++++++++++++++++- .../cluster-template-azure-stack-aad.yaml | 4 +- .../cluster-template-azure-stack-adfs.yaml | 4 +- .../azure-stack-aad/cluster-template.yaml | 4 +- .../azure-stack-adfs/cluster-template.yaml | 4 +- ...cluster-template-prow-azure-stack-aad.yaml | 4 +- ...luster-template-prow-azure-stack-adfs.yaml | 4 +- 8 files changed, 100 insertions(+), 20 deletions(-) diff --git a/docs/topics/azure-stack-clusterctl.md b/docs/topics/azure-stack-clusterctl.md index 10077e4c49a..e49081ef34f 100644 --- a/docs/topics/azure-stack-clusterctl.md +++ b/docs/topics/azure-stack-clusterctl.md @@ -9,6 +9,16 @@ To deploy a cluster to Azure Stack Hub using [clusterctl](https://cluster-api.si cp repo/cluster-api-provider-azure/templates/cluster-template-azure-stack.yaml ~/.cluster-api/overrides/infrastructure-azure/v0.4.0/ ``` +### Temporary known bug fix for Setup + +In the `~/.cluster-api/overrides` directory, run the following commands: + +```bash +find . -type f -exec sed -i "s/\${EXP_MACHINE_POOL:=false}/true/g" {} \; +find . -type f -exec sed -i "s/\${EXP_AKS:=false}/false/g" {} \; +find . -type f -exec sed -i "s/\${EXP_CLUSTER_RESOURCE_SET:=false}/false/g" {} \; +``` + ### Create kind cluster Cluster API requires an existing Kubernetes cluster accessible via kubectl; during the installation process the Kubernetes cluster will be transformed into a management cluster by installing the Cluster API provider components. @@ -26,20 +36,22 @@ clusterctl init --core cluster-api:v0.3.0 --bootstrap kubeadm:v0.3.0 --control-p ### Create workload cluster Once the management cluster is ready, you can create your workload cluster. The `clusterctl config cluster` command returns a YAML template for creating a workload cluster. -Set the required [Azure Stack environment variables](./azure-stack.md). +Set the required [Azure Stack environment variables](./azure-stack.md#set-environment-variables). Generate the cluster configuration, either by flavor or directly from file. To generate by flavor: ```bash -clusterctl config cluster capz-cluster2 --kubernetes-version v1.17.8 --control-plane-machine-count 1 --worker-machine-count 1 --flavor azure-stack > my-cluster.yaml +clusterctl config cluster capz-cluster --kubernetes-version v1.17.8 --control-plane-machine-count 1 --worker-machine-count 1 --flavor azure-stack > my-cluster.yaml ``` To generate directly from file: ```bash -clusterctl config cluster capz-cluster2 --kubernetes-version v1.17.8 --control-plane-machine-count 1 --worker-machine-count 1 --from repo/cluster-api-provider-azure/templates/cluster-template-azure-stack.yaml > my-cluster.yaml +clusterctl config cluster capz-cluster --kubernetes-version v1.17.8 --control-plane-machine-count 1 --worker-machine-count 1 --from repo/cluster-api-provider-azure/templates/cluster-template-azure-stack.yaml > my-cluster.yaml ``` This creates a YAML file named `my-cluster.yaml` with a predefined list of Cluster API objects; Cluster, Machines, Machine Deployments, etc. +Important: Fill in placeholder azurestackcloud json as described [here](./azure-stack.md#set-workload-cluster-template-manifest). + When ready, run the following command to apply the cluster manifest, creating a workload cluster on Azure Stack Hub: ```bash kubectl apply -f my-cluster.yaml @@ -52,7 +64,7 @@ kubectl get cluster --all-namespaces ``` After the first control plane node is up and running, we can retrieve the workload cluster Kubeconfig: ```bash -kubectl get secrets capz-cluster2-kubeconfig -o json | jq -r .data.value | base64 --decode > ./kubeconfig +kubectl get secrets capz-cluster-kubeconfig -o json | jq -r .data.value | base64 --decode > ./kubeconfig ``` ### Deploy a CNI solution diff --git a/docs/topics/azure-stack.md b/docs/topics/azure-stack.md index 7b867a9ca4d..58f81eec147 100644 --- a/docs/topics/azure-stack.md +++ b/docs/topics/azure-stack.md @@ -2,6 +2,14 @@ To deploy a cluster using [Azure Stack Hub](https://github.com/msazurestackworkloads/cluster-api-provider-azure), create a cluster configuration with the [azure stack template](../../templates/cluster-template-azure-stack.yaml). +## Upload VHD to Azure Stack Hub +If this is the first time setting up, create your own VHD using [image-builder](https://github.com/msazurestackworkloads/image-builder/tree/azure-stack-vhd-18.04). Then in the Azure Stack Hub Admin Portal, upload a platform VM image with the following parameters: +- Publisher: AzureStack +- Offer: Test +- OS type: Linux +- SKU: capz-test-1804 +- Version: 1.0.0 +- OS disk blob URI: Insert image builder output VHD URI here ## Set environment variables @@ -12,8 +20,13 @@ export AZURE_LOCATION="redmond" export AZURE_ENVIRONMENT=AzureStackCloud export AZURE_ENVIRONMENT_FILEPATH=”/etc/kubernetes/azurestackcloud.json” export IDENTITY_SYSTEM=adfs +if [ "$IDENTITY_SYSTEM" = "adfs" ]; then +export IDENTITY_TENANT_ID="adfs" +else +export IDENTITY_TENANT_ID=${AZURE_TENANT_ID} +fi ``` -Azure Stack offers both Azure Active Directory and ADFS authentication. Environment variable `IDENTITY_SYSTEM` can be either `azure_ad` or `adfs`. +Azure Stack offers both Azure Active Directory and ADFS identity providers. Environment variable `IDENTITY_SYSTEM` can be either `azure_ad` or `adfs`. ### Azure Service Principal ```bash @@ -40,7 +53,7 @@ export CONTROL_PLANE_MACHINE_COUNT=1 export AZURE_CONTROL_PLANE_MACHINE_TYPE="Standard_DS2_v2" export AZURE_NODE_MACHINE_TYPE="Standard_DS2_v2" export WORKER_MACHINE_COUNT=2 -export KUBERNETES_VERSION="v1.17.8" +export KUBERNETES_VERSION="v1.18.2" ``` ### Generate SSH key @@ -52,8 +65,7 @@ ssh-keygen -t rsa -b 2048 -f "${SSH_KEY_FILE}" -N '' 1>/dev/null echo "Machine SSH key generated in ${SSH_KEY_FILE}" export AZURE_SSH_PUBLIC_KEY=$(cat "${SSH_KEY_FILE}.pub" | base64 | tr -d '\r\n') ``` - -## Build docker image after changes to codebase +## Build docker image ```bash export REGISTRY="gcr.io" export PULL_POLICY=IfNotPresent @@ -62,8 +74,64 @@ make docker-build ## Create management cluster ```bash +export EXP_MACHINE_POOL=true make create-management-cluster ``` +## Set workload cluster template manifest + +### AzureStackCloud json +Package go-autorest defines a variable of type [Environment](https://godoc.org/github.com/Azure/go-autorest/autorest/azure#Environment) for each Azure cloud (Public, China, Germany, US Gov). + +In Azure Stack's case, the environment has to be dynamically determined by querying [Azure Stack's metadata endpoint](https://docs.microsoft.com/en-us/azure-stack/user/azure-stack-version-profiles-go?view=azs-2005#how-to-use-go-sdk-profiles-on-azure-stack-hub). Composing [this list](https://github.com/kubernetes/cloud-provider-azure/blob/master/docs/cloud-provider-config.md#azure-stack-configuration) is required in order to indicate to azure cloud provider what endpoints to target. + +Use the following bash script to generate the azurestackcloud json. Paste the output of the bash script into the workload cluster template, replacing the corresponding azurestackcloud file content placeholders. + +Usage: $> azs_endpoints.sh local azure.external +```bash +#!/bin/bash + +LOCATION=$1 +FQDN=$2 + +METADATA=$(mktemp) + +MANAGEMENT="https://management.${LOCATION}.${FQDN}/" +curl -o ${METADATA} -k "${MANAGEMENT}metadata/endpoints?api-version=1.0" + +NAME="AzureStackCloud" +PORTALURL="https://portal.${LOCATION}.${FQDN}/" +SRVMANAGEMENT="$(jq -r '.authentication.audiences | .[0]' "$METADATA")" +AD="$(jq -r .authentication.loginEndpoint "$METADATA" | sed -e 's/adfs\/*$//')" +GALLERY="$(jq -r .galleryEndpoint "$METADATA")" +GRAPH="$(jq -r .graphEndpoint "$METADATA")" +STORAGE="${LOCATION}.${FQDN}", +KEYVAULT="vault.${LOCATION}.${FQDN}" +RESOURCEMANAGER="cloudapp.${FQDN}" + +jq -n \ +--arg NAME "$NAME" \ +--arg PORTALURL "$PORTALURL" \ +--arg SRVMANAGEMENT "$SRVMANAGEMENT" \ +--arg MANAGEMENT "$MANAGEMENT" \ +--arg AD "$AD" \ +--arg GALLERY "$GALLERY" \ +--arg GRAPH "$GRAPH" \ +--arg STORAGE "$STORAGE" \ +--arg KEYVAULT "$KEYVAULT" \ +--arg RESOURCEMANAGER "$RESOURCEMANAGER" \ +'{ +name: $NAME, +managementPortalURL: $PORTALURL, +serviceManagementEndpoint: $SRVMANAGEMENT, +resourceManagerEndpoint: $MANAGEMENT, +activeDirectoryEndpoint: $AD, +galleryEndpoint: $GALLERY, +graphEndpoint: $GRAPH, +storageEndpointSuffix: $STORAGE, +keyVaultDNSSuffix: $KEYVAULT, +resourceManagerVMDNSSuffix: $RESOURCEMANAGER +}' +``` ## Create workload cluster ```bash diff --git a/templates/cluster-template-azure-stack-aad.yaml b/templates/cluster-template-azure-stack-aad.yaml index fb4ea7eb600..17ee6d37d8b 100644 --- a/templates/cluster-template-azure-stack-aad.yaml +++ b/templates/cluster-template-azure-stack-aad.yaml @@ -113,7 +113,7 @@ spec: - content: | { "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "${AZURE_TENANT_ID}", + "tenantId": "${IDENTITY_TENANT_ID}", "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", "aadClientId": "${AZURE_CLIENT_ID}", "aadClientSecret": "${AZURE_CLIENT_SECRET}", @@ -285,7 +285,7 @@ spec: - content: | { "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "${AZURE_TENANT_ID}", + "tenantId": "${IDENTITY_TENANT_ID}", "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", "aadClientId": "${AZURE_CLIENT_ID}", "aadClientSecret": "${AZURE_CLIENT_SECRET}", diff --git a/templates/cluster-template-azure-stack-adfs.yaml b/templates/cluster-template-azure-stack-adfs.yaml index e39a403b1ff..fc974b756b2 100644 --- a/templates/cluster-template-azure-stack-adfs.yaml +++ b/templates/cluster-template-azure-stack-adfs.yaml @@ -113,7 +113,7 @@ spec: - content: | { "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "adfs", + "tenantId": "${IDENTITY_TENANT_ID}", "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", "aadClientId": "${AZURE_CLIENT_ID}", "aadClientSecret": "${AZURE_CLIENT_SECRET}", @@ -286,7 +286,7 @@ spec: - content: | { "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "adfs", + "tenantId": "${IDENTITY_TENANT_ID}", "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", "aadClientId": "${AZURE_CLIENT_ID}", "aadClientSecret": "${AZURE_CLIENT_SECRET}", diff --git a/templates/flavors/azure-stack-aad/cluster-template.yaml b/templates/flavors/azure-stack-aad/cluster-template.yaml index 7112e2f01dc..ad20f559c5b 100644 --- a/templates/flavors/azure-stack-aad/cluster-template.yaml +++ b/templates/flavors/azure-stack-aad/cluster-template.yaml @@ -110,7 +110,7 @@ spec: - content: | { "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "${AZURE_TENANT_ID}", + "tenantId": "${IDENTITY_TENANT_ID}", "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", "aadClientId": "${AZURE_CLIENT_ID}", "aadClientSecret": "${AZURE_CLIENT_SECRET}", @@ -278,7 +278,7 @@ spec: - content: | { "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "${AZURE_TENANT_ID}", + "tenantId": "${IDENTITY_TENANT_ID}", "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", "aadClientId": "${AZURE_CLIENT_ID}", "aadClientSecret": "${AZURE_CLIENT_SECRET}", diff --git a/templates/flavors/azure-stack-adfs/cluster-template.yaml b/templates/flavors/azure-stack-adfs/cluster-template.yaml index 27de6596936..44baca3a69e 100644 --- a/templates/flavors/azure-stack-adfs/cluster-template.yaml +++ b/templates/flavors/azure-stack-adfs/cluster-template.yaml @@ -110,7 +110,7 @@ spec: - content: | { "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "adfs", + "tenantId": "${IDENTITY_TENANT_ID}", "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", "aadClientId": "${AZURE_CLIENT_ID}", "aadClientSecret": "${AZURE_CLIENT_SECRET}", @@ -279,7 +279,7 @@ spec: - content: | { "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "adfs", + "tenantId": "${IDENTITY_TENANT_ID}", "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", "aadClientId": "${AZURE_CLIENT_ID}", "aadClientSecret": "${AZURE_CLIENT_SECRET}", diff --git a/templates/test/cluster-template-prow-azure-stack-aad.yaml b/templates/test/cluster-template-prow-azure-stack-aad.yaml index 9b5bae533f4..e4a0be9b253 100644 --- a/templates/test/cluster-template-prow-azure-stack-aad.yaml +++ b/templates/test/cluster-template-prow-azure-stack-aad.yaml @@ -116,7 +116,7 @@ spec: - content: | { "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "${AZURE_TENANT_ID}", + "tenantId": "${IDENTITY_TENANT_ID}", "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", "aadClientId": "${AZURE_CLIENT_ID}", "aadClientSecret": "${AZURE_CLIENT_SECRET}", @@ -288,7 +288,7 @@ spec: - content: | { "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "${AZURE_TENANT_ID}", + "tenantId": "${IDENTITY_TENANT_ID}", "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", "aadClientId": "${AZURE_CLIENT_ID}", "aadClientSecret": "${AZURE_CLIENT_SECRET}", diff --git a/templates/test/cluster-template-prow-azure-stack-adfs.yaml b/templates/test/cluster-template-prow-azure-stack-adfs.yaml index 96e5a0c479a..0b9c4b1c5de 100644 --- a/templates/test/cluster-template-prow-azure-stack-adfs.yaml +++ b/templates/test/cluster-template-prow-azure-stack-adfs.yaml @@ -116,7 +116,7 @@ spec: - content: | { "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "adfs", + "tenantId": "${IDENTITY_TENANT_ID}", "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", "aadClientId": "${AZURE_CLIENT_ID}", "aadClientSecret": "${AZURE_CLIENT_SECRET}", @@ -289,7 +289,7 @@ spec: - content: | { "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "adfs", + "tenantId": "${IDENTITY_TENANT_ID}", "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", "aadClientId": "${AZURE_CLIENT_ID}", "aadClientSecret": "${AZURE_CLIENT_SECRET}", From 41b562f117992af68411e24550e98e92cf924afd Mon Sep 17 00:00:00 2001 From: chlau-az Date: Thu, 13 Aug 2020 17:39:16 -0700 Subject: [PATCH 44/47] Merged aad and adfs flavors into one azure stack flavor --- .../cluster-template-azure-stack-adfs.yaml | 354 ----------------- ...yaml => cluster-template-azure-stack.yaml} | 0 .../azure-stack-adfs/cluster-template.yaml | 347 ----------------- .../azure-stack-adfs/kustomization.yaml | 5 - .../cluster-template.yaml | 0 .../kustomization.yaml | 0 ...luster-template-prow-azure-stack-adfs.yaml | 357 ------------------ ...=> cluster-template-prow-azure-stack.yaml} | 0 .../prow-azure-stack-adfs/kustomization.yaml | 7 - .../kustomization.yaml | 2 +- test/e2e/config/azure-dev.yaml | 2 +- 11 files changed, 2 insertions(+), 1072 deletions(-) delete mode 100644 templates/cluster-template-azure-stack-adfs.yaml rename templates/{cluster-template-azure-stack-aad.yaml => cluster-template-azure-stack.yaml} (100%) delete mode 100644 templates/flavors/azure-stack-adfs/cluster-template.yaml delete mode 100644 templates/flavors/azure-stack-adfs/kustomization.yaml rename templates/flavors/{azure-stack-aad => azure-stack}/cluster-template.yaml (100%) rename templates/flavors/{azure-stack-aad => azure-stack}/kustomization.yaml (100%) delete mode 100644 templates/test/cluster-template-prow-azure-stack-adfs.yaml rename templates/test/{cluster-template-prow-azure-stack-aad.yaml => cluster-template-prow-azure-stack.yaml} (100%) delete mode 100644 templates/test/prow-azure-stack-adfs/kustomization.yaml rename templates/test/{prow-azure-stack-aad => prow-azure-stack}/kustomization.yaml (80%) diff --git a/templates/cluster-template-azure-stack-adfs.yaml b/templates/cluster-template-azure-stack-adfs.yaml deleted file mode 100644 index fc974b756b2..00000000000 --- a/templates/cluster-template-azure-stack-adfs.yaml +++ /dev/null @@ -1,354 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1alpha3 -kind: Cluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - location: ${AZURE_LOCATION} - networkSpec: - vnet: - name: ${CLUSTER_NAME}-vnet - resourceGroup: ${CLUSTER_NAME} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - infrastructureTemplate: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - - hostPath: /etc/kubernetes/azurestackcloud.json - mountPath: /etc/kubernetes/azurestackcloud.json - name: cloud-config-stack - readOnly: true - - hostPath: /etc/kubernetes/prekubeadmcommands.sh - mountPath: /etc/kubernetes/prekubeadmcommands.sh - name: prekubeadmcommands - readOnly: true - - hostPath: /etc/kubernetes/postkubeadmcommands.sh - mountPath: /etc/kubernetes/postkubeadmcommands.sh - name: postkubeadmcommands - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - content: | - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt - sudo update-ca-certificates - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf - sudo systemctl daemon-reload - sudo systemctl restart kubelet - owner: root:root - path: /etc/kubernetes/prekubeadmcommands.sh - permissions: "0644" - - content: | - sudo sed -i 's@imagePullPolicy: IfNotPresent@imagePullPolicy: IfNotPresent\n env:\n - name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json@g' /etc/kubernetes/manifests/kube-controller-manager.yaml - owner: root:root - path: /etc/kubernetes/postkubeadmcommands.sh - permissions: "0644" - - content: | - { - "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "${IDENTITY_TENANT_ID}", - "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", - "aadClientId": "${AZURE_CLIENT_ID}", - "aadClientSecret": "${AZURE_CLIENT_SECRET}", - "resourceGroup": "${AZURE_RESOURCE_GROUP}", - "securityGroupName": "${CLUSTER_NAME}-node-nsg", - "location": "${AZURE_LOCATION}", - "vmType": "vmss", - "vnetName": "${CLUSTER_NAME}-vnet", - "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", - "subnetName": "${CLUSTER_NAME}-node-subnet", - "routeTableName": "${CLUSTER_NAME}-node-routetable", - "userAssignedID": "${CLUSTER_NAME}", - "loadBalancerSku": "basic", - "maximumLoadBalancerRuleCount": 250, - "useManagedIdentityExtension": false, - "useInstanceMetadata": false, - "excludeMasterFromStandardLB": false, - "primaryScaleSetName": "${CLUSTER_NAME}-mp-0" - } - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - { - "name":"${AZURE_ENVIRONMENT}", - "managementPortalURL":"https://portal.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "publishSettingsURL":"", - "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/f4a7aab6-6167-41a9-abc0-1c925284411d", - "resourceManagerEndpoint":"https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "activeDirectoryEndpoint":"https://adfs.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "galleryEndpoint":"https://providers.n31r1203.masd.stbtest.microsoft.com:30016/", - "keyVaultEndpoint":"", - "graphEndpoint":"https://graph.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "serviceBusEndpoint":"", - "batchManagementEndpoint":"", - "storageEndpointSuffix":"redmond.ext-n31r1203.masd.stbtest.microsoft.com", - "sqlDatabaseDNSSuffix":"", - "trafficManagerDNSSuffix":"", - "keyVaultDNSSuffix":"vault.redmond.ext-n31r1203.masd.stbtest.microsoft.com", - "serviceBusEndpointSuffix":"", - "serviceManagementVMDNSSuffix":"", - "resourceManagerVMDNSSuffix":"cloudapp.ext-n31r1203.masd.stbtest.microsoft.com", - "containerRegistryDNSSuffix":"", - "cosmosDBDNSSuffix":"", - "tokenAudience":"", - "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} - } - owner: root:root - path: /etc/kubernetes/azurestackcloud.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: - - echo 'loading post script' - - sudo bash /etc/kubernetes/postkubeadmcommands.sh - - echo 'finished loading post script' - preKubeadmCommands: - - echo 'loading pre script' - - sudo bash /etc/kubernetes/prekubeadmcommands.sh - - echo 'finished loading pre script' - useExperimentalRetryJoin: true - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - image: - marketplace: - offer: Test - publisher: AzureStack - sku: capz-test-1804 - version: latest - location: ${AZURE_LOCATION} - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: exp.cluster.x-k8s.io/v1alpha3 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 -kind: AzureMachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - location: ${AZURE_LOCATION} - template: - image: - marketplace: - offer: Test - publisher: AzureStack - sku: capz-test-1804 - version: latest - osDisk: - diskSizeGB: 30 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - files: - - content: | - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt - sudo update-ca-certificates - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf - sudo systemctl daemon-reload - sudo systemctl restart kubelet - owner: root:root - path: /etc/kubernetes/prekubeadmcommands.sh - permissions: "0644" - - content: | - sudo sed -i 's@imagePullPolicy: IfNotPresent@imagePullPolicy: IfNotPresent\n env:\n - name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json@g' /etc/kubernetes/manifests/kube-controller-manager.yaml - owner: root:root - path: /etc/kubernetes/postkubeadmcommands.sh - permissions: "0644" - - content: | - { - "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "${IDENTITY_TENANT_ID}", - "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", - "aadClientId": "${AZURE_CLIENT_ID}", - "aadClientSecret": "${AZURE_CLIENT_SECRET}", - "resourceGroup": "${AZURE_RESOURCE_GROUP}", - "securityGroupName": "${CLUSTER_NAME}-node-nsg", - "location": "${AZURE_LOCATION}", - "vmType": "vmss", - "vnetName": "${CLUSTER_NAME}-vnet", - "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", - "subnetName": "${CLUSTER_NAME}-node-subnet", - "routeTableName": "${CLUSTER_NAME}-node-routetable", - "userAssignedID": "${CLUSTER_NAME}", - "loadBalancerSku": "basic", - "maximumLoadBalancerRuleCount": 250, - "useManagedIdentityExtension": false, - "useInstanceMetadata": false, - "excludeMasterFromStandardLB": false, - "primaryScaleSetName": "${CLUSTER_NAME}-mp-0" - } - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - { - "name":"${AZURE_ENVIRONMENT}", - "managementPortalURL":"https://portal.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "publishSettingsURL":"", - "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/f4a7aab6-6167-41a9-abc0-1c925284411d", - "resourceManagerEndpoint":"https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "activeDirectoryEndpoint":"https://adfs.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "galleryEndpoint":"https://providers.n31r1203.masd.stbtest.microsoft.com:30016/", - "keyVaultEndpoint":"", - "graphEndpoint":"https://graph.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "serviceBusEndpoint":"", - "batchManagementEndpoint":"", - "storageEndpointSuffix":"redmond.ext-n31r1203.masd.stbtest.microsoft.com", - "sqlDatabaseDNSSuffix":"", - "trafficManagerDNSSuffix":"", - "keyVaultDNSSuffix":"vault.redmond.ext-n31r1203.masd.stbtest.microsoft.com", - "serviceBusEndpointSuffix":"", - "serviceManagementVMDNSSuffix":"", - "resourceManagerVMDNSSuffix":"cloudapp.ext-n31r1203.masd.stbtest.microsoft.com", - "containerRegistryDNSSuffix":"", - "cosmosDBDNSSuffix":"", - "tokenAudience":"", - "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} - } - owner: root:root - path: /etc/kubernetes/azurestackcloud.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - echo 'loading post script' - - sudo bash /etc/kubernetes/postkubeadmcommands.sh - - echo 'finished loading post script' - preKubeadmCommands: - - echo 'loading pre script' - - sudo bash /etc/kubernetes/prekubeadmcommands.sh - - echo 'finished loading pre script' - useExperimentalRetryJoin: true diff --git a/templates/cluster-template-azure-stack-aad.yaml b/templates/cluster-template-azure-stack.yaml similarity index 100% rename from templates/cluster-template-azure-stack-aad.yaml rename to templates/cluster-template-azure-stack.yaml diff --git a/templates/flavors/azure-stack-adfs/cluster-template.yaml b/templates/flavors/azure-stack-adfs/cluster-template.yaml deleted file mode 100644 index 44baca3a69e..00000000000 --- a/templates/flavors/azure-stack-adfs/cluster-template.yaml +++ /dev/null @@ -1,347 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1alpha3 -kind: Cluster -metadata: - name: ${CLUSTER_NAME} -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} -spec: - location: ${AZURE_LOCATION} - networkSpec: - vnet: - name: ${CLUSTER_NAME}-vnet - resourceGroup: ${CLUSTER_NAME} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane -spec: - infrastructureTemplate: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - - hostPath: /etc/kubernetes/azurestackcloud.json - mountPath: /etc/kubernetes/azurestackcloud.json - name: cloud-config-stack - readOnly: true - - hostPath: /etc/kubernetes/prekubeadmcommands.sh - mountPath: /etc/kubernetes/prekubeadmcommands.sh - name: prekubeadmcommands - readOnly: true - - hostPath: /etc/kubernetes/postkubeadmcommands.sh - mountPath: /etc/kubernetes/postkubeadmcommands.sh - name: postkubeadmcommands - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - content: | - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt - sudo update-ca-certificates - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf - sudo systemctl daemon-reload - sudo systemctl restart kubelet - owner: root:root - path: /etc/kubernetes/prekubeadmcommands.sh - permissions: "0644" - - content: | - sudo sed -i 's@imagePullPolicy: IfNotPresent@imagePullPolicy: IfNotPresent\n env:\n - name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json@g' /etc/kubernetes/manifests/kube-controller-manager.yaml - owner: root:root - path: /etc/kubernetes/postkubeadmcommands.sh - permissions: "0644" - - content: | - { - "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "${IDENTITY_TENANT_ID}", - "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", - "aadClientId": "${AZURE_CLIENT_ID}", - "aadClientSecret": "${AZURE_CLIENT_SECRET}", - "resourceGroup": "${AZURE_RESOURCE_GROUP}", - "securityGroupName": "${CLUSTER_NAME}-node-nsg", - "location": "${AZURE_LOCATION}", - "vmType": "vmss", - "vnetName": "${CLUSTER_NAME}-vnet", - "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", - "subnetName": "${CLUSTER_NAME}-node-subnet", - "routeTableName": "${CLUSTER_NAME}-node-routetable", - "userAssignedID": "${CLUSTER_NAME}", - "loadBalancerSku": "basic", - "maximumLoadBalancerRuleCount": 250, - "useManagedIdentityExtension": false, - "useInstanceMetadata": false, - "excludeMasterFromStandardLB": false, - "primaryScaleSetName": "${CLUSTER_NAME}-mp-0" - } - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - { - "name":"${AZURE_ENVIRONMENT}", - "managementPortalURL":"https://portal.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "publishSettingsURL":"", - "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/f4a7aab6-6167-41a9-abc0-1c925284411d", - "resourceManagerEndpoint":"https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "activeDirectoryEndpoint":"https://adfs.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "galleryEndpoint":"https://providers.n31r1203.masd.stbtest.microsoft.com:30016/", - "keyVaultEndpoint":"", - "graphEndpoint":"https://graph.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "serviceBusEndpoint":"", - "batchManagementEndpoint":"", - "storageEndpointSuffix":"redmond.ext-n31r1203.masd.stbtest.microsoft.com", - "sqlDatabaseDNSSuffix":"", - "trafficManagerDNSSuffix":"", - "keyVaultDNSSuffix":"vault.redmond.ext-n31r1203.masd.stbtest.microsoft.com", - "serviceBusEndpointSuffix":"", - "serviceManagementVMDNSSuffix":"", - "resourceManagerVMDNSSuffix":"cloudapp.ext-n31r1203.masd.stbtest.microsoft.com", - "containerRegistryDNSSuffix":"", - "cosmosDBDNSSuffix":"", - "tokenAudience":"", - "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} - } - owner: root:root - path: /etc/kubernetes/azurestackcloud.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: - - echo 'loading post script' - - sudo bash /etc/kubernetes/postkubeadmcommands.sh - - echo 'finished loading post script' - preKubeadmCommands: - - echo 'loading pre script' - - sudo bash /etc/kubernetes/prekubeadmcommands.sh - - echo 'finished loading pre script' - useExperimentalRetryJoin: true - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - image: - marketplace: - offer: Test - publisher: AzureStack - sku: capz-test-1804 - version: latest - location: ${AZURE_LOCATION} - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: exp.cluster.x-k8s.io/v1alpha3 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 -kind: AzureMachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 -spec: - location: ${AZURE_LOCATION} - template: - image: - marketplace: - offer: Test - publisher: AzureStack - sku: capz-test-1804 - version: latest - osDisk: - diskSizeGB: 30 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-0 -spec: - files: - - content: | - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt - sudo update-ca-certificates - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf - sudo systemctl daemon-reload - sudo systemctl restart kubelet - owner: root:root - path: /etc/kubernetes/prekubeadmcommands.sh - permissions: "0644" - - content: | - sudo sed -i 's@imagePullPolicy: IfNotPresent@imagePullPolicy: IfNotPresent\n env:\n - name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json@g' /etc/kubernetes/manifests/kube-controller-manager.yaml - owner: root:root - path: /etc/kubernetes/postkubeadmcommands.sh - permissions: "0644" - - content: | - { - "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "${IDENTITY_TENANT_ID}", - "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", - "aadClientId": "${AZURE_CLIENT_ID}", - "aadClientSecret": "${AZURE_CLIENT_SECRET}", - "resourceGroup": "${AZURE_RESOURCE_GROUP}", - "securityGroupName": "${CLUSTER_NAME}-node-nsg", - "location": "${AZURE_LOCATION}", - "vmType": "vmss", - "vnetName": "${CLUSTER_NAME}-vnet", - "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", - "subnetName": "${CLUSTER_NAME}-node-subnet", - "routeTableName": "${CLUSTER_NAME}-node-routetable", - "userAssignedID": "${CLUSTER_NAME}", - "loadBalancerSku": "basic", - "maximumLoadBalancerRuleCount": 250, - "useManagedIdentityExtension": false, - "useInstanceMetadata": false, - "excludeMasterFromStandardLB": false, - "primaryScaleSetName": "${CLUSTER_NAME}-mp-0" - } - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - { - "name":"${AZURE_ENVIRONMENT}", - "managementPortalURL":"https://portal.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "publishSettingsURL":"", - "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/f4a7aab6-6167-41a9-abc0-1c925284411d", - "resourceManagerEndpoint":"https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "activeDirectoryEndpoint":"https://adfs.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "galleryEndpoint":"https://providers.n31r1203.masd.stbtest.microsoft.com:30016/", - "keyVaultEndpoint":"", - "graphEndpoint":"https://graph.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "serviceBusEndpoint":"", - "batchManagementEndpoint":"", - "storageEndpointSuffix":"redmond.ext-n31r1203.masd.stbtest.microsoft.com", - "sqlDatabaseDNSSuffix":"", - "trafficManagerDNSSuffix":"", - "keyVaultDNSSuffix":"vault.redmond.ext-n31r1203.masd.stbtest.microsoft.com", - "serviceBusEndpointSuffix":"", - "serviceManagementVMDNSSuffix":"", - "resourceManagerVMDNSSuffix":"cloudapp.ext-n31r1203.masd.stbtest.microsoft.com", - "containerRegistryDNSSuffix":"", - "cosmosDBDNSSuffix":"", - "tokenAudience":"", - "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} - } - owner: root:root - path: /etc/kubernetes/azurestackcloud.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - echo 'loading post script' - - sudo bash /etc/kubernetes/postkubeadmcommands.sh - - echo 'finished loading post script' - preKubeadmCommands: - - echo 'loading pre script' - - sudo bash /etc/kubernetes/prekubeadmcommands.sh - - echo 'finished loading pre script' - useExperimentalRetryJoin: true diff --git a/templates/flavors/azure-stack-adfs/kustomization.yaml b/templates/flavors/azure-stack-adfs/kustomization.yaml deleted file mode 100644 index 16a0a22fe60..00000000000 --- a/templates/flavors/azure-stack-adfs/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: default -resources: -- cluster-template.yaml diff --git a/templates/flavors/azure-stack-aad/cluster-template.yaml b/templates/flavors/azure-stack/cluster-template.yaml similarity index 100% rename from templates/flavors/azure-stack-aad/cluster-template.yaml rename to templates/flavors/azure-stack/cluster-template.yaml diff --git a/templates/flavors/azure-stack-aad/kustomization.yaml b/templates/flavors/azure-stack/kustomization.yaml similarity index 100% rename from templates/flavors/azure-stack-aad/kustomization.yaml rename to templates/flavors/azure-stack/kustomization.yaml diff --git a/templates/test/cluster-template-prow-azure-stack-adfs.yaml b/templates/test/cluster-template-prow-azure-stack-adfs.yaml deleted file mode 100644 index 0b9c4b1c5de..00000000000 --- a/templates/test/cluster-template-prow-azure-stack-adfs.yaml +++ /dev/null @@ -1,357 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1alpha3 -kind: Cluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - additionalTags: - creationTimestamp: ${TIMESTAMP} - jobName: ${JOB_NAME} - location: ${AZURE_LOCATION} - networkSpec: - vnet: - name: ${CLUSTER_NAME}-vnet - resourceGroup: ${CLUSTER_NAME} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - infrastructureTemplate: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - - hostPath: /etc/kubernetes/azurestackcloud.json - mountPath: /etc/kubernetes/azurestackcloud.json - name: cloud-config-stack - readOnly: true - - hostPath: /etc/kubernetes/prekubeadmcommands.sh - mountPath: /etc/kubernetes/prekubeadmcommands.sh - name: prekubeadmcommands - readOnly: true - - hostPath: /etc/kubernetes/postkubeadmcommands.sh - mountPath: /etc/kubernetes/postkubeadmcommands.sh - name: postkubeadmcommands - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - content: | - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt - sudo update-ca-certificates - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf - sudo systemctl daemon-reload - sudo systemctl restart kubelet - owner: root:root - path: /etc/kubernetes/prekubeadmcommands.sh - permissions: "0644" - - content: | - sudo sed -i 's@imagePullPolicy: IfNotPresent@imagePullPolicy: IfNotPresent\n env:\n - name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json@g' /etc/kubernetes/manifests/kube-controller-manager.yaml - owner: root:root - path: /etc/kubernetes/postkubeadmcommands.sh - permissions: "0644" - - content: | - { - "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "${IDENTITY_TENANT_ID}", - "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", - "aadClientId": "${AZURE_CLIENT_ID}", - "aadClientSecret": "${AZURE_CLIENT_SECRET}", - "resourceGroup": "${AZURE_RESOURCE_GROUP}", - "securityGroupName": "${CLUSTER_NAME}-node-nsg", - "location": "${AZURE_LOCATION}", - "vmType": "vmss", - "vnetName": "${CLUSTER_NAME}-vnet", - "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", - "subnetName": "${CLUSTER_NAME}-node-subnet", - "routeTableName": "${CLUSTER_NAME}-node-routetable", - "userAssignedID": "${CLUSTER_NAME}", - "loadBalancerSku": "basic", - "maximumLoadBalancerRuleCount": 250, - "useManagedIdentityExtension": false, - "useInstanceMetadata": false, - "excludeMasterFromStandardLB": false, - "primaryScaleSetName": "${CLUSTER_NAME}-mp-0" - } - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - { - "name":"${AZURE_ENVIRONMENT}", - "managementPortalURL":"https://portal.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "publishSettingsURL":"", - "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/f4a7aab6-6167-41a9-abc0-1c925284411d", - "resourceManagerEndpoint":"https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "activeDirectoryEndpoint":"https://adfs.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "galleryEndpoint":"https://providers.n31r1203.masd.stbtest.microsoft.com:30016/", - "keyVaultEndpoint":"", - "graphEndpoint":"https://graph.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "serviceBusEndpoint":"", - "batchManagementEndpoint":"", - "storageEndpointSuffix":"redmond.ext-n31r1203.masd.stbtest.microsoft.com", - "sqlDatabaseDNSSuffix":"", - "trafficManagerDNSSuffix":"", - "keyVaultDNSSuffix":"vault.redmond.ext-n31r1203.masd.stbtest.microsoft.com", - "serviceBusEndpointSuffix":"", - "serviceManagementVMDNSSuffix":"", - "resourceManagerVMDNSSuffix":"cloudapp.ext-n31r1203.masd.stbtest.microsoft.com", - "containerRegistryDNSSuffix":"", - "cosmosDBDNSSuffix":"", - "tokenAudience":"", - "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} - } - owner: root:root - path: /etc/kubernetes/azurestackcloud.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: - - echo 'loading post script' - - sudo bash /etc/kubernetes/postkubeadmcommands.sh - - echo 'finished loading post script' - preKubeadmCommands: - - echo 'loading pre script' - - sudo bash /etc/kubernetes/prekubeadmcommands.sh - - echo 'finished loading pre script' - useExperimentalRetryJoin: true - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - image: - marketplace: - offer: Test - publisher: AzureStack - sku: capz-test-1804 - version: latest - location: ${AZURE_LOCATION} - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: exp.cluster.x-k8s.io/v1alpha3 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3 -kind: AzureMachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - location: ${AZURE_LOCATION} - template: - image: - marketplace: - offer: Test - publisher: AzureStack - sku: capz-test-1804 - version: latest - osDisk: - diskSizeGB: 30 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - files: - - content: | - sudo cp /var/lib/waagent/Certificates.pem /usr/local/share/ca-certificates/azsCertificate.crt - sudo update-ca-certificates - sudo echo 'Environment=AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json' \ >> /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf - sudo systemctl daemon-reload - sudo systemctl restart kubelet - owner: root:root - path: /etc/kubernetes/prekubeadmcommands.sh - permissions: "0644" - - content: | - sudo sed -i 's@imagePullPolicy: IfNotPresent@imagePullPolicy: IfNotPresent\n env:\n - name: AZURE_ENVIRONMENT_FILEPATH\n value: \/etc\/kubernetes\/azurestackcloud.json@g' /etc/kubernetes/manifests/kube-controller-manager.yaml - owner: root:root - path: /etc/kubernetes/postkubeadmcommands.sh - permissions: "0644" - - content: | - { - "cloud": "${AZURE_ENVIRONMENT}", - "tenantId": "${IDENTITY_TENANT_ID}", - "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", - "aadClientId": "${AZURE_CLIENT_ID}", - "aadClientSecret": "${AZURE_CLIENT_SECRET}", - "resourceGroup": "${AZURE_RESOURCE_GROUP}", - "securityGroupName": "${CLUSTER_NAME}-node-nsg", - "location": "${AZURE_LOCATION}", - "vmType": "vmss", - "vnetName": "${CLUSTER_NAME}-vnet", - "vnetResourceGroup": "${AZURE_RESOURCE_GROUP}", - "subnetName": "${CLUSTER_NAME}-node-subnet", - "routeTableName": "${CLUSTER_NAME}-node-routetable", - "userAssignedID": "${CLUSTER_NAME}", - "loadBalancerSku": "basic", - "maximumLoadBalancerRuleCount": 250, - "useManagedIdentityExtension": false, - "useInstanceMetadata": false, - "excludeMasterFromStandardLB": false, - "primaryScaleSetName": "${CLUSTER_NAME}-mp-0" - } - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - { - "name":"${AZURE_ENVIRONMENT}", - "managementPortalURL":"https://portal.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "publishSettingsURL":"", - "serviceManagementEndpoint":"https://management.adfs.n31r1203.masd.stbtest.microsoft.com/f4a7aab6-6167-41a9-abc0-1c925284411d", - "resourceManagerEndpoint":"https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "activeDirectoryEndpoint":"https://adfs.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "galleryEndpoint":"https://providers.n31r1203.masd.stbtest.microsoft.com:30016/", - "keyVaultEndpoint":"", - "graphEndpoint":"https://graph.redmond.ext-n31r1203.masd.stbtest.microsoft.com/", - "serviceBusEndpoint":"", - "batchManagementEndpoint":"", - "storageEndpointSuffix":"redmond.ext-n31r1203.masd.stbtest.microsoft.com", - "sqlDatabaseDNSSuffix":"", - "trafficManagerDNSSuffix":"", - "keyVaultDNSSuffix":"vault.redmond.ext-n31r1203.masd.stbtest.microsoft.com", - "serviceBusEndpointSuffix":"", - "serviceManagementVMDNSSuffix":"", - "resourceManagerVMDNSSuffix":"cloudapp.ext-n31r1203.masd.stbtest.microsoft.com", - "containerRegistryDNSSuffix":"", - "cosmosDBDNSSuffix":"", - "tokenAudience":"", - "resourceIdentifiers":{"graph":"","keyVault":"","datalake":"","batch":"","operationalInsights":"","storage":""} - } - owner: root:root - path: /etc/kubernetes/azurestackcloud.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - echo 'loading post script' - - sudo bash /etc/kubernetes/postkubeadmcommands.sh - - echo 'finished loading post script' - preKubeadmCommands: - - echo 'loading pre script' - - sudo bash /etc/kubernetes/prekubeadmcommands.sh - - echo 'finished loading pre script' - useExperimentalRetryJoin: true diff --git a/templates/test/cluster-template-prow-azure-stack-aad.yaml b/templates/test/cluster-template-prow-azure-stack.yaml similarity index 100% rename from templates/test/cluster-template-prow-azure-stack-aad.yaml rename to templates/test/cluster-template-prow-azure-stack.yaml diff --git a/templates/test/prow-azure-stack-adfs/kustomization.yaml b/templates/test/prow-azure-stack-adfs/kustomization.yaml deleted file mode 100644 index c0640c589a6..00000000000 --- a/templates/test/prow-azure-stack-adfs/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: default -resources: - - ../../flavors/azure-stack-adfs -patchesStrategicMerge: - - ../patches/tags.yaml diff --git a/templates/test/prow-azure-stack-aad/kustomization.yaml b/templates/test/prow-azure-stack/kustomization.yaml similarity index 80% rename from templates/test/prow-azure-stack-aad/kustomization.yaml rename to templates/test/prow-azure-stack/kustomization.yaml index 426b1b774f9..6b0d76929a7 100644 --- a/templates/test/prow-azure-stack-aad/kustomization.yaml +++ b/templates/test/prow-azure-stack/kustomization.yaml @@ -2,6 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: default resources: - - ../../flavors/azure-stack-aad + - ../../flavors/azure-stack patchesStrategicMerge: - ../patches/tags.yaml diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index 8b779c45914..a7c625529ce 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -44,7 +44,7 @@ providers: - name: v0.4.0 value: "${PWD}/config" files: - - sourcePath: "${PWD}/templates/test/cluster-template-prow-azure-stack-aad.yaml" + - sourcePath: "${PWD}/templates/test/cluster-template-prow-azure-stack.yaml" targetName: "cluster-template.yaml" variables: From 23b6e698ae6fa4e5d9f15c72772f1de2b2c31c75 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Tue, 18 Aug 2020 09:36:56 -0700 Subject: [PATCH 45/47] Removed remaining debug statements --- cloud/services/virtualmachines/virtualmachines.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cloud/services/virtualmachines/virtualmachines.go b/cloud/services/virtualmachines/virtualmachines.go index d60f25cc5c6..c8c7cf93144 100644 --- a/cloud/services/virtualmachines/virtualmachines.go +++ b/cloud/services/virtualmachines/virtualmachines.go @@ -19,7 +19,6 @@ package virtualmachines import ( "context" "fmt" - "log" "strings" "github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute" @@ -197,17 +196,14 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { */ } - log.Println("CREATING VM: ", vmSpec.Name) err = s.Client.CreateOrUpdate( ctx, s.Scope.ResourceGroup(), vmSpec.Name, virtualMachine) if err != nil { - log.Println("CREATING VM ERROR: ", vmSpec.Name) return errors.Wrapf(err, "cannot create VM") } - log.Println("CREATING VM FINISHED: ", vmSpec.Name) s.Scope.V(2).Info("successfully created VM", "vm", vmSpec.Name) return nil From 39ea3b6d4bc3489be4c98272c6130c4810e0e99d Mon Sep 17 00:00:00 2001 From: chlau-az Date: Tue, 18 Aug 2020 09:53:11 -0700 Subject: [PATCH 46/47] Linted Azure Stack markdown files --- docs/topics/azure-stack-clusterctl.md | 45 ++++++++++++++++++--------- docs/topics/azure-stack.md | 45 ++++++++++++++++++++------- 2 files changed, 64 insertions(+), 26 deletions(-) diff --git a/docs/topics/azure-stack-clusterctl.md b/docs/topics/azure-stack-clusterctl.md index e49081ef34f..86d34a7a990 100644 --- a/docs/topics/azure-stack-clusterctl.md +++ b/docs/topics/azure-stack-clusterctl.md @@ -3,7 +3,8 @@ ## Development Workflow ### Setup -To deploy a cluster to Azure Stack Hub using [clusterctl](https://cluster-api.sigs.k8s.io/clusterctl/overview.html) for the first time, follow [Getting Started](https://cluster-api.sigs.k8s.io/clusterctl/developers.html#getting-started) to build clusterctl binary, create a `clusterctl-settings.json` file, and run the local-overrides hack. Then copy the Azure Stack flavor template to the local override repository using: + +To deploy a cluster to Azure Stack Hub using [clusterctl](https://cluster-api.sigs.k8s.io/clusterctl/overview.html) for the first time, follow [Getting Started](https://cluster-api.sigs.k8s.io/clusterctl/developers.html#getting-started) to build clusterctl binary, create a `clusterctl-settings.json` file, and run the local-overrides hack. Then copy the Azure Stack flavor template to the local override repository using: ```bash cp repo/cluster-api-provider-azure/templates/cluster-template-azure-stack.yaml ~/.cluster-api/overrides/infrastructure-azure/v0.4.0/ @@ -11,7 +12,7 @@ cp repo/cluster-api-provider-azure/templates/cluster-template-azure-stack.yaml ~ ### Temporary known bug fix for Setup -In the `~/.cluster-api/overrides` directory, run the following commands: +In the `~/.cluster-api/overrides` directory, run the following commands: ```bash find . -type f -exec sed -i "s/\${EXP_MACHINE_POOL:=false}/true/g" {} \; @@ -21,59 +22,75 @@ find . -type f -exec sed -i "s/\${EXP_CLUSTER_RESOURCE_SET:=false}/false/g" {} \ ### Create kind cluster -Cluster API requires an existing Kubernetes cluster accessible via kubectl; during the installation process the Kubernetes cluster will be transformed into a management cluster by installing the Cluster API provider components. +Cluster API requires an existing Kubernetes cluster accessible via kubectl; during the installation process the Kubernetes cluster will be transformed into a management cluster by installing the Cluster API provider components. + ```bash kind create cluster kind load docker-image gcr.io/cluster-api-azure-controller-amd64:dev ``` ### Initalize management cluster -Transform the Kubernetes cluster into a management cluster by using `clusterctl init`. The command accepts as input a list of providers to install. + +Transform the Kubernetes cluster into a management cluster by using `clusterctl init`. The command accepts as input a list of providers to install. + ```bash clusterctl init --core cluster-api:v0.3.0 --bootstrap kubeadm:v0.3.0 --control-plane kubeadm:v0.3.0 --infrastructure azure:v0.4.0 ``` ### Create workload cluster -Once the management cluster is ready, you can create your workload cluster. The `clusterctl config cluster` command returns a YAML template for creating a workload cluster. -Set the required [Azure Stack environment variables](./azure-stack.md#set-environment-variables). +Once the management cluster is ready, you can create your workload cluster. The `clusterctl config cluster` command returns a YAML template for creating a workload cluster. + +Set the required [Azure Stack environment variables](./azure-stack.md#set-environment-variables). Generate the cluster configuration, either by flavor or directly from file. -To generate by flavor: +To generate by flavor: + ```bash clusterctl config cluster capz-cluster --kubernetes-version v1.17.8 --control-plane-machine-count 1 --worker-machine-count 1 --flavor azure-stack > my-cluster.yaml ``` -To generate directly from file: + +To generate directly from file: + ```bash clusterctl config cluster capz-cluster --kubernetes-version v1.17.8 --control-plane-machine-count 1 --worker-machine-count 1 --from repo/cluster-api-provider-azure/templates/cluster-template-azure-stack.yaml > my-cluster.yaml ``` -This creates a YAML file named `my-cluster.yaml` with a predefined list of Cluster API objects; Cluster, Machines, Machine Deployments, etc. -Important: Fill in placeholder azurestackcloud json as described [here](./azure-stack.md#set-workload-cluster-template-manifest). +This creates a YAML file named `my-cluster.yaml` with a predefined list of Cluster API objects; Cluster, Machines, Machine Deployments, etc. + +Important: Fill in placeholder azurestackcloud json as described [here](./azure-stack.md#set-workload-cluster-template-manifest). When ready, run the following command to apply the cluster manifest, creating a workload cluster on Azure Stack Hub: + ```bash kubectl apply -f my-cluster.yaml ``` ### Accessing the workload cluster -The cluster will now start provisioning. You can check status with: + +The cluster will now start provisioning. You can check status with: + ```bash kubectl get cluster --all-namespaces ``` -After the first control plane node is up and running, we can retrieve the workload cluster Kubeconfig: + +After the first control plane node is up and running, we can retrieve the workload cluster Kubeconfig: + ```bash kubectl get secrets capz-cluster-kubeconfig -o json | jq -r .data.value | base64 --decode > ./kubeconfig ``` ### Deploy a CNI solution -Calico is used here as an example. + +Calico is used here as an example. + ```bash kubectl --kubeconfig=./kubeconfig apply -f https://raw.githubusercontent.com/kubernetes-sigs/cluster-api-provider-azure/master/templates/addons/calico.yaml ``` After a short while, our nodes should be running and in `Ready` state, let's check the status using `kubectl get nodes` + ```bash kubectl --kubeconfig=./kubeconfig get nodes -``` \ No newline at end of file +``` diff --git a/docs/topics/azure-stack.md b/docs/topics/azure-stack.md index 58f81eec147..2e1a9a229c5 100644 --- a/docs/topics/azure-stack.md +++ b/docs/topics/azure-stack.md @@ -3,7 +3,9 @@ To deploy a cluster using [Azure Stack Hub](https://github.com/msazurestackworkloads/cluster-api-provider-azure), create a cluster configuration with the [azure stack template](../../templates/cluster-template-azure-stack.yaml). ## Upload VHD to Azure Stack Hub -If this is the first time setting up, create your own VHD using [image-builder](https://github.com/msazurestackworkloads/image-builder/tree/azure-stack-vhd-18.04). Then in the Azure Stack Hub Admin Portal, upload a platform VM image with the following parameters: + +If this is the first time setting up, create your own VHD using [image-builder](https://github.com/msazurestackworkloads/image-builder/tree/azure-stack-vhd-18.04). Then in the Azure Stack Hub Admin Portal, upload a platform VM image with the following parameters: + - Publisher: AzureStack - Offer: Test - OS type: Linux @@ -14,6 +16,7 @@ If this is the first time setting up, create your own VHD using [image-builder]( ## Set environment variables ### Azure cloud settings + ```bash export AZURE_ARM_ENDPOINT="https://management.redmond.ext-n31r1203.masd.stbtest.microsoft.com" export AZURE_LOCATION="redmond" @@ -26,9 +29,11 @@ else export IDENTITY_TENANT_ID=${AZURE_TENANT_ID} fi ``` + Azure Stack offers both Azure Active Directory and ADFS identity providers. Environment variable `IDENTITY_SYSTEM` can be either `azure_ad` or `adfs`. -### Azure Service Principal +### Azure Service Principal + ```bash export AZURE_TENANT_ID="" export AZURE_CLIENT_ID="" @@ -42,12 +47,15 @@ export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZURE_CLIENT_SECRET" | base64 | tr - ``` ### Cluster settings + ```bash export CLUSTER_NAME="capz-cluster" export AZURE_RESOURCE_GROUP=${CLUSTER_NAME} export AZURE_VNET_NAME=${CLUSTER_NAME}-vnet ``` + ### Machine settings + ```bash export CONTROL_PLANE_MACHINE_COUNT=1 export AZURE_CONTROL_PLANE_MACHINE_TYPE="Standard_DS2_v2" @@ -57,7 +65,9 @@ export KUBERNETES_VERSION="v1.18.2" ``` ### Generate SSH key + If you want to provide your own key, skip this step and set AZURE_SSH_PUBLIC_KEY to your existing file. + ```bash SSH_KEY_FILE=.sshkey rm -f "${SSH_KEY_FILE}" 2>/dev/null @@ -65,36 +75,43 @@ ssh-keygen -t rsa -b 2048 -f "${SSH_KEY_FILE}" -N '' 1>/dev/null echo "Machine SSH key generated in ${SSH_KEY_FILE}" export AZURE_SSH_PUBLIC_KEY=$(cat "${SSH_KEY_FILE}.pub" | base64 | tr -d '\r\n') ``` -## Build docker image + +## Build docker image + ```bash export REGISTRY="gcr.io" -export PULL_POLICY=IfNotPresent +export PULL_POLICY=IfNotPresent make docker-build ``` -## Create management cluster +## Create management cluster + ```bash + export EXP_MACHINE_POOL=true make create-management-cluster ``` + ## Set workload cluster template manifest ### AzureStackCloud json + Package go-autorest defines a variable of type [Environment](https://godoc.org/github.com/Azure/go-autorest/autorest/azure#Environment) for each Azure cloud (Public, China, Germany, US Gov). - + In Azure Stack's case, the environment has to be dynamically determined by querying [Azure Stack's metadata endpoint](https://docs.microsoft.com/en-us/azure-stack/user/azure-stack-version-profiles-go?view=azs-2005#how-to-use-go-sdk-profiles-on-azure-stack-hub). Composing [this list](https://github.com/kubernetes/cloud-provider-azure/blob/master/docs/cloud-provider-config.md#azure-stack-configuration) is required in order to indicate to azure cloud provider what endpoints to target. Use the following bash script to generate the azurestackcloud json. Paste the output of the bash script into the workload cluster template, replacing the corresponding azurestackcloud file content placeholders. -Usage: $> azs_endpoints.sh local azure.external +Usage: $> azs_endpoints.sh local azure.external + ```bash #!/bin/bash LOCATION=$1 FQDN=$2 - + METADATA=$(mktemp) - + MANAGEMENT="https://management.${LOCATION}.${FQDN}/" curl -o ${METADATA} -k "${MANAGEMENT}metadata/endpoints?api-version=1.0" @@ -107,7 +124,7 @@ GRAPH="$(jq -r .graphEndpoint "$METADATA")" STORAGE="${LOCATION}.${FQDN}", KEYVAULT="vault.${LOCATION}.${FQDN}" RESOURCEMANAGER="cloudapp.${FQDN}" - + jq -n \ --arg NAME "$NAME" \ --arg PORTALURL "$PORTALURL" \ @@ -133,21 +150,25 @@ resourceManagerVMDNSSuffix: $RESOURCEMANAGER }' ``` -## Create workload cluster +## Create workload cluster + ```bash export CLUSTER_TEMPLATE=cluster-template-azure-stack.yaml make create-workload-cluster ``` ## Debug + ### Retrieve CAPZ logs + ```bash kubectl logs deployment/capz-controller-manager -n capz-system --all-containers=true > stack.log ``` ### ssh into virtual machine + ```bash ssh -i repo/cluster-api-provider-azure/.sshkey capi@ cat /var/log/cloud-init-output.log > init.log sudo journalctl -u kubelet -l --no-pager > kubelet.log -``` \ No newline at end of file +``` From 8398d7780040ac2b3039ba81f405fa68568042a5 Mon Sep 17 00:00:00 2001 From: chlau-az Date: Thu, 20 Aug 2020 11:00:36 -0700 Subject: [PATCH 47/47] Create Dockerfile dev copy for certs --- Dockerfile | 5 --- Dockerfile.dev | 50 ++++++++++++++++++++++++++ Makefile | 18 +++++++++- scripts/ci-e2e-dev.sh | 82 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 149 insertions(+), 6 deletions(-) create mode 100644 Dockerfile.dev create mode 100755 scripts/ci-e2e-dev.sh diff --git a/Dockerfile b/Dockerfile index 4f01278dda6..19ecc35585a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,11 +36,6 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \ go build -a -ldflags '-extldflags "-static"' \ -o manager . - -COPY .vscode/azs.crt /usr/local/share/ca-certificates/ -COPY .vscode/azs02.crt /usr/local/share/ca-certificates/ -RUN update-ca-certificates - # Copy the controller-manager into a thin image FROM gcr.io/distroless/static:latest WORKDIR / diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 00000000000..4f01278dda6 --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,50 @@ +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build the manager binary +FROM golang:1.13.8 as builder +WORKDIR /workspace + +# Run this with docker build --build_arg $(go env GOPROXY) to override the goproxy +ARG goproxy=https://proxy.golang.org +ENV GOPROXY=$goproxy + +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# Cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the sources +COPY ./ ./ + +# Build +ARG ARCH +RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \ + go build -a -ldflags '-extldflags "-static"' \ + -o manager . + + +COPY .vscode/azs.crt /usr/local/share/ca-certificates/ +COPY .vscode/azs02.crt /usr/local/share/ca-certificates/ +RUN update-ca-certificates + +# Copy the controller-manager into a thin image +FROM gcr.io/distroless/static:latest +WORKDIR / +COPY --from=builder /etc/ssl/certs /etc/ssl/certs +COPY --from=builder /workspace/manager . +USER nobody +ENTRYPOINT ["/manager"] diff --git a/Makefile b/Makefile index 3240857ffe4..7247b0d2385 100644 --- a/Makefile +++ b/Makefile @@ -158,6 +158,16 @@ test-e2e: $(ENVSUBST) $(GINKGO) ## Run e2e tests -e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \ -e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) +.PHONY: test-e2e-dev +test-e2e-dev: $(ENVSUBST) $(GINKGO) ## Run e2e tests + PULL_POLICY=IfNotPresent $(MAKE) docker-build-dev + MANAGER_IMAGE=$(CONTROLLER_IMG)-$(ARCH):$(TAG) \ + $(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \ + $(GINKGO) -v -trace -tags=e2e -focus=$(GINKGO_FOCUS) -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) ./test/e2e -- \ + -e2e.artifacts-folder="$(ARTIFACTS)" \ + -e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \ + -e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) + $(KUBECTL) $(KUBE_APISERVER) $(ETCD): ## install test asset kubectl, kube-apiserver, etcd source ./scripts/fetch_ext_bins.sh && fetch_tools @@ -272,7 +282,13 @@ generate-flavors: $(KUSTOMIZE) docker-build: ## Build the docker image for controller-manager docker build --pull --build-arg ARCH=$(ARCH) . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG) MANIFEST_IMG=$(CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) $(MAKE) set-manifest-image - $(MAKE) set-manifest-pull-policy + $(MAKE) set-manifest-pull-policy + +.PHONY: docker-build-dev +docker-build-dev: ## Build the docker image for controller-manager + docker build -f Dockerfile.dev --pull --build-arg ARCH=$(ARCH) . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG) + MANIFEST_IMG=$(CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) $(MAKE) set-manifest-image + $(MAKE) set-manifest-pull-policy .PHONY: docker-push docker-push: ## Push the docker image diff --git a/scripts/ci-e2e-dev.sh b/scripts/ci-e2e-dev.sh new file mode 100755 index 00000000000..b90a5e4e32f --- /dev/null +++ b/scripts/ci-e2e-dev.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +############################################################################### + +# This script is executed by presubmit `pull-cluster-api-provider-azure-e2e` +# To run locally, set AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_SUBSCRIPTION_ID, AZURE_TENANT_ID + +set -o errexit +set -o nounset +set -o pipefail + +REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +cd "${REPO_ROOT}" || exit 1 + +# shellcheck source=../hack/ensure-go.sh +source "${REPO_ROOT}/hack/ensure-go.sh" +# shellcheck source=../hack/ensure-kind.sh +source "${REPO_ROOT}/hack/ensure-kind.sh" +# shellcheck source=../hack/ensure-kubectl.sh +source "${REPO_ROOT}/hack/ensure-kubectl.sh" +# shellcheck source=../hack/ensure-kustomize.sh +source "${REPO_ROOT}/hack/ensure-kustomize.sh" +# shellcheck source=../hack/parse-prow-creds.sh +source "${REPO_ROOT}/hack/parse-prow-creds.sh" + +# Verify the required Environment Variables are present. +: "${AZURE_SUBSCRIPTION_ID:?Environment variable empty or not defined.}" +: "${AZURE_TENANT_ID:?Environment variable empty or not defined.}" +: "${AZURE_CLIENT_ID:?Environment variable empty or not defined.}" +: "${AZURE_CLIENT_SECRET:?Environment variable empty or not defined.}" + +get_random_region() { + local REGIONS=("eastus" "eastus2" "southcentralus" "westus2" "westeurope") + echo "${REGIONS[${RANDOM} % ${#REGIONS[@]}]}" +} + +export REGISTRY="e2e" +export AZURE_ENVIRONMENT="${AZURE_ENVIRONMENT}" +export GINKGO_NODES=3 +export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZURE_SUBSCRIPTION_ID" | base64 | tr -d '\n')" +export AZURE_TENANT_ID_B64="$(echo -n "$AZURE_TENANT_ID" | base64 | tr -d '\n')" +export AZURE_CLIENT_ID_B64="$(echo -n "$AZURE_CLIENT_ID" | base64 | tr -d '\n')" +export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZURE_CLIENT_SECRET" | base64 | tr -d '\n')" +export AZURE_LOCATION="${AZURE_LOCATION:-$(get_random_region)}" +export AZURE_CONTROL_PLANE_MACHINE_TYPE="${AZURE_CONTROL_PLANE_MACHINE_TYPE:-"Standard_D2s_v3"}" +export AZURE_NODE_MACHINE_TYPE="${AZURE_NODE_MACHINE_TYPE:-"Standard_D2s_v3"}" + +# Generate SSH key. +AZURE_SSH_PUBLIC_KEY_FILE=${AZURE_SSH_PUBLIC_KEY_FILE:-""} +if ! [ -n "${AZURE_SSH_PUBLIC_KEY_FILE}" ]; then + SSH_KEY_FILE=.sshkey + rm -f "${SSH_KEY_FILE}" 2>/dev/null + ssh-keygen -t rsa -b 2048 -f "${SSH_KEY_FILE}" -N '' 1>/dev/null + AZURE_SSH_PUBLIC_KEY_FILE="${SSH_KEY_FILE}.pub" +fi +export AZURE_SSH_PUBLIC_KEY=$(cat "${AZURE_SSH_PUBLIC_KEY_FILE}" | base64 | tr -d '\r\n') + +# timestamp is in RFC-3339 format to match kubetest +export TIMESTAMP="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" +export JOB_NAME="${JOB_NAME:-"cluster-api-provider-azure-e2e"}" + +cleanup() { + ${REPO_ROOT}/hack/log/redact.sh || true +} + +trap cleanup EXIT + +make test-e2e-dev