Merge pull request #78242 from feiskyer/configurable-provider

staging/src/k8s.io/legacy-cloud-providers/azure/BUILD

@@ -14,6 +14,7 @@ go_library(
         "azure_blobDiskController.go",
         "azure_cache.go",
         "azure_client.go",
+        "azure_config.go",
         "azure_controller_common.go",
         "azure_controller_standard.go",
         "azure_controller_vmss.go",
@@ -39,6 +40,7 @@ go_library(
     deps = [
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
@@ -80,6 +82,7 @@ go_test(
     srcs = [
         "azure_backoff_test.go",
         "azure_cache_test.go",
+        "azure_config_test.go",
         "azure_controller_common_test.go",
         "azure_instances_test.go",
         "azure_loadbalancer_test.go",
@@ -100,6 +103,7 @@ go_test(
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
         "//staging/src/k8s.io/client-go/tools/record:go_default_library",
         "//staging/src/k8s.io/cloud-provider:go_default_library",
         "//staging/src/k8s.io/cloud-provider/service/helpers:go_default_library",
@@ -110,6 +114,7 @@ go_test(
         "//vendor/github.com/Azure/go-autorest/autorest:go_default_library",
         "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/sigs.k8s.io/yaml:go_default_library",
     ],
 )
 

staging/src/k8s.io/legacy-cloud-providers/azure/auth/azure_auth.go

@@ -36,25 +36,25 @@ var (
 // AzureAuthConfig holds auth related part of cloud config
 type AzureAuthConfig struct {
 	// The cloud environment identifier. Takes values from https://github.com/Azure/go-autorest/blob/ec5f4903f77ed9927ac95b19ab8e44ada64c1356/autorest/azure/environments.go#L13
-	Cloud string `json:"cloud" yaml:"cloud"`
+	Cloud string `json:"cloud,omitempty" yaml:"cloud,omitempty"`
 	// The AAD Tenant ID for the Subscription that the cluster is deployed in
-	TenantID string `json:"tenantId" yaml:"tenantId"`
+	TenantID string `json:"tenantId,omitempty" yaml:"tenantId,omitempty"`
 	// The ClientID for an AAD application with RBAC access to talk to Azure RM APIs
-	AADClientID string `json:"aadClientId" yaml:"aadClientId"`
+	AADClientID string `json:"aadClientId,omitempty" yaml:"aadClientId,omitempty"`
 	// The ClientSecret for an AAD application with RBAC access to talk to Azure RM APIs
-	AADClientSecret string `json:"aadClientSecret" yaml:"aadClientSecret"`
+	AADClientSecret string `json:"aadClientSecret,omitempty" yaml:"aadClientSecret,omitempty"`
 	// The path of a client certificate for an AAD application with RBAC access to talk to Azure RM APIs
-	AADClientCertPath string `json:"aadClientCertPath" yaml:"aadClientCertPath"`
+	AADClientCertPath string `json:"aadClientCertPath,omitempty" yaml:"aadClientCertPath,omitempty"`
 	// The password of the client certificate for an AAD application with RBAC access to talk to Azure RM APIs
-	AADClientCertPassword string `json:"aadClientCertPassword" yaml:"aadClientCertPassword"`
+	AADClientCertPassword string `json:"aadClientCertPassword,omitempty" yaml:"aadClientCertPassword,omitempty"`
 	// Use managed service identity for the virtual machine to access Azure ARM APIs
-	UseManagedIdentityExtension bool `json:"useManagedIdentityExtension" yaml:"useManagedIdentityExtension"`
+	UseManagedIdentityExtension bool `json:"useManagedIdentityExtension,omitempty" yaml:"useManagedIdentityExtension,omitempty"`
 	// UserAssignedIdentityID contains the Client ID of the user assigned MSI which is assigned to the underlying VMs. If empty the user assigned identity is not used.
 	// More details of the user assigned identity can be found at: https://docs.microsoft.com/en-us/azure/active-directory/managed-service-identity/overview
 	// For the user assigned identity specified here to be used, the UseManagedIdentityExtension has to be set to true.
-	UserAssignedIdentityID string `json:"userAssignedIdentityID" yaml:"userAssignedIdentityID"`
+	UserAssignedIdentityID string `json:"userAssignedIdentityID,omitempty" yaml:"userAssignedIdentityID,omitempty"`
 	// The ID of the Azure Subscription that the cluster is deployed in
-	SubscriptionID string `json:"subscriptionId" yaml:"subscriptionId"`
+	SubscriptionID string `json:"subscriptionId,omitempty" yaml:"subscriptionId,omitempty"`
 }
 
 // GetServicePrincipalToken creates a new service principal token based on the configuration

staging/src/k8s.io/legacy-cloud-providers/azure/azure_config.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"fmt"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/klog"
+	"sigs.k8s.io/yaml"
+)
+
+const (
+	cloudConfigNamespace  = "kube-system"
+	cloudConfigKey        = "cloud-config"
+	cloudConfigSecretName = "azure-cloud-provider"
+)
+
+// The config type for Azure cloud provider secret. Supported values are:
+// * file   : The values are read from local cloud-config file.
+// * secret : The values from secret would override all configures from local cloud-config file.
+// * merge  : The values from secret would override only configurations that are explicitly set in the secret. This is the default value.
+type cloudConfigType string
+
+const (
+	cloudConfigTypeFile   cloudConfigType = "file"
+	cloudConfigTypeSecret cloudConfigType = "secret"
+	cloudConfigTypeMerge  cloudConfigType = "merge"
+)
+
+// InitializeCloudFromSecret initializes Azure cloud provider from Kubernetes secret.
+func (az *Cloud) InitializeCloudFromSecret() {
+	config, err := az.getConfigFromSecret()
+	if err != nil {
+		klog.Warningf("Failed to get cloud-config from secret: %v, skip initializing from secret", err)
+		return
+	}
+
+	if config == nil {
+		// Skip re-initialization if the config is not override.
+		return
+	}
+
+	if err := az.InitializeCloudFromConfig(config, true); err != nil {
+		klog.Errorf("Failed to initialize Azure cloud provider: %v", err)
+	}
+}
+
+func (az *Cloud) getConfigFromSecret() (*Config, error) {
+	// Read config from file and no override, return nil.
+	if az.Config.CloudConfigType == cloudConfigTypeFile {
+		return nil, nil
+	}
+
+	secret, err := az.kubeClient.CoreV1().Secrets(cloudConfigNamespace).Get(cloudConfigSecretName, metav1.GetOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("Failed to get secret %s: %v", cloudConfigSecretName, err)
+	}
+
+	cloudConfigData, ok := secret.Data[cloudConfigKey]
+	if !ok {
+		return nil, fmt.Errorf("cloud-config is not set in the secret (%s)", cloudConfigSecretName)
+	}
+
+	config := Config{}
+	if az.Config.CloudConfigType == "" || az.Config.CloudConfigType == cloudConfigTypeMerge {
+		// Merge cloud config, set default value to existing config.
+		config = az.Config
+	}
+
+	err = yaml.Unmarshal(cloudConfigData, &config)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to parse Azure cloud-config: %v", err)
+	}
+
+	return &config, nil
+}

staging/src/k8s.io/legacy-cloud-providers/azure/azure_config_test.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"testing"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	fakeclient "k8s.io/client-go/kubernetes/fake"
+	"k8s.io/legacy-cloud-providers/azure/auth"
+	"sigs.k8s.io/yaml"
+
+	"github.com/Azure/go-autorest/autorest/to"
+	"github.com/stretchr/testify/assert"
+)
+
+func getTestConfig() *Config {
+	return &Config{
+		AzureAuthConfig: auth.AzureAuthConfig{
+			TenantID:        "TenantID",
+			SubscriptionID:  "SubscriptionID",
+			AADClientID:     "AADClientID",
+			AADClientSecret: "AADClientSecret",
+		},
+		ResourceGroup:               "ResourceGroup",
+		RouteTableName:              "RouteTableName",
+		RouteTableResourceGroup:     "RouteTableResourceGroup",
+		Location:                    "Location",
+		SubnetName:                  "SubnetName",
+		VnetName:                    "VnetName",
+		PrimaryAvailabilitySetName:  "PrimaryAvailabilitySetName",
+		PrimaryScaleSetName:         "PrimaryScaleSetName",
+		LoadBalancerSku:             "LoadBalancerSku",
+		ExcludeMasterFromStandardLB: to.BoolPtr(true),
+	}
+}
+
+func getTestCloudConfigTypeSecretConfig() *Config {
+	return &Config{
+		AzureAuthConfig: auth.AzureAuthConfig{
+			TenantID:       "TenantID",
+			SubscriptionID: "SubscriptionID",
+		},
+		ResourceGroup:           "ResourceGroup",
+		RouteTableName:          "RouteTableName",
+		RouteTableResourceGroup: "RouteTableResourceGroup",
+		SecurityGroupName:       "SecurityGroupName",
+		CloudConfigType:         cloudConfigTypeSecret,
+	}
+}
+
+func getTestCloudConfigTypeMergeConfig() *Config {
+	return &Config{
+		AzureAuthConfig: auth.AzureAuthConfig{
+			TenantID:       "TenantID",
+			SubscriptionID: "SubscriptionID",
+		},
+		ResourceGroup:           "ResourceGroup",
+		RouteTableName:          "RouteTableName",
+		RouteTableResourceGroup: "RouteTableResourceGroup",
+		SecurityGroupName:       "SecurityGroupName",
+		CloudConfigType:         cloudConfigTypeMerge,
+	}
+}
+
+func getTestCloudConfigTypeMergeConfigExpected() *Config {
+	config := getTestConfig()
+	config.SecurityGroupName = "SecurityGroupName"
+	config.CloudConfigType = cloudConfigTypeMerge
+	return config
+}
+
+func TestGetConfigFromSecret(t *testing.T) {
+	emptyConfig := &Config{}
+	tests := []struct {
+		name           string
+		existingConfig *Config
+		secretConfig   *Config
+		expected       *Config
+		expectErr      bool
+	}{
+		{
+			name: "Azure config shouldn't be override when cloud config type is file",
+			existingConfig: &Config{
+				ResourceGroup:   "ResourceGroup1",
+				CloudConfigType: cloudConfigTypeFile,
+			},
+			secretConfig: getTestConfig(),
+			expected:     nil,
+		},
+		{
+			name:           "Azure config should be override when cloud config type is secret",
+			existingConfig: getTestCloudConfigTypeSecretConfig(),
+			secretConfig:   getTestConfig(),
+			expected:       getTestConfig(),
+		},
+		{
+			name:           "Azure config should be override when cloud config type is merge",
+			existingConfig: getTestCloudConfigTypeMergeConfig(),
+			secretConfig:   getTestConfig(),
+			expected:       getTestCloudConfigTypeMergeConfigExpected(),
+		},
+		{
+			name:           "Error should be reported when secret doesn't exists",
+			existingConfig: getTestCloudConfigTypeMergeConfig(),
+			expectErr:      true,
+		},
+		{
+			name:           "Error should be reported when secret exists but cloud-config data is not provided",
+			existingConfig: getTestCloudConfigTypeMergeConfig(),
+			secretConfig:   emptyConfig,
+			expectErr:      true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			az := &Cloud{
+				kubeClient: fakeclient.NewSimpleClientset(),
+			}
+			if test.existingConfig != nil {
+				az.Config = *test.existingConfig
+			}
+			if test.secretConfig != nil {
+				secret := &v1.Secret{
+					Type: v1.SecretTypeOpaque,
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "azure-cloud-provider",
+						Namespace: "kube-system",
+					},
+				}
+				if test.secretConfig != emptyConfig {
+					secretData, err := yaml.Marshal(test.secretConfig)
+					assert.NoError(t, err, test.name)
+					secret.Data = map[string][]byte{
+						"cloud-config": secretData,
+					}
+				}
+				_, err := az.kubeClient.CoreV1().Secrets(cloudConfigNamespace).Create(secret)
+				assert.NoError(t, err, test.name)
+			}
+
+			real, err := az.getConfigFromSecret()
+			if test.expectErr {
+				assert.Error(t, err, test.name)
+				return
+			}
+
+			assert.NoError(t, err, test.name)
+			assert.Equal(t, test.expected, real, test.name)
+		})
+	}
+}