Unverified Commit 71e1a58c authored by Erik Wilson's avatar Erik Wilson Committed by GitHub

Merge pull request #1831 from erikwilson/release-1.17

Update k8s 1.17.6
parents 58ebdb2a dae68679
FROM golang:1.13.8-alpine3.10 FROM golang:1.13.11-alpine3.10
RUN apk -U --no-cache add bash git gcc musl-dev docker vim less file curl wget ca-certificates jq linux-headers zlib-dev tar zip squashfs-tools npm coreutils \ RUN apk -U --no-cache add bash git gcc musl-dev docker vim less file curl wget ca-certificates jq linux-headers zlib-dev tar zip squashfs-tools npm coreutils \
python2 openssl-dev libffi-dev libseccomp libseccomp-dev make libuv-static python2 openssl-dev libffi-dev libseccomp libseccomp-dev make libuv-static
......
...@@ -32,31 +32,31 @@ replace ( ...@@ -32,31 +32,31 @@ replace (
github.com/prometheus/client_model => github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 github.com/prometheus/client_model => github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910
github.com/prometheus/common => github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 github.com/prometheus/common => github.com/prometheus/common v0.0.0-20181126121408-4724e9255275
github.com/prometheus/procfs => github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a github.com/prometheus/procfs => github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a
k8s.io/api => github.com/rancher/kubernetes/staging/src/k8s.io/api v1.17.5-k3s1 k8s.io/api => github.com/rancher/kubernetes/staging/src/k8s.io/api v1.17.6-k3s1
k8s.io/apiextensions-apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.17.5-k3s1 k8s.io/apiextensions-apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.17.6-k3s1
k8s.io/apimachinery => github.com/rancher/kubernetes/staging/src/k8s.io/apimachinery v1.17.5-k3s1 k8s.io/apimachinery => github.com/rancher/kubernetes/staging/src/k8s.io/apimachinery v1.17.6-k3s1
k8s.io/apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/apiserver v1.17.5-k3s1 k8s.io/apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/apiserver v1.17.6-k3s1
k8s.io/cli-runtime => github.com/rancher/kubernetes/staging/src/k8s.io/cli-runtime v1.17.5-k3s1 k8s.io/cli-runtime => github.com/rancher/kubernetes/staging/src/k8s.io/cli-runtime v1.17.6-k3s1
k8s.io/client-go => github.com/rancher/kubernetes/staging/src/k8s.io/client-go v1.17.5-k3s1 k8s.io/client-go => github.com/rancher/kubernetes/staging/src/k8s.io/client-go v1.17.6-k3s1
k8s.io/cloud-provider => github.com/rancher/kubernetes/staging/src/k8s.io/cloud-provider v1.17.5-k3s1 k8s.io/cloud-provider => github.com/rancher/kubernetes/staging/src/k8s.io/cloud-provider v1.17.6-k3s1
k8s.io/cluster-bootstrap => github.com/rancher/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.17.5-k3s1 k8s.io/cluster-bootstrap => github.com/rancher/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.17.6-k3s1
k8s.io/code-generator => github.com/rancher/kubernetes/staging/src/k8s.io/code-generator v1.17.5-k3s1 k8s.io/code-generator => github.com/rancher/kubernetes/staging/src/k8s.io/code-generator v1.17.6-k3s1
k8s.io/component-base => github.com/rancher/kubernetes/staging/src/k8s.io/component-base v1.17.5-k3s1 k8s.io/component-base => github.com/rancher/kubernetes/staging/src/k8s.io/component-base v1.17.6-k3s1
k8s.io/cri-api => github.com/rancher/kubernetes/staging/src/k8s.io/cri-api v1.17.5-k3s1 k8s.io/cri-api => github.com/rancher/kubernetes/staging/src/k8s.io/cri-api v1.17.6-k3s1
k8s.io/csi-translation-lib => github.com/rancher/kubernetes/staging/src/k8s.io/csi-translation-lib v1.17.5-k3s1 k8s.io/csi-translation-lib => github.com/rancher/kubernetes/staging/src/k8s.io/csi-translation-lib v1.17.6-k3s1
k8s.io/kube-aggregator => github.com/rancher/kubernetes/staging/src/k8s.io/kube-aggregator v1.17.5-k3s1 k8s.io/kube-aggregator => github.com/rancher/kubernetes/staging/src/k8s.io/kube-aggregator v1.17.6-k3s1
k8s.io/kube-controller-manager => github.com/rancher/kubernetes/staging/src/k8s.io/kube-controller-manager v1.17.5-k3s1 k8s.io/kube-controller-manager => github.com/rancher/kubernetes/staging/src/k8s.io/kube-controller-manager v1.17.6-k3s1
k8s.io/kube-proxy => github.com/rancher/kubernetes/staging/src/k8s.io/kube-proxy v1.17.5-k3s1 k8s.io/kube-proxy => github.com/rancher/kubernetes/staging/src/k8s.io/kube-proxy v1.17.6-k3s1
k8s.io/kube-scheduler => github.com/rancher/kubernetes/staging/src/k8s.io/kube-scheduler v1.17.5-k3s1 k8s.io/kube-scheduler => github.com/rancher/kubernetes/staging/src/k8s.io/kube-scheduler v1.17.6-k3s1
k8s.io/kubectl => github.com/rancher/kubernetes/staging/src/k8s.io/kubectl v1.17.5-k3s1 k8s.io/kubectl => github.com/rancher/kubernetes/staging/src/k8s.io/kubectl v1.17.6-k3s1
k8s.io/kubelet => github.com/rancher/kubernetes/staging/src/k8s.io/kubelet v1.17.5-k3s1 k8s.io/kubelet => github.com/rancher/kubernetes/staging/src/k8s.io/kubelet v1.17.6-k3s1
k8s.io/kubernetes => github.com/rancher/kubernetes v1.17.5-k3s1 k8s.io/kubernetes => github.com/rancher/kubernetes v1.17.6-k3s1
k8s.io/legacy-cloud-providers => github.com/rancher/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.17.5-k3s1 k8s.io/legacy-cloud-providers => github.com/rancher/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.17.6-k3s1
k8s.io/metrics => github.com/rancher/kubernetes/staging/src/k8s.io/metrics v1.17.5-k3s1 k8s.io/metrics => github.com/rancher/kubernetes/staging/src/k8s.io/metrics v1.17.6-k3s1
k8s.io/node-api => github.com/rancher/kubernetes/staging/src/k8s.io/node-api v1.17.5-k3s1 k8s.io/node-api => github.com/rancher/kubernetes/staging/src/k8s.io/node-api v1.17.6-k3s1
k8s.io/sample-apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/sample-apiserver v1.17.5-k3s1 k8s.io/sample-apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/sample-apiserver v1.17.6-k3s1
k8s.io/sample-cli-plugin => github.com/rancher/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.17.5-k3s1 k8s.io/sample-cli-plugin => github.com/rancher/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.17.6-k3s1
k8s.io/sample-controller => github.com/rancher/kubernetes/staging/src/k8s.io/sample-controller v1.17.5-k3s1 k8s.io/sample-controller => github.com/rancher/kubernetes/staging/src/k8s.io/sample-controller v1.17.6-k3s1
mvdan.cc/unparam => mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34 mvdan.cc/unparam => mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34
) )
......
...@@ -38,6 +38,10 @@ func (a statusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.O ...@@ -38,6 +38,10 @@ func (a statusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.O
newCustomResource := newCustomResourceObject.UnstructuredContent() newCustomResource := newCustomResourceObject.UnstructuredContent()
status, ok := newCustomResource["status"] status, ok := newCustomResource["status"]
// managedFields must be preserved since it's been modified to
// track changed fields in the status update.
managedFields := newCustomResourceObject.GetManagedFields()
// copy old object into new object // copy old object into new object
oldCustomResourceObject := old.(*unstructured.Unstructured) oldCustomResourceObject := old.(*unstructured.Unstructured)
// overridding the resourceVersion in metadata is safe here, we have already checked that // overridding the resourceVersion in metadata is safe here, we have already checked that
...@@ -45,6 +49,7 @@ func (a statusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.O ...@@ -45,6 +49,7 @@ func (a statusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.O
*newCustomResourceObject = *oldCustomResourceObject.DeepCopy() *newCustomResourceObject = *oldCustomResourceObject.DeepCopy()
// set status // set status
newCustomResourceObject.SetManagedFields(managedFields)
newCustomResource = newCustomResourceObject.UnstructuredContent() newCustomResource = newCustomResourceObject.UnstructuredContent()
if ok { if ok {
newCustomResource["status"] = status newCustomResource["status"] = status
......
...@@ -252,7 +252,9 @@ func ResetObjectMetaForStatus(meta, existingMeta Object) { ...@@ -252,7 +252,9 @@ func ResetObjectMetaForStatus(meta, existingMeta Object) {
meta.SetAnnotations(existingMeta.GetAnnotations()) meta.SetAnnotations(existingMeta.GetAnnotations())
meta.SetFinalizers(existingMeta.GetFinalizers()) meta.SetFinalizers(existingMeta.GetFinalizers())
meta.SetOwnerReferences(existingMeta.GetOwnerReferences()) meta.SetOwnerReferences(existingMeta.GetOwnerReferences())
meta.SetManagedFields(existingMeta.GetManagedFields()) // managedFields must be preserved since it's been modified to
// track changed fields in the status update.
//meta.SetManagedFields(existingMeta.GetManagedFields())
} }
// MarshalJSON implements json.Marshaler // MarshalJSON implements json.Marshaler
......
...@@ -66,11 +66,36 @@ func Unmarshal(data []byte, v interface{}) error { ...@@ -66,11 +66,36 @@ func Unmarshal(data []byte, v interface{}) error {
// If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64 // If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64
return convertSliceNumbers(*v, 0) return convertSliceNumbers(*v, 0)
case *interface{}:
// Build a decoder from the given data
decoder := json.NewDecoder(bytes.NewBuffer(data))
// Preserve numbers, rather than casting to float64 automatically
decoder.UseNumber()
// Run the decode
if err := decoder.Decode(v); err != nil {
return err
}
// If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64
return convertInterfaceNumbers(v, 0)
default: default:
return json.Unmarshal(data, v) return json.Unmarshal(data, v)
} }
} }
func convertInterfaceNumbers(v *interface{}, depth int) error {
var err error
switch v2 := (*v).(type) {
case json.Number:
*v, err = convertNumber(v2)
case map[string]interface{}:
err = convertMapNumbers(v2, depth+1)
case []interface{}:
err = convertSliceNumbers(v2, depth+1)
}
return err
}
// convertMapNumbers traverses the map, converting any json.Number values to int64 or float64. // convertMapNumbers traverses the map, converting any json.Number values to int64 or float64.
// values which are map[string]interface{} or []interface{} are recursively visited // values which are map[string]interface{} or []interface{} are recursively visited
func convertMapNumbers(m map[string]interface{}, depth int) error { func convertMapNumbers(m map[string]interface{}, depth int) error {
......
...@@ -650,6 +650,7 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler { ...@@ -650,6 +650,7 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
handler = genericfilters.WithTimeoutForNonLongRunningRequests(handler, c.LongRunningFunc, c.RequestTimeout) handler = genericfilters.WithTimeoutForNonLongRunningRequests(handler, c.LongRunningFunc, c.RequestTimeout)
handler = genericfilters.WithWaitGroup(handler, c.LongRunningFunc, c.HandlerChainWaitGroup) handler = genericfilters.WithWaitGroup(handler, c.LongRunningFunc, c.HandlerChainWaitGroup)
handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver) handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver)
handler = genericapifilters.WithCacheControl(handler)
handler = genericfilters.WithPanicRecovery(handler) handler = genericfilters.WithPanicRecovery(handler)
return handler return handler
} }
......
...@@ -3,8 +3,8 @@ package version ...@@ -3,8 +3,8 @@ package version
var ( var (
gitMajor = "1" gitMajor = "1"
gitMinor = "17" gitMinor = "17"
gitVersion = "v1.17.5-k3s1" gitVersion = "v1.17.6-k3s1"
gitCommit = "4e0e5ac0e78825663e85b065ba8bbd358c0661b6" gitCommit = "1e1abaed8e851a7354fc128a32c7cb1f5433e3bb"
gitTreeState = "clean" gitTreeState = "clean"
buildDate = "2020-04-23T07:57:14Z" buildDate = "2020-05-26T21:16:48Z"
) )
...@@ -175,8 +175,8 @@ k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUc ...@@ -175,8 +175,8 @@ k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUc
k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
k8s.io/kube-openapi v0.0.0-20200316234421-82d701f24f9d h1:jocF7XFucw2pEiv2wS7wk2FRFCjDFGV1oa4TMs0SAT0= k8s.io/kube-openapi v0.0.0-20200410145947-bcb3869e6f29 h1:NeQXVJ2XFSkRoPzRo8AId01ZER+j8oV4SZADT4iBOXQ=
k8s.io/kube-openapi v0.0.0-20200316234421-82d701f24f9d/go.mod h1:F+5wygcW0wmRTnM3cOgIqGivxkwSWIWT5YdsDbeAOaU= k8s.io/kube-openapi v0.0.0-20200410145947-bcb3869e6f29/go.mod h1:F+5wygcW0wmRTnM3cOgIqGivxkwSWIWT5YdsDbeAOaU=
k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo=
k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
sigs.k8s.io/structured-merge-diff/v2 v2.0.1/go.mod h1:Wb7vfKAodbKgf6tn1Kl0VvGj7mRH6DGaRcixXEJXTsE= sigs.k8s.io/structured-merge-diff/v2 v2.0.1/go.mod h1:Wb7vfKAodbKgf6tn1Kl0VvGj7mRH6DGaRcixXEJXTsE=
......
...@@ -3,8 +3,8 @@ package version ...@@ -3,8 +3,8 @@ package version
var ( var (
gitMajor = "1" gitMajor = "1"
gitMinor = "17" gitMinor = "17"
gitVersion = "v1.17.5-k3s1" gitVersion = "v1.17.6-k3s1"
gitCommit = "4e0e5ac0e78825663e85b065ba8bbd358c0661b6" gitCommit = "1e1abaed8e851a7354fc128a32c7cb1f5433e3bb"
gitTreeState = "clean" gitTreeState = "clean"
buildDate = "2020-04-23T07:57:14Z" buildDate = "2020-05-26T21:16:48Z"
) )
...@@ -2,6 +2,6 @@ ...@@ -2,6 +2,6 @@
Do not open pull requests directly against this repository, they will be ignored. Instead, please open pull requests against [kubernetes/kubernetes](https://git.k8s.io/kubernetes/). Please follow the same [contributing guide](https://git.k8s.io/kubernetes/CONTRIBUTING.md) you would follow for any other pull request made to kubernetes/kubernetes. Do not open pull requests directly against this repository, they will be ignored. Instead, please open pull requests against [kubernetes/kubernetes](https://git.k8s.io/kubernetes/). Please follow the same [contributing guide](https://git.k8s.io/kubernetes/CONTRIBUTING.md) you would follow for any other pull request made to kubernetes/kubernetes.
This repository is published from [kubernetes/kubernetes/staging/src/k8s.io/csi-api](https://git.k8s.io/kubernetes/staging/src/k8s.io/csi-api) by the [kubernetes publishing-bot](https://git.k8s.io/publishing-bot). This repository is published from [kubernetes/kubernetes/staging/src/k8s.io/csi-translation-lib](https://git.k8s.io/kubernetes/staging/src/k8s.io/csi-translation-lib) by the [kubernetes publishing-bot](https://git.k8s.io/publishing-bot).
Please see [Staging Directory and Publishing](https://git.k8s.io/community/contributors/devel/sig-architecture/staging.md) for more information. Please see [Staging Directory and Publishing](https://git.k8s.io/community/contributors/devel/sig-architecture/staging.md) for more information.
...@@ -5,9 +5,11 @@ module k8s.io/csi-translation-lib ...@@ -5,9 +5,11 @@ module k8s.io/csi-translation-lib
go 1.12 go 1.12
require ( require (
github.com/stretchr/testify v1.4.0
k8s.io/api v0.0.0 k8s.io/api v0.0.0
k8s.io/apimachinery v0.0.0 k8s.io/apimachinery v0.0.0
k8s.io/cloud-provider v0.0.0 k8s.io/cloud-provider v0.0.0
k8s.io/klog v1.0.0
) )
replace ( replace (
......
...@@ -158,7 +158,7 @@ k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUc ...@@ -158,7 +158,7 @@ k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUc
k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
k8s.io/kube-openapi v0.0.0-20200316234421-82d701f24f9d/go.mod h1:F+5wygcW0wmRTnM3cOgIqGivxkwSWIWT5YdsDbeAOaU= k8s.io/kube-openapi v0.0.0-20200410145947-bcb3869e6f29/go.mod h1:F+5wygcW0wmRTnM3cOgIqGivxkwSWIWT5YdsDbeAOaU=
k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
sigs.k8s.io/structured-merge-diff/v2 v2.0.1/go.mod h1:Wb7vfKAodbKgf6tn1Kl0VvGj7mRH6DGaRcixXEJXTsE= sigs.k8s.io/structured-merge-diff/v2 v2.0.1/go.mod h1:Wb7vfKAodbKgf6tn1Kl0VvGj7mRH6DGaRcixXEJXTsE=
sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
......
...@@ -19,6 +19,7 @@ go_library( ...@@ -19,6 +19,7 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/cloud-provider/volume:go_default_library", "//staging/src/k8s.io/cloud-provider/volume:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
], ],
) )
...@@ -49,5 +50,7 @@ go_test( ...@@ -49,5 +50,7 @@ go_test(
deps = [ deps = [
"//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/api/storage/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
], ],
) )
...@@ -110,22 +110,23 @@ func (t *azureDiskCSITranslator) TranslateInTreePVToCSI(pv *v1.PersistentVolume) ...@@ -110,22 +110,23 @@ func (t *azureDiskCSITranslator) TranslateInTreePVToCSI(pv *v1.PersistentVolume)
return nil, fmt.Errorf("pv is nil or Azure Disk source not defined on pv") return nil, fmt.Errorf("pv is nil or Azure Disk source not defined on pv")
} }
azureSource := pv.Spec.PersistentVolumeSource.AzureDisk var (
azureSource = pv.Spec.PersistentVolumeSource.AzureDisk
// refer to https://github.com/kubernetes-sigs/azuredisk-csi-driver/blob/master/docs/driver-parameters.md // refer to https://github.com/kubernetes-sigs/azuredisk-csi-driver/blob/master/docs/driver-parameters.md
csiSource := &v1.CSIPersistentVolumeSource{ csiSource = &v1.CSIPersistentVolumeSource{
Driver: AzureDiskDriverName, Driver: AzureDiskDriverName,
VolumeHandle: azureSource.DataDiskURI,
ReadOnly: *azureSource.ReadOnly,
FSType: *azureSource.FSType,
VolumeAttributes: map[string]string{azureDiskKind: "Managed"}, VolumeAttributes: map[string]string{azureDiskKind: "Managed"},
VolumeHandle: azureSource.DataDiskURI,
} }
)
if azureSource.CachingMode != nil { if azureSource.CachingMode != nil {
csiSource.VolumeAttributes[azureDiskCachingMode] = string(*azureSource.CachingMode) csiSource.VolumeAttributes[azureDiskCachingMode] = string(*azureSource.CachingMode)
} }
if azureSource.FSType != nil { if azureSource.FSType != nil {
csiSource.FSType = *azureSource.FSType
csiSource.VolumeAttributes[azureDiskFSType] = *azureSource.FSType csiSource.VolumeAttributes[azureDiskFSType] = *azureSource.FSType
} }
...@@ -133,9 +134,12 @@ func (t *azureDiskCSITranslator) TranslateInTreePVToCSI(pv *v1.PersistentVolume) ...@@ -133,9 +134,12 @@ func (t *azureDiskCSITranslator) TranslateInTreePVToCSI(pv *v1.PersistentVolume)
csiSource.VolumeAttributes[azureDiskKind] = string(*azureSource.Kind) csiSource.VolumeAttributes[azureDiskKind] = string(*azureSource.Kind)
} }
if azureSource.ReadOnly != nil {
csiSource.ReadOnly = *azureSource.ReadOnly
}
pv.Spec.PersistentVolumeSource.AzureDisk = nil pv.Spec.PersistentVolumeSource.AzureDisk = nil
pv.Spec.PersistentVolumeSource.CSI = csiSource pv.Spec.PersistentVolumeSource.CSI = csiSource
pv.Spec.AccessModes = backwardCompatibleAccessModes(pv.Spec.AccessModes)
return pv, nil return pv, nil
} }
......
...@@ -18,11 +18,13 @@ package plugins ...@@ -18,11 +18,13 @@ package plugins
import ( import (
"fmt" "fmt"
"regexp"
"strings" "strings"
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
storage "k8s.io/api/storage/v1" storage "k8s.io/api/storage/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/klog"
) )
const ( const (
...@@ -32,14 +34,19 @@ const ( ...@@ -32,14 +34,19 @@ const (
AzureFileInTreePluginName = "kubernetes.io/azure-file" AzureFileInTreePluginName = "kubernetes.io/azure-file"
separator = "#" separator = "#"
volumeIDTemplate = "%s#%s#%s" volumeIDTemplate = "%s#%s#%s#%s"
// Parameter names defined in azure file CSI driver, refer to // Parameter names defined in azure file CSI driver, refer to
// https://github.com/kubernetes-sigs/azurefile-csi-driver/blob/master/docs/driver-parameters.md // https://github.com/kubernetes-sigs/azurefile-csi-driver/blob/master/docs/driver-parameters.md
azureFileShareName = "shareName" azureFileShareName = "shareName"
secretNameTemplate = "azure-storage-account-%s-secret"
defaultSecretNamespace = "default"
) )
var _ InTreePlugin = &azureFileCSITranslator{} var _ InTreePlugin = &azureFileCSITranslator{}
var secretNameFormatRE = regexp.MustCompile(`azure-storage-account-(.+)-secret`)
// azureFileCSITranslator handles translation of PV spec from In-tree // azureFileCSITranslator handles translation of PV spec from In-tree
// Azure File to CSI Azure File and vice versa // Azure File to CSI Azure File and vice versa
type azureFileCSITranslator struct{} type azureFileCSITranslator struct{}
...@@ -58,12 +65,18 @@ func (t *azureFileCSITranslator) TranslateInTreeStorageClassToCSI(sc *storage.St ...@@ -58,12 +65,18 @@ func (t *azureFileCSITranslator) TranslateInTreeStorageClassToCSI(sc *storage.St
// and converts the AzureFile source to a CSIPersistentVolumeSource // and converts the AzureFile source to a CSIPersistentVolumeSource
func (t *azureFileCSITranslator) TranslateInTreeInlineVolumeToCSI(volume *v1.Volume) (*v1.PersistentVolume, error) { func (t *azureFileCSITranslator) TranslateInTreeInlineVolumeToCSI(volume *v1.Volume) (*v1.PersistentVolume, error) {
if volume == nil || volume.AzureFile == nil { if volume == nil || volume.AzureFile == nil {
return nil, fmt.Errorf("volume is nil or AWS EBS not defined on volume") return nil, fmt.Errorf("volume is nil or Azure File not defined on volume")
} }
azureSource := volume.AzureFile azureSource := volume.AzureFile
accountName, err := getStorageAccountName(azureSource.SecretName)
if err != nil {
klog.Warningf("getStorageAccountName(%s) returned with error: %v", azureSource.SecretName, err)
accountName = azureSource.SecretName
}
pv := &v1.PersistentVolume{ var (
pv = &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{ ObjectMeta: metav1.ObjectMeta{
// Must be unique per disk as it is used as the unique part of the // Must be unique per disk as it is used as the unique part of the
// staging path // staging path
...@@ -72,18 +85,21 @@ func (t *azureFileCSITranslator) TranslateInTreeInlineVolumeToCSI(volume *v1.Vol ...@@ -72,18 +85,21 @@ func (t *azureFileCSITranslator) TranslateInTreeInlineVolumeToCSI(volume *v1.Vol
Spec: v1.PersistentVolumeSpec{ Spec: v1.PersistentVolumeSpec{
PersistentVolumeSource: v1.PersistentVolumeSource{ PersistentVolumeSource: v1.PersistentVolumeSource{
CSI: &v1.CSIPersistentVolumeSource{ CSI: &v1.CSIPersistentVolumeSource{
VolumeHandle: fmt.Sprintf(volumeIDTemplate, "", azureSource.SecretName, azureSource.ShareName), Driver: AzureFileDriverName,
VolumeHandle: fmt.Sprintf(volumeIDTemplate, "", accountName, azureSource.ShareName, ""),
ReadOnly: azureSource.ReadOnly, ReadOnly: azureSource.ReadOnly,
VolumeAttributes: map[string]string{azureFileShareName: azureSource.ShareName}, VolumeAttributes: map[string]string{azureFileShareName: azureSource.ShareName},
NodePublishSecretRef: &v1.SecretReference{ NodeStageSecretRef: &v1.SecretReference{
Name: azureSource.ShareName, Name: azureSource.SecretName,
Namespace: "default", Namespace: defaultSecretNamespace,
}, },
}, },
}, },
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteMany},
}, },
} }
)
return pv, nil return pv, nil
} }
...@@ -95,23 +111,33 @@ func (t *azureFileCSITranslator) TranslateInTreePVToCSI(pv *v1.PersistentVolume) ...@@ -95,23 +111,33 @@ func (t *azureFileCSITranslator) TranslateInTreePVToCSI(pv *v1.PersistentVolume)
} }
azureSource := pv.Spec.PersistentVolumeSource.AzureFile azureSource := pv.Spec.PersistentVolumeSource.AzureFile
accountName, err := getStorageAccountName(azureSource.SecretName)
if err != nil {
klog.Warningf("getStorageAccountName(%s) returned with error: %v", azureSource.SecretName, err)
accountName = azureSource.SecretName
}
volumeID := fmt.Sprintf(volumeIDTemplate, "", accountName, azureSource.ShareName, "")
volumeID := fmt.Sprintf(volumeIDTemplate, "", azureSource.SecretName, azureSource.ShareName) var (
// refer to https://github.com/kubernetes-sigs/azurefile-csi-driver/blob/master/docs/driver-parameters.md // refer to https://github.com/kubernetes-sigs/azurefile-csi-driver/blob/master/docs/driver-parameters.md
csiSource := &v1.CSIPersistentVolumeSource{ csiSource = &v1.CSIPersistentVolumeSource{
VolumeHandle: volumeID, Driver: AzureFileDriverName,
NodeStageSecretRef: &v1.SecretReference{
Name: azureSource.SecretName,
Namespace: defaultSecretNamespace,
},
ReadOnly: azureSource.ReadOnly, ReadOnly: azureSource.ReadOnly,
VolumeAttributes: map[string]string{azureFileShareName: azureSource.ShareName}, VolumeAttributes: map[string]string{azureFileShareName: azureSource.ShareName},
VolumeHandle: volumeID,
} }
)
csiSource.NodePublishSecretRef = &v1.SecretReference{ if azureSource.SecretNamespace != nil {
Name: azureSource.ShareName, csiSource.NodeStageSecretRef.Namespace = *azureSource.SecretNamespace
Namespace: *azureSource.SecretNamespace,
} }
pv.Spec.PersistentVolumeSource.AzureFile = nil pv.Spec.PersistentVolumeSource.AzureFile = nil
pv.Spec.PersistentVolumeSource.CSI = csiSource pv.Spec.PersistentVolumeSource.CSI = csiSource
pv.Spec.AccessModes = backwardCompatibleAccessModes(pv.Spec.AccessModes)
return pv, nil return pv, nil
} }
...@@ -129,22 +155,21 @@ func (t *azureFileCSITranslator) TranslateCSIPVToInTree(pv *v1.PersistentVolume) ...@@ -129,22 +155,21 @@ func (t *azureFileCSITranslator) TranslateCSIPVToInTree(pv *v1.PersistentVolume)
ReadOnly: csiSource.ReadOnly, ReadOnly: csiSource.ReadOnly,
} }
if csiSource.NodePublishSecretRef != nil && csiSource.NodePublishSecretRef.Name != "" { if csiSource.NodeStageSecretRef != nil && csiSource.NodeStageSecretRef.Name != "" {
azureSource.SecretName = csiSource.NodePublishSecretRef.Name azureSource.SecretName = csiSource.NodeStageSecretRef.Name
azureSource.SecretNamespace = &csiSource.NodePublishSecretRef.Namespace azureSource.SecretNamespace = &csiSource.NodeStageSecretRef.Namespace
if csiSource.VolumeAttributes != nil { if csiSource.VolumeAttributes != nil {
if shareName, ok := csiSource.VolumeAttributes[azureFileShareName]; ok { if shareName, ok := csiSource.VolumeAttributes[azureFileShareName]; ok {
azureSource.ShareName = shareName azureSource.ShareName = shareName
} }
} }
} else { } else {
_, _, fileShareName, err := getFileShareInfo(csiSource.VolumeHandle) _, storageAccount, fileShareName, _, err := getFileShareInfo(csiSource.VolumeHandle)
if err != nil { if err != nil {
return nil, err return nil, err
} }
azureSource.ShareName = fileShareName azureSource.ShareName = fileShareName
// to-do: for dynamic provision scenario in CSI, it uses cluster's identity to get storage account key azureSource.SecretName = fmt.Sprintf(secretNameTemplate, storageAccount)
// secret for the file share is not created, we may create a serect here
} }
pv.Spec.CSI = nil pv.Spec.CSI = nil
...@@ -182,12 +207,25 @@ func (t *azureFileCSITranslator) RepairVolumeHandle(volumeHandle, nodeID string) ...@@ -182,12 +207,25 @@ func (t *azureFileCSITranslator) RepairVolumeHandle(volumeHandle, nodeID string)
} }
// get file share info according to volume id, e.g. // get file share info according to volume id, e.g.
// input: "rg#f5713de20cde511e8ba4900#pvc-file-dynamic-17e43f84-f474-11e8-acd0-000d3a00df41" // input: "rg#f5713de20cde511e8ba4900#pvc-file-dynamic-17e43f84-f474-11e8-acd0-000d3a00df41#diskname.vhd"
// output: rg, f5713de20cde511e8ba4900, pvc-file-dynamic-17e43f84-f474-11e8-acd0-000d3a00df41 // output: rg, f5713de20cde511e8ba4900, pvc-file-dynamic-17e43f84-f474-11e8-acd0-000d3a00df41, diskname.vhd
func getFileShareInfo(id string) (string, string, string, error) { func getFileShareInfo(id string) (string, string, string, string, error) {
segments := strings.Split(id, separator) segments := strings.Split(id, separator)
if len(segments) < 3 { if len(segments) < 3 {
return "", "", "", fmt.Errorf("error parsing volume id: %q, should at least contain two #", id) return "", "", "", "", fmt.Errorf("error parsing volume id: %q, should at least contain two #", id)
}
var diskName string
if len(segments) > 3 {
diskName = segments[3]
}
return segments[0], segments[1], segments[2], diskName, nil
}
// get storage account name from secret name
func getStorageAccountName(secretName string) (string, error) {
matches := secretNameFormatRE.FindStringSubmatch(secretName)
if len(matches) != 2 {
return "", fmt.Errorf("could not get account name from %s, correct format: %s", secretName, secretNameFormatRE)
} }
return segments[0], segments[1], segments[2], nil return matches[1], nil
} }
...@@ -210,7 +210,7 @@ func (item *primitiveItem) VisitPrimitive(schema *proto.Primitive) { ...@@ -210,7 +210,7 @@ func (item *primitiveItem) VisitPrimitive(schema *proto.Primitive) {
} }
case proto.Number: case proto.Number:
switch item.Kind { switch item.Kind {
case proto.Number: case proto.Integer, proto.Number:
return return
} }
case proto.String: case proto.String:
......
This source diff could not be displayed because it is too large. You can view the blob instead.
...@@ -40,10 +40,12 @@ func cadvisorInfoToCPUandMemoryStats(info *cadvisorapiv2.ContainerInfo) (*statsa ...@@ -40,10 +40,12 @@ func cadvisorInfoToCPUandMemoryStats(info *cadvisorapiv2.ContainerInfo) (*statsa
} }
var cpuStats *statsapi.CPUStats var cpuStats *statsapi.CPUStats
var memoryStats *statsapi.MemoryStats var memoryStats *statsapi.MemoryStats
if info.Spec.HasCpu {
cpuStats = &statsapi.CPUStats{ cpuStats = &statsapi.CPUStats{
Time: metav1.NewTime(cstat.Timestamp), Time: metav1.NewTime(cstat.Timestamp),
UsageNanoCores: uint64Ptr(0),
UsageCoreNanoSeconds: uint64Ptr(0),
} }
if info.Spec.HasCpu {
if cstat.CpuInst != nil { if cstat.CpuInst != nil {
cpuStats.UsageNanoCores = &cstat.CpuInst.Usage.Total cpuStats.UsageNanoCores = &cstat.CpuInst.Usage.Total
} }
...@@ -67,6 +69,11 @@ func cadvisorInfoToCPUandMemoryStats(info *cadvisorapiv2.ContainerInfo) (*statsa ...@@ -67,6 +69,11 @@ func cadvisorInfoToCPUandMemoryStats(info *cadvisorapiv2.ContainerInfo) (*statsa
availableBytes := info.Spec.Memory.Limit - cstat.Memory.WorkingSet availableBytes := info.Spec.Memory.Limit - cstat.Memory.WorkingSet
memoryStats.AvailableBytes = &availableBytes memoryStats.AvailableBytes = &availableBytes
} }
} else {
memoryStats = &statsapi.MemoryStats{
Time: metav1.NewTime(cstat.Timestamp),
WorkingSetBytes: uint64Ptr(0),
}
} }
return cpuStats, memoryStats return cpuStats, memoryStats
} }
......
...@@ -94,7 +94,12 @@ var StatusStrategy = flowSchemaStatusStrategy{Strategy} ...@@ -94,7 +94,12 @@ var StatusStrategy = flowSchemaStatusStrategy{Strategy}
func (flowSchemaStatusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) { func (flowSchemaStatusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) {
newFlowSchema := obj.(*flowcontrol.FlowSchema) newFlowSchema := obj.(*flowcontrol.FlowSchema)
oldFlowSchema := old.(*flowcontrol.FlowSchema) oldFlowSchema := old.(*flowcontrol.FlowSchema)
// managedFields must be preserved since it's been modified to
// track changed fields in the status update.
managedFields := newFlowSchema.ManagedFields
newFlowSchema.ObjectMeta = oldFlowSchema.ObjectMeta newFlowSchema.ObjectMeta = oldFlowSchema.ObjectMeta
newFlowSchema.ManagedFields = managedFields
newFlowSchema.Spec = oldFlowSchema.Spec newFlowSchema.Spec = oldFlowSchema.Spec
} }
......
...@@ -94,7 +94,12 @@ var StatusStrategy = priorityLevelConfigurationStatusStrategy{Strategy} ...@@ -94,7 +94,12 @@ var StatusStrategy = priorityLevelConfigurationStatusStrategy{Strategy}
func (priorityLevelConfigurationStatusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) { func (priorityLevelConfigurationStatusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) {
newPriorityLevelConfiguration := obj.(*flowcontrol.PriorityLevelConfiguration) newPriorityLevelConfiguration := obj.(*flowcontrol.PriorityLevelConfiguration)
oldPriorityLevelConfiguration := old.(*flowcontrol.PriorityLevelConfiguration) oldPriorityLevelConfiguration := old.(*flowcontrol.PriorityLevelConfiguration)
// managedFields must be preserved since it's been modified to
// track changed fields in the status update.
managedFields := newPriorityLevelConfiguration.ManagedFields
newPriorityLevelConfiguration.ObjectMeta = oldPriorityLevelConfiguration.ObjectMeta newPriorityLevelConfiguration.ObjectMeta = oldPriorityLevelConfiguration.ObjectMeta
newPriorityLevelConfiguration.ManagedFields = managedFields
newPriorityLevelConfiguration.Spec = oldPriorityLevelConfiguration.Spec newPriorityLevelConfiguration.Spec = oldPriorityLevelConfiguration.Spec
} }
......
...@@ -124,6 +124,7 @@ func calculateResourceAllocatableRequest(nodeInfo *schedulernodeinfo.NodeInfo, p ...@@ -124,6 +124,7 @@ func calculateResourceAllocatableRequest(nodeInfo *schedulernodeinfo.NodeInfo, p
// calculatePodResourceRequest returns the total non-zero requests. If Overhead is defined for the pod and the // calculatePodResourceRequest returns the total non-zero requests. If Overhead is defined for the pod and the
// PodOverhead feature is enabled, the Overhead is added to the result. // PodOverhead feature is enabled, the Overhead is added to the result.
// podResourceRequest = max(sum(podSpec.Containers), podSpec.InitContainers) + overHead
func calculatePodResourceRequest(pod *v1.Pod, resource v1.ResourceName) int64 { func calculatePodResourceRequest(pod *v1.Pod, resource v1.ResourceName) int64 {
var podRequest int64 var podRequest int64
for i := range pod.Spec.Containers { for i := range pod.Spec.Containers {
...@@ -132,11 +133,20 @@ func calculatePodResourceRequest(pod *v1.Pod, resource v1.ResourceName) int64 { ...@@ -132,11 +133,20 @@ func calculatePodResourceRequest(pod *v1.Pod, resource v1.ResourceName) int64 {
podRequest += value podRequest += value
} }
for i := range pod.Spec.InitContainers {
initContainer := &pod.Spec.InitContainers[i]
value := priorityutil.GetNonzeroRequestForResource(resource, &initContainer.Resources.Requests)
if podRequest < value {
podRequest = value
}
}
// If Overhead is being utilized, add to the total requests for the pod // If Overhead is being utilized, add to the total requests for the pod
if pod.Spec.Overhead != nil && utilfeature.DefaultFeatureGate.Enabled(features.PodOverhead) { if pod.Spec.Overhead != nil && utilfeature.DefaultFeatureGate.Enabled(features.PodOverhead) {
if quantity, found := pod.Spec.Overhead[resource]; found { if quantity, found := pod.Spec.Overhead[resource]; found {
podRequest += quantity.Value() podRequest += quantity.Value()
} }
} }
return podRequest return podRequest
} }
...@@ -555,6 +555,7 @@ func (n *NodeInfo) RemovePod(pod *v1.Pod) error { ...@@ -555,6 +555,7 @@ func (n *NodeInfo) RemovePod(pod *v1.Pod) error {
return fmt.Errorf("no corresponding pod %s in pods of node %s", pod.Name, n.node.Name) return fmt.Errorf("no corresponding pod %s in pods of node %s", pod.Name, n.node.Name)
} }
// resourceRequest = max(sum(podSpec.Containers), podSpec.InitContainers) + overHead
func calculateResource(pod *v1.Pod) (res Resource, non0CPU int64, non0Mem int64) { func calculateResource(pod *v1.Pod) (res Resource, non0CPU int64, non0Mem int64) {
resPtr := &res resPtr := &res
for _, c := range pod.Spec.Containers { for _, c := range pod.Spec.Containers {
...@@ -566,6 +567,18 @@ func calculateResource(pod *v1.Pod) (res Resource, non0CPU int64, non0Mem int64) ...@@ -566,6 +567,18 @@ func calculateResource(pod *v1.Pod) (res Resource, non0CPU int64, non0Mem int64)
// No non-zero resources for GPUs or opaque resources. // No non-zero resources for GPUs or opaque resources.
} }
for _, ic := range pod.Spec.InitContainers {
resPtr.SetMaxResource(ic.Resources.Requests)
non0CPUReq, non0MemReq := priorityutil.GetNonzeroRequests(&ic.Resources.Requests)
if non0CPU < non0CPUReq {
non0CPU = non0CPUReq
}
if non0Mem < non0MemReq {
non0Mem = non0MemReq
}
}
// If Overhead is being utilized, add to the total requests for the pod // If Overhead is being utilized, add to the total requests for the pod
if pod.Spec.Overhead != nil && utilfeature.DefaultFeatureGate.Enabled(features.PodOverhead) { if pod.Spec.Overhead != nil && utilfeature.DefaultFeatureGate.Enabled(features.PodOverhead) {
resPtr.Add(pod.Spec.Overhead) resPtr.Add(pod.Spec.Overhead)
......
...@@ -635,6 +635,9 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) { ...@@ -635,6 +635,9 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) {
// succeeds, the pod should get counted as a success the next time we try to // succeeds, the pod should get counted as a success the next time we try to
// schedule it. (hopefully) // schedule it. (hopefully)
metrics.PodScheduleFailures.Inc() metrics.PodScheduleFailures.Inc()
} else if err == core.ErrNoNodesAvailable {
// No nodes available is counted as unschedulable rather than an error.
metrics.PodScheduleFailures.Inc()
} else { } else {
klog.Errorf("error selecting node for pod: %v", err) klog.Errorf("error selecting node for pod: %v", err)
metrics.PodScheduleErrors.Inc() metrics.PodScheduleErrors.Inc()
......
...@@ -45,9 +45,12 @@ var maxDataDiskCountMap = map[string]int64{ ...@@ -45,9 +45,12 @@ var maxDataDiskCountMap = map[string]int64{
"STANDARD_A8M_V2": 16, "STANDARD_A8M_V2": 16,
"STANDARD_A8_V2": 16, "STANDARD_A8_V2": 16,
"STANDARD_A9": 64, "STANDARD_A9": 64,
"STANDARD_B12MS": 16,
"STANDARD_B16MS": 32,
"STANDARD_B1LS": 2, "STANDARD_B1LS": 2,
"STANDARD_B1MS": 2, "STANDARD_B1MS": 2,
"STANDARD_B1S": 2, "STANDARD_B1S": 2,
"STANDARD_B20MS": 32,
"STANDARD_B2MS": 4, "STANDARD_B2MS": 4,
"STANDARD_B2S": 4, "STANDARD_B2S": 4,
"STANDARD_B4MS": 8, "STANDARD_B4MS": 8,
...@@ -66,32 +69,54 @@ var maxDataDiskCountMap = map[string]int64{ ...@@ -66,32 +69,54 @@ var maxDataDiskCountMap = map[string]int64{
"STANDARD_D14_V2": 64, "STANDARD_D14_V2": 64,
"STANDARD_D14_V2_PROMO": 64, "STANDARD_D14_V2_PROMO": 64,
"STANDARD_D15_V2": 64, "STANDARD_D15_V2": 64,
"STANDARD_D16AS_V4": 32,
"STANDARD_D16A_V4": 32,
"STANDARD_D16S_V3": 32, "STANDARD_D16S_V3": 32,
"STANDARD_D16_V3": 32, "STANDARD_D16_V3": 32,
"STANDARD_D1_V2": 4, "STANDARD_D1_V2": 4,
"STANDARD_D2": 8, "STANDARD_D2": 8,
"STANDARD_D2AS_V4": 4,
"STANDARD_D2A_V4": 4,
"STANDARD_D2S_V3": 4, "STANDARD_D2S_V3": 4,
"STANDARD_D2_V2": 8, "STANDARD_D2_V2": 8,
"STANDARD_D2_V2_PROMO": 8, "STANDARD_D2_V2_PROMO": 8,
"STANDARD_D2_V3": 4, "STANDARD_D2_V3": 4,
"STANDARD_D3": 16, "STANDARD_D3": 16,
"STANDARD_D32AS_V4": 32,
"STANDARD_D32A_V4": 32,
"STANDARD_D32S_V3": 32, "STANDARD_D32S_V3": 32,
"STANDARD_D32_V3": 32, "STANDARD_D32_V3": 32,
"STANDARD_D3_V2": 16, "STANDARD_D3_V2": 16,
"STANDARD_D3_V2_PROMO": 16, "STANDARD_D3_V2_PROMO": 16,
"STANDARD_D4": 32, "STANDARD_D4": 32,
"STANDARD_D48AS_V4": 32,
"STANDARD_D48A_V4": 32,
"STANDARD_D48S_V3": 32,
"STANDARD_D48_V3": 32,
"STANDARD_D4AS_V4": 8,
"STANDARD_D4A_V4": 8,
"STANDARD_D4S_V3": 8, "STANDARD_D4S_V3": 8,
"STANDARD_D4_V2": 32, "STANDARD_D4_V2": 32,
"STANDARD_D4_V2_PROMO": 32, "STANDARD_D4_V2_PROMO": 32,
"STANDARD_D4_V3": 8, "STANDARD_D4_V3": 8,
"STANDARD_D5_V2": 64, "STANDARD_D5_V2": 64,
"STANDARD_D5_V2_PROMO": 64, "STANDARD_D5_V2_PROMO": 64,
"STANDARD_D64AS_V4": 32,
"STANDARD_D64A_V4": 32,
"STANDARD_D64S_V3": 32, "STANDARD_D64S_V3": 32,
"STANDARD_D64_V3": 32, "STANDARD_D64_V3": 32,
"STANDARD_D8AS_V4": 16,
"STANDARD_D8A_V4": 16,
"STANDARD_D8S_V3": 16, "STANDARD_D8S_V3": 16,
"STANDARD_D8_V3": 16, "STANDARD_D8_V3": 16,
"STANDARD_D96AS_V4": 32,
"STANDARD_D96A_V4": 32,
"STANDARD_DC1S_V2": 1,
"STANDARD_DC2S": 2, "STANDARD_DC2S": 2,
"STANDARD_DC2S_V2": 2,
"STANDARD_DC4S": 4, "STANDARD_DC4S": 4,
"STANDARD_DC4S_V2": 4,
"STANDARD_DC8_V2": 8,
"STANDARD_DS11-1_V2": 8, "STANDARD_DS11-1_V2": 8,
"STANDARD_DS11": 8, "STANDARD_DS11": 8,
"STANDARD_DS11_V2": 8, "STANDARD_DS11_V2": 8,
...@@ -127,29 +152,49 @@ var maxDataDiskCountMap = map[string]int64{ ...@@ -127,29 +152,49 @@ var maxDataDiskCountMap = map[string]int64{
"STANDARD_DS5_V2_PROMO": 64, "STANDARD_DS5_V2_PROMO": 64,
"STANDARD_E16-4S_V3": 32, "STANDARD_E16-4S_V3": 32,
"STANDARD_E16-8S_V3": 32, "STANDARD_E16-8S_V3": 32,
"STANDARD_E16AS_V4": 32,
"STANDARD_E16A_V4": 32,
"STANDARD_E16S_V3": 32, "STANDARD_E16S_V3": 32,
"STANDARD_E16_V3": 32, "STANDARD_E16_V3": 32,
"STANDARD_E20AS_V4": 32,
"STANDARD_E20A_V4": 32,
"STANDARD_E20S_V3": 32, "STANDARD_E20S_V3": 32,
"STANDARD_E20_V3": 32, "STANDARD_E20_V3": 32,
"STANDARD_E2AS_V4": 4,
"STANDARD_E2A_V4": 4,
"STANDARD_E2S_V3": 4, "STANDARD_E2S_V3": 4,
"STANDARD_E2_V3": 4, "STANDARD_E2_V3": 4,
"STANDARD_E32-16S_V3": 32, "STANDARD_E32-16S_V3": 32,
"STANDARD_E32-8S_V3": 32, "STANDARD_E32-8S_V3": 32,
"STANDARD_E32AS_V4": 32,
"STANDARD_E32A_V4": 32,
"STANDARD_E32S_V3": 32, "STANDARD_E32S_V3": 32,
"STANDARD_E32_V3": 32, "STANDARD_E32_V3": 32,
"STANDARD_E4-2S_V3": 8, "STANDARD_E4-2S_V3": 8,
"STANDARD_E48AS_V4": 32,
"STANDARD_E48A_V4": 32,
"STANDARD_E48S_V3": 32,
"STANDARD_E48_V3": 32,
"STANDARD_E4AS_V4": 8,
"STANDARD_E4A_V4": 8,
"STANDARD_E4S_V3": 8, "STANDARD_E4S_V3": 8,
"STANDARD_E4_V3": 8, "STANDARD_E4_V3": 8,
"STANDARD_E64-16S_V3": 32, "STANDARD_E64-16S_V3": 32,
"STANDARD_E64-32S_V3": 32, "STANDARD_E64-32S_V3": 32,
"STANDARD_E64AS_V4": 32,
"STANDARD_E64A_V4": 32,
"STANDARD_E64IS_V3": 32, "STANDARD_E64IS_V3": 32,
"STANDARD_E64I_V3": 32, "STANDARD_E64I_V3": 32,
"STANDARD_E64S_V3": 32, "STANDARD_E64S_V3": 32,
"STANDARD_E64_V3": 32, "STANDARD_E64_V3": 32,
"STANDARD_E8-2S_V3": 16, "STANDARD_E8-2S_V3": 16,
"STANDARD_E8-4S_V3": 16, "STANDARD_E8-4S_V3": 16,
"STANDARD_E8AS_V4": 16,
"STANDARD_E8A_V4": 16,
"STANDARD_E8S_V3": 16, "STANDARD_E8S_V3": 16,
"STANDARD_E8_V3": 16, "STANDARD_E8_V3": 16,
"STANDARD_E96AS_V4": 32,
"STANDARD_E96A_V4": 32,
"STANDARD_F1": 4, "STANDARD_F1": 4,
"STANDARD_F16": 64, "STANDARD_F16": 64,
"STANDARD_F16S": 64, "STANDARD_F16S": 64,
...@@ -160,6 +205,7 @@ var maxDataDiskCountMap = map[string]int64{ ...@@ -160,6 +205,7 @@ var maxDataDiskCountMap = map[string]int64{
"STANDARD_F2S_V2": 4, "STANDARD_F2S_V2": 4,
"STANDARD_F32S_V2": 32, "STANDARD_F32S_V2": 32,
"STANDARD_F4": 16, "STANDARD_F4": 16,
"STANDARD_F48S_V2": 32,
"STANDARD_F4S": 16, "STANDARD_F4S": 16,
"STANDARD_F4S_V2": 8, "STANDARD_F4S_V2": 8,
"STANDARD_F64S_V2": 32, "STANDARD_F64S_V2": 32,
...@@ -193,12 +239,14 @@ var maxDataDiskCountMap = map[string]int64{ ...@@ -193,12 +239,14 @@ var maxDataDiskCountMap = map[string]int64{
"STANDARD_H8M": 32, "STANDARD_H8M": 32,
"STANDARD_H8M_PROMO": 32, "STANDARD_H8M_PROMO": 32,
"STANDARD_H8_PROMO": 32, "STANDARD_H8_PROMO": 32,
"STANDARD_HB120RS_V2": 8,
"STANDARD_HB60RS": 4, "STANDARD_HB60RS": 4,
"STANDARD_HC44RS": 4, "STANDARD_HC44RS": 4,
"STANDARD_L16S": 64, "STANDARD_L16S": 64,
"STANDARD_L16S_V2": 32, "STANDARD_L16S_V2": 32,
"STANDARD_L32S": 64, "STANDARD_L32S": 64,
"STANDARD_L32S_V2": 32, "STANDARD_L32S_V2": 32,
"STANDARD_L48S_V2": 32,
"STANDARD_L4S": 16, "STANDARD_L4S": 16,
"STANDARD_L64S_V2": 32, "STANDARD_L64S_V2": 32,
"STANDARD_L80S_V2": 32, "STANDARD_L80S_V2": 32,
...@@ -220,6 +268,8 @@ var maxDataDiskCountMap = map[string]int64{ ...@@ -220,6 +268,8 @@ var maxDataDiskCountMap = map[string]int64{
"STANDARD_M32LS": 32, "STANDARD_M32LS": 32,
"STANDARD_M32MS": 32, "STANDARD_M32MS": 32,
"STANDARD_M32TS": 32, "STANDARD_M32TS": 32,
"STANDARD_M416MS_V2": 64,
"STANDARD_M416S_V2": 64,
"STANDARD_M64-16MS": 64, "STANDARD_M64-16MS": 64,
"STANDARD_M64-32MS": 64, "STANDARD_M64-32MS": 64,
"STANDARD_M64": 64, "STANDARD_M64": 64,
...@@ -249,18 +299,27 @@ var maxDataDiskCountMap = map[string]int64{ ...@@ -249,18 +299,27 @@ var maxDataDiskCountMap = map[string]int64{
"STANDARD_ND12S": 24, "STANDARD_ND12S": 24,
"STANDARD_ND24RS": 32, "STANDARD_ND24RS": 32,
"STANDARD_ND24S": 32, "STANDARD_ND24S": 32,
"STANDARD_ND40RS_V2": 8,
"STANDARD_ND40S_V3": 32,
"STANDARD_ND6S": 12, "STANDARD_ND6S": 12,
"STANDARD_NP10S": 8,
"STANDARD_NP20S": 16,
"STANDARD_NP40S": 32,
"STANDARD_NV12": 48, "STANDARD_NV12": 48,
"STANDARD_NV12_PROMO": 48, "STANDARD_NV12_PROMO": 48,
"STANDARD_NV12S_V2": 24, "STANDARD_NV12S_V2": 24,
"STANDARD_NV12S_V3": 12, "STANDARD_NV12S_V3": 12,
"STANDARD_NV16AS_V4": 32,
"STANDARD_NV24": 64, "STANDARD_NV24": 64,
"STANDARD_NV24_PROMO": 64, "STANDARD_NV24_PROMO": 64,
"STANDARD_NV24S_V2": 32, "STANDARD_NV24S_V2": 32,
"STANDARD_NV24S_V3": 24, "STANDARD_NV24S_V3": 24,
"STANDARD_NV32AS_V4": 32,
"STANDARD_NV48S_V3": 32, "STANDARD_NV48S_V3": 32,
"STANDARD_NV4AS_V4": 8,
"STANDARD_NV6": 24, "STANDARD_NV6": 24,
"STANDARD_NV6_PROMO": 24, "STANDARD_NV6_PROMO": 24,
"STANDARD_NV6S_V2": 12, "STANDARD_NV6S_V2": 12,
"STANDARD_NV8AS_V4": 16,
"STANDARD_PB6S": 12, "STANDARD_PB6S": 12,
} }
...@@ -17,6 +17,7 @@ limitations under the License. ...@@ -17,6 +17,7 @@ limitations under the License.
package csi package csi
import ( import (
"context"
"errors" "errors"
"fmt" "fmt"
"os" "os"
...@@ -24,8 +25,6 @@ import ( ...@@ -24,8 +25,6 @@ import (
"strings" "strings"
"time" "time"
"context"
"k8s.io/klog" "k8s.io/klog"
api "k8s.io/api/core/v1" api "k8s.io/api/core/v1"
...@@ -229,7 +228,7 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error { ...@@ -229,7 +228,7 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error {
if utilfeature.DefaultFeatureGate.Enabled(features.CSINodeInfo) && if utilfeature.DefaultFeatureGate.Enabled(features.CSINodeInfo) &&
utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) { utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) {
// This function prevents Kubelet from posting Ready status until CSINodeInfo // This function prevents Kubelet from posting Ready status until CSINode
// is both installed and initialized // is both installed and initialized
if err := initializeCSINode(host, localNim); err != nil { if err := initializeCSINode(host, localNim); err != nil {
return errors.New(log("failed to initialize CSINodeInfo: %v", err)) return errors.New(log("failed to initialize CSINodeInfo: %v", err))
...@@ -246,21 +245,28 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error { ...@@ -246,21 +245,28 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error {
func initializeCSINode(host volume.VolumeHost, nim nodeinfomanager.Interface) error { func initializeCSINode(host volume.VolumeHost, nim nodeinfomanager.Interface) error {
kvh, ok := host.(volume.KubeletVolumeHost) kvh, ok := host.(volume.KubeletVolumeHost)
if !ok { if !ok {
klog.V(4).Info("Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINodeInfo initialization, not running on kubelet") klog.V(4).Info("Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINode initialization, not running on kubelet")
return nil return nil
} }
kubeClient := host.GetKubeClient() kubeClient := host.GetKubeClient()
if kubeClient == nil { if kubeClient == nil {
// Kubelet running in standalone mode. Skip CSINodeInfo initialization // Kubelet running in standalone mode. Skip CSINode initialization
klog.Warning("Skipping CSINodeInfo initialization, kubelet running in standalone mode") klog.Warning("Skipping CSINode initialization, kubelet running in standalone mode")
return nil return nil
} }
kvh.SetKubeletError(errors.New("CSINodeInfo is not yet initialized")) kvh.SetKubeletError(errors.New("CSINode is not yet initialized"))
go func() { go func() {
defer utilruntime.HandleCrash() defer utilruntime.HandleCrash()
// First wait indefinitely to talk to Kube APIServer
nodeName := host.GetNodeName()
err := waitForAPIServerForever(kubeClient, nodeName)
if err != nil {
klog.Fatalf("Failed to initialize CSINode while waiting for API server to report ok: %v", err)
}
// Backoff parameters tuned to retry over 140 seconds. Will fail and restart the Kubelet // Backoff parameters tuned to retry over 140 seconds. Will fail and restart the Kubelet
// after max retry steps. // after max retry steps.
initBackoff := wait.Backoff{ initBackoff := wait.Backoff{
...@@ -269,12 +275,12 @@ func initializeCSINode(host volume.VolumeHost, nim nodeinfomanager.Interface) er ...@@ -269,12 +275,12 @@ func initializeCSINode(host volume.VolumeHost, nim nodeinfomanager.Interface) er
Factor: 6.0, Factor: 6.0,
Jitter: 0.1, Jitter: 0.1,
} }
err := wait.ExponentialBackoff(initBackoff, func() (bool, error) { err = wait.ExponentialBackoff(initBackoff, func() (bool, error) {
klog.V(4).Infof("Initializing migrated drivers on CSINodeInfo") klog.V(4).Infof("Initializing migrated drivers on CSINode")
err := nim.InitializeCSINodeWithAnnotation() err := nim.InitializeCSINodeWithAnnotation()
if err != nil { if err != nil {
kvh.SetKubeletError(fmt.Errorf("Failed to initialize CSINodeInfo: %v", err)) kvh.SetKubeletError(fmt.Errorf("Failed to initialize CSINode: %v", err))
klog.Errorf("Failed to initialize CSINodeInfo: %v", err) klog.Errorf("Failed to initialize CSINode: %v", err)
return false, nil return false, nil
} }
...@@ -288,7 +294,7 @@ func initializeCSINode(host volume.VolumeHost, nim nodeinfomanager.Interface) er ...@@ -288,7 +294,7 @@ func initializeCSINode(host volume.VolumeHost, nim nodeinfomanager.Interface) er
// using CSI for all Migrated volume plugins. Then all the CSINode initialization // using CSI for all Migrated volume plugins. Then all the CSINode initialization
// code can be dropped from Kubelet. // code can be dropped from Kubelet.
// Kill the Kubelet process and allow it to restart to retry initialization // Kill the Kubelet process and allow it to restart to retry initialization
klog.Fatalf("Failed to initialize CSINodeInfo after retrying") klog.Fatalf("Failed to initialize CSINode after retrying: %v", err)
} }
}() }()
return nil return nil
...@@ -933,3 +939,28 @@ func isV0Version(version string) bool { ...@@ -933,3 +939,28 @@ func isV0Version(version string) bool {
return parsedVersion.Major() == 0 return parsedVersion.Major() == 0
} }
// waitForAPIServerForever waits forever to get a CSINode instance as a proxy
// for a healthy APIServer
func waitForAPIServerForever(client clientset.Interface, nodeName types.NodeName) error {
var lastErr error
err := wait.PollImmediateInfinite(time.Second, func() (bool, error) {
// Get a CSINode from API server to make sure 1) kubelet can reach API server
// and 2) it has enough permissions. Kubelet may have restricted permissions
// when it's bootstrapping TLS.
// https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/
_, lastErr = client.StorageV1().CSINodes().Get(string(nodeName), meta.GetOptions{})
if lastErr == nil || apierrs.IsNotFound(lastErr) {
// API server contacted
return true, nil
}
klog.V(2).Infof("Failed to contact API server when waiting for CSINode publishing: %s", lastErr)
return false, nil
})
if err != nil {
// In theory this is unreachable, but just in case:
return fmt.Errorf("%v: %v", err, lastErr)
}
return nil
}
...@@ -396,16 +396,16 @@ func (nim *nodeInfoManager) InitializeCSINodeWithAnnotation() error { ...@@ -396,16 +396,16 @@ func (nim *nodeInfoManager) InitializeCSINodeWithAnnotation() error {
return goerrors.New("error getting CSI client") return goerrors.New("error getting CSI client")
} }
var updateErrs []error var lastErr error
err := wait.ExponentialBackoff(updateBackoff, func() (bool, error) { err := wait.ExponentialBackoff(updateBackoff, func() (bool, error) {
if err := nim.tryInitializeCSINodeWithAnnotation(csiKubeClient); err != nil { if lastErr = nim.tryInitializeCSINodeWithAnnotation(csiKubeClient); lastErr != nil {
updateErrs = append(updateErrs, err) klog.V(2).Infof("Failed to publish CSINode: %v", lastErr)
return false, nil return false, nil
} }
return true, nil return true, nil
}) })
if err != nil { if err != nil {
return fmt.Errorf("error updating CSINode annotation: %v; caused by: %v", err, utilerrors.NewAggregate(updateErrs)) return fmt.Errorf("error updating CSINode annotation: %v; caused by: %v", err, lastErr)
} }
return nil return nil
......
...@@ -2105,17 +2105,18 @@ func (c *Cloud) applyUnSchedulableTaint(nodeName types.NodeName, reason string) ...@@ -2105,17 +2105,18 @@ func (c *Cloud) applyUnSchedulableTaint(nodeName types.NodeName, reason string)
// waitForAttachmentStatus polls until the attachment status is the expected value // waitForAttachmentStatus polls until the attachment status is the expected value
// On success, it returns the last attachment state. // On success, it returns the last attachment state.
func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment, error) { func (d *awsDisk) waitForAttachmentStatus(status string, expectedInstance, expectedDevice string) (*ec2.VolumeAttachment, error) {
backoff := wait.Backoff{ backoff := wait.Backoff{
Duration: volumeAttachmentStatusPollDelay, Duration: volumeAttachmentStatusPollDelay,
Factor: volumeAttachmentStatusFactor, Factor: volumeAttachmentStatusFactor,
Steps: volumeAttachmentStatusSteps, Steps: volumeAttachmentStatusSteps,
} }
// Because of rate limiting, we often see errors from describeVolume // Because of rate limiting, we often see errors from describeVolume.
// Or AWS eventual consistency returns unexpected data.
// So we tolerate a limited number of failures. // So we tolerate a limited number of failures.
// But once we see more than 10 errors in a row, we return the error // But once we see more than 10 errors in a row, we return the error.
describeErrorCount := 0 errorCount := 0
// Attach/detach usually takes time. It does not make sense to start // Attach/detach usually takes time. It does not make sense to start
// polling DescribeVolumes before some initial delay to let AWS // polling DescribeVolumes before some initial delay to let AWS
...@@ -2144,8 +2145,8 @@ func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment, ...@@ -2144,8 +2145,8 @@ func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment,
return false, err return false, err
} }
} }
describeErrorCount++ errorCount++
if describeErrorCount > volumeAttachmentStatusConsecutiveErrorLimit { if errorCount > volumeAttachmentStatusConsecutiveErrorLimit {
// report the error // report the error
return false, err return false, err
} }
...@@ -2154,8 +2155,6 @@ func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment, ...@@ -2154,8 +2155,6 @@ func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment,
return false, nil return false, nil
} }
describeErrorCount = 0
if len(info.Attachments) > 1 { if len(info.Attachments) > 1 {
// Shouldn't happen; log so we know if it is // Shouldn't happen; log so we know if it is
klog.Warningf("Found multiple attachments for volume %q: %v", d.awsID, info) klog.Warningf("Found multiple attachments for volume %q: %v", d.awsID, info)
...@@ -2177,11 +2176,39 @@ func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment, ...@@ -2177,11 +2176,39 @@ func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment,
if attachmentStatus == "" { if attachmentStatus == "" {
attachmentStatus = "detached" attachmentStatus = "detached"
} }
if attachment != nil {
// AWS eventual consistency can go back in time.
// For example, we're waiting for a volume to be attached as /dev/xvdba, but AWS can tell us it's
// attached as /dev/xvdbb, where it was attached before and it was already detached.
// Retry couple of times, hoping AWS starts reporting the right status.
device := aws.StringValue(attachment.Device)
if expectedDevice != "" && device != "" && device != expectedDevice {
klog.Warningf("Expected device %s %s for volume %s, but found device %s %s", expectedDevice, status, d.name, device, attachmentStatus)
errorCount++
if errorCount > volumeAttachmentStatusConsecutiveErrorLimit {
// report the error
return false, fmt.Errorf("attachment of disk %q failed: requested device %q but found %q", d.name, expectedDevice, device)
}
return false, nil
}
instanceID := aws.StringValue(attachment.InstanceId)
if expectedInstance != "" && instanceID != "" && instanceID != expectedInstance {
klog.Warningf("Expected instance %s/%s for volume %s, but found instance %s/%s", expectedInstance, status, d.name, instanceID, attachmentStatus)
errorCount++
if errorCount > volumeAttachmentStatusConsecutiveErrorLimit {
// report the error
return false, fmt.Errorf("attachment of disk %q failed: requested device %q but found %q", d.name, expectedDevice, device)
}
return false, nil
}
}
if attachmentStatus == status { if attachmentStatus == status {
// Attachment is in requested state, finish waiting // Attachment is in requested state, finish waiting
return true, nil return true, nil
} }
// continue waiting // continue waiting
errorCount = 0
klog.V(2).Infof("Waiting for volume %q state: actual=%s, desired=%s", d.awsID, attachmentStatus, status) klog.V(2).Infof("Waiting for volume %q state: actual=%s, desired=%s", d.awsID, attachmentStatus, status)
return false, nil return false, nil
}) })
...@@ -2321,7 +2348,7 @@ func (c *Cloud) AttachDisk(diskName KubernetesVolumeID, nodeName types.NodeName) ...@@ -2321,7 +2348,7 @@ func (c *Cloud) AttachDisk(diskName KubernetesVolumeID, nodeName types.NodeName)
klog.V(2).Infof("AttachVolume volume=%q instance=%q request returned %v", disk.awsID, awsInstance.awsID, attachResponse) klog.V(2).Infof("AttachVolume volume=%q instance=%q request returned %v", disk.awsID, awsInstance.awsID, attachResponse)
} }
attachment, err := disk.waitForAttachmentStatus("attached") attachment, err := disk.waitForAttachmentStatus("attached", awsInstance.awsID, ec2Device)
if err != nil { if err != nil {
if err == wait.ErrWaitTimeout { if err == wait.ErrWaitTimeout {
...@@ -2341,6 +2368,7 @@ func (c *Cloud) AttachDisk(diskName KubernetesVolumeID, nodeName types.NodeName) ...@@ -2341,6 +2368,7 @@ func (c *Cloud) AttachDisk(diskName KubernetesVolumeID, nodeName types.NodeName)
return "", fmt.Errorf("unexpected state: attachment nil after attached %q to %q", diskName, nodeName) return "", fmt.Errorf("unexpected state: attachment nil after attached %q to %q", diskName, nodeName)
} }
if ec2Device != aws.StringValue(attachment.Device) { if ec2Device != aws.StringValue(attachment.Device) {
// Already checked in waitForAttachmentStatus(), but just to be sure...
return "", fmt.Errorf("disk attachment of %q to %q failed: requested device %q but found %q", diskName, nodeName, ec2Device, aws.StringValue(attachment.Device)) return "", fmt.Errorf("disk attachment of %q to %q failed: requested device %q but found %q", diskName, nodeName, ec2Device, aws.StringValue(attachment.Device))
} }
if awsInstance.awsID != aws.StringValue(attachment.InstanceId) { if awsInstance.awsID != aws.StringValue(attachment.InstanceId) {
...@@ -2398,7 +2426,7 @@ func (c *Cloud) DetachDisk(diskName KubernetesVolumeID, nodeName types.NodeName) ...@@ -2398,7 +2426,7 @@ func (c *Cloud) DetachDisk(diskName KubernetesVolumeID, nodeName types.NodeName)
return "", errors.New("no response from DetachVolume") return "", errors.New("no response from DetachVolume")
} }
attachment, err := diskInfo.disk.waitForAttachmentStatus("detached") attachment, err := diskInfo.disk.waitForAttachmentStatus("detached", awsInstance.awsID, "")
if err != nil { if err != nil {
return "", err return "", err
} }
......
...@@ -100,6 +100,11 @@ func (c *controllerCommon) getNodeVMSet(nodeName types.NodeName, crt cacheReadTy ...@@ -100,6 +100,11 @@ func (c *controllerCommon) getNodeVMSet(nodeName types.NodeName, crt cacheReadTy
// return (lun, error) // return (lun, error)
func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, cachingMode compute.CachingTypes) (int32, error) { func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, cachingMode compute.CachingTypes) (int32, error) {
diskEncryptionSetID := "" diskEncryptionSetID := ""
vmset, err := c.getNodeVMSet(nodeName, cacheReadTypeUnsafe)
if err != nil {
return -1, err
}
if isManagedDisk { if isManagedDisk {
diskName := path.Base(diskURI) diskName := path.Base(diskURI)
resourceGroup, err := getResourceGroupFromDiskURI(diskURI) resourceGroup, err := getResourceGroupFromDiskURI(diskURI)
...@@ -119,9 +124,12 @@ func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI stri ...@@ -119,9 +124,12 @@ func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI stri
attachErr := fmt.Sprintf( attachErr := fmt.Sprintf(
"disk(%s) already attached to node(%s), could not be attached to node(%s)", "disk(%s) already attached to node(%s), could not be attached to node(%s)",
diskURI, *disk.ManagedBy, nodeName) diskURI, *disk.ManagedBy, nodeName)
attachedNode := path.Base(*disk.ManagedBy) attachedNode, err := vmset.GetNodeNameByProviderID(*disk.ManagedBy)
if err != nil {
return -1, err
}
klog.V(2).Infof("found dangling volume %s attached to node %s", diskURI, attachedNode) klog.V(2).Infof("found dangling volume %s attached to node %s", diskURI, attachedNode)
danglingErr := volerr.NewDanglingError(attachErr, types.NodeName(attachedNode), "") danglingErr := volerr.NewDanglingError(attachErr, attachedNode, "")
return -1, danglingErr return -1, danglingErr
} }
...@@ -131,11 +139,6 @@ func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI stri ...@@ -131,11 +139,6 @@ func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI stri
} }
} }
vmset, err := c.getNodeVMSet(nodeName, cacheReadTypeUnsafe)
if err != nil {
return -1, err
}
instanceid, err := c.cloud.InstanceID(context.TODO(), nodeName) instanceid, err := c.cloud.InstanceID(context.TODO(), nodeName)
if err != nil { if err != nil {
klog.Warningf("failed to get azure instance id (%v) for node %s", err, nodeName) klog.Warningf("failed to get azure instance id (%v) for node %s", err, nodeName)
......
...@@ -420,7 +420,7 @@ func (az *Cloud) getServiceLoadBalancerStatus(service *v1.Service, lb *network.L ...@@ -420,7 +420,7 @@ func (az *Cloud) getServiceLoadBalancerStatus(service *v1.Service, lb *network.L
if pipID == nil { if pipID == nil {
return nil, fmt.Errorf("get(%s): lb(%s) - failed to get LB PublicIPAddress ID is Nil", serviceName, *lb.Name) return nil, fmt.Errorf("get(%s): lb(%s) - failed to get LB PublicIPAddress ID is Nil", serviceName, *lb.Name)
} }
pipName, err := getLastSegment(*pipID) pipName, err := getLastSegment(*pipID, "/")
if err != nil { if err != nil {
return nil, fmt.Errorf("get(%s): lb(%s) - failed to get LB PublicIPAddress Name from ID(%s)", serviceName, *lb.Name, *pipID) return nil, fmt.Errorf("get(%s): lb(%s) - failed to get LB PublicIPAddress Name from ID(%s)", serviceName, *lb.Name, *pipID)
} }
......
...@@ -69,7 +69,7 @@ const ( ...@@ -69,7 +69,7 @@ const (
) )
var errNotInVMSet = errors.New("vm is not in the vmset") var errNotInVMSet = errors.New("vm is not in the vmset")
var providerIDRE = regexp.MustCompile(`^` + CloudProviderName + `://(?:.*)/Microsoft.Compute/virtualMachines/(.+)$`) var providerIDRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/Microsoft.Compute/virtualMachines/(.+)$`)
var backendPoolIDRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/backendAddressPools/(?:.*)`) var backendPoolIDRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/backendAddressPools/(?:.*)`)
var nicResourceGroupRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Network/networkInterfaces/(?:.*)`) var nicResourceGroupRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Network/networkInterfaces/(?:.*)`)
var publicIPResourceGroupRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Network/publicIPAddresses/(?:.*)`) var publicIPResourceGroupRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Network/publicIPAddresses/(?:.*)`)
...@@ -161,8 +161,8 @@ func isMasterNode(node *v1.Node) bool { ...@@ -161,8 +161,8 @@ func isMasterNode(node *v1.Node) bool {
} }
// returns the deepest child's identifier from a full identifier string. // returns the deepest child's identifier from a full identifier string.
func getLastSegment(ID string) (string, error) { func getLastSegment(ID, separator string) (string, error) {
parts := strings.Split(ID, "/") parts := strings.Split(ID, separator)
name := parts[len(parts)-1] name := parts[len(parts)-1]
if len(name) == 0 { if len(name) == 0 {
return "", fmt.Errorf("resource name was missing from identifier") return "", fmt.Errorf("resource name was missing from identifier")
...@@ -497,7 +497,7 @@ func (as *availabilitySet) GetIPByNodeName(name string) (string, string, error) ...@@ -497,7 +497,7 @@ func (as *availabilitySet) GetIPByNodeName(name string) (string, string, error)
publicIP := "" publicIP := ""
if ipConfig.PublicIPAddress != nil && ipConfig.PublicIPAddress.ID != nil { if ipConfig.PublicIPAddress != nil && ipConfig.PublicIPAddress.ID != nil {
pipID := *ipConfig.PublicIPAddress.ID pipID := *ipConfig.PublicIPAddress.ID
pipName, err := getLastSegment(pipID) pipName, err := getLastSegment(pipID, "/")
if err != nil { if err != nil {
return "", "", fmt.Errorf("failed to publicIP name for node %q with pipID %q", name, pipID) return "", "", fmt.Errorf("failed to publicIP name for node %q with pipID %q", name, pipID)
} }
...@@ -567,7 +567,7 @@ func (as *availabilitySet) getAgentPoolAvailabiliySets(nodes []*v1.Node) (agentP ...@@ -567,7 +567,7 @@ func (as *availabilitySet) getAgentPoolAvailabiliySets(nodes []*v1.Node) (agentP
// already added in the list // already added in the list
continue continue
} }
asName, err := getLastSegment(asID) asName, err := getLastSegment(asID, "/")
if err != nil { if err != nil {
klog.Errorf("as.getNodeAvailabilitySet - Node (%s)- getLastSegment(%s), err=%v", nodeName, asID, err) klog.Errorf("as.getNodeAvailabilitySet - Node (%s)- getLastSegment(%s), err=%v", nodeName, asID, err)
return nil, err return nil, err
...@@ -658,7 +658,7 @@ func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName stri ...@@ -658,7 +658,7 @@ func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName stri
if err != nil { if err != nil {
return network.Interface{}, err return network.Interface{}, err
} }
nicName, err := getLastSegment(primaryNicID) nicName, err := getLastSegment(primaryNicID, "/")
if err != nil { if err != nil {
return network.Interface{}, err return network.Interface{}, err
} }
......
...@@ -186,6 +186,16 @@ func (ss *scaleSet) getVmssVM(nodeName string, crt cacheReadType) (string, strin ...@@ -186,6 +186,16 @@ func (ss *scaleSet) getVmssVM(nodeName string, crt cacheReadType) (string, strin
// GetPowerStatusByNodeName returns the power state of the specified node. // GetPowerStatusByNodeName returns the power state of the specified node.
func (ss *scaleSet) GetPowerStatusByNodeName(name string) (powerState string, err error) { func (ss *scaleSet) GetPowerStatusByNodeName(name string) (powerState string, err error) {
managedByAS, err := ss.isNodeManagedByAvailabilitySet(name, cacheReadTypeUnsafe)
if err != nil {
klog.Errorf("Failed to check isNodeManagedByAvailabilitySet: %v", err)
return "", err
}
if managedByAS {
// vm is managed by availability set.
return ss.availabilitySet.GetPowerStatusByNodeName(name)
}
_, _, vm, err := ss.getVmssVM(name, cacheReadTypeDefault) _, _, vm, err := ss.getVmssVM(name, cacheReadTypeDefault)
if err != nil { if err != nil {
return powerState, err return powerState, err
...@@ -282,6 +292,11 @@ func (ss *scaleSet) GetInstanceIDByNodeName(name string) (string, error) { ...@@ -282,6 +292,11 @@ func (ss *scaleSet) GetInstanceIDByNodeName(name string) (string, error) {
} }
// GetNodeNameByProviderID gets the node name by provider ID. // GetNodeNameByProviderID gets the node name by provider ID.
// providerID example:
// 1. vmas providerID: azure:///subscriptions/subsid/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/aks-nodepool1-27053986-0
// 2. vmss providerID:
// azure:///subscriptions/subsid/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/aks-agentpool-22126781-vmss/virtualMachines/1
// /subscriptions/subsid/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/aks-agentpool-22126781-vmss/virtualMachines/k8s-agentpool-36841236-vmss_1
func (ss *scaleSet) GetNodeNameByProviderID(providerID string) (types.NodeName, error) { func (ss *scaleSet) GetNodeNameByProviderID(providerID string) (types.NodeName, error) {
// NodeName is not part of providerID for vmss instances. // NodeName is not part of providerID for vmss instances.
scaleSetName, err := extractScaleSetNameByProviderID(providerID) scaleSetName, err := extractScaleSetNameByProviderID(providerID)
...@@ -295,12 +310,20 @@ func (ss *scaleSet) GetNodeNameByProviderID(providerID string) (types.NodeName, ...@@ -295,12 +310,20 @@ func (ss *scaleSet) GetNodeNameByProviderID(providerID string) (types.NodeName,
return "", fmt.Errorf("error of extracting resource group for node %q", providerID) return "", fmt.Errorf("error of extracting resource group for node %q", providerID)
} }
instanceID, err := getLastSegment(providerID) instanceID, err := getLastSegment(providerID, "/")
if err != nil { if err != nil {
klog.V(4).Infof("Can not extract instanceID from providerID (%s), assuming it is mananaged by availability set: %v", providerID, err) klog.V(4).Infof("Can not extract instanceID from providerID (%s), assuming it is mananaged by availability set: %v", providerID, err)
return ss.availabilitySet.GetNodeNameByProviderID(providerID) return ss.availabilitySet.GetNodeNameByProviderID(providerID)
} }
// instanceID contains scaleSetName (returned by disk.ManagedBy), e.g. k8s-agentpool-36841236-vmss_1
if strings.HasPrefix(strings.ToLower(instanceID), strings.ToLower(scaleSetName)) {
instanceID, err = getLastSegment(instanceID, "_")
if err != nil {
return "", err
}
}
vm, err := ss.getVmssVMByInstanceID(resourceGroup, scaleSetName, instanceID, cacheReadTypeUnsafe) vm, err := ss.getVmssVMByInstanceID(resourceGroup, scaleSetName, instanceID, cacheReadTypeUnsafe)
if err != nil { if err != nil {
return "", err return "", err
...@@ -695,7 +718,7 @@ func (ss *scaleSet) GetPrimaryInterface(nodeName string) (network.Interface, err ...@@ -695,7 +718,7 @@ func (ss *scaleSet) GetPrimaryInterface(nodeName string) (network.Interface, err
return network.Interface{}, err return network.Interface{}, err
} }
nicName, err := getLastSegment(primaryInterfaceID) nicName, err := getLastSegment(primaryInterfaceID, "/")
if err != nil { if err != nil {
klog.Errorf("error: ss.GetPrimaryInterface(%s), getLastSegment(%s), err=%v", nodeName, primaryInterfaceID, err) klog.Errorf("error: ss.GetPrimaryInterface(%s), getLastSegment(%s), err=%v", nodeName, primaryInterfaceID, err)
return network.Interface{}, err return network.Interface{}, err
......
...@@ -1132,7 +1132,7 @@ gopkg.in/square/go-jose.v2/jwt ...@@ -1132,7 +1132,7 @@ gopkg.in/square/go-jose.v2/jwt
gopkg.in/warnings.v0 gopkg.in/warnings.v0
# gopkg.in/yaml.v2 v2.2.8 # gopkg.in/yaml.v2 v2.2.8
gopkg.in/yaml.v2 gopkg.in/yaml.v2
# k8s.io/api v0.17.0 => github.com/rancher/kubernetes/staging/src/k8s.io/api v1.17.5-k3s1 # k8s.io/api v0.17.0 => github.com/rancher/kubernetes/staging/src/k8s.io/api v1.17.6-k3s1
k8s.io/api/admission/v1 k8s.io/api/admission/v1
k8s.io/api/admission/v1beta1 k8s.io/api/admission/v1beta1
k8s.io/api/admissionregistration/v1 k8s.io/api/admissionregistration/v1
...@@ -1176,7 +1176,7 @@ k8s.io/api/settings/v1alpha1 ...@@ -1176,7 +1176,7 @@ k8s.io/api/settings/v1alpha1
k8s.io/api/storage/v1 k8s.io/api/storage/v1
k8s.io/api/storage/v1alpha1 k8s.io/api/storage/v1alpha1
k8s.io/api/storage/v1beta1 k8s.io/api/storage/v1beta1
# k8s.io/apiextensions-apiserver v0.17.0 => github.com/rancher/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.17.5-k3s1 # k8s.io/apiextensions-apiserver v0.17.0 => github.com/rancher/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.17.6-k3s1
k8s.io/apiextensions-apiserver/pkg/apihelpers k8s.io/apiextensions-apiserver/pkg/apihelpers
k8s.io/apiextensions-apiserver/pkg/apis/apiextensions k8s.io/apiextensions-apiserver/pkg/apis/apiextensions
k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install
...@@ -1224,7 +1224,7 @@ k8s.io/apiextensions-apiserver/pkg/generated/openapi ...@@ -1224,7 +1224,7 @@ k8s.io/apiextensions-apiserver/pkg/generated/openapi
k8s.io/apiextensions-apiserver/pkg/registry/customresource k8s.io/apiextensions-apiserver/pkg/registry/customresource
k8s.io/apiextensions-apiserver/pkg/registry/customresource/tableconvertor k8s.io/apiextensions-apiserver/pkg/registry/customresource/tableconvertor
k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition
# k8s.io/apimachinery v0.17.0 => github.com/rancher/kubernetes/staging/src/k8s.io/apimachinery v1.17.5-k3s1 # k8s.io/apimachinery v0.17.0 => github.com/rancher/kubernetes/staging/src/k8s.io/apimachinery v1.17.6-k3s1
k8s.io/apimachinery/pkg/api/equality k8s.io/apimachinery/pkg/api/equality
k8s.io/apimachinery/pkg/api/errors k8s.io/apimachinery/pkg/api/errors
k8s.io/apimachinery/pkg/api/meta k8s.io/apimachinery/pkg/api/meta
...@@ -1286,7 +1286,7 @@ k8s.io/apimachinery/pkg/watch ...@@ -1286,7 +1286,7 @@ k8s.io/apimachinery/pkg/watch
k8s.io/apimachinery/third_party/forked/golang/json k8s.io/apimachinery/third_party/forked/golang/json
k8s.io/apimachinery/third_party/forked/golang/netutil k8s.io/apimachinery/third_party/forked/golang/netutil
k8s.io/apimachinery/third_party/forked/golang/reflect k8s.io/apimachinery/third_party/forked/golang/reflect
# k8s.io/apiserver v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/apiserver v1.17.5-k3s1 # k8s.io/apiserver v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/apiserver v1.17.6-k3s1
k8s.io/apiserver/pkg/admission k8s.io/apiserver/pkg/admission
k8s.io/apiserver/pkg/admission/configuration k8s.io/apiserver/pkg/admission/configuration
k8s.io/apiserver/pkg/admission/initializer k8s.io/apiserver/pkg/admission/initializer
...@@ -1403,7 +1403,7 @@ k8s.io/apiserver/plugin/pkg/authenticator/request/basicauth ...@@ -1403,7 +1403,7 @@ k8s.io/apiserver/plugin/pkg/authenticator/request/basicauth
k8s.io/apiserver/plugin/pkg/authenticator/token/oidc k8s.io/apiserver/plugin/pkg/authenticator/token/oidc
k8s.io/apiserver/plugin/pkg/authenticator/token/webhook k8s.io/apiserver/plugin/pkg/authenticator/token/webhook
k8s.io/apiserver/plugin/pkg/authorizer/webhook k8s.io/apiserver/plugin/pkg/authorizer/webhook
# k8s.io/cli-runtime v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/cli-runtime v1.17.5-k3s1 # k8s.io/cli-runtime v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/cli-runtime v1.17.6-k3s1
k8s.io/cli-runtime/pkg/genericclioptions k8s.io/cli-runtime/pkg/genericclioptions
k8s.io/cli-runtime/pkg/kustomize k8s.io/cli-runtime/pkg/kustomize
k8s.io/cli-runtime/pkg/kustomize/k8sdeps k8s.io/cli-runtime/pkg/kustomize/k8sdeps
...@@ -1416,7 +1416,7 @@ k8s.io/cli-runtime/pkg/kustomize/k8sdeps/transformer/patch ...@@ -1416,7 +1416,7 @@ k8s.io/cli-runtime/pkg/kustomize/k8sdeps/transformer/patch
k8s.io/cli-runtime/pkg/kustomize/k8sdeps/validator k8s.io/cli-runtime/pkg/kustomize/k8sdeps/validator
k8s.io/cli-runtime/pkg/printers k8s.io/cli-runtime/pkg/printers
k8s.io/cli-runtime/pkg/resource k8s.io/cli-runtime/pkg/resource
# k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible => github.com/rancher/kubernetes/staging/src/k8s.io/client-go v1.17.5-k3s1 # k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible => github.com/rancher/kubernetes/staging/src/k8s.io/client-go v1.17.6-k3s1
k8s.io/client-go/discovery k8s.io/client-go/discovery
k8s.io/client-go/discovery/cached k8s.io/client-go/discovery/cached
k8s.io/client-go/discovery/cached/disk k8s.io/client-go/discovery/cached/disk
...@@ -1609,20 +1609,20 @@ k8s.io/client-go/util/jsonpath ...@@ -1609,20 +1609,20 @@ k8s.io/client-go/util/jsonpath
k8s.io/client-go/util/keyutil k8s.io/client-go/util/keyutil
k8s.io/client-go/util/retry k8s.io/client-go/util/retry
k8s.io/client-go/util/workqueue k8s.io/client-go/util/workqueue
# k8s.io/cloud-provider v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/cloud-provider v1.17.5-k3s1 # k8s.io/cloud-provider v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/cloud-provider v1.17.6-k3s1
k8s.io/cloud-provider k8s.io/cloud-provider
k8s.io/cloud-provider/node/helpers k8s.io/cloud-provider/node/helpers
k8s.io/cloud-provider/service/helpers k8s.io/cloud-provider/service/helpers
k8s.io/cloud-provider/volume k8s.io/cloud-provider/volume
k8s.io/cloud-provider/volume/errors k8s.io/cloud-provider/volume/errors
k8s.io/cloud-provider/volume/helpers k8s.io/cloud-provider/volume/helpers
# k8s.io/cluster-bootstrap v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.17.5-k3s1 # k8s.io/cluster-bootstrap v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.17.6-k3s1
k8s.io/cluster-bootstrap/token/api k8s.io/cluster-bootstrap/token/api
k8s.io/cluster-bootstrap/token/jws k8s.io/cluster-bootstrap/token/jws
k8s.io/cluster-bootstrap/token/util k8s.io/cluster-bootstrap/token/util
k8s.io/cluster-bootstrap/util/secrets k8s.io/cluster-bootstrap/util/secrets
k8s.io/cluster-bootstrap/util/tokens k8s.io/cluster-bootstrap/util/tokens
# k8s.io/code-generator v0.17.0 => github.com/rancher/kubernetes/staging/src/k8s.io/code-generator v1.17.5-k3s1 # k8s.io/code-generator v0.17.0 => github.com/rancher/kubernetes/staging/src/k8s.io/code-generator v1.17.6-k3s1
k8s.io/code-generator/cmd/client-gen/args k8s.io/code-generator/cmd/client-gen/args
k8s.io/code-generator/cmd/client-gen/generators k8s.io/code-generator/cmd/client-gen/generators
k8s.io/code-generator/cmd/client-gen/generators/fake k8s.io/code-generator/cmd/client-gen/generators/fake
...@@ -1637,7 +1637,7 @@ k8s.io/code-generator/cmd/lister-gen/args ...@@ -1637,7 +1637,7 @@ k8s.io/code-generator/cmd/lister-gen/args
k8s.io/code-generator/cmd/lister-gen/generators k8s.io/code-generator/cmd/lister-gen/generators
k8s.io/code-generator/pkg/namer k8s.io/code-generator/pkg/namer
k8s.io/code-generator/pkg/util k8s.io/code-generator/pkg/util
# k8s.io/component-base v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/component-base v1.17.5-k3s1 # k8s.io/component-base v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/component-base v1.17.6-k3s1
k8s.io/component-base/cli/flag k8s.io/component-base/cli/flag
k8s.io/component-base/cli/globalflag k8s.io/component-base/cli/globalflag
k8s.io/component-base/codec k8s.io/component-base/codec
...@@ -1654,10 +1654,10 @@ k8s.io/component-base/metrics/prometheus/version ...@@ -1654,10 +1654,10 @@ k8s.io/component-base/metrics/prometheus/version
k8s.io/component-base/metrics/prometheus/workqueue k8s.io/component-base/metrics/prometheus/workqueue
k8s.io/component-base/version k8s.io/component-base/version
k8s.io/component-base/version/verflag k8s.io/component-base/version/verflag
# k8s.io/cri-api v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/cri-api v1.17.5-k3s1 # k8s.io/cri-api v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/cri-api v1.17.6-k3s1
k8s.io/cri-api/pkg/apis k8s.io/cri-api/pkg/apis
k8s.io/cri-api/pkg/apis/runtime/v1alpha2 k8s.io/cri-api/pkg/apis/runtime/v1alpha2
# k8s.io/csi-translation-lib v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/csi-translation-lib v1.17.5-k3s1 # k8s.io/csi-translation-lib v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/csi-translation-lib v1.17.6-k3s1
k8s.io/csi-translation-lib k8s.io/csi-translation-lib
k8s.io/csi-translation-lib/plugins k8s.io/csi-translation-lib/plugins
# k8s.io/gengo v0.0.0-20191120174120-e74f70b9b27e # k8s.io/gengo v0.0.0-20191120174120-e74f70b9b27e
...@@ -1672,7 +1672,7 @@ k8s.io/gengo/types ...@@ -1672,7 +1672,7 @@ k8s.io/gengo/types
k8s.io/heapster/metrics/api/v1/types k8s.io/heapster/metrics/api/v1/types
# k8s.io/klog v1.0.0 # k8s.io/klog v1.0.0
k8s.io/klog k8s.io/klog
# k8s.io/kube-aggregator v0.17.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kube-aggregator v1.17.5-k3s1 # k8s.io/kube-aggregator v0.17.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kube-aggregator v1.17.6-k3s1
k8s.io/kube-aggregator/pkg/apis/apiregistration k8s.io/kube-aggregator/pkg/apis/apiregistration
k8s.io/kube-aggregator/pkg/apis/apiregistration/install k8s.io/kube-aggregator/pkg/apis/apiregistration/install
k8s.io/kube-aggregator/pkg/apis/apiregistration/v1 k8s.io/kube-aggregator/pkg/apis/apiregistration/v1
...@@ -1700,9 +1700,9 @@ k8s.io/kube-aggregator/pkg/controllers/status ...@@ -1700,9 +1700,9 @@ k8s.io/kube-aggregator/pkg/controllers/status
k8s.io/kube-aggregator/pkg/registry/apiservice k8s.io/kube-aggregator/pkg/registry/apiservice
k8s.io/kube-aggregator/pkg/registry/apiservice/etcd k8s.io/kube-aggregator/pkg/registry/apiservice/etcd
k8s.io/kube-aggregator/pkg/registry/apiservice/rest k8s.io/kube-aggregator/pkg/registry/apiservice/rest
# k8s.io/kube-controller-manager v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kube-controller-manager v1.17.5-k3s1 # k8s.io/kube-controller-manager v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kube-controller-manager v1.17.6-k3s1
k8s.io/kube-controller-manager/config/v1alpha1 k8s.io/kube-controller-manager/config/v1alpha1
# k8s.io/kube-openapi v0.0.0-20200316234421-82d701f24f9d # k8s.io/kube-openapi v0.0.0-20200410145947-bcb3869e6f29
k8s.io/kube-openapi/pkg/aggregator k8s.io/kube-openapi/pkg/aggregator
k8s.io/kube-openapi/pkg/builder k8s.io/kube-openapi/pkg/builder
k8s.io/kube-openapi/pkg/common k8s.io/kube-openapi/pkg/common
...@@ -1711,12 +1711,12 @@ k8s.io/kube-openapi/pkg/schemaconv ...@@ -1711,12 +1711,12 @@ k8s.io/kube-openapi/pkg/schemaconv
k8s.io/kube-openapi/pkg/util k8s.io/kube-openapi/pkg/util
k8s.io/kube-openapi/pkg/util/proto k8s.io/kube-openapi/pkg/util/proto
k8s.io/kube-openapi/pkg/util/proto/validation k8s.io/kube-openapi/pkg/util/proto/validation
# k8s.io/kube-proxy v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kube-proxy v1.17.5-k3s1 # k8s.io/kube-proxy v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kube-proxy v1.17.6-k3s1
k8s.io/kube-proxy/config/v1alpha1 k8s.io/kube-proxy/config/v1alpha1
# k8s.io/kube-scheduler v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kube-scheduler v1.17.5-k3s1 # k8s.io/kube-scheduler v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kube-scheduler v1.17.6-k3s1
k8s.io/kube-scheduler/config/v1 k8s.io/kube-scheduler/config/v1
k8s.io/kube-scheduler/config/v1alpha1 k8s.io/kube-scheduler/config/v1alpha1
# k8s.io/kubectl v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kubectl v1.17.5-k3s1 # k8s.io/kubectl v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kubectl v1.17.6-k3s1
k8s.io/kubectl/pkg/apps k8s.io/kubectl/pkg/apps
k8s.io/kubectl/pkg/cmd k8s.io/kubectl/pkg/cmd
k8s.io/kubectl/pkg/cmd/annotate k8s.io/kubectl/pkg/cmd/annotate
...@@ -1792,11 +1792,11 @@ k8s.io/kubectl/pkg/util/storage ...@@ -1792,11 +1792,11 @@ k8s.io/kubectl/pkg/util/storage
k8s.io/kubectl/pkg/util/templates k8s.io/kubectl/pkg/util/templates
k8s.io/kubectl/pkg/util/term k8s.io/kubectl/pkg/util/term
k8s.io/kubectl/pkg/validation k8s.io/kubectl/pkg/validation
# k8s.io/kubelet v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kubelet v1.17.5-k3s1 # k8s.io/kubelet v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kubelet v1.17.6-k3s1
k8s.io/kubelet/config/v1beta1 k8s.io/kubelet/config/v1beta1
k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1 k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1
k8s.io/kubelet/pkg/apis/pluginregistration/v1 k8s.io/kubelet/pkg/apis/pluginregistration/v1
# k8s.io/kubernetes v1.17.0 => github.com/rancher/kubernetes v1.17.5-k3s1 # k8s.io/kubernetes v1.17.0 => github.com/rancher/kubernetes v1.17.6-k3s1
k8s.io/kubernetes/cmd/cloud-controller-manager/app k8s.io/kubernetes/cmd/cloud-controller-manager/app
k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config
k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/scheme k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/scheme
...@@ -2534,7 +2534,7 @@ k8s.io/kubernetes/third_party/forked/gonum/graph ...@@ -2534,7 +2534,7 @@ k8s.io/kubernetes/third_party/forked/gonum/graph
k8s.io/kubernetes/third_party/forked/gonum/graph/internal/linear k8s.io/kubernetes/third_party/forked/gonum/graph/internal/linear
k8s.io/kubernetes/third_party/forked/gonum/graph/simple k8s.io/kubernetes/third_party/forked/gonum/graph/simple
k8s.io/kubernetes/third_party/forked/gonum/graph/traverse k8s.io/kubernetes/third_party/forked/gonum/graph/traverse
# k8s.io/legacy-cloud-providers v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.17.5-k3s1 # k8s.io/legacy-cloud-providers v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.17.6-k3s1
k8s.io/legacy-cloud-providers/aws k8s.io/legacy-cloud-providers/aws
k8s.io/legacy-cloud-providers/azure k8s.io/legacy-cloud-providers/azure
k8s.io/legacy-cloud-providers/azure/auth k8s.io/legacy-cloud-providers/azure/auth
...@@ -2543,7 +2543,7 @@ k8s.io/legacy-cloud-providers/openstack ...@@ -2543,7 +2543,7 @@ k8s.io/legacy-cloud-providers/openstack
k8s.io/legacy-cloud-providers/vsphere k8s.io/legacy-cloud-providers/vsphere
k8s.io/legacy-cloud-providers/vsphere/vclib k8s.io/legacy-cloud-providers/vsphere/vclib
k8s.io/legacy-cloud-providers/vsphere/vclib/diskmanagers k8s.io/legacy-cloud-providers/vsphere/vclib/diskmanagers
# k8s.io/metrics v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/metrics v1.17.5-k3s1 # k8s.io/metrics v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/metrics v1.17.6-k3s1
k8s.io/metrics/pkg/apis/custom_metrics k8s.io/metrics/pkg/apis/custom_metrics
k8s.io/metrics/pkg/apis/custom_metrics/v1beta1 k8s.io/metrics/pkg/apis/custom_metrics/v1beta1
k8s.io/metrics/pkg/apis/custom_metrics/v1beta2 k8s.io/metrics/pkg/apis/custom_metrics/v1beta2
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment