Unverified Commit e58c2c9c authored by Erik Wilson's avatar Erik Wilson Committed by GitHub

Merge pull request #1830 from erikwilson/release-1.16

Update k8s 1.16.10
parents eb4ad1b3 2250ae67
FROM golang:1.13.6-alpine3.10 FROM golang:1.13.11-alpine3.10
RUN apk -U --no-cache add bash git gcc musl-dev docker vim less file curl wget ca-certificates jq linux-headers zlib-dev tar zip squashfs-tools npm coreutils \ RUN apk -U --no-cache add bash git gcc musl-dev docker vim less file curl wget ca-certificates jq linux-headers zlib-dev tar zip squashfs-tools npm coreutils \
python2 python3 py3-pip python3-dev openssl-dev libffi-dev libseccomp libseccomp-dev make libuv-static python2 python3 py3-pip python3-dev openssl-dev libffi-dev libseccomp libseccomp-dev make libuv-static
......
...@@ -33,31 +33,31 @@ replace ( ...@@ -33,31 +33,31 @@ replace (
github.com/prometheus/client_model => github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 github.com/prometheus/client_model => github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910
github.com/prometheus/common => github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 github.com/prometheus/common => github.com/prometheus/common v0.0.0-20181126121408-4724e9255275
github.com/prometheus/procfs => github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a github.com/prometheus/procfs => github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a
k8s.io/api => github.com/rancher/kubernetes/staging/src/k8s.io/api v1.16.9-k3s1 k8s.io/api => github.com/rancher/kubernetes/staging/src/k8s.io/api v1.16.10-k3s1
k8s.io/apiextensions-apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.16.9-k3s1 k8s.io/apiextensions-apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.16.10-k3s1
k8s.io/apimachinery => github.com/rancher/kubernetes/staging/src/k8s.io/apimachinery v1.16.9-k3s1 k8s.io/apimachinery => github.com/rancher/kubernetes/staging/src/k8s.io/apimachinery v1.16.10-k3s1
k8s.io/apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/apiserver v1.16.9-k3s1 k8s.io/apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/apiserver v1.16.10-k3s1
k8s.io/cli-runtime => github.com/rancher/kubernetes/staging/src/k8s.io/cli-runtime v1.16.9-k3s1 k8s.io/cli-runtime => github.com/rancher/kubernetes/staging/src/k8s.io/cli-runtime v1.16.10-k3s1
k8s.io/client-go => github.com/rancher/kubernetes/staging/src/k8s.io/client-go v1.16.9-k3s1 k8s.io/client-go => github.com/rancher/kubernetes/staging/src/k8s.io/client-go v1.16.10-k3s1
k8s.io/cloud-provider => github.com/rancher/kubernetes/staging/src/k8s.io/cloud-provider v1.16.9-k3s1 k8s.io/cloud-provider => github.com/rancher/kubernetes/staging/src/k8s.io/cloud-provider v1.16.10-k3s1
k8s.io/cluster-bootstrap => github.com/rancher/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.16.9-k3s1 k8s.io/cluster-bootstrap => github.com/rancher/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.16.10-k3s1
k8s.io/code-generator => github.com/rancher/kubernetes/staging/src/k8s.io/code-generator v1.16.9-k3s1 k8s.io/code-generator => github.com/rancher/kubernetes/staging/src/k8s.io/code-generator v1.16.10-k3s1
k8s.io/component-base => github.com/rancher/kubernetes/staging/src/k8s.io/component-base v1.16.9-k3s1 k8s.io/component-base => github.com/rancher/kubernetes/staging/src/k8s.io/component-base v1.16.10-k3s1
k8s.io/cri-api => github.com/rancher/kubernetes/staging/src/k8s.io/cri-api v1.16.9-k3s1 k8s.io/cri-api => github.com/rancher/kubernetes/staging/src/k8s.io/cri-api v1.16.10-k3s1
k8s.io/csi-translation-lib => github.com/rancher/kubernetes/staging/src/k8s.io/csi-translation-lib v1.16.9-k3s1 k8s.io/csi-translation-lib => github.com/rancher/kubernetes/staging/src/k8s.io/csi-translation-lib v1.16.10-k3s1
k8s.io/kube-aggregator => github.com/rancher/kubernetes/staging/src/k8s.io/kube-aggregator v1.16.9-k3s1 k8s.io/kube-aggregator => github.com/rancher/kubernetes/staging/src/k8s.io/kube-aggregator v1.16.10-k3s1
k8s.io/kube-controller-manager => github.com/rancher/kubernetes/staging/src/k8s.io/kube-controller-manager v1.16.9-k3s1 k8s.io/kube-controller-manager => github.com/rancher/kubernetes/staging/src/k8s.io/kube-controller-manager v1.16.10-k3s1
k8s.io/kube-proxy => github.com/rancher/kubernetes/staging/src/k8s.io/kube-proxy v1.16.9-k3s1 k8s.io/kube-proxy => github.com/rancher/kubernetes/staging/src/k8s.io/kube-proxy v1.16.10-k3s1
k8s.io/kube-scheduler => github.com/rancher/kubernetes/staging/src/k8s.io/kube-scheduler v1.16.9-k3s1 k8s.io/kube-scheduler => github.com/rancher/kubernetes/staging/src/k8s.io/kube-scheduler v1.16.10-k3s1
k8s.io/kubectl => github.com/rancher/kubernetes/staging/src/k8s.io/kubectl v1.16.9-k3s1 k8s.io/kubectl => github.com/rancher/kubernetes/staging/src/k8s.io/kubectl v1.16.10-k3s1
k8s.io/kubelet => github.com/rancher/kubernetes/staging/src/k8s.io/kubelet v1.16.9-k3s1 k8s.io/kubelet => github.com/rancher/kubernetes/staging/src/k8s.io/kubelet v1.16.10-k3s1
k8s.io/kubernetes => github.com/rancher/kubernetes v1.16.9-k3s1 k8s.io/kubernetes => github.com/rancher/kubernetes v1.16.10-k3s1
k8s.io/legacy-cloud-providers => github.com/rancher/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.16.9-k3s1 k8s.io/legacy-cloud-providers => github.com/rancher/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.16.10-k3s1
k8s.io/metrics => github.com/rancher/kubernetes/staging/src/k8s.io/metrics v1.16.9-k3s1 k8s.io/metrics => github.com/rancher/kubernetes/staging/src/k8s.io/metrics v1.16.10-k3s1
k8s.io/node-api => github.com/rancher/kubernetes/staging/src/k8s.io/node-api v1.16.9-k3s1 k8s.io/node-api => github.com/rancher/kubernetes/staging/src/k8s.io/node-api v1.16.10-k3s1
k8s.io/sample-apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/sample-apiserver v1.16.9-k3s1 k8s.io/sample-apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/sample-apiserver v1.16.10-k3s1
k8s.io/sample-cli-plugin => github.com/rancher/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.16.9-k3s1 k8s.io/sample-cli-plugin => github.com/rancher/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.16.10-k3s1
k8s.io/sample-controller => github.com/rancher/kubernetes/staging/src/k8s.io/sample-controller v1.16.9-k3s1 k8s.io/sample-controller => github.com/rancher/kubernetes/staging/src/k8s.io/sample-controller v1.16.10-k3s1
mvdan.cc/unparam => mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34 mvdan.cc/unparam => mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34
) )
......
...@@ -38,6 +38,10 @@ func (a statusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.O ...@@ -38,6 +38,10 @@ func (a statusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.O
newCustomResource := newCustomResourceObject.UnstructuredContent() newCustomResource := newCustomResourceObject.UnstructuredContent()
status, ok := newCustomResource["status"] status, ok := newCustomResource["status"]
// managedFields must be preserved since it's been modified to
// track changed fields in the status update.
managedFields := newCustomResourceObject.GetManagedFields()
// copy old object into new object // copy old object into new object
oldCustomResourceObject := old.(*unstructured.Unstructured) oldCustomResourceObject := old.(*unstructured.Unstructured)
// overridding the resourceVersion in metadata is safe here, we have already checked that // overridding the resourceVersion in metadata is safe here, we have already checked that
...@@ -45,6 +49,7 @@ func (a statusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.O ...@@ -45,6 +49,7 @@ func (a statusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.O
*newCustomResourceObject = *oldCustomResourceObject.DeepCopy() *newCustomResourceObject = *oldCustomResourceObject.DeepCopy()
// set status // set status
newCustomResourceObject.SetManagedFields(managedFields)
newCustomResource = newCustomResourceObject.UnstructuredContent() newCustomResource = newCustomResourceObject.UnstructuredContent()
if ok { if ok {
newCustomResource["status"] = status newCustomResource["status"] = status
......
...@@ -252,7 +252,9 @@ func ResetObjectMetaForStatus(meta, existingMeta Object) { ...@@ -252,7 +252,9 @@ func ResetObjectMetaForStatus(meta, existingMeta Object) {
meta.SetAnnotations(existingMeta.GetAnnotations()) meta.SetAnnotations(existingMeta.GetAnnotations())
meta.SetFinalizers(existingMeta.GetFinalizers()) meta.SetFinalizers(existingMeta.GetFinalizers())
meta.SetOwnerReferences(existingMeta.GetOwnerReferences()) meta.SetOwnerReferences(existingMeta.GetOwnerReferences())
meta.SetManagedFields(existingMeta.GetManagedFields()) // managedFields must be preserved since it's been modified to
// track changed fields in the status update.
//meta.SetManagedFields(existingMeta.GetManagedFields())
} }
// MarshalJSON implements json.Marshaler // MarshalJSON implements json.Marshaler
......
...@@ -66,11 +66,36 @@ func Unmarshal(data []byte, v interface{}) error { ...@@ -66,11 +66,36 @@ func Unmarshal(data []byte, v interface{}) error {
// If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64 // If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64
return convertSliceNumbers(*v, 0) return convertSliceNumbers(*v, 0)
case *interface{}:
// Build a decoder from the given data
decoder := json.NewDecoder(bytes.NewBuffer(data))
// Preserve numbers, rather than casting to float64 automatically
decoder.UseNumber()
// Run the decode
if err := decoder.Decode(v); err != nil {
return err
}
// If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64
return convertInterfaceNumbers(v, 0)
default: default:
return json.Unmarshal(data, v) return json.Unmarshal(data, v)
} }
} }
func convertInterfaceNumbers(v *interface{}, depth int) error {
var err error
switch v2 := (*v).(type) {
case json.Number:
*v, err = convertNumber(v2)
case map[string]interface{}:
err = convertMapNumbers(v2, depth+1)
case []interface{}:
err = convertSliceNumbers(v2, depth+1)
}
return err
}
// convertMapNumbers traverses the map, converting any json.Number values to int64 or float64. // convertMapNumbers traverses the map, converting any json.Number values to int64 or float64.
// values which are map[string]interface{} or []interface{} are recursively visited // values which are map[string]interface{} or []interface{} are recursively visited
func convertMapNumbers(m map[string]interface{}, depth int) error { func convertMapNumbers(m map[string]interface{}, depth int) error {
......
...@@ -3,8 +3,8 @@ package version ...@@ -3,8 +3,8 @@ package version
var ( var (
gitMajor = "1" gitMajor = "1"
gitMinor = "16" gitMinor = "16"
gitVersion = "v1.16.9-k3s1" gitVersion = "v1.16.10-k3s1"
gitCommit = "70e76ba34b743fda5a960a421e94d8ae96b51ac9" gitCommit = "4b7f81026e6e88cbfdfe8a56a624eb35f883e8de"
gitTreeState = "clean" gitTreeState = "clean"
buildDate = "2020-04-27T22:49:01Z" buildDate = "2020-05-26T20:41:39Z"
) )
...@@ -184,8 +184,8 @@ k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUc ...@@ -184,8 +184,8 @@ k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUc
k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf h1:EYm5AW/UUDbnmnI+gK0TJDVK9qPLhM+sRHYanNKw0EQ= k8s.io/kube-openapi v0.0.0-20200410163147-594e756bea31 h1:PsbYeEz2x7ll6JYUzBEG+DT78910DDTlvn5Ma10F5/E=
k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kube-openapi v0.0.0-20200410163147-594e756bea31/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
k8s.io/utils v0.0.0-20190801114015-581e00157fb1 h1:+ySTxfHnfzZb9ys375PXNlLhkJPLKgHajBU0N62BDvE= k8s.io/utils v0.0.0-20190801114015-581e00157fb1 h1:+ySTxfHnfzZb9ys375PXNlLhkJPLKgHajBU0N62BDvE=
k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
......
...@@ -3,8 +3,8 @@ package version ...@@ -3,8 +3,8 @@ package version
var ( var (
gitMajor = "1" gitMajor = "1"
gitMinor = "16" gitMinor = "16"
gitVersion = "v1.16.9-k3s1" gitVersion = "v1.16.10-k3s1"
gitCommit = "70e76ba34b743fda5a960a421e94d8ae96b51ac9" gitCommit = "4b7f81026e6e88cbfdfe8a56a624eb35f883e8de"
gitTreeState = "clean" gitTreeState = "clean"
buildDate = "2020-04-27T22:49:01Z" buildDate = "2020-05-26T20:41:39Z"
) )
...@@ -165,7 +165,7 @@ k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUc ...@@ -165,7 +165,7 @@ k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUc
k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kube-openapi v0.0.0-20200410163147-594e756bea31/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
......
...@@ -210,7 +210,7 @@ func (item *primitiveItem) VisitPrimitive(schema *proto.Primitive) { ...@@ -210,7 +210,7 @@ func (item *primitiveItem) VisitPrimitive(schema *proto.Primitive) {
} }
case proto.Number: case proto.Number:
switch item.Kind { switch item.Kind {
case proto.Number: case proto.Integer, proto.Number:
return return
} }
case proto.String: case proto.String:
......
...@@ -3,8 +3,8 @@ package version ...@@ -3,8 +3,8 @@ package version
var ( var (
gitMajor = "1" gitMajor = "1"
gitMinor = "16" gitMinor = "16"
gitVersion = "v1.16.9-k3s1" gitVersion = "v1.16.10-k3s1"
gitCommit = "70e76ba34b743fda5a960a421e94d8ae96b51ac9" gitCommit = "4b7f81026e6e88cbfdfe8a56a624eb35f883e8de"
gitTreeState = "clean" gitTreeState = "clean"
buildDate = "2020-04-27T22:49:01Z" buildDate = "2020-05-26T20:41:39Z"
) )
...@@ -40,10 +40,12 @@ func cadvisorInfoToCPUandMemoryStats(info *cadvisorapiv2.ContainerInfo) (*statsa ...@@ -40,10 +40,12 @@ func cadvisorInfoToCPUandMemoryStats(info *cadvisorapiv2.ContainerInfo) (*statsa
} }
var cpuStats *statsapi.CPUStats var cpuStats *statsapi.CPUStats
var memoryStats *statsapi.MemoryStats var memoryStats *statsapi.MemoryStats
cpuStats = &statsapi.CPUStats{
Time: metav1.NewTime(cstat.Timestamp),
UsageNanoCores: uint64Ptr(0),
UsageCoreNanoSeconds: uint64Ptr(0),
}
if info.Spec.HasCpu { if info.Spec.HasCpu {
cpuStats = &statsapi.CPUStats{
Time: metav1.NewTime(cstat.Timestamp),
}
if cstat.CpuInst != nil { if cstat.CpuInst != nil {
cpuStats.UsageNanoCores = &cstat.CpuInst.Usage.Total cpuStats.UsageNanoCores = &cstat.CpuInst.Usage.Total
} }
...@@ -67,6 +69,11 @@ func cadvisorInfoToCPUandMemoryStats(info *cadvisorapiv2.ContainerInfo) (*statsa ...@@ -67,6 +69,11 @@ func cadvisorInfoToCPUandMemoryStats(info *cadvisorapiv2.ContainerInfo) (*statsa
availableBytes := info.Spec.Memory.Limit - cstat.Memory.WorkingSet availableBytes := info.Spec.Memory.Limit - cstat.Memory.WorkingSet
memoryStats.AvailableBytes = &availableBytes memoryStats.AvailableBytes = &availableBytes
} }
} else {
memoryStats = &statsapi.MemoryStats{
Time: metav1.NewTime(cstat.Timestamp),
WorkingSetBytes: uint64Ptr(0),
}
} }
return cpuStats, memoryStats return cpuStats, memoryStats
} }
......
...@@ -124,6 +124,7 @@ func calculateResourceAllocatableRequest(nodeInfo *schedulernodeinfo.NodeInfo, p ...@@ -124,6 +124,7 @@ func calculateResourceAllocatableRequest(nodeInfo *schedulernodeinfo.NodeInfo, p
// calculatePodResourceRequest returns the total non-zero requests. If Overhead is defined for the pod and the // calculatePodResourceRequest returns the total non-zero requests. If Overhead is defined for the pod and the
// PodOverhead feature is enabled, the Overhead is added to the result. // PodOverhead feature is enabled, the Overhead is added to the result.
// podResourceRequest = max(sum(podSpec.Containers), podSpec.InitContainers) + overHead
func calculatePodResourceRequest(pod *v1.Pod, resource v1.ResourceName) int64 { func calculatePodResourceRequest(pod *v1.Pod, resource v1.ResourceName) int64 {
var podRequest int64 var podRequest int64
for i := range pod.Spec.Containers { for i := range pod.Spec.Containers {
...@@ -132,11 +133,20 @@ func calculatePodResourceRequest(pod *v1.Pod, resource v1.ResourceName) int64 { ...@@ -132,11 +133,20 @@ func calculatePodResourceRequest(pod *v1.Pod, resource v1.ResourceName) int64 {
podRequest += value podRequest += value
} }
for i := range pod.Spec.InitContainers {
initContainer := &pod.Spec.InitContainers[i]
value := priorityutil.GetNonzeroRequestForResource(resource, &initContainer.Resources.Requests)
if podRequest < value {
podRequest = value
}
}
// If Overhead is being utilized, add to the total requests for the pod // If Overhead is being utilized, add to the total requests for the pod
if pod.Spec.Overhead != nil && utilfeature.DefaultFeatureGate.Enabled(features.PodOverhead) { if pod.Spec.Overhead != nil && utilfeature.DefaultFeatureGate.Enabled(features.PodOverhead) {
if quantity, found := pod.Spec.Overhead[resource]; found { if quantity, found := pod.Spec.Overhead[resource]; found {
podRequest += quantity.Value() podRequest += quantity.Value()
} }
} }
return podRequest return podRequest
} }
...@@ -579,6 +579,7 @@ func (n *NodeInfo) RemovePod(pod *v1.Pod) error { ...@@ -579,6 +579,7 @@ func (n *NodeInfo) RemovePod(pod *v1.Pod) error {
return fmt.Errorf("no corresponding pod %s in pods of node %s", pod.Name, n.node.Name) return fmt.Errorf("no corresponding pod %s in pods of node %s", pod.Name, n.node.Name)
} }
// resourceRequest = max(sum(podSpec.Containers), podSpec.InitContainers) + overHead
func calculateResource(pod *v1.Pod) (res Resource, non0CPU int64, non0Mem int64) { func calculateResource(pod *v1.Pod) (res Resource, non0CPU int64, non0Mem int64) {
resPtr := &res resPtr := &res
for _, c := range pod.Spec.Containers { for _, c := range pod.Spec.Containers {
...@@ -590,6 +591,18 @@ func calculateResource(pod *v1.Pod) (res Resource, non0CPU int64, non0Mem int64) ...@@ -590,6 +591,18 @@ func calculateResource(pod *v1.Pod) (res Resource, non0CPU int64, non0Mem int64)
// No non-zero resources for GPUs or opaque resources. // No non-zero resources for GPUs or opaque resources.
} }
for _, ic := range pod.Spec.InitContainers {
resPtr.SetMaxResource(ic.Resources.Requests)
non0CPUReq, non0MemReq := priorityutil.GetNonzeroRequests(&ic.Resources.Requests)
if non0CPU < non0CPUReq {
non0CPU = non0CPUReq
}
if non0Mem < non0MemReq {
non0Mem = non0MemReq
}
}
// If Overhead is being utilized, add to the total requests for the pod // If Overhead is being utilized, add to the total requests for the pod
if pod.Spec.Overhead != nil && utilfeature.DefaultFeatureGate.Enabled(features.PodOverhead) { if pod.Spec.Overhead != nil && utilfeature.DefaultFeatureGate.Enabled(features.PodOverhead) {
resPtr.Add(pod.Spec.Overhead) resPtr.Add(pod.Spec.Overhead)
......
...@@ -557,6 +557,9 @@ func (sched *Scheduler) scheduleOne() { ...@@ -557,6 +557,9 @@ func (sched *Scheduler) scheduleOne() {
// succeeds, the pod should get counted as a success the next time we try to // succeeds, the pod should get counted as a success the next time we try to
// schedule it. (hopefully) // schedule it. (hopefully)
metrics.PodScheduleFailures.Inc() metrics.PodScheduleFailures.Inc()
} else if err == core.ErrNoNodesAvailable {
// No nodes available is counted as unschedulable rather than an error.
metrics.PodScheduleFailures.Inc()
} else { } else {
klog.Errorf("error selecting node for pod: %v", err) klog.Errorf("error selecting node for pod: %v", err)
metrics.PodScheduleErrors.Inc() metrics.PodScheduleErrors.Inc()
......
...@@ -60,16 +60,13 @@ func GetPodPriority(pod *v1.Pod) int32 { ...@@ -60,16 +60,13 @@ func GetPodPriority(pod *v1.Pod) int32 {
return scheduling.DefaultPriorityWhenNoDefaultClassExists return scheduling.DefaultPriorityWhenNoDefaultClassExists
} }
// GetPodStartTime returns start time of the given pod. // GetPodStartTime returns start time of the given pod or current timestamp
// if it hasn't started yet.
func GetPodStartTime(pod *v1.Pod) *metav1.Time { func GetPodStartTime(pod *v1.Pod) *metav1.Time {
if pod.Status.StartTime != nil { if pod.Status.StartTime != nil {
return pod.Status.StartTime return pod.Status.StartTime
} }
// Should not reach here as the start time of a running time should not be nil // Assumed pods and bound pods that haven't started don't have a StartTime yet.
// Return current timestamp as the default value.
// This will not affect the calculation of earliest timestamp of all the pods on one node,
// because current timestamp is always after the StartTime of any pod in good state.
klog.Errorf("pod.Status.StartTime is nil for pod %s. Should not reach here.", pod.Name)
return &metav1.Time{Time: time.Now()} return &metav1.Time{Time: time.Now()}
} }
......
...@@ -3,8 +3,8 @@ package version ...@@ -3,8 +3,8 @@ package version
var ( var (
gitMajor = "1" gitMajor = "1"
gitMinor = "16" gitMinor = "16"
gitVersion = "v1.16.9-k3s1" gitVersion = "v1.16.10-k3s1"
gitCommit = "70e76ba34b743fda5a960a421e94d8ae96b51ac9" gitCommit = "4b7f81026e6e88cbfdfe8a56a624eb35f883e8de"
gitTreeState = "clean" gitTreeState = "clean"
buildDate = "2020-04-27T22:49:01Z" buildDate = "2020-05-26T20:41:39Z"
) )
...@@ -45,9 +45,12 @@ var maxDataDiskCountMap = map[string]int64{ ...@@ -45,9 +45,12 @@ var maxDataDiskCountMap = map[string]int64{
"STANDARD_A8M_V2": 16, "STANDARD_A8M_V2": 16,
"STANDARD_A8_V2": 16, "STANDARD_A8_V2": 16,
"STANDARD_A9": 64, "STANDARD_A9": 64,
"STANDARD_B12MS": 16,
"STANDARD_B16MS": 32,
"STANDARD_B1LS": 2, "STANDARD_B1LS": 2,
"STANDARD_B1MS": 2, "STANDARD_B1MS": 2,
"STANDARD_B1S": 2, "STANDARD_B1S": 2,
"STANDARD_B20MS": 32,
"STANDARD_B2MS": 4, "STANDARD_B2MS": 4,
"STANDARD_B2S": 4, "STANDARD_B2S": 4,
"STANDARD_B4MS": 8, "STANDARD_B4MS": 8,
...@@ -66,32 +69,54 @@ var maxDataDiskCountMap = map[string]int64{ ...@@ -66,32 +69,54 @@ var maxDataDiskCountMap = map[string]int64{
"STANDARD_D14_V2": 64, "STANDARD_D14_V2": 64,
"STANDARD_D14_V2_PROMO": 64, "STANDARD_D14_V2_PROMO": 64,
"STANDARD_D15_V2": 64, "STANDARD_D15_V2": 64,
"STANDARD_D16AS_V4": 32,
"STANDARD_D16A_V4": 32,
"STANDARD_D16S_V3": 32, "STANDARD_D16S_V3": 32,
"STANDARD_D16_V3": 32, "STANDARD_D16_V3": 32,
"STANDARD_D1_V2": 4, "STANDARD_D1_V2": 4,
"STANDARD_D2": 8, "STANDARD_D2": 8,
"STANDARD_D2AS_V4": 4,
"STANDARD_D2A_V4": 4,
"STANDARD_D2S_V3": 4, "STANDARD_D2S_V3": 4,
"STANDARD_D2_V2": 8, "STANDARD_D2_V2": 8,
"STANDARD_D2_V2_PROMO": 8, "STANDARD_D2_V2_PROMO": 8,
"STANDARD_D2_V3": 4, "STANDARD_D2_V3": 4,
"STANDARD_D3": 16, "STANDARD_D3": 16,
"STANDARD_D32AS_V4": 32,
"STANDARD_D32A_V4": 32,
"STANDARD_D32S_V3": 32, "STANDARD_D32S_V3": 32,
"STANDARD_D32_V3": 32, "STANDARD_D32_V3": 32,
"STANDARD_D3_V2": 16, "STANDARD_D3_V2": 16,
"STANDARD_D3_V2_PROMO": 16, "STANDARD_D3_V2_PROMO": 16,
"STANDARD_D4": 32, "STANDARD_D4": 32,
"STANDARD_D48AS_V4": 32,
"STANDARD_D48A_V4": 32,
"STANDARD_D48S_V3": 32,
"STANDARD_D48_V3": 32,
"STANDARD_D4AS_V4": 8,
"STANDARD_D4A_V4": 8,
"STANDARD_D4S_V3": 8, "STANDARD_D4S_V3": 8,
"STANDARD_D4_V2": 32, "STANDARD_D4_V2": 32,
"STANDARD_D4_V2_PROMO": 32, "STANDARD_D4_V2_PROMO": 32,
"STANDARD_D4_V3": 8, "STANDARD_D4_V3": 8,
"STANDARD_D5_V2": 64, "STANDARD_D5_V2": 64,
"STANDARD_D5_V2_PROMO": 64, "STANDARD_D5_V2_PROMO": 64,
"STANDARD_D64AS_V4": 32,
"STANDARD_D64A_V4": 32,
"STANDARD_D64S_V3": 32, "STANDARD_D64S_V3": 32,
"STANDARD_D64_V3": 32, "STANDARD_D64_V3": 32,
"STANDARD_D8AS_V4": 16,
"STANDARD_D8A_V4": 16,
"STANDARD_D8S_V3": 16, "STANDARD_D8S_V3": 16,
"STANDARD_D8_V3": 16, "STANDARD_D8_V3": 16,
"STANDARD_D96AS_V4": 32,
"STANDARD_D96A_V4": 32,
"STANDARD_DC1S_V2": 1,
"STANDARD_DC2S": 2, "STANDARD_DC2S": 2,
"STANDARD_DC2S_V2": 2,
"STANDARD_DC4S": 4, "STANDARD_DC4S": 4,
"STANDARD_DC4S_V2": 4,
"STANDARD_DC8_V2": 8,
"STANDARD_DS11-1_V2": 8, "STANDARD_DS11-1_V2": 8,
"STANDARD_DS11": 8, "STANDARD_DS11": 8,
"STANDARD_DS11_V2": 8, "STANDARD_DS11_V2": 8,
...@@ -127,29 +152,49 @@ var maxDataDiskCountMap = map[string]int64{ ...@@ -127,29 +152,49 @@ var maxDataDiskCountMap = map[string]int64{
"STANDARD_DS5_V2_PROMO": 64, "STANDARD_DS5_V2_PROMO": 64,
"STANDARD_E16-4S_V3": 32, "STANDARD_E16-4S_V3": 32,
"STANDARD_E16-8S_V3": 32, "STANDARD_E16-8S_V3": 32,
"STANDARD_E16AS_V4": 32,
"STANDARD_E16A_V4": 32,
"STANDARD_E16S_V3": 32, "STANDARD_E16S_V3": 32,
"STANDARD_E16_V3": 32, "STANDARD_E16_V3": 32,
"STANDARD_E20AS_V4": 32,
"STANDARD_E20A_V4": 32,
"STANDARD_E20S_V3": 32, "STANDARD_E20S_V3": 32,
"STANDARD_E20_V3": 32, "STANDARD_E20_V3": 32,
"STANDARD_E2AS_V4": 4,
"STANDARD_E2A_V4": 4,
"STANDARD_E2S_V3": 4, "STANDARD_E2S_V3": 4,
"STANDARD_E2_V3": 4, "STANDARD_E2_V3": 4,
"STANDARD_E32-16S_V3": 32, "STANDARD_E32-16S_V3": 32,
"STANDARD_E32-8S_V3": 32, "STANDARD_E32-8S_V3": 32,
"STANDARD_E32AS_V4": 32,
"STANDARD_E32A_V4": 32,
"STANDARD_E32S_V3": 32, "STANDARD_E32S_V3": 32,
"STANDARD_E32_V3": 32, "STANDARD_E32_V3": 32,
"STANDARD_E4-2S_V3": 8, "STANDARD_E4-2S_V3": 8,
"STANDARD_E48AS_V4": 32,
"STANDARD_E48A_V4": 32,
"STANDARD_E48S_V3": 32,
"STANDARD_E48_V3": 32,
"STANDARD_E4AS_V4": 8,
"STANDARD_E4A_V4": 8,
"STANDARD_E4S_V3": 8, "STANDARD_E4S_V3": 8,
"STANDARD_E4_V3": 8, "STANDARD_E4_V3": 8,
"STANDARD_E64-16S_V3": 32, "STANDARD_E64-16S_V3": 32,
"STANDARD_E64-32S_V3": 32, "STANDARD_E64-32S_V3": 32,
"STANDARD_E64AS_V4": 32,
"STANDARD_E64A_V4": 32,
"STANDARD_E64IS_V3": 32, "STANDARD_E64IS_V3": 32,
"STANDARD_E64I_V3": 32, "STANDARD_E64I_V3": 32,
"STANDARD_E64S_V3": 32, "STANDARD_E64S_V3": 32,
"STANDARD_E64_V3": 32, "STANDARD_E64_V3": 32,
"STANDARD_E8-2S_V3": 16, "STANDARD_E8-2S_V3": 16,
"STANDARD_E8-4S_V3": 16, "STANDARD_E8-4S_V3": 16,
"STANDARD_E8AS_V4": 16,
"STANDARD_E8A_V4": 16,
"STANDARD_E8S_V3": 16, "STANDARD_E8S_V3": 16,
"STANDARD_E8_V3": 16, "STANDARD_E8_V3": 16,
"STANDARD_E96AS_V4": 32,
"STANDARD_E96A_V4": 32,
"STANDARD_F1": 4, "STANDARD_F1": 4,
"STANDARD_F16": 64, "STANDARD_F16": 64,
"STANDARD_F16S": 64, "STANDARD_F16S": 64,
...@@ -160,6 +205,7 @@ var maxDataDiskCountMap = map[string]int64{ ...@@ -160,6 +205,7 @@ var maxDataDiskCountMap = map[string]int64{
"STANDARD_F2S_V2": 4, "STANDARD_F2S_V2": 4,
"STANDARD_F32S_V2": 32, "STANDARD_F32S_V2": 32,
"STANDARD_F4": 16, "STANDARD_F4": 16,
"STANDARD_F48S_V2": 32,
"STANDARD_F4S": 16, "STANDARD_F4S": 16,
"STANDARD_F4S_V2": 8, "STANDARD_F4S_V2": 8,
"STANDARD_F64S_V2": 32, "STANDARD_F64S_V2": 32,
...@@ -193,12 +239,14 @@ var maxDataDiskCountMap = map[string]int64{ ...@@ -193,12 +239,14 @@ var maxDataDiskCountMap = map[string]int64{
"STANDARD_H8M": 32, "STANDARD_H8M": 32,
"STANDARD_H8M_PROMO": 32, "STANDARD_H8M_PROMO": 32,
"STANDARD_H8_PROMO": 32, "STANDARD_H8_PROMO": 32,
"STANDARD_HB120RS_V2": 8,
"STANDARD_HB60RS": 4, "STANDARD_HB60RS": 4,
"STANDARD_HC44RS": 4, "STANDARD_HC44RS": 4,
"STANDARD_L16S": 64, "STANDARD_L16S": 64,
"STANDARD_L16S_V2": 32, "STANDARD_L16S_V2": 32,
"STANDARD_L32S": 64, "STANDARD_L32S": 64,
"STANDARD_L32S_V2": 32, "STANDARD_L32S_V2": 32,
"STANDARD_L48S_V2": 32,
"STANDARD_L4S": 16, "STANDARD_L4S": 16,
"STANDARD_L64S_V2": 32, "STANDARD_L64S_V2": 32,
"STANDARD_L80S_V2": 32, "STANDARD_L80S_V2": 32,
...@@ -220,6 +268,8 @@ var maxDataDiskCountMap = map[string]int64{ ...@@ -220,6 +268,8 @@ var maxDataDiskCountMap = map[string]int64{
"STANDARD_M32LS": 32, "STANDARD_M32LS": 32,
"STANDARD_M32MS": 32, "STANDARD_M32MS": 32,
"STANDARD_M32TS": 32, "STANDARD_M32TS": 32,
"STANDARD_M416MS_V2": 64,
"STANDARD_M416S_V2": 64,
"STANDARD_M64-16MS": 64, "STANDARD_M64-16MS": 64,
"STANDARD_M64-32MS": 64, "STANDARD_M64-32MS": 64,
"STANDARD_M64": 64, "STANDARD_M64": 64,
...@@ -249,18 +299,27 @@ var maxDataDiskCountMap = map[string]int64{ ...@@ -249,18 +299,27 @@ var maxDataDiskCountMap = map[string]int64{
"STANDARD_ND12S": 24, "STANDARD_ND12S": 24,
"STANDARD_ND24RS": 32, "STANDARD_ND24RS": 32,
"STANDARD_ND24S": 32, "STANDARD_ND24S": 32,
"STANDARD_ND40RS_V2": 8,
"STANDARD_ND40S_V3": 32,
"STANDARD_ND6S": 12, "STANDARD_ND6S": 12,
"STANDARD_NP10S": 8,
"STANDARD_NP20S": 16,
"STANDARD_NP40S": 32,
"STANDARD_NV12": 48, "STANDARD_NV12": 48,
"STANDARD_NV12_PROMO": 48, "STANDARD_NV12_PROMO": 48,
"STANDARD_NV12S_V2": 24, "STANDARD_NV12S_V2": 24,
"STANDARD_NV12S_V3": 12, "STANDARD_NV12S_V3": 12,
"STANDARD_NV16AS_V4": 32,
"STANDARD_NV24": 64, "STANDARD_NV24": 64,
"STANDARD_NV24_PROMO": 64, "STANDARD_NV24_PROMO": 64,
"STANDARD_NV24S_V2": 32, "STANDARD_NV24S_V2": 32,
"STANDARD_NV24S_V3": 24, "STANDARD_NV24S_V3": 24,
"STANDARD_NV32AS_V4": 32,
"STANDARD_NV48S_V3": 32, "STANDARD_NV48S_V3": 32,
"STANDARD_NV4AS_V4": 8,
"STANDARD_NV6": 24, "STANDARD_NV6": 24,
"STANDARD_NV6_PROMO": 24, "STANDARD_NV6_PROMO": 24,
"STANDARD_NV6S_V2": 12, "STANDARD_NV6S_V2": 12,
"STANDARD_NV8AS_V4": 16,
"STANDARD_PB6S": 12, "STANDARD_PB6S": 12,
} }
...@@ -219,13 +219,15 @@ const nodeWithImpairedVolumes = "NodeWithImpairedVolumes" ...@@ -219,13 +219,15 @@ const nodeWithImpairedVolumes = "NodeWithImpairedVolumes"
const ( const (
// volumeAttachmentConsecutiveErrorLimit is the number of consecutive errors we will ignore when waiting for a volume to attach/detach // volumeAttachmentConsecutiveErrorLimit is the number of consecutive errors we will ignore when waiting for a volume to attach/detach
volumeAttachmentStatusConsecutiveErrorLimit = 10 volumeAttachmentStatusConsecutiveErrorLimit = 10
// most attach/detach operations on AWS finish within 1-4 seconds
// By using 1 second starting interval with a backoff of 1.8 // Attach typically takes 2-5 seconds (average is 2). Asking before 2 seconds is just waste of API quota.
// we get - [1, 1.8, 3.24, 5.832000000000001, 10.4976] volumeAttachmentStatusInitialDelay = 2 * time.Second
// in total we wait for 2601 seconds // Detach typically takes 5-10 seconds (average is 6). Asking before 5 seconds is just waste of API quota.
volumeAttachmentStatusInitialDelay = 1 * time.Second volumeDetachmentStatusInitialDelay = 5 * time.Second
volumeAttachmentStatusFactor = 1.8 // After the initial delay, poll attach/detach with exponential backoff (2046 seconds total)
volumeAttachmentStatusSteps = 13 volumeAttachmentStatusPollDelay = 2 * time.Second
volumeAttachmentStatusFactor = 2
volumeAttachmentStatusSteps = 11
// createTag* is configuration of exponential backoff for CreateTag call. We // createTag* is configuration of exponential backoff for CreateTag call. We
// retry mainly because if we create an object, we cannot tag it until it is // retry mainly because if we create an object, we cannot tag it until it is
...@@ -2069,17 +2071,24 @@ func (c *Cloud) applyUnSchedulableTaint(nodeName types.NodeName, reason string) ...@@ -2069,17 +2071,24 @@ func (c *Cloud) applyUnSchedulableTaint(nodeName types.NodeName, reason string)
// waitForAttachmentStatus polls until the attachment status is the expected value // waitForAttachmentStatus polls until the attachment status is the expected value
// On success, it returns the last attachment state. // On success, it returns the last attachment state.
func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment, error) { func (d *awsDisk) waitForAttachmentStatus(status string, expectedInstance, expectedDevice string) (*ec2.VolumeAttachment, error) {
backoff := wait.Backoff{ backoff := wait.Backoff{
Duration: volumeAttachmentStatusInitialDelay, Duration: volumeAttachmentStatusPollDelay,
Factor: volumeAttachmentStatusFactor, Factor: volumeAttachmentStatusFactor,
Steps: volumeAttachmentStatusSteps, Steps: volumeAttachmentStatusSteps,
} }
// Because of rate limiting, we often see errors from describeVolume // Because of rate limiting, we often see errors from describeVolume.
// Or AWS eventual consistency returns unexpected data.
// So we tolerate a limited number of failures. // So we tolerate a limited number of failures.
// But once we see more than 10 errors in a row, we return the error // But once we see more than 10 errors in a row, we return the error.
describeErrorCount := 0 errorCount := 0
// Attach/detach usually takes time. It does not make sense to start
// polling DescribeVolumes before some initial delay to let AWS
// process the request.
time.Sleep(getInitialAttachDetachDelay(status))
var attachment *ec2.VolumeAttachment var attachment *ec2.VolumeAttachment
err := wait.ExponentialBackoff(backoff, func() (bool, error) { err := wait.ExponentialBackoff(backoff, func() (bool, error) {
...@@ -2102,8 +2111,8 @@ func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment, ...@@ -2102,8 +2111,8 @@ func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment,
return false, err return false, err
} }
} }
describeErrorCount++ errorCount++
if describeErrorCount > volumeAttachmentStatusConsecutiveErrorLimit { if errorCount > volumeAttachmentStatusConsecutiveErrorLimit {
// report the error // report the error
return false, err return false, err
} }
...@@ -2112,8 +2121,6 @@ func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment, ...@@ -2112,8 +2121,6 @@ func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment,
return false, nil return false, nil
} }
describeErrorCount = 0
if len(info.Attachments) > 1 { if len(info.Attachments) > 1 {
// Shouldn't happen; log so we know if it is // Shouldn't happen; log so we know if it is
klog.Warningf("Found multiple attachments for volume %q: %v", d.awsID, info) klog.Warningf("Found multiple attachments for volume %q: %v", d.awsID, info)
...@@ -2135,15 +2142,42 @@ func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment, ...@@ -2135,15 +2142,42 @@ func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment,
if attachmentStatus == "" { if attachmentStatus == "" {
attachmentStatus = "detached" attachmentStatus = "detached"
} }
if attachment != nil {
// AWS eventual consistency can go back in time.
// For example, we're waiting for a volume to be attached as /dev/xvdba, but AWS can tell us it's
// attached as /dev/xvdbb, where it was attached before and it was already detached.
// Retry couple of times, hoping AWS starts reporting the right status.
device := aws.StringValue(attachment.Device)
if expectedDevice != "" && device != "" && device != expectedDevice {
klog.Warningf("Expected device %s %s for volume %s, but found device %s %s", expectedDevice, status, d.name, device, attachmentStatus)
errorCount++
if errorCount > volumeAttachmentStatusConsecutiveErrorLimit {
// report the error
return false, fmt.Errorf("attachment of disk %q failed: requested device %q but found %q", d.name, expectedDevice, device)
}
return false, nil
}
instanceID := aws.StringValue(attachment.InstanceId)
if expectedInstance != "" && instanceID != "" && instanceID != expectedInstance {
klog.Warningf("Expected instance %s/%s for volume %s, but found instance %s/%s", expectedInstance, status, d.name, instanceID, attachmentStatus)
errorCount++
if errorCount > volumeAttachmentStatusConsecutiveErrorLimit {
// report the error
return false, fmt.Errorf("attachment of disk %q failed: requested device %q but found %q", d.name, expectedDevice, device)
}
return false, nil
}
}
if attachmentStatus == status { if attachmentStatus == status {
// Attachment is in requested state, finish waiting // Attachment is in requested state, finish waiting
return true, nil return true, nil
} }
// continue waiting // continue waiting
errorCount = 0
klog.V(2).Infof("Waiting for volume %q state: actual=%s, desired=%s", d.awsID, attachmentStatus, status) klog.V(2).Infof("Waiting for volume %q state: actual=%s, desired=%s", d.awsID, attachmentStatus, status)
return false, nil return false, nil
}) })
return attachment, err return attachment, err
} }
...@@ -2280,7 +2314,7 @@ func (c *Cloud) AttachDisk(diskName KubernetesVolumeID, nodeName types.NodeName) ...@@ -2280,7 +2314,7 @@ func (c *Cloud) AttachDisk(diskName KubernetesVolumeID, nodeName types.NodeName)
klog.V(2).Infof("AttachVolume volume=%q instance=%q request returned %v", disk.awsID, awsInstance.awsID, attachResponse) klog.V(2).Infof("AttachVolume volume=%q instance=%q request returned %v", disk.awsID, awsInstance.awsID, attachResponse)
} }
attachment, err := disk.waitForAttachmentStatus("attached") attachment, err := disk.waitForAttachmentStatus("attached", awsInstance.awsID, ec2Device)
if err != nil { if err != nil {
if err == wait.ErrWaitTimeout { if err == wait.ErrWaitTimeout {
...@@ -2300,6 +2334,7 @@ func (c *Cloud) AttachDisk(diskName KubernetesVolumeID, nodeName types.NodeName) ...@@ -2300,6 +2334,7 @@ func (c *Cloud) AttachDisk(diskName KubernetesVolumeID, nodeName types.NodeName)
return "", fmt.Errorf("unexpected state: attachment nil after attached %q to %q", diskName, nodeName) return "", fmt.Errorf("unexpected state: attachment nil after attached %q to %q", diskName, nodeName)
} }
if ec2Device != aws.StringValue(attachment.Device) { if ec2Device != aws.StringValue(attachment.Device) {
// Already checked in waitForAttachmentStatus(), but just to be sure...
return "", fmt.Errorf("disk attachment of %q to %q failed: requested device %q but found %q", diskName, nodeName, ec2Device, aws.StringValue(attachment.Device)) return "", fmt.Errorf("disk attachment of %q to %q failed: requested device %q but found %q", diskName, nodeName, ec2Device, aws.StringValue(attachment.Device))
} }
if awsInstance.awsID != aws.StringValue(attachment.InstanceId) { if awsInstance.awsID != aws.StringValue(attachment.InstanceId) {
...@@ -2357,7 +2392,7 @@ func (c *Cloud) DetachDisk(diskName KubernetesVolumeID, nodeName types.NodeName) ...@@ -2357,7 +2392,7 @@ func (c *Cloud) DetachDisk(diskName KubernetesVolumeID, nodeName types.NodeName)
return "", errors.New("no response from DetachVolume") return "", errors.New("no response from DetachVolume")
} }
attachment, err := diskInfo.disk.waitForAttachmentStatus("detached") attachment, err := diskInfo.disk.waitForAttachmentStatus("detached", awsInstance.awsID, "")
if err != nil { if err != nil {
return "", err return "", err
} }
...@@ -4619,3 +4654,10 @@ func setNodeDisk( ...@@ -4619,3 +4654,10 @@ func setNodeDisk(
} }
volumeMap[volumeID] = check volumeMap[volumeID] = check
} }
func getInitialAttachDetachDelay(status string) time.Duration {
if status == "detached" {
return volumeDetachmentStatusInitialDelay
}
return volumeAttachmentStatusInitialDelay
}
...@@ -99,6 +99,11 @@ func (c *controllerCommon) getNodeVMSet(nodeName types.NodeName, crt cacheReadTy ...@@ -99,6 +99,11 @@ func (c *controllerCommon) getNodeVMSet(nodeName types.NodeName, crt cacheReadTy
// AttachDisk attaches a vhd to vm. The vhd must exist, can be identified by diskName, diskURI. // AttachDisk attaches a vhd to vm. The vhd must exist, can be identified by diskName, diskURI.
// return (lun, error) // return (lun, error)
func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, cachingMode compute.CachingTypes) (int32, error) { func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, cachingMode compute.CachingTypes) (int32, error) {
vmset, err := c.getNodeVMSet(nodeName, cacheReadTypeUnsafe)
if err != nil {
return -1, err
}
if isManagedDisk { if isManagedDisk {
diskName := path.Base(diskURI) diskName := path.Base(diskURI)
resourceGroup, err := getResourceGroupFromDiskURI(diskURI) resourceGroup, err := getResourceGroupFromDiskURI(diskURI)
...@@ -118,18 +123,16 @@ func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI stri ...@@ -118,18 +123,16 @@ func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI stri
attachErr := fmt.Sprintf( attachErr := fmt.Sprintf(
"disk(%s) already attached to node(%s), could not be attached to node(%s)", "disk(%s) already attached to node(%s), could not be attached to node(%s)",
diskURI, *disk.ManagedBy, nodeName) diskURI, *disk.ManagedBy, nodeName)
attachedNode := path.Base(*disk.ManagedBy) attachedNode, err := vmset.GetNodeNameByProviderID(*disk.ManagedBy)
if err != nil {
return -1, err
}
klog.V(2).Infof("found dangling volume %s attached to node %s", diskURI, attachedNode) klog.V(2).Infof("found dangling volume %s attached to node %s", diskURI, attachedNode)
danglingErr := volerr.NewDanglingError(attachErr, types.NodeName(attachedNode), "") danglingErr := volerr.NewDanglingError(attachErr, attachedNode, "")
return -1, danglingErr return -1, danglingErr
} }
} }
vmset, err := c.getNodeVMSet(nodeName, cacheReadTypeUnsafe)
if err != nil {
return -1, err
}
instanceid, err := c.cloud.InstanceID(context.TODO(), nodeName) instanceid, err := c.cloud.InstanceID(context.TODO(), nodeName)
if err != nil { if err != nil {
klog.Warningf("failed to get azure instance id (%v) for node %s", err, nodeName) klog.Warningf("failed to get azure instance id (%v) for node %s", err, nodeName)
......
...@@ -422,7 +422,7 @@ func (az *Cloud) getServiceLoadBalancerStatus(service *v1.Service, lb *network.L ...@@ -422,7 +422,7 @@ func (az *Cloud) getServiceLoadBalancerStatus(service *v1.Service, lb *network.L
if pipID == nil { if pipID == nil {
return nil, fmt.Errorf("get(%s): lb(%s) - failed to get LB PublicIPAddress ID is Nil", serviceName, *lb.Name) return nil, fmt.Errorf("get(%s): lb(%s) - failed to get LB PublicIPAddress ID is Nil", serviceName, *lb.Name)
} }
pipName, err := getLastSegment(*pipID) pipName, err := getLastSegment(*pipID, "/")
if err != nil { if err != nil {
return nil, fmt.Errorf("get(%s): lb(%s) - failed to get LB PublicIPAddress Name from ID(%s)", serviceName, *lb.Name, *pipID) return nil, fmt.Errorf("get(%s): lb(%s) - failed to get LB PublicIPAddress Name from ID(%s)", serviceName, *lb.Name, *pipID)
} }
......
...@@ -70,7 +70,7 @@ const ( ...@@ -70,7 +70,7 @@ const (
) )
var errNotInVMSet = errors.New("vm is not in the vmset") var errNotInVMSet = errors.New("vm is not in the vmset")
var providerIDRE = regexp.MustCompile(`^` + CloudProviderName + `://(?:.*)/Microsoft.Compute/virtualMachines/(.+)$`) var providerIDRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/Microsoft.Compute/virtualMachines/(.+)$`)
var backendPoolIDRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/backendAddressPools/(?:.*)`) var backendPoolIDRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/backendAddressPools/(?:.*)`)
var nicResourceGroupRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Network/networkInterfaces/(?:.*)`) var nicResourceGroupRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Network/networkInterfaces/(?:.*)`)
var publicIPResourceGroupRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Network/publicIPAddresses/(?:.*)`) var publicIPResourceGroupRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Network/publicIPAddresses/(?:.*)`)
...@@ -162,8 +162,8 @@ func isMasterNode(node *v1.Node) bool { ...@@ -162,8 +162,8 @@ func isMasterNode(node *v1.Node) bool {
} }
// returns the deepest child's identifier from a full identifier string. // returns the deepest child's identifier from a full identifier string.
func getLastSegment(ID string) (string, error) { func getLastSegment(ID, separator string) (string, error) {
parts := strings.Split(ID, "/") parts := strings.Split(ID, separator)
name := parts[len(parts)-1] name := parts[len(parts)-1]
if len(name) == 0 { if len(name) == 0 {
return "", fmt.Errorf("resource name was missing from identifier") return "", fmt.Errorf("resource name was missing from identifier")
...@@ -498,7 +498,7 @@ func (as *availabilitySet) GetIPByNodeName(name string) (string, string, error) ...@@ -498,7 +498,7 @@ func (as *availabilitySet) GetIPByNodeName(name string) (string, string, error)
publicIP := "" publicIP := ""
if ipConfig.PublicIPAddress != nil && ipConfig.PublicIPAddress.ID != nil { if ipConfig.PublicIPAddress != nil && ipConfig.PublicIPAddress.ID != nil {
pipID := *ipConfig.PublicIPAddress.ID pipID := *ipConfig.PublicIPAddress.ID
pipName, err := getLastSegment(pipID) pipName, err := getLastSegment(pipID, "/")
if err != nil { if err != nil {
return "", "", fmt.Errorf("failed to publicIP name for node %q with pipID %q", name, pipID) return "", "", fmt.Errorf("failed to publicIP name for node %q with pipID %q", name, pipID)
} }
...@@ -568,7 +568,7 @@ func (as *availabilitySet) getAgentPoolAvailabiliySets(nodes []*v1.Node) (agentP ...@@ -568,7 +568,7 @@ func (as *availabilitySet) getAgentPoolAvailabiliySets(nodes []*v1.Node) (agentP
// already added in the list // already added in the list
continue continue
} }
asName, err := getLastSegment(asID) asName, err := getLastSegment(asID, "/")
if err != nil { if err != nil {
klog.Errorf("as.getNodeAvailabilitySet - Node (%s)- getLastSegment(%s), err=%v", nodeName, asID, err) klog.Errorf("as.getNodeAvailabilitySet - Node (%s)- getLastSegment(%s), err=%v", nodeName, asID, err)
return nil, err return nil, err
...@@ -659,7 +659,7 @@ func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName stri ...@@ -659,7 +659,7 @@ func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName stri
if err != nil { if err != nil {
return network.Interface{}, err return network.Interface{}, err
} }
nicName, err := getLastSegment(primaryNicID) nicName, err := getLastSegment(primaryNicID, "/")
if err != nil { if err != nil {
return network.Interface{}, err return network.Interface{}, err
} }
......
...@@ -187,6 +187,16 @@ func (ss *scaleSet) getVmssVM(nodeName string, crt cacheReadType) (string, strin ...@@ -187,6 +187,16 @@ func (ss *scaleSet) getVmssVM(nodeName string, crt cacheReadType) (string, strin
// GetPowerStatusByNodeName returns the power state of the specified node. // GetPowerStatusByNodeName returns the power state of the specified node.
func (ss *scaleSet) GetPowerStatusByNodeName(name string) (powerState string, err error) { func (ss *scaleSet) GetPowerStatusByNodeName(name string) (powerState string, err error) {
managedByAS, err := ss.isNodeManagedByAvailabilitySet(name, cacheReadTypeUnsafe)
if err != nil {
klog.Errorf("Failed to check isNodeManagedByAvailabilitySet: %v", err)
return "", err
}
if managedByAS {
// vm is managed by availability set.
return ss.availabilitySet.GetPowerStatusByNodeName(name)
}
_, _, vm, err := ss.getVmssVM(name, cacheReadTypeDefault) _, _, vm, err := ss.getVmssVM(name, cacheReadTypeDefault)
if err != nil { if err != nil {
return powerState, err return powerState, err
...@@ -283,6 +293,11 @@ func (ss *scaleSet) GetInstanceIDByNodeName(name string) (string, error) { ...@@ -283,6 +293,11 @@ func (ss *scaleSet) GetInstanceIDByNodeName(name string) (string, error) {
} }
// GetNodeNameByProviderID gets the node name by provider ID. // GetNodeNameByProviderID gets the node name by provider ID.
// providerID example:
// 1. vmas providerID: azure:///subscriptions/subsid/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/aks-nodepool1-27053986-0
// 2. vmss providerID:
// azure:///subscriptions/subsid/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/aks-agentpool-22126781-vmss/virtualMachines/1
// /subscriptions/subsid/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/aks-agentpool-22126781-vmss/virtualMachines/k8s-agentpool-36841236-vmss_1
func (ss *scaleSet) GetNodeNameByProviderID(providerID string) (types.NodeName, error) { func (ss *scaleSet) GetNodeNameByProviderID(providerID string) (types.NodeName, error) {
// NodeName is not part of providerID for vmss instances. // NodeName is not part of providerID for vmss instances.
scaleSetName, err := extractScaleSetNameByProviderID(providerID) scaleSetName, err := extractScaleSetNameByProviderID(providerID)
...@@ -296,12 +311,20 @@ func (ss *scaleSet) GetNodeNameByProviderID(providerID string) (types.NodeName, ...@@ -296,12 +311,20 @@ func (ss *scaleSet) GetNodeNameByProviderID(providerID string) (types.NodeName,
return "", fmt.Errorf("error of extracting resource group for node %q", providerID) return "", fmt.Errorf("error of extracting resource group for node %q", providerID)
} }
instanceID, err := getLastSegment(providerID) instanceID, err := getLastSegment(providerID, "/")
if err != nil { if err != nil {
klog.V(4).Infof("Can not extract instanceID from providerID (%s), assuming it is mananaged by availability set: %v", providerID, err) klog.V(4).Infof("Can not extract instanceID from providerID (%s), assuming it is mananaged by availability set: %v", providerID, err)
return ss.availabilitySet.GetNodeNameByProviderID(providerID) return ss.availabilitySet.GetNodeNameByProviderID(providerID)
} }
// instanceID contains scaleSetName (returned by disk.ManagedBy), e.g. k8s-agentpool-36841236-vmss_1
if strings.HasPrefix(strings.ToLower(instanceID), strings.ToLower(scaleSetName)) {
instanceID, err = getLastSegment(instanceID, "_")
if err != nil {
return "", err
}
}
vm, err := ss.getVmssVMByInstanceID(resourceGroup, scaleSetName, instanceID, cacheReadTypeUnsafe) vm, err := ss.getVmssVMByInstanceID(resourceGroup, scaleSetName, instanceID, cacheReadTypeUnsafe)
if err != nil { if err != nil {
return "", err return "", err
...@@ -696,7 +719,7 @@ func (ss *scaleSet) GetPrimaryInterface(nodeName string) (network.Interface, err ...@@ -696,7 +719,7 @@ func (ss *scaleSet) GetPrimaryInterface(nodeName string) (network.Interface, err
return network.Interface{}, err return network.Interface{}, err
} }
nicName, err := getLastSegment(primaryInterfaceID) nicName, err := getLastSegment(primaryInterfaceID, "/")
if err != nil { if err != nil {
klog.Errorf("error: ss.GetPrimaryInterface(%s), getLastSegment(%s), err=%v", nodeName, primaryInterfaceID, err) klog.Errorf("error: ss.GetPrimaryInterface(%s), getLastSegment(%s), err=%v", nodeName, primaryInterfaceID, err)
return network.Interface{}, err return network.Interface{}, err
......
...@@ -1130,7 +1130,7 @@ gopkg.in/square/go-jose.v2/jwt ...@@ -1130,7 +1130,7 @@ gopkg.in/square/go-jose.v2/jwt
gopkg.in/warnings.v0 gopkg.in/warnings.v0
# gopkg.in/yaml.v2 v2.2.8 # gopkg.in/yaml.v2 v2.2.8
gopkg.in/yaml.v2 gopkg.in/yaml.v2
# k8s.io/api v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/api v1.16.9-k3s1 # k8s.io/api v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/api v1.16.10-k3s1
k8s.io/api/admission/v1 k8s.io/api/admission/v1
k8s.io/api/admission/v1beta1 k8s.io/api/admission/v1beta1
k8s.io/api/admissionregistration/v1 k8s.io/api/admissionregistration/v1
...@@ -1172,7 +1172,7 @@ k8s.io/api/settings/v1alpha1 ...@@ -1172,7 +1172,7 @@ k8s.io/api/settings/v1alpha1
k8s.io/api/storage/v1 k8s.io/api/storage/v1
k8s.io/api/storage/v1alpha1 k8s.io/api/storage/v1alpha1
k8s.io/api/storage/v1beta1 k8s.io/api/storage/v1beta1
# k8s.io/apiextensions-apiserver v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.16.9-k3s1 # k8s.io/apiextensions-apiserver v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.16.10-k3s1
k8s.io/apiextensions-apiserver/pkg/apihelpers k8s.io/apiextensions-apiserver/pkg/apihelpers
k8s.io/apiextensions-apiserver/pkg/apis/apiextensions k8s.io/apiextensions-apiserver/pkg/apis/apiextensions
k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install
...@@ -1220,7 +1220,7 @@ k8s.io/apiextensions-apiserver/pkg/generated/openapi ...@@ -1220,7 +1220,7 @@ k8s.io/apiextensions-apiserver/pkg/generated/openapi
k8s.io/apiextensions-apiserver/pkg/registry/customresource k8s.io/apiextensions-apiserver/pkg/registry/customresource
k8s.io/apiextensions-apiserver/pkg/registry/customresource/tableconvertor k8s.io/apiextensions-apiserver/pkg/registry/customresource/tableconvertor
k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition
# k8s.io/apimachinery v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/apimachinery v1.16.9-k3s1 # k8s.io/apimachinery v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/apimachinery v1.16.10-k3s1
k8s.io/apimachinery/pkg/api/equality k8s.io/apimachinery/pkg/api/equality
k8s.io/apimachinery/pkg/api/errors k8s.io/apimachinery/pkg/api/errors
k8s.io/apimachinery/pkg/api/meta k8s.io/apimachinery/pkg/api/meta
...@@ -1282,7 +1282,7 @@ k8s.io/apimachinery/pkg/watch ...@@ -1282,7 +1282,7 @@ k8s.io/apimachinery/pkg/watch
k8s.io/apimachinery/third_party/forked/golang/json k8s.io/apimachinery/third_party/forked/golang/json
k8s.io/apimachinery/third_party/forked/golang/netutil k8s.io/apimachinery/third_party/forked/golang/netutil
k8s.io/apimachinery/third_party/forked/golang/reflect k8s.io/apimachinery/third_party/forked/golang/reflect
# k8s.io/apiserver v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/apiserver v1.16.9-k3s1 # k8s.io/apiserver v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/apiserver v1.16.10-k3s1
k8s.io/apiserver/pkg/admission k8s.io/apiserver/pkg/admission
k8s.io/apiserver/pkg/admission/configuration k8s.io/apiserver/pkg/admission/configuration
k8s.io/apiserver/pkg/admission/initializer k8s.io/apiserver/pkg/admission/initializer
...@@ -1395,7 +1395,7 @@ k8s.io/apiserver/plugin/pkg/authenticator/request/basicauth ...@@ -1395,7 +1395,7 @@ k8s.io/apiserver/plugin/pkg/authenticator/request/basicauth
k8s.io/apiserver/plugin/pkg/authenticator/token/oidc k8s.io/apiserver/plugin/pkg/authenticator/token/oidc
k8s.io/apiserver/plugin/pkg/authenticator/token/webhook k8s.io/apiserver/plugin/pkg/authenticator/token/webhook
k8s.io/apiserver/plugin/pkg/authorizer/webhook k8s.io/apiserver/plugin/pkg/authorizer/webhook
# k8s.io/cli-runtime v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/cli-runtime v1.16.9-k3s1 # k8s.io/cli-runtime v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/cli-runtime v1.16.10-k3s1
k8s.io/cli-runtime/pkg/genericclioptions k8s.io/cli-runtime/pkg/genericclioptions
k8s.io/cli-runtime/pkg/kustomize k8s.io/cli-runtime/pkg/kustomize
k8s.io/cli-runtime/pkg/kustomize/k8sdeps k8s.io/cli-runtime/pkg/kustomize/k8sdeps
...@@ -1408,7 +1408,7 @@ k8s.io/cli-runtime/pkg/kustomize/k8sdeps/transformer/patch ...@@ -1408,7 +1408,7 @@ k8s.io/cli-runtime/pkg/kustomize/k8sdeps/transformer/patch
k8s.io/cli-runtime/pkg/kustomize/k8sdeps/validator k8s.io/cli-runtime/pkg/kustomize/k8sdeps/validator
k8s.io/cli-runtime/pkg/printers k8s.io/cli-runtime/pkg/printers
k8s.io/cli-runtime/pkg/resource k8s.io/cli-runtime/pkg/resource
# k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible => github.com/rancher/kubernetes/staging/src/k8s.io/client-go v1.16.9-k3s1 # k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible => github.com/rancher/kubernetes/staging/src/k8s.io/client-go v1.16.10-k3s1
k8s.io/client-go/discovery k8s.io/client-go/discovery
k8s.io/client-go/discovery/cached k8s.io/client-go/discovery/cached
k8s.io/client-go/discovery/cached/disk k8s.io/client-go/discovery/cached/disk
...@@ -1594,20 +1594,20 @@ k8s.io/client-go/util/jsonpath ...@@ -1594,20 +1594,20 @@ k8s.io/client-go/util/jsonpath
k8s.io/client-go/util/keyutil k8s.io/client-go/util/keyutil
k8s.io/client-go/util/retry k8s.io/client-go/util/retry
k8s.io/client-go/util/workqueue k8s.io/client-go/util/workqueue
# k8s.io/cloud-provider v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/cloud-provider v1.16.9-k3s1 # k8s.io/cloud-provider v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/cloud-provider v1.16.10-k3s1
k8s.io/cloud-provider k8s.io/cloud-provider
k8s.io/cloud-provider/node/helpers k8s.io/cloud-provider/node/helpers
k8s.io/cloud-provider/service/helpers k8s.io/cloud-provider/service/helpers
k8s.io/cloud-provider/volume k8s.io/cloud-provider/volume
k8s.io/cloud-provider/volume/errors k8s.io/cloud-provider/volume/errors
k8s.io/cloud-provider/volume/helpers k8s.io/cloud-provider/volume/helpers
# k8s.io/cluster-bootstrap v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.16.9-k3s1 # k8s.io/cluster-bootstrap v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.16.10-k3s1
k8s.io/cluster-bootstrap/token/api k8s.io/cluster-bootstrap/token/api
k8s.io/cluster-bootstrap/token/jws k8s.io/cluster-bootstrap/token/jws
k8s.io/cluster-bootstrap/token/util k8s.io/cluster-bootstrap/token/util
k8s.io/cluster-bootstrap/util/secrets k8s.io/cluster-bootstrap/util/secrets
k8s.io/cluster-bootstrap/util/tokens k8s.io/cluster-bootstrap/util/tokens
# k8s.io/code-generator v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/code-generator v1.16.9-k3s1 # k8s.io/code-generator v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/code-generator v1.16.10-k3s1
k8s.io/code-generator/cmd/client-gen/args k8s.io/code-generator/cmd/client-gen/args
k8s.io/code-generator/cmd/client-gen/generators k8s.io/code-generator/cmd/client-gen/generators
k8s.io/code-generator/cmd/client-gen/generators/fake k8s.io/code-generator/cmd/client-gen/generators/fake
...@@ -1622,7 +1622,7 @@ k8s.io/code-generator/cmd/lister-gen/args ...@@ -1622,7 +1622,7 @@ k8s.io/code-generator/cmd/lister-gen/args
k8s.io/code-generator/cmd/lister-gen/generators k8s.io/code-generator/cmd/lister-gen/generators
k8s.io/code-generator/pkg/namer k8s.io/code-generator/pkg/namer
k8s.io/code-generator/pkg/util k8s.io/code-generator/pkg/util
# k8s.io/component-base v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/component-base v1.16.9-k3s1 # k8s.io/component-base v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/component-base v1.16.10-k3s1
k8s.io/component-base/cli/flag k8s.io/component-base/cli/flag
k8s.io/component-base/cli/globalflag k8s.io/component-base/cli/globalflag
k8s.io/component-base/config k8s.io/component-base/config
...@@ -1635,10 +1635,10 @@ k8s.io/component-base/metrics/legacyregistry ...@@ -1635,10 +1635,10 @@ k8s.io/component-base/metrics/legacyregistry
k8s.io/component-base/metrics/prometheus/restclient k8s.io/component-base/metrics/prometheus/restclient
k8s.io/component-base/metrics/prometheus/workqueue k8s.io/component-base/metrics/prometheus/workqueue
k8s.io/component-base/version k8s.io/component-base/version
# k8s.io/cri-api v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/cri-api v1.16.9-k3s1 # k8s.io/cri-api v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/cri-api v1.16.10-k3s1
k8s.io/cri-api/pkg/apis k8s.io/cri-api/pkg/apis
k8s.io/cri-api/pkg/apis/runtime/v1alpha2 k8s.io/cri-api/pkg/apis/runtime/v1alpha2
# k8s.io/csi-translation-lib v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/csi-translation-lib v1.16.9-k3s1 # k8s.io/csi-translation-lib v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/csi-translation-lib v1.16.10-k3s1
k8s.io/csi-translation-lib k8s.io/csi-translation-lib
k8s.io/csi-translation-lib/plugins k8s.io/csi-translation-lib/plugins
# k8s.io/gengo v0.0.0-20190822140433-26a664648505 # k8s.io/gengo v0.0.0-20190822140433-26a664648505
...@@ -1653,7 +1653,7 @@ k8s.io/gengo/types ...@@ -1653,7 +1653,7 @@ k8s.io/gengo/types
k8s.io/heapster/metrics/api/v1/types k8s.io/heapster/metrics/api/v1/types
# k8s.io/klog v1.0.0 # k8s.io/klog v1.0.0
k8s.io/klog k8s.io/klog
# k8s.io/kube-aggregator v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kube-aggregator v1.16.9-k3s1 # k8s.io/kube-aggregator v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kube-aggregator v1.16.10-k3s1
k8s.io/kube-aggregator/pkg/apis/apiregistration k8s.io/kube-aggregator/pkg/apis/apiregistration
k8s.io/kube-aggregator/pkg/apis/apiregistration/install k8s.io/kube-aggregator/pkg/apis/apiregistration/install
k8s.io/kube-aggregator/pkg/apis/apiregistration/v1 k8s.io/kube-aggregator/pkg/apis/apiregistration/v1
...@@ -1681,9 +1681,9 @@ k8s.io/kube-aggregator/pkg/controllers/status ...@@ -1681,9 +1681,9 @@ k8s.io/kube-aggregator/pkg/controllers/status
k8s.io/kube-aggregator/pkg/registry/apiservice k8s.io/kube-aggregator/pkg/registry/apiservice
k8s.io/kube-aggregator/pkg/registry/apiservice/etcd k8s.io/kube-aggregator/pkg/registry/apiservice/etcd
k8s.io/kube-aggregator/pkg/registry/apiservice/rest k8s.io/kube-aggregator/pkg/registry/apiservice/rest
# k8s.io/kube-controller-manager v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kube-controller-manager v1.16.9-k3s1 # k8s.io/kube-controller-manager v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kube-controller-manager v1.16.10-k3s1
k8s.io/kube-controller-manager/config/v1alpha1 k8s.io/kube-controller-manager/config/v1alpha1
# k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf # k8s.io/kube-openapi v0.0.0-20200410163147-594e756bea31
k8s.io/kube-openapi/pkg/aggregator k8s.io/kube-openapi/pkg/aggregator
k8s.io/kube-openapi/pkg/builder k8s.io/kube-openapi/pkg/builder
k8s.io/kube-openapi/pkg/common k8s.io/kube-openapi/pkg/common
...@@ -1692,11 +1692,11 @@ k8s.io/kube-openapi/pkg/schemaconv ...@@ -1692,11 +1692,11 @@ k8s.io/kube-openapi/pkg/schemaconv
k8s.io/kube-openapi/pkg/util k8s.io/kube-openapi/pkg/util
k8s.io/kube-openapi/pkg/util/proto k8s.io/kube-openapi/pkg/util/proto
k8s.io/kube-openapi/pkg/util/proto/validation k8s.io/kube-openapi/pkg/util/proto/validation
# k8s.io/kube-proxy v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kube-proxy v1.16.9-k3s1 # k8s.io/kube-proxy v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kube-proxy v1.16.10-k3s1
k8s.io/kube-proxy/config/v1alpha1 k8s.io/kube-proxy/config/v1alpha1
# k8s.io/kube-scheduler v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kube-scheduler v1.16.9-k3s1 # k8s.io/kube-scheduler v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kube-scheduler v1.16.10-k3s1
k8s.io/kube-scheduler/config/v1alpha1 k8s.io/kube-scheduler/config/v1alpha1
# k8s.io/kubectl v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kubectl v1.16.9-k3s1 # k8s.io/kubectl v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kubectl v1.16.10-k3s1
k8s.io/kubectl/pkg/apps k8s.io/kubectl/pkg/apps
k8s.io/kubectl/pkg/cmd k8s.io/kubectl/pkg/cmd
k8s.io/kubectl/pkg/cmd/annotate k8s.io/kubectl/pkg/cmd/annotate
...@@ -1773,9 +1773,9 @@ k8s.io/kubectl/pkg/util/templates ...@@ -1773,9 +1773,9 @@ k8s.io/kubectl/pkg/util/templates
k8s.io/kubectl/pkg/util/term k8s.io/kubectl/pkg/util/term
k8s.io/kubectl/pkg/validation k8s.io/kubectl/pkg/validation
k8s.io/kubectl/pkg/version k8s.io/kubectl/pkg/version
# k8s.io/kubelet v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kubelet v1.16.9-k3s1 # k8s.io/kubelet v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/kubelet v1.16.10-k3s1
k8s.io/kubelet/config/v1beta1 k8s.io/kubelet/config/v1beta1
# k8s.io/kubernetes v1.16.0 => github.com/rancher/kubernetes v1.16.9-k3s1 # k8s.io/kubernetes v1.16.0 => github.com/rancher/kubernetes v1.16.10-k3s1
k8s.io/kubernetes/cmd/cloud-controller-manager/app k8s.io/kubernetes/cmd/cloud-controller-manager/app
k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config
k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/scheme k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/scheme
...@@ -2491,7 +2491,7 @@ k8s.io/kubernetes/third_party/forked/gonum/graph ...@@ -2491,7 +2491,7 @@ k8s.io/kubernetes/third_party/forked/gonum/graph
k8s.io/kubernetes/third_party/forked/gonum/graph/internal/linear k8s.io/kubernetes/third_party/forked/gonum/graph/internal/linear
k8s.io/kubernetes/third_party/forked/gonum/graph/simple k8s.io/kubernetes/third_party/forked/gonum/graph/simple
k8s.io/kubernetes/third_party/forked/gonum/graph/traverse k8s.io/kubernetes/third_party/forked/gonum/graph/traverse
# k8s.io/legacy-cloud-providers v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.16.9-k3s1 # k8s.io/legacy-cloud-providers v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.16.10-k3s1
k8s.io/legacy-cloud-providers/aws k8s.io/legacy-cloud-providers/aws
k8s.io/legacy-cloud-providers/azure k8s.io/legacy-cloud-providers/azure
k8s.io/legacy-cloud-providers/azure/auth k8s.io/legacy-cloud-providers/azure/auth
...@@ -2501,7 +2501,7 @@ k8s.io/legacy-cloud-providers/openstack/util/mount ...@@ -2501,7 +2501,7 @@ k8s.io/legacy-cloud-providers/openstack/util/mount
k8s.io/legacy-cloud-providers/vsphere k8s.io/legacy-cloud-providers/vsphere
k8s.io/legacy-cloud-providers/vsphere/vclib k8s.io/legacy-cloud-providers/vsphere/vclib
k8s.io/legacy-cloud-providers/vsphere/vclib/diskmanagers k8s.io/legacy-cloud-providers/vsphere/vclib/diskmanagers
# k8s.io/metrics v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/metrics v1.16.9-k3s1 # k8s.io/metrics v0.0.0 => github.com/rancher/kubernetes/staging/src/k8s.io/metrics v1.16.10-k3s1
k8s.io/metrics/pkg/apis/custom_metrics k8s.io/metrics/pkg/apis/custom_metrics
k8s.io/metrics/pkg/apis/custom_metrics/v1beta1 k8s.io/metrics/pkg/apis/custom_metrics/v1beta1
k8s.io/metrics/pkg/apis/custom_metrics/v1beta2 k8s.io/metrics/pkg/apis/custom_metrics/v1beta2
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment