v1.17.12-k3s1

parent 3a2fa61b
......@@ -33,31 +33,31 @@ replace (
github.com/prometheus/client_model => github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910
github.com/prometheus/common => github.com/prometheus/common v0.0.0-20181126121408-4724e9255275
github.com/prometheus/procfs => github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a
k8s.io/api => github.com/rancher/kubernetes/staging/src/k8s.io/api v1.17.11-k3s1
k8s.io/apiextensions-apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.17.11-k3s1
k8s.io/apimachinery => github.com/rancher/kubernetes/staging/src/k8s.io/apimachinery v1.17.11-k3s1
k8s.io/apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/apiserver v1.17.11-k3s1
k8s.io/cli-runtime => github.com/rancher/kubernetes/staging/src/k8s.io/cli-runtime v1.17.11-k3s1
k8s.io/client-go => github.com/rancher/kubernetes/staging/src/k8s.io/client-go v1.17.11-k3s1
k8s.io/cloud-provider => github.com/rancher/kubernetes/staging/src/k8s.io/cloud-provider v1.17.11-k3s1
k8s.io/cluster-bootstrap => github.com/rancher/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.17.11-k3s1
k8s.io/code-generator => github.com/rancher/kubernetes/staging/src/k8s.io/code-generator v1.17.11-k3s1
k8s.io/component-base => github.com/rancher/kubernetes/staging/src/k8s.io/component-base v1.17.11-k3s1
k8s.io/cri-api => github.com/rancher/kubernetes/staging/src/k8s.io/cri-api v1.17.11-k3s1
k8s.io/csi-translation-lib => github.com/rancher/kubernetes/staging/src/k8s.io/csi-translation-lib v1.17.11-k3s1
k8s.io/kube-aggregator => github.com/rancher/kubernetes/staging/src/k8s.io/kube-aggregator v1.17.11-k3s1
k8s.io/kube-controller-manager => github.com/rancher/kubernetes/staging/src/k8s.io/kube-controller-manager v1.17.11-k3s1
k8s.io/kube-proxy => github.com/rancher/kubernetes/staging/src/k8s.io/kube-proxy v1.17.11-k3s1
k8s.io/kube-scheduler => github.com/rancher/kubernetes/staging/src/k8s.io/kube-scheduler v1.17.11-k3s1
k8s.io/kubectl => github.com/rancher/kubernetes/staging/src/k8s.io/kubectl v1.17.11-k3s1
k8s.io/kubelet => github.com/rancher/kubernetes/staging/src/k8s.io/kubelet v1.17.11-k3s1
k8s.io/kubernetes => github.com/rancher/kubernetes v1.17.11-k3s1
k8s.io/legacy-cloud-providers => github.com/rancher/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.17.11-k3s1
k8s.io/metrics => github.com/rancher/kubernetes/staging/src/k8s.io/metrics v1.17.11-k3s1
k8s.io/node-api => github.com/rancher/kubernetes/staging/src/k8s.io/node-api v1.17.11-k3s1
k8s.io/sample-apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/sample-apiserver v1.17.11-k3s1
k8s.io/sample-cli-plugin => github.com/rancher/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.17.11-k3s1
k8s.io/sample-controller => github.com/rancher/kubernetes/staging/src/k8s.io/sample-controller v1.17.11-k3s1
k8s.io/api => github.com/rancher/kubernetes/staging/src/k8s.io/api v1.17.12-k3s1
k8s.io/apiextensions-apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.17.12-k3s1
k8s.io/apimachinery => github.com/rancher/kubernetes/staging/src/k8s.io/apimachinery v1.17.12-k3s1
k8s.io/apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/apiserver v1.17.12-k3s1
k8s.io/cli-runtime => github.com/rancher/kubernetes/staging/src/k8s.io/cli-runtime v1.17.12-k3s1
k8s.io/client-go => github.com/rancher/kubernetes/staging/src/k8s.io/client-go v1.17.12-k3s1
k8s.io/cloud-provider => github.com/rancher/kubernetes/staging/src/k8s.io/cloud-provider v1.17.12-k3s1
k8s.io/cluster-bootstrap => github.com/rancher/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.17.12-k3s1
k8s.io/code-generator => github.com/rancher/kubernetes/staging/src/k8s.io/code-generator v1.17.12-k3s1
k8s.io/component-base => github.com/rancher/kubernetes/staging/src/k8s.io/component-base v1.17.12-k3s1
k8s.io/cri-api => github.com/rancher/kubernetes/staging/src/k8s.io/cri-api v1.17.12-k3s1
k8s.io/csi-translation-lib => github.com/rancher/kubernetes/staging/src/k8s.io/csi-translation-lib v1.17.12-k3s1
k8s.io/kube-aggregator => github.com/rancher/kubernetes/staging/src/k8s.io/kube-aggregator v1.17.12-k3s1
k8s.io/kube-controller-manager => github.com/rancher/kubernetes/staging/src/k8s.io/kube-controller-manager v1.17.12-k3s1
k8s.io/kube-proxy => github.com/rancher/kubernetes/staging/src/k8s.io/kube-proxy v1.17.12-k3s1
k8s.io/kube-scheduler => github.com/rancher/kubernetes/staging/src/k8s.io/kube-scheduler v1.17.12-k3s1
k8s.io/kubectl => github.com/rancher/kubernetes/staging/src/k8s.io/kubectl v1.17.12-k3s1
k8s.io/kubelet => github.com/rancher/kubernetes/staging/src/k8s.io/kubelet v1.17.12-k3s1
k8s.io/kubernetes => github.com/rancher/kubernetes v1.17.12-k3s1
k8s.io/legacy-cloud-providers => github.com/rancher/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.17.12-k3s1
k8s.io/metrics => github.com/rancher/kubernetes/staging/src/k8s.io/metrics v1.17.12-k3s1
k8s.io/node-api => github.com/rancher/kubernetes/staging/src/k8s.io/node-api v1.17.12-k3s1
k8s.io/sample-apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/sample-apiserver v1.17.12-k3s1
k8s.io/sample-cli-plugin => github.com/rancher/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.17.12-k3s1
k8s.io/sample-controller => github.com/rancher/kubernetes/staging/src/k8s.io/sample-controller v1.17.12-k3s1
mvdan.cc/unparam => mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34
)
......
language: go
go:
- 1.8
- 1.7
- 1.14
- 1.13
install:
- if ! go get code.google.com/p/go.tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; fi
......@@ -11,6 +11,9 @@ install:
script:
- go get
- go test -cover ./...
- cd ./v5
- go get
- go test -cover ./...
notifications:
email: false
......@@ -6,7 +6,7 @@ modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of the Evan Phoenix nor the names of its contributors
......
# JSON-Patch
`jsonpatch` is a library which provides functionallity for both applying
`jsonpatch` is a library which provides functionality for both applying
[RFC6902 JSON patches](http://tools.ietf.org/html/rfc6902) against documents, as
well as for calculating & applying [RFC7396 JSON merge patches](https://tools.ietf.org/html/rfc7396).
......@@ -11,10 +11,11 @@ well as for calculating & applying [RFC7396 JSON merge patches](https://tools.ie
**Latest and greatest**:
```bash
go get -u github.com/evanphx/json-patch
go get -u github.com/evanphx/json-patch/v5
```
**Stable Versions**:
* Version 5: `go get -u gopkg.in/evanphx/json-patch.v5`
* Version 4: `go get -u gopkg.in/evanphx/json-patch.v4`
(previous versions below `v3` are unavailable)
......@@ -82,7 +83,7 @@ When ran, you get the following output:
```bash
$ go run main.go
patch document: {"height":null,"name":"Jane"}
updated tina doc: {"age":28,"name":"Jane"}
updated alternative doc: {"age":28,"name":"Jane"}
```
## Create and apply a JSON Patch
......@@ -164,7 +165,7 @@ func main() {
}
if !jsonpatch.Equal(original, different) {
fmt.Println(`"original" is _not_ structurally equal to "similar"`)
fmt.Println(`"original" is _not_ structurally equal to "different"`)
}
}
```
......@@ -173,7 +174,7 @@ When ran, you get the following output:
```bash
$ go run main.go
"original" is structurally equal to "similar"
"original" is _not_ structurally equal to "similar"
"original" is _not_ structurally equal to "different"
```
## Combine merge patches
......
......@@ -307,13 +307,16 @@ func matchesValue(av, bv interface{}) bool {
return true
case map[string]interface{}:
bt := bv.(map[string]interface{})
for key := range at {
if !matchesValue(at[key], bt[key]) {
return false
}
if len(bt) != len(at) {
return false
}
for key := range bt {
if !matchesValue(at[key], bt[key]) {
av, aOK := at[key]
bv, bOK := bt[key]
if aOK != bOK {
return false
}
if !matchesValue(av, bv) {
return false
}
}
......
......@@ -202,6 +202,10 @@ func (n *lazyNode) equal(o *lazyNode) bool {
return false
}
if len(n.doc) != len(o.doc) {
return false
}
for k, v := range n.doc {
ov, ok := o.doc[k]
......@@ -209,6 +213,10 @@ func (n *lazyNode) equal(o *lazyNode) bool {
return false
}
if (v == nil) != (ov == nil) {
return false
}
if v == nil && ov == nil {
continue
}
......@@ -429,14 +437,14 @@ func (d *partialArray) add(key string, val *lazyNode) error {
return errors.Wrapf(ErrInvalidIndex, "Unable to access invalid index: %d", idx)
}
if SupportNegativeIndices {
if idx < -len(ary) {
if idx < 0 {
if !SupportNegativeIndices {
return errors.Wrapf(ErrInvalidIndex, "Unable to access invalid index: %d", idx)
}
if idx < 0 {
idx += len(ary)
if idx < -len(ary) {
return errors.Wrapf(ErrInvalidIndex, "Unable to access invalid index: %d", idx)
}
idx += len(ary)
}
copy(ary[0:idx], cur[0:idx])
......@@ -473,14 +481,14 @@ func (d *partialArray) remove(key string) error {
return errors.Wrapf(ErrInvalidIndex, "Unable to access invalid index: %d", idx)
}
if SupportNegativeIndices {
if idx < -len(cur) {
if idx < 0 {
if !SupportNegativeIndices {
return errors.Wrapf(ErrInvalidIndex, "Unable to access invalid index: %d", idx)
}
if idx < 0 {
idx += len(cur)
if idx < -len(cur) {
return errors.Wrapf(ErrInvalidIndex, "Unable to access invalid index: %d", idx)
}
idx += len(cur)
}
ary := make([]*lazyNode, len(cur)-1)
......
......@@ -212,13 +212,17 @@ func GetHTTPClient(req *http.Request) string {
return "unknown"
}
// SourceIPs splits the comma separated X-Forwarded-For header or returns the X-Real-Ip header or req.RemoteAddr,
// in that order, ignoring invalid IPs. It returns nil if all of these are empty or invalid.
// SourceIPs splits the comma separated X-Forwarded-For header and joins it with
// the X-Real-Ip header and/or req.RemoteAddr, ignoring invalid IPs.
// The X-Real-Ip is omitted if it's already present in the X-Forwarded-For chain.
// The req.RemoteAddr is always the last IP in the returned list.
// It returns nil if all of these are empty or invalid.
func SourceIPs(req *http.Request) []net.IP {
var srcIPs []net.IP
hdr := req.Header
// First check the X-Forwarded-For header for requests via proxy.
hdrForwardedFor := hdr.Get("X-Forwarded-For")
forwardedForIPs := []net.IP{}
if hdrForwardedFor != "" {
// X-Forwarded-For can be a csv of IPs in case of multiple proxies.
// Use the first valid one.
......@@ -226,38 +230,49 @@ func SourceIPs(req *http.Request) []net.IP {
for _, part := range parts {
ip := net.ParseIP(strings.TrimSpace(part))
if ip != nil {
forwardedForIPs = append(forwardedForIPs, ip)
srcIPs = append(srcIPs, ip)
}
}
}
if len(forwardedForIPs) > 0 {
return forwardedForIPs
}
// Try the X-Real-Ip header.
hdrRealIp := hdr.Get("X-Real-Ip")
if hdrRealIp != "" {
ip := net.ParseIP(hdrRealIp)
if ip != nil {
return []net.IP{ip}
// Only append the X-Real-Ip if it's not already contained in the X-Forwarded-For chain.
if ip != nil && !containsIP(srcIPs, ip) {
srcIPs = append(srcIPs, ip)
}
}
// Fallback to Remote Address in request, which will give the correct client IP when there is no proxy.
// Always include the request Remote Address as it cannot be easily spoofed.
var remoteIP net.IP
// Remote Address in Go's HTTP server is in the form host:port so we need to split that first.
host, _, err := net.SplitHostPort(req.RemoteAddr)
if err == nil {
if remoteIP := net.ParseIP(host); remoteIP != nil {
return []net.IP{remoteIP}
}
remoteIP = net.ParseIP(host)
}
// Fallback if Remote Address was just IP.
if remoteIP := net.ParseIP(req.RemoteAddr); remoteIP != nil {
return []net.IP{remoteIP}
if remoteIP == nil {
remoteIP = net.ParseIP(req.RemoteAddr)
}
// Don't duplicate remote IP if it's already the last address in the chain.
if remoteIP != nil && (len(srcIPs) == 0 || !remoteIP.Equal(srcIPs[len(srcIPs)-1])) {
srcIPs = append(srcIPs, remoteIP)
}
return nil
return srcIPs
}
// Checks whether the given IP address is contained in the list of IPs.
func containsIP(ips []net.IP, ip net.IP) bool {
for _, v := range ips {
if v.Equal(ip) {
return true
}
}
return false
}
// Extracts and returns the clients IP from the given request.
......
......@@ -117,10 +117,37 @@ func WithImpersonation(handler http.Handler, a authorizer.Authorizer, s runtime.
}
}
if !groupsSpecified && username != user.Anonymous {
// When impersonating a non-anonymous user, if no groups were specified
// include the system:authenticated group in the impersonated user info
groups = append(groups, user.AllAuthenticated)
if username != user.Anonymous {
// When impersonating a non-anonymous user, include the 'system:authenticated' group
// in the impersonated user info:
// - if no groups were specified
// - if a group has been specified other than 'system:authenticated'
//
// If 'system:unauthenticated' group has been specified we should not include
// the 'system:authenticated' group.
addAuthenticated := true
for _, group := range groups {
if group == user.AllAuthenticated || group == user.AllUnauthenticated {
addAuthenticated = false
break
}
}
if addAuthenticated {
groups = append(groups, user.AllAuthenticated)
}
} else {
addUnauthenticated := true
for _, group := range groups {
if group == user.AllUnauthenticated {
addUnauthenticated = false
break
}
}
if addUnauthenticated {
groups = append(groups, user.AllUnauthenticated)
}
}
newUser := &user.DefaultInfo{
......
......@@ -64,6 +64,8 @@ func (w *realTimeoutFactory) TimeoutCh() (<-chan time.Time, func() bool) {
// serveWatch will serve a watch response.
// TODO: the functionality in this method and in WatchServer.Serve is not cleanly decoupled.
func serveWatch(watcher watch.Interface, scope *RequestScope, mediaTypeOptions negotiation.MediaTypeOptions, req *http.Request, w http.ResponseWriter, timeout time.Duration) {
defer watcher.Stop()
options, err := optionsForTransform(mediaTypeOptions, req)
if err != nil {
scope.err(err, w, req)
......@@ -201,7 +203,6 @@ func (s *WatchServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// ensure the connection times out
timeoutCh, cleanup := s.TimeoutFactory.TimeoutCh()
defer cleanup()
defer s.Watching.Stop()
// begin the stream
w.Header().Set("Content-Type", s.MediaType)
......@@ -286,8 +287,6 @@ func (s *WatchServer) HandleWS(ws *websocket.Conn) {
streamBuf := &bytes.Buffer{}
ch := s.Watching.ResultChan()
defer s.Watching.Stop()
for {
select {
case <-done:
......
......@@ -547,7 +547,7 @@ func (s *store) List(ctx context.Context, key, resourceVersion string, pred stor
newItemFunc := getNewItemFunc(listObj, v)
var returnedRV, continueRV int64
var returnedRV, continueRV, withRev int64
var continueKey string
switch {
case s.pagingEnabled && len(pred.Continue) > 0:
......@@ -568,7 +568,7 @@ func (s *store) List(ctx context.Context, key, resourceVersion string, pred stor
// continueRV==0 is invalid.
// If continueRV < 0, the request is for the latest resource version.
if continueRV > 0 {
options = append(options, clientv3.WithRev(continueRV))
withRev = continueRV
returnedRV = continueRV
}
case s.pagingEnabled && pred.Limit > 0:
......@@ -578,7 +578,7 @@ func (s *store) List(ctx context.Context, key, resourceVersion string, pred stor
return apierrors.NewBadRequest(fmt.Sprintf("invalid resource version: %v", err))
}
if fromRV > 0 {
options = append(options, clientv3.WithRev(int64(fromRV)))
withRev = int64(fromRV)
}
returnedRV = int64(fromRV)
}
......@@ -589,6 +589,9 @@ func (s *store) List(ctx context.Context, key, resourceVersion string, pred stor
default:
options = append(options, clientv3.WithPrefix())
}
if withRev != 0 {
options = append(options, clientv3.WithRev(withRev))
}
// loop until we have filled the requested limit from etcd or there are no more results
var lastKey []byte
......@@ -654,6 +657,10 @@ func (s *store) List(ctx context.Context, key, resourceVersion string, pred stor
break
}
key = string(lastKey) + "\x00"
if withRev == 0 {
withRev = returnedRV
options = append(options, clientv3.WithRev(withRev))
}
}
// instruct the client to begin querying from immediately after the last key we returned
......
......@@ -3,8 +3,8 @@ package version
var (
gitMajor = "1"
gitMinor = "17"
gitVersion = "v1.17.11-k3s1"
gitCommit = "51cfdf7fa2caf99d6ee48a3a5257d4177b8b4665"
gitVersion = "v1.17.12-k3s1"
gitCommit = "3a47527bc3f8a5ea50a5332060ebad8809b7bef8"
gitTreeState = "clean"
buildDate = "2020-08-13T18:43:40Z"
buildDate = "2020-09-16T20:26:31Z"
)
......@@ -118,11 +118,11 @@ func (c *controller) Run(stopCh <-chan struct{}) {
c.reflectorMutex.Unlock()
var wg wait.Group
defer wg.Wait()
wg.StartWithChannel(stopCh, r.Run)
wait.Until(c.processLoop, time.Second, stopCh)
wg.Wait()
}
// Returns true once this controller has completed an initial resource listing
......
......@@ -20,7 +20,7 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/evanphx/json-patch v0.0.0-20200808040245-162e5629780b/go.mod h1:NAJj0yf/KaRKURN6nyi7A9IZydMivZEm9oQLWNjfKDc=
github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
......@@ -63,7 +63,6 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
......
......@@ -3,8 +3,8 @@ package version
var (
gitMajor = "1"
gitMinor = "17"
gitVersion = "v1.17.11-k3s1"
gitCommit = "51cfdf7fa2caf99d6ee48a3a5257d4177b8b4665"
gitVersion = "v1.17.12-k3s1"
gitCommit = "3a47527bc3f8a5ea50a5332060ebad8809b7bef8"
gitTreeState = "clean"
buildDate = "2020-08-13T18:43:40Z"
buildDate = "2020-09-16T20:26:31Z"
)
......@@ -20,7 +20,7 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/evanphx/json-patch v0.0.0-20200808040245-162e5629780b/go.mod h1:NAJj0yf/KaRKURN6nyi7A9IZydMivZEm9oQLWNjfKDc=
github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
......@@ -56,7 +56,6 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
......
......@@ -21,6 +21,7 @@ go_test(
"//staging/src/k8s.io/api/apps/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
......
......@@ -601,7 +601,11 @@ func (r RealPodControl) DeletePod(namespace string, podID string, object runtime
return fmt.Errorf("object does not have ObjectMeta, %v", err)
}
klog.V(2).Infof("Controller %v deleting pod %v/%v", accessor.GetName(), namespace, podID)
if err := r.KubeClient.CoreV1().Pods(namespace).Delete(podID, nil); err != nil && !apierrors.IsNotFound(err) {
if err := r.KubeClient.CoreV1().Pods(namespace).Delete(podID, nil); err != nil {
if apierrors.IsNotFound(err) {
klog.V(4).Infof("pod %v/%v has already been deleted.", namespace, podID)
return err
}
r.Recorder.Eventf(object, v1.EventTypeWarning, FailedDeletePodReason, "Error deleting: %v", err)
return fmt.Errorf("unable to delete pods: %v", err)
}
......
......@@ -29,6 +29,7 @@ import (
v1 "k8s.io/api/core/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
"k8s.io/apimachinery/pkg/api/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
......@@ -1013,10 +1014,12 @@ func (dsc *DaemonSetsController) syncNodes(ds *apps.DaemonSet, podsToDelete, nod
go func(ix int) {
defer deleteWait.Done()
if err := dsc.podControl.DeletePod(ds.Namespace, podsToDelete[ix], ds); err != nil {
klog.V(2).Infof("Failed deletion, decrementing expectations for set %q/%q", ds.Namespace, ds.Name)
dsc.expectations.DeletionObserved(dsKey)
errCh <- err
utilruntime.HandleError(err)
if !apierrors.IsNotFound(err) {
klog.V(2).Infof("Failed deletion, decremented expectations for set %q/%q", ds.Namespace, ds.Name)
errCh <- err
utilruntime.HandleError(err)
}
}
}(i)
}
......
......@@ -207,7 +207,7 @@ func (c *Controller) Run(workers int, stopCh <-chan struct{}) {
klog.Infof("Starting endpoint slice controller")
defer klog.Infof("Shutting down endpoint slice controller")
if !cache.WaitForNamedCacheSync("endpoint_slice", stopCh, c.podsSynced, c.servicesSynced) {
if !cache.WaitForNamedCacheSync("endpoint_slice", stopCh, c.podsSynced, c.servicesSynced, c.endpointSlicesSynced, c.nodesSynced) {
return
}
......
......@@ -27,6 +27,7 @@ import (
batch "k8s.io/api/batch/v1"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
......@@ -612,7 +613,7 @@ func (jm *JobController) deleteJobPods(job *batch.Job, pods []*v1.Pod, errCh cha
for i := int32(0); i < int32(nbPods); i++ {
go func(ix int32) {
defer wait.Done()
if err := jm.podControl.DeletePod(job.Namespace, pods[ix].Name, job); err != nil {
if err := jm.podControl.DeletePod(job.Namespace, pods[ix].Name, job); err != nil && !apierrors.IsNotFound(err) {
defer utilruntime.HandleError(err)
klog.V(2).Infof("Failed to delete %v, job %q/%q deadline exceeded", pods[ix].Name, job.Namespace, job.Name)
errCh <- err
......@@ -709,14 +710,17 @@ func (jm *JobController) manageJob(activePods []*v1.Pod, succeeded int32, job *b
go func(ix int32) {
defer wait.Done()
if err := jm.podControl.DeletePod(job.Namespace, activePods[ix].Name, job); err != nil {
defer utilruntime.HandleError(err)
// Decrement the expected number of deletes because the informer won't observe this deletion
klog.V(2).Infof("Failed to delete %v, decrementing expectations for job %q/%q", activePods[ix].Name, job.Namespace, job.Name)
jm.expectations.DeletionObserved(jobKey)
activeLock.Lock()
active++
activeLock.Unlock()
errCh <- err
if !apierrors.IsNotFound(err) {
klog.V(2).Infof("Failed to delete %v, decremented expectations for job %q/%q", activePods[ix].Name, job.Namespace, job.Name)
activeLock.Lock()
active++
activeLock.Unlock()
errCh <- err
utilruntime.HandleError(err)
}
}
}(i)
}
......
......@@ -46,7 +46,9 @@ go_test(
"//staging/src/k8s.io/client-go/discovery:go_default_library",
"//staging/src/k8s.io/client-go/dynamic:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
"//staging/src/k8s.io/client-go/metadata:go_default_library",
"//staging/src/k8s.io/client-go/metadata/fake:go_default_library",
"//staging/src/k8s.io/client-go/rest:go_default_library",
"//staging/src/k8s.io/client-go/testing:go_default_library",
],
......
......@@ -265,11 +265,9 @@ func (d *namespacedResourcesDeleter) updateNamespaceStatusFunc(namespace *v1.Nam
if namespace.DeletionTimestamp.IsZero() || namespace.Status.Phase == v1.NamespaceTerminating {
return namespace, nil
}
newNamespace := v1.Namespace{}
newNamespace.ObjectMeta = namespace.ObjectMeta
newNamespace.Status = *namespace.Status.DeepCopy()
newNamespace := namespace.DeepCopy()
newNamespace.Status.Phase = v1.NamespaceTerminating
return d.nsClient.UpdateStatus(&newNamespace)
return d.nsClient.UpdateStatus(newNamespace)
}
// finalized returns true if the namespace.Spec.Finalizers is an empty list
......@@ -330,10 +328,8 @@ func (d *namespacedResourcesDeleter) deleteCollection(gvr schema.GroupVersionRes
// we have a resource returned in the discovery API that supports no top-level verbs:
// /apis/extensions/v1beta1/namespaces/default/replicationcontrollers
// when working with this resource type, we will get a literal not found error rather than expected method not supported
// remember next time that this resource does not support delete collection...
if errors.IsMethodNotSupported(err) || errors.IsNotFound(err) {
klog.V(5).Infof("namespace controller - deleteCollection not supported - namespace: %s, gvr: %v", namespace, gvr)
d.opCache.setNotSupported(key)
return false, nil
}
......@@ -365,10 +361,8 @@ func (d *namespacedResourcesDeleter) listCollection(gvr schema.GroupVersionResou
// we have a resource returned in the discovery API that supports no top-level verbs:
// /apis/extensions/v1beta1/namespaces/default/replicationcontrollers
// when working with this resource type, we will get a literal not found error rather than expected method not supported
// remember next time that this resource does not support delete collection...
if errors.IsMethodNotSupported(err) || errors.IsNotFound(err) {
klog.V(5).Infof("namespace controller - listCollection not supported - namespace: %s, gvr: %v", namespace, gvr)
d.opCache.setNotSupported(key)
return nil, false, nil
}
......
......@@ -42,6 +42,7 @@ go_test(
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
"//staging/src/k8s.io/client-go/testing:go_default_library",
......
......@@ -357,7 +357,8 @@ func (tc *NoExecuteTaintManager) processPodOnNode(
minTolerationTime := getMinTolerationTime(usedTolerations)
// getMinTolerationTime returns negative value to denote infinite toleration.
if minTolerationTime < 0 {
klog.V(4).Infof("New tolerations for %v tolerate forever. Scheduled deletion won't be cancelled if already scheduled.", podNamespacedName.String())
klog.V(4).Infof("Current tolerations for %v tolerate forever, cancelling any scheduled deletion.", podNamespacedName.String())
tc.cancelWorkWithEvent(podNamespacedName)
return
}
......
......@@ -38,6 +38,7 @@ import (
apps "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime/schema"
......@@ -619,9 +620,11 @@ func (rsc *ReplicaSetController) manageReplicas(filteredPods []*v1.Pod, rs *apps
if err := rsc.podControl.DeletePod(rs.Namespace, targetPod.Name, rs); err != nil {
// Decrement the expected number of deletes because the informer won't observe this deletion
podKey := controller.PodKey(targetPod)
klog.V(2).Infof("Failed to delete %v, decrementing expectations for %v %s/%s", podKey, rsc.Kind, rs.Namespace, rs.Name)
rsc.expectations.DeletionObserved(rsKey, podKey)
errCh <- err
if !apierrors.IsNotFound(err) {
klog.V(2).Infof("Failed to delete %v, decremented expectations for %v %s/%s", podKey, rsc.Kind, rs.Namespace, rs.Name)
errCh <- err
}
}
}(pod)
}
......
......@@ -8,6 +8,7 @@ approvers:
- vishh
- yujuhong
- dashpole
- sjenning
reviewers:
- sig-node-reviewers
labels:
......
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- sjenning
......@@ -4,10 +4,10 @@ approvers:
- Random-Liu
- dchen1107
- derekwaynecarr
- tallclair
- vishh
- yujuhong
- ConnorDoyle
- sjenning
- klueska
reviewers:
- sig-node-reviewers
......@@ -4,7 +4,6 @@ approvers:
- derekwaynecarr
- vishh
- ConnorDoyle
- sjenning
- balajismaniam
reviewers:
- klueska
......@@ -4,4 +4,3 @@ approvers:
- derekwaynecarr
- vishh
- ConnorDoyle
- sjenning
......@@ -38,10 +38,17 @@ const (
//createNodeAllocatableCgroups creates Node Allocatable Cgroup when CgroupsPerQOS flag is specified as true
func (cm *containerManagerImpl) createNodeAllocatableCgroups() error {
nodeAllocatable := cm.internalCapacity
// Use Node Allocatable limits instead of capacity if the user requested enforcing node allocatable.
nc := cm.NodeConfig.NodeAllocatableConfig
if cm.CgroupsPerQOS && nc.EnforceNodeAllocatable.Has(kubetypes.NodeAllocatableEnforcementKey) {
nodeAllocatable = cm.getNodeAllocatableInternalAbsolute()
}
cgroupConfig := &CgroupConfig{
Name: cm.cgroupRoot,
// The default limits for cpu shares can be very low which can lead to CPU starvation for pods.
ResourceParameters: getCgroupConfig(cm.internalCapacity),
ResourceParameters: getCgroupConfig(nodeAllocatable),
}
if cm.cgroupManager.Exists(cgroupConfig.Name) {
return nil
......
......@@ -38,6 +38,9 @@ type OSInterface interface {
Pipe() (r *os.File, w *os.File, err error)
ReadDir(dirname string) ([]os.FileInfo, error)
Glob(pattern string) ([]string, error)
Open(name string) (*os.File, error)
OpenFile(name string, flag int, perm os.FileMode) (*os.File, error)
Rename(oldpath, newpath string) error
}
// RealOS is used to dispatch the real system level operations.
......@@ -105,3 +108,18 @@ func (RealOS) ReadDir(dirname string) ([]os.FileInfo, error) {
func (RealOS) Glob(pattern string) ([]string, error) {
return filepath.Glob(pattern)
}
// Open will call os.Open to return the file.
func (RealOS) Open(name string) (*os.File, error) {
return os.Open(name)
}
// OpenFile will call os.OpenFile to return the file.
func (RealOS) OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) {
return os.OpenFile(name, flag, perm)
}
// Rename will call os.Rename to rename a file.
func (RealOS) Rename(oldpath, newpath string) error {
return os.Rename(oldpath, newpath)
}
......@@ -81,6 +81,25 @@ func (ds *dockerService) ListContainers(_ context.Context, r *runtimeapi.ListCon
return &runtimeapi.ListContainersResponse{Containers: result}, nil
}
func (ds *dockerService) getContainerCleanupInfo(containerID string) (*containerCleanupInfo, bool) {
ds.cleanupInfosLock.RLock()
defer ds.cleanupInfosLock.RUnlock()
info, ok := ds.containerCleanupInfos[containerID]
return info, ok
}
func (ds *dockerService) setContainerCleanupInfo(containerID string, info *containerCleanupInfo) {
ds.cleanupInfosLock.Lock()
defer ds.cleanupInfosLock.Unlock()
ds.containerCleanupInfos[containerID] = info
}
func (ds *dockerService) clearContainerCleanupInfo(containerID string) {
ds.cleanupInfosLock.Lock()
defer ds.cleanupInfosLock.Unlock()
delete(ds.containerCleanupInfos, containerID)
}
// CreateContainer creates a new container in the given PodSandbox
// Docker cannot store the log to an arbitrary location (yet), so we create an
// symlink at LogPath, linking to the actual path of the log.
......@@ -183,7 +202,7 @@ func (ds *dockerService) CreateContainer(_ context.Context, r *runtimeapi.Create
// we don't perform the clean up just yet at that could destroy information
// needed for the container to start (e.g. Windows credentials stored in
// registry keys); instead, we'll clean up when the container gets removed
ds.containerCleanupInfos[containerID] = cleanupInfo
ds.setContainerCleanupInfo(containerID, cleanupInfo)
}
return &runtimeapi.CreateContainerResponse{ContainerId: containerID}, nil
}
......@@ -456,11 +475,11 @@ func (ds *dockerService) UpdateContainerResources(_ context.Context, r *runtimea
}
func (ds *dockerService) performPlatformSpecificContainerForContainer(containerID string) (errors []error) {
if cleanupInfo, present := ds.containerCleanupInfos[containerID]; present {
if cleanupInfo, present := ds.getContainerCleanupInfo(containerID); present {
errors = ds.performPlatformSpecificContainerCleanupAndLogErrors(containerID, cleanupInfo)
if len(errors) == 0 {
delete(ds.containerCleanupInfos, containerID)
ds.clearContainerCleanupInfo(containerID)
}
}
......
......@@ -29,7 +29,7 @@ import (
dockertypes "github.com/docker/docker/api/types"
"k8s.io/klog"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
"k8s.io/kubernetes/pkg/kubelet/checkpointmanager"
......@@ -316,6 +316,7 @@ type dockerService struct {
// (see `applyPlatformSpecificDockerConfig` and `performPlatformSpecificContainerCleanup`
// methods for more info).
containerCleanupInfos map[string]*containerCleanupInfo
cleanupInfosLock sync.RWMutex
}
// TODO: handle context.
......
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- sjenning
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- sjenning
......@@ -670,6 +670,22 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
klet.runtimeClassManager = runtimeclass.NewManager(kubeDeps.KubeClient)
}
if containerRuntime == kubetypes.RemoteContainerRuntime && utilfeature.DefaultFeatureGate.Enabled(features.CRIContainerLogRotation) {
// setup containerLogManager for CRI container runtime
containerLogManager, err := logs.NewContainerLogManager(
klet.runtimeService,
kubeDeps.OSInterface,
kubeCfg.ContainerLogMaxSize,
int(kubeCfg.ContainerLogMaxFiles),
)
if err != nil {
return nil, fmt.Errorf("failed to initialize container log manager: %v", err)
}
klet.containerLogManager = containerLogManager
} else {
klet.containerLogManager = logs.NewStubContainerLogManager()
}
runtime, err := kuberuntime.NewKubeGenericRuntimeManager(
kubecontainer.FilterEventRecorder(kubeDeps.Recorder),
klet.livenessManager,
......@@ -691,6 +707,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
imageService,
kubeDeps.ContainerManager.InternalContainerLifecycle(),
legacyLogProvider,
klet.containerLogManager,
klet.runtimeClassManager,
)
if err != nil {
......@@ -748,21 +765,6 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
}
klet.imageManager = imageManager
if containerRuntime == kubetypes.RemoteContainerRuntime && utilfeature.DefaultFeatureGate.Enabled(features.CRIContainerLogRotation) {
// setup containerLogManager for CRI container runtime
containerLogManager, err := logs.NewContainerLogManager(
klet.runtimeService,
kubeCfg.ContainerLogMaxSize,
int(kubeCfg.ContainerLogMaxFiles),
)
if err != nil {
return nil, fmt.Errorf("failed to initialize container log manager: %v", err)
}
klet.containerLogManager = containerLogManager
} else {
klet.containerLogManager = logs.NewStubContainerLogManager()
}
if kubeCfg.ServerTLSBootstrap && kubeDeps.TLSOptions != nil && utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) {
klet.serverCertificateManager, err = kubeletcertificate.NewKubeletServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, klet.getLastObservedNodeAddresses, certDirectory)
if err != nil {
......
......@@ -41,6 +41,7 @@ go_library(
"//pkg/kubelet/images:go_default_library",
"//pkg/kubelet/kuberuntime/logs:go_default_library",
"//pkg/kubelet/lifecycle:go_default_library",
"//pkg/kubelet/logs:go_default_library",
"//pkg/kubelet/metrics:go_default_library",
"//pkg/kubelet/prober/results:go_default_library",
"//pkg/kubelet/runtimeclass:go_default_library",
......
......@@ -31,6 +31,7 @@ import (
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/images"
"k8s.io/kubernetes/pkg/kubelet/lifecycle"
"k8s.io/kubernetes/pkg/kubelet/logs"
proberesults "k8s.io/kubernetes/pkg/kubelet/prober/results"
"k8s.io/kubernetes/pkg/kubelet/util/logreduction"
)
......@@ -73,6 +74,10 @@ func (f *fakePodStateProvider) IsPodTerminated(uid types.UID) bool {
func newFakeKubeRuntimeManager(runtimeService internalapi.RuntimeService, imageService internalapi.ImageManagerService, machineInfo *cadvisorapi.MachineInfo, osInterface kubecontainer.OSInterface, runtimeHelper kubecontainer.RuntimeHelper, keyring credentialprovider.DockerKeyring) (*kubeGenericRuntimeManager, error) {
recorder := &record.FakeRecorder{}
logManager, err := logs.NewContainerLogManager(runtimeService, osInterface, "1", 2)
if err != nil {
return nil, err
}
kubeRuntimeManager := &kubeGenericRuntimeManager{
recorder: recorder,
cpuCFSQuota: false,
......@@ -89,6 +94,7 @@ func newFakeKubeRuntimeManager(runtimeService internalapi.RuntimeService, imageS
seccompProfileRoot: fakeSeccompProfileRoot,
internalLifecycle: cm.NewFakeInternalContainerLifecycle(),
logReduction: logreduction.NewLogReduction(identicalErrorDelay),
logManager: logManager,
}
typedVersion, err := runtimeService.Version(kubeRuntimeAPIVersion)
......
......@@ -835,19 +835,19 @@ func (m *kubeGenericRuntimeManager) removeContainer(containerID string) error {
// removeContainerLog removes the container log.
func (m *kubeGenericRuntimeManager) removeContainerLog(containerID string) error {
// Remove the container log.
// Use log manager to remove rotated logs.
err := m.logManager.Clean(containerID)
if err != nil {
return err
}
status, err := m.runtimeService.ContainerStatus(containerID)
if err != nil {
return fmt.Errorf("failed to get container status %q: %v", containerID, err)
}
labeledInfo := getContainerInfoFromLabels(status.Labels)
path := status.GetLogPath()
if err := m.osInterface.Remove(path); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to remove container %q log %q: %v", containerID, path, err)
}
// Remove the legacy container log symlink.
// TODO(random-liu): Remove this after cluster logging supports CRI container log path.
labeledInfo := getContainerInfoFromLabels(status.Labels)
legacySymlink := legacyLogSymlink(containerID, labeledInfo.ContainerName, labeledInfo.PodName,
labeledInfo.PodNamespace)
if err := m.osInterface.Remove(legacySymlink); err != nil && !os.IsNotExist(err) {
......
......@@ -45,6 +45,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/events"
"k8s.io/kubernetes/pkg/kubelet/images"
"k8s.io/kubernetes/pkg/kubelet/lifecycle"
"k8s.io/kubernetes/pkg/kubelet/logs"
proberesults "k8s.io/kubernetes/pkg/kubelet/prober/results"
"k8s.io/kubernetes/pkg/kubelet/runtimeclass"
"k8s.io/kubernetes/pkg/kubelet/types"
......@@ -128,6 +129,9 @@ type kubeGenericRuntimeManager struct {
// A shim to legacy functions for backward compatibility.
legacyLogProvider LegacyLogProvider
// Manage container logs.
logManager logs.ContainerLogManager
// Manage RuntimeClass resources.
runtimeClassManager *runtimeclass.Manager
......@@ -170,6 +174,7 @@ func NewKubeGenericRuntimeManager(
imageService internalapi.ImageManagerService,
internalLifecycle cm.InternalContainerLifecycle,
legacyLogProvider LegacyLogProvider,
logManager logs.ContainerLogManager,
runtimeClassManager *runtimeclass.Manager,
) (KubeGenericRuntime, error) {
kubeRuntimeManager := &kubeGenericRuntimeManager{
......@@ -188,6 +193,7 @@ func NewKubeGenericRuntimeManager(
keyring: credentialprovider.NewDockerKeyring(),
internalLifecycle: internalLifecycle,
legacyLogProvider: legacyLogProvider,
logManager: logManager,
runtimeClassManager: runtimeClassManager,
logReduction: logreduction.NewLogReduction(identicalErrorDelay),
}
......
......@@ -9,6 +9,7 @@ go_library(
importpath = "k8s.io/kubernetes/pkg/kubelet/logs",
visibility = ["//visibility:public"],
deps = [
"//pkg/kubelet/container:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
......@@ -23,6 +24,7 @@ go_test(
srcs = ["container_log_manager_test.go"],
embed = [":go_default_library"],
deps = [
"//pkg/kubelet/container:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library",
"//staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2:go_default_library",
"//staging/src/k8s.io/cri-api/pkg/apis/testing:go_default_library",
......
......@@ -24,6 +24,7 @@ import (
"path/filepath"
"sort"
"strings"
"sync"
"time"
"k8s.io/klog"
......@@ -33,6 +34,7 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
internalapi "k8s.io/cri-api/pkg/apis"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
)
const (
......@@ -55,6 +57,8 @@ type ContainerLogManager interface {
// TODO(random-liu): Add RotateLogs function and call it under disk pressure.
// Start container log manager.
Start()
// Clean removes all logs of specified container.
Clean(containerID string) error
}
// LogRotatePolicy is a policy for container log rotation. The policy applies to all
......@@ -142,12 +146,14 @@ func parseMaxSize(size string) (int64, error) {
type containerLogManager struct {
runtimeService internalapi.RuntimeService
osInterface kubecontainer.OSInterface
policy LogRotatePolicy
clock clock.Clock
mutex sync.Mutex
}
// NewContainerLogManager creates a new container log manager.
func NewContainerLogManager(runtimeService internalapi.RuntimeService, maxSize string, maxFiles int) (ContainerLogManager, error) {
func NewContainerLogManager(runtimeService internalapi.RuntimeService, osInterface kubecontainer.OSInterface, maxSize string, maxFiles int) (ContainerLogManager, error) {
if maxFiles <= 1 {
return nil, fmt.Errorf("invalid MaxFiles %d, must be > 1", maxFiles)
}
......@@ -157,12 +163,14 @@ func NewContainerLogManager(runtimeService internalapi.RuntimeService, maxSize s
}
// policy LogRotatePolicy
return &containerLogManager{
osInterface: osInterface,
runtimeService: runtimeService,
policy: LogRotatePolicy{
MaxSize: parsedMaxSize,
MaxFiles: maxFiles,
},
clock: clock.RealClock{},
mutex: sync.Mutex{},
}, nil
}
......@@ -176,7 +184,32 @@ func (c *containerLogManager) Start() {
}, logMonitorPeriod)
}
// Clean removes all logs of specified container (including rotated one).
func (c *containerLogManager) Clean(containerID string) error {
c.mutex.Lock()
defer c.mutex.Unlock()
status, err := c.runtimeService.ContainerStatus(containerID)
if err != nil {
return fmt.Errorf("failed to get container status %q: %v", containerID, err)
}
pattern := fmt.Sprintf("%s*", status.GetLogPath())
logs, err := c.osInterface.Glob(pattern)
if err != nil {
return fmt.Errorf("failed to list all log files with pattern %q: %v", pattern, err)
}
for _, l := range logs {
if err := c.osInterface.Remove(l); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to remove container %q log %q: %v", containerID, l, err)
}
}
return nil
}
func (c *containerLogManager) rotateLogs() error {
c.mutex.Lock()
defer c.mutex.Unlock()
// TODO(#59998): Use kubelet pod cache.
containers, err := c.runtimeService.ListContainers(&runtimeapi.ContainerFilter{})
if err != nil {
......@@ -197,7 +230,7 @@ func (c *containerLogManager) rotateLogs() error {
continue
}
path := status.GetLogPath()
info, err := os.Stat(path)
info, err := c.osInterface.Stat(path)
if err != nil {
if !os.IsNotExist(err) {
klog.Errorf("Failed to stat container log %q: %v", path, err)
......@@ -211,7 +244,7 @@ func (c *containerLogManager) rotateLogs() error {
continue
}
// The container log should be recovered.
info, err = os.Stat(path)
info, err = c.osInterface.Stat(path)
if err != nil {
klog.Errorf("Failed to stat container log %q after reopen: %v", path, err)
continue
......@@ -269,7 +302,7 @@ func (c *containerLogManager) rotateLog(id, log string) error {
func (c *containerLogManager) cleanupUnusedLogs(logs []string) ([]string, error) {
inuse, unused := filterUnusedLogs(logs)
for _, l := range unused {
if err := os.Remove(l); err != nil {
if err := c.osInterface.Remove(l); err != nil {
return nil, fmt.Errorf("failed to remove unused log %q: %v", l, err)
}
}
......@@ -322,7 +355,7 @@ func (c *containerLogManager) removeExcessLogs(logs []string) ([]string, error)
}
i := 0
for ; i < len(logs)-maxRotatedFiles; i++ {
if err := os.Remove(logs[i]); err != nil {
if err := c.osInterface.Remove(logs[i]); err != nil {
return nil, fmt.Errorf("failed to remove old log %q: %v", logs[i], err)
}
}
......@@ -332,19 +365,19 @@ func (c *containerLogManager) removeExcessLogs(logs []string) ([]string, error)
// compressLog compresses a log to log.gz with gzip.
func (c *containerLogManager) compressLog(log string) error {
r, err := os.Open(log)
r, err := c.osInterface.Open(log)
if err != nil {
return fmt.Errorf("failed to open log %q: %v", log, err)
}
defer r.Close()
tmpLog := log + tmpSuffix
f, err := os.OpenFile(tmpLog, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
f, err := c.osInterface.OpenFile(tmpLog, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return fmt.Errorf("failed to create temporary log %q: %v", tmpLog, err)
}
defer func() {
// Best effort cleanup of tmpLog.
os.Remove(tmpLog)
c.osInterface.Remove(tmpLog)
}()
defer f.Close()
w := gzip.NewWriter(f)
......@@ -353,11 +386,11 @@ func (c *containerLogManager) compressLog(log string) error {
return fmt.Errorf("failed to compress %q to %q: %v", log, tmpLog, err)
}
compressedLog := log + compressSuffix
if err := os.Rename(tmpLog, compressedLog); err != nil {
if err := c.osInterface.Rename(tmpLog, compressedLog); err != nil {
return fmt.Errorf("failed to rename %q to %q: %v", tmpLog, compressedLog, err)
}
// Remove old log file.
if err := os.Remove(log); err != nil {
if err := c.osInterface.Remove(log); err != nil {
return fmt.Errorf("failed to remove log %q after compress: %v", log, err)
}
return nil
......@@ -368,14 +401,14 @@ func (c *containerLogManager) compressLog(log string) error {
func (c *containerLogManager) rotateLatestLog(id, log string) error {
timestamp := c.clock.Now().Format(timestampFormat)
rotated := fmt.Sprintf("%s.%s", log, timestamp)
if err := os.Rename(log, rotated); err != nil {
if err := c.osInterface.Rename(log, rotated); err != nil {
return fmt.Errorf("failed to rotate log %q to %q: %v", log, rotated, err)
}
if err := c.runtimeService.ReopenContainerLog(id); err != nil {
// Rename the rotated log back, so that we can try rotating it again
// next round.
// If kubelet gets restarted at this point, we'll lose original log.
if renameErr := os.Rename(rotated, log); renameErr != nil {
if renameErr := c.osInterface.Rename(rotated, log); renameErr != nil {
// This shouldn't happen.
// Report an error if this happens, because we will lose original
// log.
......
......@@ -20,6 +20,10 @@ type containerLogManagerStub struct{}
func (*containerLogManagerStub) Start() {}
func (*containerLogManagerStub) Clean(containerID string) error {
return nil
}
// NewStubContainerLogManager returns an empty ContainerLogManager which does nothing.
func NewStubContainerLogManager() ContainerLogManager {
return &containerLogManagerStub{}
......
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- sjenning
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
......@@ -60,25 +60,6 @@ go_library(
}),
)
go_test(
name = "go_default_test",
srcs = ["oom_watcher_linux_test.go"],
embed = [":go_default_library"],
deps = select({
"@io_bazel_rules_go//go/platform:android": [
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
],
"@io_bazel_rules_go//go/platform:linux": [
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
],
"//conditions:default": [],
}),
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
......
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- sjenning
......@@ -649,7 +649,7 @@ func (p *criStatsProvider) getAndUpdateContainerUsageNanoCores(stats *runtimeapi
defer p.mutex.Unlock()
cached, ok := p.cpuUsageCache[id]
if !ok || cached.stats.UsageCoreNanoSeconds == nil {
if !ok || cached.stats.UsageCoreNanoSeconds == nil || stats.Cpu.UsageCoreNanoSeconds.Value < cached.stats.UsageCoreNanoSeconds.Value {
// Cannot compute the usage now, but update the cached stats anyway
p.cpuUsageCache[id] = &cpuUsageRecord{stats: stats.Cpu, usageNanoCores: nil}
return nil, nil
......
......@@ -71,12 +71,16 @@ var maxDataDiskCountMap = map[string]int64{
"STANDARD_D15_V2": 64,
"STANDARD_D16AS_V4": 32,
"STANDARD_D16A_V4": 32,
"STANDARD_D16DS_V4": 32,
"STANDARD_D16D_V4": 32,
"STANDARD_D16S_V3": 32,
"STANDARD_D16_V3": 32,
"STANDARD_D1_V2": 4,
"STANDARD_D2": 8,
"STANDARD_D2AS_V4": 4,
"STANDARD_D2A_V4": 4,
"STANDARD_D2DS_V4": 4,
"STANDARD_D2D_V4": 4,
"STANDARD_D2S_V3": 4,
"STANDARD_D2_V2": 8,
"STANDARD_D2_V2_PROMO": 8,
......@@ -84,6 +88,8 @@ var maxDataDiskCountMap = map[string]int64{
"STANDARD_D3": 16,
"STANDARD_D32AS_V4": 32,
"STANDARD_D32A_V4": 32,
"STANDARD_D32DS_V4": 32,
"STANDARD_D32D_V4": 32,
"STANDARD_D32S_V3": 32,
"STANDARD_D32_V3": 32,
"STANDARD_D3_V2": 16,
......@@ -91,10 +97,14 @@ var maxDataDiskCountMap = map[string]int64{
"STANDARD_D4": 32,
"STANDARD_D48AS_V4": 32,
"STANDARD_D48A_V4": 32,
"STANDARD_D48DS_V4": 32,
"STANDARD_D48D_V4": 32,
"STANDARD_D48S_V3": 32,
"STANDARD_D48_V3": 32,
"STANDARD_D4AS_V4": 8,
"STANDARD_D4A_V4": 8,
"STANDARD_D4DS_V4": 8,
"STANDARD_D4D_V4": 8,
"STANDARD_D4S_V3": 8,
"STANDARD_D4_V2": 32,
"STANDARD_D4_V2_PROMO": 32,
......@@ -103,10 +113,14 @@ var maxDataDiskCountMap = map[string]int64{
"STANDARD_D5_V2_PROMO": 64,
"STANDARD_D64AS_V4": 32,
"STANDARD_D64A_V4": 32,
"STANDARD_D64DS_V4": 32,
"STANDARD_D64D_V4": 32,
"STANDARD_D64S_V3": 32,
"STANDARD_D64_V3": 32,
"STANDARD_D8AS_V4": 16,
"STANDARD_D8A_V4": 16,
"STANDARD_D8DS_V4": 16,
"STANDARD_D8D_V4": 16,
"STANDARD_D8S_V3": 16,
"STANDARD_D8_V3": 16,
"STANDARD_D96AS_V4": 32,
......@@ -150,47 +164,71 @@ var maxDataDiskCountMap = map[string]int64{
"STANDARD_DS4_V2_PROMO": 32,
"STANDARD_DS5_V2": 64,
"STANDARD_DS5_V2_PROMO": 64,
"STANDARD_E16-4DS_V4": 32,
"STANDARD_E16-4S_V3": 32,
"STANDARD_E16-8DS_V4": 32,
"STANDARD_E16-8S_V3": 32,
"STANDARD_E16AS_V4": 32,
"STANDARD_E16A_V4": 32,
"STANDARD_E16DS_V4": 32,
"STANDARD_E16D_V4": 32,
"STANDARD_E16S_V3": 32,
"STANDARD_E16_V3": 32,
"STANDARD_E20AS_V4": 32,
"STANDARD_E20A_V4": 32,
"STANDARD_E20DS_V4": 32,
"STANDARD_E20D_V4": 32,
"STANDARD_E20S_V3": 32,
"STANDARD_E20_V3": 32,
"STANDARD_E2AS_V4": 4,
"STANDARD_E2A_V4": 4,
"STANDARD_E2DS_V4": 4,
"STANDARD_E2D_V4": 4,
"STANDARD_E2S_V3": 4,
"STANDARD_E2_V3": 4,
"STANDARD_E32-16DS_V4": 32,
"STANDARD_E32-16S_V3": 32,
"STANDARD_E32-8DS_V4": 32,
"STANDARD_E32-8S_V3": 32,
"STANDARD_E32AS_V4": 32,
"STANDARD_E32A_V4": 32,
"STANDARD_E32DS_V4": 32,
"STANDARD_E32D_V4": 32,
"STANDARD_E32S_V3": 32,
"STANDARD_E32_V3": 32,
"STANDARD_E4-2DS_V4": 8,
"STANDARD_E4-2S_V3": 8,
"STANDARD_E48AS_V4": 32,
"STANDARD_E48A_V4": 32,
"STANDARD_E48DS_V4": 32,
"STANDARD_E48D_V4": 32,
"STANDARD_E48S_V3": 32,
"STANDARD_E48_V3": 32,
"STANDARD_E4AS_V4": 8,
"STANDARD_E4A_V4": 8,
"STANDARD_E4DS_V4": 8,
"STANDARD_E4D_V4": 8,
"STANDARD_E4S_V3": 8,
"STANDARD_E4_V3": 8,
"STANDARD_E64-16DS_V4": 32,
"STANDARD_E64-16S_V3": 32,
"STANDARD_E64-32S_V3": 32,
"STANDARD_E64AS_V4": 32,
"STANDARD_E64A_V4": 32,
"STANDARD_E64DS_V4": 32,
"STANDARD_E64D_V4": 32,
"STANDARD_E64IS_V3": 32,
"STANDARD_E64I_V3": 32,
"STANDARD_E64S_V3": 32,
"STANDARD_E64_V3": 32,
"STANDARD_E8-2DS_V4": 16,
"STANDARD_E8-2S_V3": 16,
"STANDARD_E8-4DS_V4": 16,
"STANDARD_E8-4S_V3": 16,
"STANDARD_E8AS_V4": 16,
"STANDARD_E8A_V4": 16,
"STANDARD_E8DS_V4": 16,
"STANDARD_E8D_V4": 16,
"STANDARD_E8S_V3": 16,
"STANDARD_E8_V3": 16,
"STANDARD_E96AS_V4": 32,
......@@ -268,6 +306,8 @@ var maxDataDiskCountMap = map[string]int64{
"STANDARD_M32LS": 32,
"STANDARD_M32MS": 32,
"STANDARD_M32TS": 32,
"STANDARD_M416-208MS_V2": 64,
"STANDARD_M416-208S_V2": 64,
"STANDARD_M416MS_V2": 64,
"STANDARD_M416S_V2": 64,
"STANDARD_M64-16MS": 64,
......
......@@ -23,6 +23,7 @@ import (
"syscall"
"os"
"time"
"k8s.io/klog"
)
......@@ -42,7 +43,10 @@ func SetVolumeOwnership(mounter Mounter, fsGroup *int64) error {
return nil
}
klog.Warningf("Setting volume ownership for %s and fsGroup set. If the volume has a lot of files then setting volume ownership could be slow, see https://github.com/kubernetes/kubernetes/issues/69699", mounter.GetPath())
timer := time.AfterFunc(30*time.Second, func() {
klog.Warningf("Setting volume ownership for %s and fsGroup set. If the volume has a lot of files then setting volume ownership could be slow, see https://github.com/kubernetes/kubernetes/issues/69699", mounter.GetPath())
})
defer timer.Stop()
return filepath.Walk(mounter.GetPath(), func(path string, info os.FileInfo, err error) error {
if err != nil {
......
......@@ -251,6 +251,14 @@ const (
filterNodeLimit = 150
)
const (
// represents expected attachment status of a volume after attach
volumeAttachedStatus = "attached"
// represents expected attachment status of a volume after detach
volumeDetachedStatus = "detached"
)
// awsTagNameMasterRoles is a set of well-known AWS tag names that indicate the instance is a master
// The major consequence is that it is then not considered for AWS zone discovery for dynamic volume creation.
var awsTagNameMasterRoles = sets.NewString("kubernetes.io/role/master", "k8s.io/role/master")
......@@ -1911,7 +1919,6 @@ func (c *Cloud) getMountDevice(
// AWS API returns consistent result next time (i.e. the volume is detached).
status := volumeStatus[mappingVolumeID]
klog.Warningf("Got assignment call for already-assigned volume: %s@%s, volume status: %s", mountDevice, mappingVolumeID, status)
return mountDevice, false, fmt.Errorf("volume is still being detached from the node")
}
return mountDevice, true, nil
}
......@@ -2112,7 +2119,7 @@ func (c *Cloud) applyUnSchedulableTaint(nodeName types.NodeName, reason string)
// waitForAttachmentStatus polls until the attachment status is the expected value
// On success, it returns the last attachment state.
func (d *awsDisk) waitForAttachmentStatus(status string, expectedInstance, expectedDevice string) (*ec2.VolumeAttachment, error) {
func (d *awsDisk) waitForAttachmentStatus(status string, expectedInstance, expectedDevice string, alreadyAttached bool) (*ec2.VolumeAttachment, error) {
backoff := wait.Backoff{
Duration: volumeAttachmentStatusPollDelay,
Factor: volumeAttachmentStatusFactor,
......@@ -2137,7 +2144,7 @@ func (d *awsDisk) waitForAttachmentStatus(status string, expectedInstance, expec
if err != nil {
// The VolumeNotFound error is special -- we don't need to wait for it to repeat
if isAWSErrorVolumeNotFound(err) {
if status == "detached" {
if status == volumeDetachedStatus {
// The disk doesn't exist, assume it's detached, log warning and stop waiting
klog.Warningf("Waiting for volume %q to be detached but the volume does not exist", d.awsID)
stateStr := "detached"
......@@ -2146,7 +2153,7 @@ func (d *awsDisk) waitForAttachmentStatus(status string, expectedInstance, expec
}
return true, nil
}
if status == "attached" {
if status == volumeAttachedStatus {
// The disk doesn't exist, complain, give up waiting and report error
klog.Warningf("Waiting for volume %q to be attached but the volume does not exist", d.awsID)
return false, err
......@@ -2181,7 +2188,7 @@ func (d *awsDisk) waitForAttachmentStatus(status string, expectedInstance, expec
}
}
if attachmentStatus == "" {
attachmentStatus = "detached"
attachmentStatus = volumeDetachedStatus
}
if attachment != nil {
// AWS eventual consistency can go back in time.
......@@ -2210,6 +2217,13 @@ func (d *awsDisk) waitForAttachmentStatus(status string, expectedInstance, expec
}
}
// if we expected volume to be attached and it was reported as already attached via DescribeInstance call
// but DescribeVolume told us volume is detached, we will short-circuit this long wait loop and return error
// so as AttachDisk can be retried without waiting for 20 minutes.
if (status == volumeAttachedStatus) && alreadyAttached && (attachmentStatus != status) {
return false, fmt.Errorf("attachment of disk %q failed, expected device to be attached but was %s", d.name, attachmentStatus)
}
if attachmentStatus == status {
// Attachment is in requested state, finish waiting
return true, nil
......@@ -2355,7 +2369,7 @@ func (c *Cloud) AttachDisk(diskName KubernetesVolumeID, nodeName types.NodeName)
klog.V(2).Infof("AttachVolume volume=%q instance=%q request returned %v", disk.awsID, awsInstance.awsID, attachResponse)
}
attachment, err := disk.waitForAttachmentStatus("attached", awsInstance.awsID, ec2Device)
attachment, err := disk.waitForAttachmentStatus("attached", awsInstance.awsID, ec2Device, alreadyAttached)
if err != nil {
if err == wait.ErrWaitTimeout {
......@@ -2433,7 +2447,7 @@ func (c *Cloud) DetachDisk(diskName KubernetesVolumeID, nodeName types.NodeName)
return "", errors.New("no response from DetachVolume")
}
attachment, err := diskInfo.disk.waitForAttachmentStatus("detached", awsInstance.awsID, "")
attachment, err := diskInfo.disk.waitForAttachmentStatus("detached", awsInstance.awsID, "", false)
if err != nil {
return "", err
}
......@@ -3581,6 +3595,27 @@ func buildListener(port v1.ServicePort, annotations map[string]string, sslPorts
return listener, nil
}
func (c *Cloud) getSubnetCidrs(subnetIDs []string) ([]string, error) {
request := &ec2.DescribeSubnetsInput{}
for _, subnetID := range subnetIDs {
request.SubnetIds = append(request.SubnetIds, aws.String(subnetID))
}
subnets, err := c.ec2.DescribeSubnets(request)
if err != nil {
return nil, fmt.Errorf("error querying Subnet for ELB: %q", err)
}
if len(subnets) != len(subnetIDs) {
return nil, fmt.Errorf("error querying Subnet for ELB, got %d subnets for %v", len(subnets), subnetIDs)
}
cidrs := make([]string, 0, len(subnets))
for _, subnet := range subnets {
cidrs = append(cidrs, aws.StringValue(subnet.CidrBlock))
}
return cidrs, nil
}
// EnsureLoadBalancer implements LoadBalancer.EnsureLoadBalancer
func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiService *v1.Service, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) {
annotations := apiService.Annotations
......@@ -3708,6 +3743,12 @@ func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiS
return nil, err
}
subnetCidrs, err := c.getSubnetCidrs(subnetIDs)
if err != nil {
klog.Errorf("Error getting subnet cidrs: %q", err)
return nil, err
}
sourceRangeCidrs := []string{}
for cidr := range sourceRanges {
sourceRangeCidrs = append(sourceRangeCidrs, cidr)
......@@ -3716,7 +3757,7 @@ func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiS
sourceRangeCidrs = append(sourceRangeCidrs, "0.0.0.0/0")
}
err = c.updateInstanceSecurityGroupsForNLB(loadBalancerName, instances, sourceRangeCidrs, v2Mappings)
err = c.updateInstanceSecurityGroupsForNLB(loadBalancerName, instances, subnetCidrs, sourceRangeCidrs, v2Mappings)
if err != nil {
klog.Warningf("Error opening ingress rules for the load balancer to the instances: %q", err)
return nil, err
......@@ -4295,7 +4336,7 @@ func (c *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName strin
}
}
return c.updateInstanceSecurityGroupsForNLB(loadBalancerName, nil, nil, nil)
return c.updateInstanceSecurityGroupsForNLB(loadBalancerName, nil, nil, nil, nil)
}
lb, err := c.describeLoadBalancer(loadBalancerName)
......@@ -4697,7 +4738,7 @@ func setNodeDisk(
}
func getInitialAttachDetachDelay(status string) time.Duration {
if status == "detached" {
if status == volumeDetachedStatus {
return volumeDetachmentStatusInitialDelay
}
return volumeAttachmentStatusInitialDelay
......
......@@ -701,27 +701,9 @@ func (c *Cloud) ensureTargetGroup(targetGroup *elbv2.TargetGroup, serviceName ty
return targetGroup, nil
}
func (c *Cloud) getVpcCidrBlocks() ([]string, error) {
vpcs, err := c.ec2.DescribeVpcs(&ec2.DescribeVpcsInput{
VpcIds: []*string{aws.String(c.vpcID)},
})
if err != nil {
return nil, fmt.Errorf("error querying VPC for ELB: %q", err)
}
if len(vpcs.Vpcs) != 1 {
return nil, fmt.Errorf("error querying VPC for ELB, got %d vpcs for %s", len(vpcs.Vpcs), c.vpcID)
}
cidrBlocks := make([]string, 0, len(vpcs.Vpcs[0].CidrBlockAssociationSet))
for _, cidr := range vpcs.Vpcs[0].CidrBlockAssociationSet {
cidrBlocks = append(cidrBlocks, aws.StringValue(cidr.CidrBlock))
}
return cidrBlocks, nil
}
// updateInstanceSecurityGroupsForNLB will adjust securityGroup's settings to allow inbound traffic into instances from clientCIDRs and portMappings.
// TIP: if either instances or clientCIDRs or portMappings are nil, then the securityGroup rules for lbName are cleared.
func (c *Cloud) updateInstanceSecurityGroupsForNLB(lbName string, instances map[InstanceID]*ec2.Instance, clientCIDRs []string, portMappings []nlbPortMapping) error {
func (c *Cloud) updateInstanceSecurityGroupsForNLB(lbName string, instances map[InstanceID]*ec2.Instance, subnetCIDRs []string, clientCIDRs []string, portMappings []nlbPortMapping) error {
if c.cfg.Global.DisableSecurityGroupIngress {
return nil
}
......@@ -765,14 +747,10 @@ func (c *Cloud) updateInstanceSecurityGroupsForNLB(lbName string, instances map[
}
clientRuleAnnotation := fmt.Sprintf("%s=%s", NLBClientRuleDescription, lbName)
healthRuleAnnotation := fmt.Sprintf("%s=%s", NLBHealthCheckRuleDescription, lbName)
vpcCIDRs, err := c.getVpcCidrBlocks()
if err != nil {
return err
}
for sgID, sg := range clusterSGs {
sgPerms := NewIPPermissionSet(sg.IpPermissions...).Ungroup()
if desiredSGIDs.Has(sgID) {
if err := c.updateInstanceSecurityGroupForNLBTraffic(sgID, sgPerms, healthRuleAnnotation, "tcp", healthCheckPorts, vpcCIDRs); err != nil {
if err := c.updateInstanceSecurityGroupForNLBTraffic(sgID, sgPerms, healthRuleAnnotation, "tcp", healthCheckPorts, subnetCIDRs); err != nil {
return err
}
if err := c.updateInstanceSecurityGroupForNLBTraffic(sgID, sgPerms, clientRuleAnnotation, "tcp", clientPorts, clientCIDRs); err != nil {
......
......@@ -346,7 +346,7 @@ func (az *Cloud) InitializeCloudFromConfig(config *Config, fromSecret bool) erro
// No credentials provided, useInstanceMetadata should be enabled for Kubelet.
// TODO(feiskyer): print different error message for Kubelet and controller-manager, as they're
// requiring different credential settings.
if !config.UseInstanceMetadata && az.Config.CloudConfigType == cloudConfigTypeFile {
if !config.UseInstanceMetadata && config.CloudConfigType == cloudConfigTypeFile {
return fmt.Errorf("useInstanceMetadata must be enabled without Azure credentials")
}
......
......@@ -1386,6 +1386,9 @@ func (ss *scaleSet) ensureBackendPoolDeletedFromVMSS(service *v1.Service, backen
for vmssName := range vmssNamesMap {
vmss, err := ss.getVMSS(vmssName, cacheReadTypeDefault)
if err != nil {
return err
}
// When vmss is being deleted, CreateOrUpdate API would report "the vmss is being deleted" error.
// Since it is being deleted, we shouldn't send more CreateOrUpdate requests for it.
......@@ -1393,10 +1396,6 @@ func (ss *scaleSet) ensureBackendPoolDeletedFromVMSS(service *v1.Service, backen
klog.V(3).Infof("ensureVMSSInPool: found vmss %s being deleted, skipping", vmssName)
continue
}
if err != nil {
return err
}
if vmss.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations == nil {
klog.V(4).Infof("EnsureHostInPool: cannot obtain the primary network interface configuration, of vmss %s", vmssName)
continue
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment