Commit fdc65025 authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub

Merge pull request #50337 from bowei/node-controller-repackage

Automatic merge from submit-queue (batch tested with PRs 50016, 49583, 49930, 46254, 50337) Break up node controller into packages This change does NO actual code changes other than moving constituent parts into packages. ```release-note NONE ```
parents 458cc043 61c43f64
...@@ -63,6 +63,7 @@ go_library( ...@@ -63,6 +63,7 @@ go_library(
"//pkg/controller/job:go_default_library", "//pkg/controller/job:go_default_library",
"//pkg/controller/namespace:go_default_library", "//pkg/controller/namespace:go_default_library",
"//pkg/controller/node:go_default_library", "//pkg/controller/node:go_default_library",
"//pkg/controller/node/ipam:go_default_library",
"//pkg/controller/podautoscaler:go_default_library", "//pkg/controller/podautoscaler:go_default_library",
"//pkg/controller/podautoscaler/metrics:go_default_library", "//pkg/controller/podautoscaler/metrics:go_default_library",
"//pkg/controller/podgc:go_default_library", "//pkg/controller/podgc:go_default_library",
......
...@@ -42,6 +42,7 @@ import ( ...@@ -42,6 +42,7 @@ import (
"k8s.io/kubernetes/pkg/controller/garbagecollector" "k8s.io/kubernetes/pkg/controller/garbagecollector"
namespacecontroller "k8s.io/kubernetes/pkg/controller/namespace" namespacecontroller "k8s.io/kubernetes/pkg/controller/namespace"
nodecontroller "k8s.io/kubernetes/pkg/controller/node" nodecontroller "k8s.io/kubernetes/pkg/controller/node"
"k8s.io/kubernetes/pkg/controller/node/ipam"
"k8s.io/kubernetes/pkg/controller/podgc" "k8s.io/kubernetes/pkg/controller/podgc"
replicationcontroller "k8s.io/kubernetes/pkg/controller/replication" replicationcontroller "k8s.io/kubernetes/pkg/controller/replication"
resourcequotacontroller "k8s.io/kubernetes/pkg/controller/resourcequota" resourcequotacontroller "k8s.io/kubernetes/pkg/controller/resourcequota"
...@@ -108,7 +109,7 @@ func startNodeController(ctx ControllerContext) (bool, error) { ...@@ -108,7 +109,7 @@ func startNodeController(ctx ControllerContext) (bool, error) {
serviceCIDR, serviceCIDR,
int(ctx.Options.NodeCIDRMaskSize), int(ctx.Options.NodeCIDRMaskSize),
ctx.Options.AllocateNodeCIDRs, ctx.Options.AllocateNodeCIDRs,
nodecontroller.CIDRAllocatorType(ctx.Options.CIDRAllocatorType), ipam.CIDRAllocatorType(ctx.Options.CIDRAllocatorType),
ctx.Options.EnableTaintManager, ctx.Options.EnableTaintManager,
utilfeature.DefaultFeatureGate.Enabled(features.TaintBasedEvictions), utilfeature.DefaultFeatureGate.Enabled(features.TaintBasedEvictions),
) )
......
...@@ -10,25 +10,20 @@ load( ...@@ -10,25 +10,20 @@ load(
go_test( go_test(
name = "go_default_test", name = "go_default_test",
srcs = [ srcs = ["nodecontroller_test.go"],
"cidr_allocator_test.go",
"cidr_set_test.go",
"nodecontroller_test.go",
"rate_limited_queue_test.go",
"taint_controller_test.go",
"timed_workers_test.go",
],
library = ":go_default_library", library = ":go_default_library",
tags = ["automanaged"], tags = ["automanaged"],
deps = [ deps = [
"//pkg/cloudprovider:go_default_library", "//pkg/cloudprovider:go_default_library",
"//pkg/cloudprovider/providers/fake:go_default_library", "//pkg/cloudprovider/providers/fake:go_default_library",
"//pkg/controller:go_default_library", "//pkg/controller:go_default_library",
"//pkg/controller/node/ipam:go_default_library",
"//pkg/controller/node/scheduler:go_default_library",
"//pkg/controller/node/util:go_default_library",
"//pkg/controller/testutil:go_default_library", "//pkg/controller/testutil:go_default_library",
"//pkg/kubelet/apis:go_default_library", "//pkg/kubelet/apis:go_default_library",
"//pkg/util/node:go_default_library", "//pkg/util/node:go_default_library",
"//pkg/util/taints:go_default_library", "//pkg/util/taints:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/api/extensions/v1beta1:go_default_library", "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
...@@ -36,7 +31,6 @@ go_test( ...@@ -36,7 +31,6 @@ go_test(
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/client-go/informers:go_default_library", "//vendor/k8s.io/client-go/informers:go_default_library",
"//vendor/k8s.io/client-go/informers/core/v1:go_default_library", "//vendor/k8s.io/client-go/informers/core/v1:go_default_library",
...@@ -44,35 +38,24 @@ go_test( ...@@ -44,35 +38,24 @@ go_test(
"//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library", "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
"//vendor/k8s.io/client-go/testing:go_default_library", "//vendor/k8s.io/client-go/testing:go_default_library",
"//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
], ],
) )
go_library( go_library(
name = "go_default_library", name = "go_default_library",
srcs = [ srcs = [
"cidr_allocator.go",
"cidr_set.go",
"cloud_cidr_allocator.go",
"controller_utils.go",
"doc.go", "doc.go",
"metrics.go", "metrics.go",
"node_controller.go", "node_controller.go",
"range_allocator.go",
"rate_limited_queue.go",
"taint_controller.go",
"timed_workers.go",
], ],
tags = ["automanaged"], tags = ["automanaged"],
deps = [ deps = [
"//pkg/api:go_default_library",
"//pkg/api/helper:go_default_library",
"//pkg/api/v1/helper:go_default_library",
"//pkg/api/v1/node:go_default_library", "//pkg/api/v1/node:go_default_library",
"//pkg/cloudprovider:go_default_library", "//pkg/cloudprovider:go_default_library",
"//pkg/cloudprovider/providers/gce:go_default_library",
"//pkg/controller:go_default_library", "//pkg/controller:go_default_library",
"//pkg/kubelet/util/format:go_default_library", "//pkg/controller/node/ipam:go_default_library",
"//pkg/controller/node/scheduler:go_default_library",
"//pkg/controller/node/util:go_default_library",
"//pkg/util/metrics:go_default_library", "//pkg/util/metrics:go_default_library",
"//pkg/util/node:go_default_library", "//pkg/util/node:go_default_library",
"//pkg/util/system:go_default_library", "//pkg/util/system:go_default_library",
...@@ -88,9 +71,7 @@ go_library( ...@@ -88,9 +71,7 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/fields:go_default_library", "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/client-go/informers/core/v1:go_default_library", "//vendor/k8s.io/client-go/informers/core/v1:go_default_library",
"//vendor/k8s.io/client-go/informers/extensions/v1beta1:go_default_library", "//vendor/k8s.io/client-go/informers/extensions/v1beta1:go_default_library",
...@@ -102,7 +83,6 @@ go_library( ...@@ -102,7 +83,6 @@ go_library(
"//vendor/k8s.io/client-go/tools/cache:go_default_library", "//vendor/k8s.io/client-go/tools/cache:go_default_library",
"//vendor/k8s.io/client-go/tools/record:go_default_library", "//vendor/k8s.io/client-go/tools/record:go_default_library",
"//vendor/k8s.io/client-go/util/flowcontrol:go_default_library", "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
"//vendor/k8s.io/client-go/util/workqueue:go_default_library",
], ],
) )
...@@ -115,6 +95,11 @@ filegroup( ...@@ -115,6 +95,11 @@ filegroup(
filegroup( filegroup(
name = "all-srcs", name = "all-srcs",
srcs = [":package-srcs"], srcs = [
":package-srcs",
"//pkg/controller/node/ipam:all-srcs",
"//pkg/controller/node/scheduler:all-srcs",
"//pkg/controller/node/util:all-srcs",
],
tags = ["automanaged"], tags = ["automanaged"],
) )
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_test(
name = "go_default_test",
srcs = ["cidr_allocator_test.go"],
library = ":go_default_library",
tags = ["automanaged"],
deps = [
"//pkg/controller/testutil:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"cidr_allocator.go",
"cloud_cidr_allocator.go",
"range_allocator.go",
],
tags = ["automanaged"],
deps = [
"//pkg/api:go_default_library",
"//pkg/cloudprovider:go_default_library",
"//pkg/cloudprovider/providers/gce:go_default_library",
"//pkg/controller/node/ipam/cidrset:go_default_library",
"//pkg/controller/node/util:go_default_library",
"//pkg/util/node:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
"//vendor/k8s.io/client-go/tools/record:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//pkg/controller/node/ipam/cidrset:all-srcs",
],
tags = ["automanaged"],
)
approvers:
- bowei
- dnardo
reviewers:
- bowei
- dnardo
- freehan
...@@ -14,18 +14,14 @@ See the License for the specific language governing permissions and ...@@ -14,18 +14,14 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package node package ipam
import ( import (
"errors"
"net" "net"
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
) )
var errCIDRRangeNoCIDRsRemaining = errors.New(
"CIDR allocation failed; there are no remaining CIDRs left to allocate in the accepted range")
type nodeAndCIDR struct { type nodeAndCIDR struct {
cidr *net.IPNet cidr *net.IPNet
nodeName string nodeName string
...@@ -35,7 +31,11 @@ type nodeAndCIDR struct { ...@@ -35,7 +31,11 @@ type nodeAndCIDR struct {
type CIDRAllocatorType string type CIDRAllocatorType string
const ( const (
// RangeAllocatorType is the allocator that uses an internal CIDR
// range allocator to do node CIDR range allocations.
RangeAllocatorType CIDRAllocatorType = "RangeAllocator" RangeAllocatorType CIDRAllocatorType = "RangeAllocator"
// CloudAllocatorType is the allocator that uses cloud platform
// support to do node CIDR range allocations.
CloudAllocatorType CIDRAllocatorType = "CloudAllocator" CloudAllocatorType CIDRAllocatorType = "CloudAllocator"
) )
......
...@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and ...@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package node package ipam
import ( import (
"net" "net"
...@@ -143,7 +143,7 @@ func TestAllocateOrOccupyCIDRSuccess(t *testing.T) { ...@@ -143,7 +143,7 @@ func TestAllocateOrOccupyCIDRSuccess(t *testing.T) {
return return
} }
rangeAllocator.recorder = testutil.NewFakeRecorder() rangeAllocator.recorder = testutil.NewFakeRecorder()
if err = rangeAllocator.cidrs.occupy(cidr); err != nil { if err = rangeAllocator.cidrs.Occupy(cidr); err != nil {
t.Fatalf("%v: unexpected error when occupying CIDR %v: %v", tc.description, allocated, err) t.Fatalf("%v: unexpected error when occupying CIDR %v: %v", tc.description, allocated, err)
} }
} }
...@@ -225,7 +225,7 @@ func TestAllocateOrOccupyCIDRFailure(t *testing.T) { ...@@ -225,7 +225,7 @@ func TestAllocateOrOccupyCIDRFailure(t *testing.T) {
return return
} }
rangeAllocator.recorder = testutil.NewFakeRecorder() rangeAllocator.recorder = testutil.NewFakeRecorder()
err = rangeAllocator.cidrs.occupy(cidr) err = rangeAllocator.cidrs.Occupy(cidr)
if err != nil { if err != nil {
t.Fatalf("%v: unexpected error when occupying CIDR %v: %v", tc.description, allocated, err) t.Fatalf("%v: unexpected error when occupying CIDR %v: %v", tc.description, allocated, err)
} }
...@@ -337,7 +337,7 @@ func TestReleaseCIDRSuccess(t *testing.T) { ...@@ -337,7 +337,7 @@ func TestReleaseCIDRSuccess(t *testing.T) {
return return
} }
rangeAllocator.recorder = testutil.NewFakeRecorder() rangeAllocator.recorder = testutil.NewFakeRecorder()
err = rangeAllocator.cidrs.occupy(cidr) err = rangeAllocator.cidrs.Occupy(cidr)
if err != nil { if err != nil {
t.Fatalf("%v: unexpected error when occupying CIDR %v: %v", tc.description, allocated, err) t.Fatalf("%v: unexpected error when occupying CIDR %v: %v", tc.description, allocated, err)
} }
......
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_test(
name = "go_default_test",
srcs = ["cidr_set_test.go"],
library = ":go_default_library",
tags = ["automanaged"],
deps = ["//vendor/github.com/golang/glog:go_default_library"],
)
go_library(
name = "go_default_library",
srcs = ["cidr_set.go"],
tags = ["automanaged"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)
...@@ -14,17 +14,20 @@ See the License for the specific language governing permissions and ...@@ -14,17 +14,20 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package node package cidrset
import ( import (
"encoding/binary" "encoding/binary"
"errors"
"fmt" "fmt"
"math/big" "math/big"
"net" "net"
"sync" "sync"
) )
type cidrSet struct { // CidrSet manages a set of CIDR ranges from which blocks of IPs can
// be allocated from.
type CidrSet struct {
sync.Mutex sync.Mutex
clusterCIDR *net.IPNet clusterCIDR *net.IPNet
clusterIP net.IP clusterIP net.IP
...@@ -44,7 +47,15 @@ const ( ...@@ -44,7 +47,15 @@ const (
maxPrefixLength = 64 maxPrefixLength = 64
) )
func newCIDRSet(clusterCIDR *net.IPNet, subNetMaskSize int) *cidrSet { var (
// ErrCIDRRangeNoCIDRsRemaining occurs when there are no more space
// to allocate CIDR ranges.
ErrCIDRRangeNoCIDRsRemaining = errors.New(
"CIDR allocation failed; there are no remaining CIDRs left to allocate in the accepted range")
)
// NewCIDRSet creates a new CidrSet.
func NewCIDRSet(clusterCIDR *net.IPNet, subNetMaskSize int) *CidrSet {
clusterMask := clusterCIDR.Mask clusterMask := clusterCIDR.Mask
clusterMaskSize, _ := clusterMask.Size() clusterMaskSize, _ := clusterMask.Size()
...@@ -54,7 +65,7 @@ func newCIDRSet(clusterCIDR *net.IPNet, subNetMaskSize int) *cidrSet { ...@@ -54,7 +65,7 @@ func newCIDRSet(clusterCIDR *net.IPNet, subNetMaskSize int) *cidrSet {
} else { } else {
maxCIDRs = 1 << uint32(subNetMaskSize-clusterMaskSize) maxCIDRs = 1 << uint32(subNetMaskSize-clusterMaskSize)
} }
return &cidrSet{ return &CidrSet{
clusterCIDR: clusterCIDR, clusterCIDR: clusterCIDR,
clusterIP: clusterCIDR.IP, clusterIP: clusterCIDR.IP,
clusterMaskSize: clusterMaskSize, clusterMaskSize: clusterMaskSize,
...@@ -63,7 +74,7 @@ func newCIDRSet(clusterCIDR *net.IPNet, subNetMaskSize int) *cidrSet { ...@@ -63,7 +74,7 @@ func newCIDRSet(clusterCIDR *net.IPNet, subNetMaskSize int) *cidrSet {
} }
} }
func (s *cidrSet) indexToCIDRBlock(index int) *net.IPNet { func (s *CidrSet) indexToCIDRBlock(index int) *net.IPNet {
var ip []byte var ip []byte
var mask int var mask int
switch /*v4 or v6*/ { switch /*v4 or v6*/ {
...@@ -91,7 +102,8 @@ func (s *cidrSet) indexToCIDRBlock(index int) *net.IPNet { ...@@ -91,7 +102,8 @@ func (s *cidrSet) indexToCIDRBlock(index int) *net.IPNet {
} }
} }
func (s *cidrSet) allocateNext() (*net.IPNet, error) { // AllocateNext allocates the next free CIDR range.
func (s *CidrSet) AllocateNext() (*net.IPNet, error) {
s.Lock() s.Lock()
defer s.Unlock() defer s.Unlock()
...@@ -104,7 +116,7 @@ func (s *cidrSet) allocateNext() (*net.IPNet, error) { ...@@ -104,7 +116,7 @@ func (s *cidrSet) allocateNext() (*net.IPNet, error) {
} }
} }
if nextUnused == -1 { if nextUnused == -1 {
return nil, errCIDRRangeNoCIDRsRemaining return nil, ErrCIDRRangeNoCIDRsRemaining
} }
s.nextCandidate = (nextUnused + 1) % s.maxCIDRs s.nextCandidate = (nextUnused + 1) % s.maxCIDRs
...@@ -113,7 +125,7 @@ func (s *cidrSet) allocateNext() (*net.IPNet, error) { ...@@ -113,7 +125,7 @@ func (s *cidrSet) allocateNext() (*net.IPNet, error) {
return s.indexToCIDRBlock(nextUnused), nil return s.indexToCIDRBlock(nextUnused), nil
} }
func (s *cidrSet) getBeginingAndEndIndices(cidr *net.IPNet) (begin, end int, err error) { func (s *CidrSet) getBeginingAndEndIndices(cidr *net.IPNet) (begin, end int, err error) {
begin, end = 0, s.maxCIDRs-1 begin, end = 0, s.maxCIDRs-1
cidrMask := cidr.Mask cidrMask := cidr.Mask
maskSize, _ := cidrMask.Size() maskSize, _ := cidrMask.Size()
...@@ -160,7 +172,8 @@ func (s *cidrSet) getBeginingAndEndIndices(cidr *net.IPNet) (begin, end int, err ...@@ -160,7 +172,8 @@ func (s *cidrSet) getBeginingAndEndIndices(cidr *net.IPNet) (begin, end int, err
return begin, end, nil return begin, end, nil
} }
func (s *cidrSet) release(cidr *net.IPNet) error { // Release releases the given CIDR range.
func (s *CidrSet) Release(cidr *net.IPNet) error {
begin, end, err := s.getBeginingAndEndIndices(cidr) begin, end, err := s.getBeginingAndEndIndices(cidr)
if err != nil { if err != nil {
return err return err
...@@ -173,7 +186,8 @@ func (s *cidrSet) release(cidr *net.IPNet) error { ...@@ -173,7 +186,8 @@ func (s *cidrSet) release(cidr *net.IPNet) error {
return nil return nil
} }
func (s *cidrSet) occupy(cidr *net.IPNet) (err error) { // Occupy marks the given CIDR range as used.
func (s *CidrSet) Occupy(cidr *net.IPNet) (err error) {
begin, end, err := s.getBeginingAndEndIndices(cidr) begin, end, err := s.getBeginingAndEndIndices(cidr)
if err != nil { if err != nil {
return err return err
...@@ -188,7 +202,7 @@ func (s *cidrSet) occupy(cidr *net.IPNet) (err error) { ...@@ -188,7 +202,7 @@ func (s *cidrSet) occupy(cidr *net.IPNet) (err error) {
return nil return nil
} }
func (s *cidrSet) getIndexForCIDR(cidr *net.IPNet) (int, error) { func (s *CidrSet) getIndexForCIDR(cidr *net.IPNet) (int, error) {
var cidrIndex uint32 var cidrIndex uint32
if cidr.IP.To4() != nil { if cidr.IP.To4() != nil {
cidrIndex = (binary.BigEndian.Uint32(s.clusterIP) ^ binary.BigEndian.Uint32(cidr.IP.To4())) >> uint32(32-s.subNetMaskSize) cidrIndex = (binary.BigEndian.Uint32(s.clusterIP) ^ binary.BigEndian.Uint32(cidr.IP.To4())) >> uint32(32-s.subNetMaskSize)
......
...@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and ...@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package node package cidrset
import ( import (
"math/big" "math/big"
...@@ -47,9 +47,9 @@ func TestCIDRSetFullyAllocated(t *testing.T) { ...@@ -47,9 +47,9 @@ func TestCIDRSetFullyAllocated(t *testing.T) {
} }
for _, tc := range cases { for _, tc := range cases {
_, clusterCIDR, _ := net.ParseCIDR(tc.clusterCIDRStr) _, clusterCIDR, _ := net.ParseCIDR(tc.clusterCIDRStr)
a := newCIDRSet(clusterCIDR, tc.subNetMaskSize) a := NewCIDRSet(clusterCIDR, tc.subNetMaskSize)
p, err := a.allocateNext() p, err := a.AllocateNext()
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v for %v", err, tc.description) t.Fatalf("unexpected error: %v for %v", err, tc.description)
} }
...@@ -58,14 +58,14 @@ func TestCIDRSetFullyAllocated(t *testing.T) { ...@@ -58,14 +58,14 @@ func TestCIDRSetFullyAllocated(t *testing.T) {
p.String(), tc.expectedCIDR, tc.description) p.String(), tc.expectedCIDR, tc.description)
} }
_, err = a.allocateNext() _, err = a.AllocateNext()
if err == nil { if err == nil {
t.Fatalf("expected error because of fully-allocated range for %v", tc.description) t.Fatalf("expected error because of fully-allocated range for %v", tc.description)
} }
a.release(p) a.Release(p)
p, err = a.allocateNext() p, err = a.AllocateNext()
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v for %v", err, tc.description) t.Fatalf("unexpected error: %v for %v", err, tc.description)
} }
...@@ -73,7 +73,7 @@ func TestCIDRSetFullyAllocated(t *testing.T) { ...@@ -73,7 +73,7 @@ func TestCIDRSetFullyAllocated(t *testing.T) {
t.Fatalf("unexpected allocated cidr: %v, expecting %v for %v", t.Fatalf("unexpected allocated cidr: %v, expecting %v for %v",
p.String(), tc.expectedCIDR, tc.description) p.String(), tc.expectedCIDR, tc.description)
} }
_, err = a.allocateNext() _, err = a.AllocateNext()
if err == nil { if err == nil {
t.Fatalf("expected error because of fully-allocated range for %v", tc.description) t.Fatalf("expected error because of fully-allocated range for %v", tc.description)
} }
...@@ -133,7 +133,7 @@ func TestIndexToCIDRBlock(t *testing.T) { ...@@ -133,7 +133,7 @@ func TestIndexToCIDRBlock(t *testing.T) {
} }
for _, tc := range cases { for _, tc := range cases {
_, clusterCIDR, _ := net.ParseCIDR(tc.clusterCIDRStr) _, clusterCIDR, _ := net.ParseCIDR(tc.clusterCIDRStr)
a := newCIDRSet(clusterCIDR, tc.subnetMaskSize) a := NewCIDRSet(clusterCIDR, tc.subnetMaskSize)
cidr := a.indexToCIDRBlock(tc.index) cidr := a.indexToCIDRBlock(tc.index)
if cidr.String() != tc.CIDRBlock { if cidr.String() != tc.CIDRBlock {
t.Fatalf("error for %v index %d %s", tc.description, tc.index, cidr.String()) t.Fatalf("error for %v index %d %s", tc.description, tc.index, cidr.String())
...@@ -157,12 +157,12 @@ func TestCIDRSet_RandomishAllocation(t *testing.T) { ...@@ -157,12 +157,12 @@ func TestCIDRSet_RandomishAllocation(t *testing.T) {
} }
for _, tc := range cases { for _, tc := range cases {
_, clusterCIDR, _ := net.ParseCIDR(tc.clusterCIDRStr) _, clusterCIDR, _ := net.ParseCIDR(tc.clusterCIDRStr)
a := newCIDRSet(clusterCIDR, 24) a := NewCIDRSet(clusterCIDR, 24)
// allocate all the CIDRs // allocate all the CIDRs
var cidrs []*net.IPNet var cidrs []*net.IPNet
for i := 0; i < 256; i++ { for i := 0; i < 256; i++ {
if c, err := a.allocateNext(); err == nil { if c, err := a.AllocateNext(); err == nil {
cidrs = append(cidrs, c) cidrs = append(cidrs, c)
} else { } else {
t.Fatalf("unexpected error: %v for %v", err, tc.description) t.Fatalf("unexpected error: %v for %v", err, tc.description)
...@@ -170,25 +170,25 @@ func TestCIDRSet_RandomishAllocation(t *testing.T) { ...@@ -170,25 +170,25 @@ func TestCIDRSet_RandomishAllocation(t *testing.T) {
} }
var err error var err error
_, err = a.allocateNext() _, err = a.AllocateNext()
if err == nil { if err == nil {
t.Fatalf("expected error because of fully-allocated range for %v", tc.description) t.Fatalf("expected error because of fully-allocated range for %v", tc.description)
} }
// release them all // release them all
for i := 0; i < len(cidrs); i++ { for i := 0; i < len(cidrs); i++ {
a.release(cidrs[i]) a.Release(cidrs[i])
} }
// allocate the CIDRs again // allocate the CIDRs again
var rcidrs []*net.IPNet var rcidrs []*net.IPNet
for i := 0; i < 256; i++ { for i := 0; i < 256; i++ {
if c, err := a.allocateNext(); err == nil { if c, err := a.AllocateNext(); err == nil {
rcidrs = append(rcidrs, c) rcidrs = append(rcidrs, c)
} else { } else {
t.Fatalf("unexpected error: %d, %v for %v", i, err, tc.description) t.Fatalf("unexpected error: %d, %v for %v", i, err, tc.description)
} }
} }
_, err = a.allocateNext() _, err = a.AllocateNext()
if err == nil { if err == nil {
t.Fatalf("expected error because of fully-allocated range for %v", tc.description) t.Fatalf("expected error because of fully-allocated range for %v", tc.description)
} }
...@@ -215,14 +215,14 @@ func TestCIDRSet_AllocationOccupied(t *testing.T) { ...@@ -215,14 +215,14 @@ func TestCIDRSet_AllocationOccupied(t *testing.T) {
} }
for _, tc := range cases { for _, tc := range cases {
_, clusterCIDR, _ := net.ParseCIDR(tc.clusterCIDRStr) _, clusterCIDR, _ := net.ParseCIDR(tc.clusterCIDRStr)
a := newCIDRSet(clusterCIDR, 24) a := NewCIDRSet(clusterCIDR, 24)
// allocate all the CIDRs // allocate all the CIDRs
var cidrs []*net.IPNet var cidrs []*net.IPNet
var num_cidrs = 256 var numCIDRs = 256
for i := 0; i < num_cidrs; i++ { for i := 0; i < numCIDRs; i++ {
if c, err := a.allocateNext(); err == nil { if c, err := a.AllocateNext(); err == nil {
cidrs = append(cidrs, c) cidrs = append(cidrs, c)
} else { } else {
t.Fatalf("unexpected error: %v for %v", err, tc.description) t.Fatalf("unexpected error: %v for %v", err, tc.description)
...@@ -230,35 +230,35 @@ func TestCIDRSet_AllocationOccupied(t *testing.T) { ...@@ -230,35 +230,35 @@ func TestCIDRSet_AllocationOccupied(t *testing.T) {
} }
var err error var err error
_, err = a.allocateNext() _, err = a.AllocateNext()
if err == nil { if err == nil {
t.Fatalf("expected error because of fully-allocated range for %v", tc.description) t.Fatalf("expected error because of fully-allocated range for %v", tc.description)
} }
// release them all // release them all
for i := 0; i < len(cidrs); i++ { for i := 0; i < len(cidrs); i++ {
a.release(cidrs[i]) a.Release(cidrs[i])
} }
// occupy the last 128 CIDRs // occupy the last 128 CIDRs
for i := num_cidrs / 2; i < num_cidrs; i++ { for i := numCIDRs / 2; i < numCIDRs; i++ {
a.occupy(cidrs[i]) a.Occupy(cidrs[i])
} }
// allocate the first 128 CIDRs again // allocate the first 128 CIDRs again
var rcidrs []*net.IPNet var rcidrs []*net.IPNet
for i := 0; i < num_cidrs/2; i++ { for i := 0; i < numCIDRs/2; i++ {
if c, err := a.allocateNext(); err == nil { if c, err := a.AllocateNext(); err == nil {
rcidrs = append(rcidrs, c) rcidrs = append(rcidrs, c)
} else { } else {
t.Fatalf("unexpected error: %d, %v for %v", i, err, tc.description) t.Fatalf("unexpected error: %d, %v for %v", i, err, tc.description)
} }
} }
_, err = a.allocateNext() _, err = a.AllocateNext()
if err == nil { if err == nil {
t.Fatalf("expected error because of fully-allocated range for %v", tc.description) t.Fatalf("expected error because of fully-allocated range for %v", tc.description)
} }
// check Occupy() work properly // check Occupy() work properly
for i := num_cidrs / 2; i < num_cidrs; i++ { for i := numCIDRs / 2; i < numCIDRs; i++ {
rcidrs = append(rcidrs, cidrs[i]) rcidrs = append(rcidrs, cidrs[i])
} }
if !reflect.DeepEqual(cidrs, rcidrs) { if !reflect.DeepEqual(cidrs, rcidrs) {
...@@ -394,7 +394,7 @@ func TestGetBitforCIDR(t *testing.T) { ...@@ -394,7 +394,7 @@ func TestGetBitforCIDR(t *testing.T) {
t.Fatalf("unexpected error: %v for %v", err, tc.description) t.Fatalf("unexpected error: %v for %v", err, tc.description)
} }
cs := newCIDRSet(clusterCIDR, tc.subNetMaskSize) cs := NewCIDRSet(clusterCIDR, tc.subNetMaskSize)
_, subnetCIDR, err := net.ParseCIDR(tc.subNetCIDRStr) _, subnetCIDR, err := net.ParseCIDR(tc.subNetCIDRStr)
if err != nil { if err != nil {
...@@ -562,14 +562,14 @@ func TestOccupy(t *testing.T) { ...@@ -562,14 +562,14 @@ func TestOccupy(t *testing.T) {
t.Fatalf("unexpected error: %v for %v", err, tc.description) t.Fatalf("unexpected error: %v for %v", err, tc.description)
} }
cs := newCIDRSet(clusterCIDR, tc.subNetMaskSize) cs := NewCIDRSet(clusterCIDR, tc.subNetMaskSize)
_, subnetCIDR, err := net.ParseCIDR(tc.subNetCIDRStr) _, subnetCIDR, err := net.ParseCIDR(tc.subNetCIDRStr)
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v for %v", err, tc.description) t.Fatalf("unexpected error: %v for %v", err, tc.description)
} }
err = cs.occupy(subnetCIDR) err = cs.Occupy(subnetCIDR)
if err == nil && tc.expectErr { if err == nil && tc.expectErr {
t.Errorf("expected error but got none for %v", tc.description) t.Errorf("expected error but got none for %v", tc.description)
continue continue
...@@ -629,9 +629,9 @@ func TestCIDRSetv6(t *testing.T) { ...@@ -629,9 +629,9 @@ func TestCIDRSetv6(t *testing.T) {
} }
for _, tc := range cases { for _, tc := range cases {
_, clusterCIDR, _ := net.ParseCIDR(tc.clusterCIDRStr) _, clusterCIDR, _ := net.ParseCIDR(tc.clusterCIDRStr)
a := newCIDRSet(clusterCIDR, tc.subNetMaskSize) a := NewCIDRSet(clusterCIDR, tc.subNetMaskSize)
p, err := a.allocateNext() p, err := a.AllocateNext()
if err == nil && tc.expectErr { if err == nil && tc.expectErr {
t.Errorf("expected error but got none for %v", tc.description) t.Errorf("expected error but got none for %v", tc.description)
continue continue
...@@ -645,7 +645,7 @@ func TestCIDRSetv6(t *testing.T) { ...@@ -645,7 +645,7 @@ func TestCIDRSetv6(t *testing.T) {
t.Fatalf("unexpected allocated cidr: %s for %v", p.String(), tc.description) t.Fatalf("unexpected allocated cidr: %s for %v", p.String(), tc.description)
} }
} }
p2, err := a.allocateNext() p2, err := a.AllocateNext()
if !tc.expectErr { if !tc.expectErr {
if p2.String() != tc.expectedCIDR2 { if p2.String() != tc.expectedCIDR2 {
t.Fatalf("unexpected allocated cidr: %s for %v", p2.String(), tc.description) t.Fatalf("unexpected allocated cidr: %s for %v", p2.String(), tc.description)
......
...@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and ...@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package node package ipam
import ( import (
"fmt" "fmt"
...@@ -32,6 +32,7 @@ import ( ...@@ -32,6 +32,7 @@ import (
"k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/cloudprovider"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
"k8s.io/kubernetes/pkg/controller/node/util"
nodeutil "k8s.io/kubernetes/pkg/util/node" nodeutil "k8s.io/kubernetes/pkg/util/node"
) )
...@@ -50,6 +51,7 @@ type cloudCIDRAllocator struct { ...@@ -50,6 +51,7 @@ type cloudCIDRAllocator struct {
var _ CIDRAllocator = (*cloudCIDRAllocator)(nil) var _ CIDRAllocator = (*cloudCIDRAllocator)(nil)
// NewCloudCIDRAllocator creates a new cloud CIDR allocator.
func NewCloudCIDRAllocator( func NewCloudCIDRAllocator(
client clientset.Interface, client clientset.Interface,
cloud cloudprovider.Interface) (ca CIDRAllocator, err error) { cloud cloudprovider.Interface) (ca CIDRAllocator, err error) {
...@@ -79,12 +81,12 @@ func (ca *cloudCIDRAllocator) AllocateOrOccupyCIDR(node *v1.Node) error { ...@@ -79,12 +81,12 @@ func (ca *cloudCIDRAllocator) AllocateOrOccupyCIDR(node *v1.Node) error {
cidrs, err := ca.cloud.AliasRanges(types.NodeName(node.Name)) cidrs, err := ca.cloud.AliasRanges(types.NodeName(node.Name))
if err != nil { if err != nil {
recordNodeStatusChange(ca.recorder, node, "CIDRNotAvailable") util.RecordNodeStatusChange(ca.recorder, node, "CIDRNotAvailable")
return fmt.Errorf("failed to allocate cidr: %v", err) return fmt.Errorf("failed to allocate cidr: %v", err)
} }
if len(cidrs) == 0 { if len(cidrs) == 0 {
recordNodeStatusChange(ca.recorder, node, "CIDRNotAvailable") util.RecordNodeStatusChange(ca.recorder, node, "CIDRNotAvailable")
glog.V(2).Infof("Node %v has no CIDRs", node.Name) glog.V(2).Infof("Node %v has no CIDRs", node.Name)
return fmt.Errorf("failed to allocate cidr (none exist)") return fmt.Errorf("failed to allocate cidr (none exist)")
} }
......
...@@ -14,13 +14,15 @@ See the License for the specific language governing permissions and ...@@ -14,13 +14,15 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package node package ipam
import ( import (
"fmt" "fmt"
"net" "net"
"sync" "sync"
"github.com/golang/glog"
"k8s.io/api/core/v1" "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors" apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
...@@ -31,7 +33,8 @@ import ( ...@@ -31,7 +33,8 @@ import (
v1core "k8s.io/client-go/kubernetes/typed/core/v1" v1core "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/tools/record" "k8s.io/client-go/tools/record"
"github.com/golang/glog" "k8s.io/kubernetes/pkg/controller/node/ipam/cidrset"
"k8s.io/kubernetes/pkg/controller/node/util"
) )
// TODO: figure out the good setting for those constants. // TODO: figure out the good setting for those constants.
...@@ -45,7 +48,7 @@ const ( ...@@ -45,7 +48,7 @@ const (
type rangeAllocator struct { type rangeAllocator struct {
client clientset.Interface client clientset.Interface
cidrs *cidrSet cidrs *cidrset.CidrSet
clusterCIDR *net.IPNet clusterCIDR *net.IPNet
maxCIDRs int maxCIDRs int
// Channel that is used to pass updating Nodes with assigned CIDRs to the background // Channel that is used to pass updating Nodes with assigned CIDRs to the background
...@@ -74,7 +77,7 @@ func NewCIDRRangeAllocator(client clientset.Interface, clusterCIDR *net.IPNet, s ...@@ -74,7 +77,7 @@ func NewCIDRRangeAllocator(client clientset.Interface, clusterCIDR *net.IPNet, s
ra := &rangeAllocator{ ra := &rangeAllocator{
client: client, client: client,
cidrs: newCIDRSet(clusterCIDR, subNetMaskSize), cidrs: cidrset.NewCIDRSet(clusterCIDR, subNetMaskSize),
clusterCIDR: clusterCIDR, clusterCIDR: clusterCIDR,
nodeCIDRUpdateChannel: make(chan nodeAndCIDR, cidrUpdateQueueSize), nodeCIDRUpdateChannel: make(chan nodeAndCIDR, cidrUpdateQueueSize),
recorder: recorder, recorder: recorder,
...@@ -150,7 +153,7 @@ func (r *rangeAllocator) occupyCIDR(node *v1.Node) error { ...@@ -150,7 +153,7 @@ func (r *rangeAllocator) occupyCIDR(node *v1.Node) error {
if err != nil { if err != nil {
return fmt.Errorf("failed to parse node %s, CIDR %s", node.Name, node.Spec.PodCIDR) return fmt.Errorf("failed to parse node %s, CIDR %s", node.Name, node.Spec.PodCIDR)
} }
if err := r.cidrs.occupy(podCIDR); err != nil { if err := r.cidrs.Occupy(podCIDR); err != nil {
return fmt.Errorf("failed to mark cidr as occupied: %v", err) return fmt.Errorf("failed to mark cidr as occupied: %v", err)
} }
return nil return nil
...@@ -169,10 +172,10 @@ func (r *rangeAllocator) AllocateOrOccupyCIDR(node *v1.Node) error { ...@@ -169,10 +172,10 @@ func (r *rangeAllocator) AllocateOrOccupyCIDR(node *v1.Node) error {
if node.Spec.PodCIDR != "" { if node.Spec.PodCIDR != "" {
return r.occupyCIDR(node) return r.occupyCIDR(node)
} }
podCIDR, err := r.cidrs.allocateNext() podCIDR, err := r.cidrs.AllocateNext()
if err != nil { if err != nil {
r.removeNodeFromProcessing(node.Name) r.removeNodeFromProcessing(node.Name)
recordNodeStatusChange(r.recorder, node, "CIDRNotAvailable") util.RecordNodeStatusChange(r.recorder, node, "CIDRNotAvailable")
return fmt.Errorf("failed to allocate cidr: %v", err) return fmt.Errorf("failed to allocate cidr: %v", err)
} }
...@@ -194,7 +197,7 @@ func (r *rangeAllocator) ReleaseCIDR(node *v1.Node) error { ...@@ -194,7 +197,7 @@ func (r *rangeAllocator) ReleaseCIDR(node *v1.Node) error {
} }
glog.V(4).Infof("release CIDR %s", node.Spec.PodCIDR) glog.V(4).Infof("release CIDR %s", node.Spec.PodCIDR)
if err = r.cidrs.release(podCIDR); err != nil { if err = r.cidrs.Release(podCIDR); err != nil {
return fmt.Errorf("Error when releasing CIDR %v: %v", node.Spec.PodCIDR, err) return fmt.Errorf("Error when releasing CIDR %v: %v", node.Spec.PodCIDR, err)
} }
return err return err
...@@ -212,7 +215,7 @@ func (r *rangeAllocator) filterOutServiceRange(serviceCIDR *net.IPNet) { ...@@ -212,7 +215,7 @@ func (r *rangeAllocator) filterOutServiceRange(serviceCIDR *net.IPNet) {
return return
} }
if err := r.cidrs.occupy(serviceCIDR); err != nil { if err := r.cidrs.Occupy(serviceCIDR); err != nil {
glog.Errorf("Error filtering out service cidr %v: %v", serviceCIDR, err) glog.Errorf("Error filtering out service cidr %v: %v", serviceCIDR, err)
} }
} }
...@@ -232,7 +235,7 @@ func (r *rangeAllocator) updateCIDRAllocation(data nodeAndCIDR) error { ...@@ -232,7 +235,7 @@ func (r *rangeAllocator) updateCIDRAllocation(data nodeAndCIDR) error {
if node.Spec.PodCIDR != "" { if node.Spec.PodCIDR != "" {
glog.Errorf("Node %v already has allocated CIDR %v. Releasing assigned one if different.", node.Name, node.Spec.PodCIDR) glog.Errorf("Node %v already has allocated CIDR %v. Releasing assigned one if different.", node.Name, node.Spec.PodCIDR)
if node.Spec.PodCIDR != data.cidr.String() { if node.Spec.PodCIDR != data.cidr.String() {
if err := r.cidrs.release(data.cidr); err != nil { if err := r.cidrs.Release(data.cidr); err != nil {
glog.Errorf("Error when releasing CIDR %v", data.cidr.String()) glog.Errorf("Error when releasing CIDR %v", data.cidr.String())
} }
} }
...@@ -246,13 +249,13 @@ func (r *rangeAllocator) updateCIDRAllocation(data nodeAndCIDR) error { ...@@ -246,13 +249,13 @@ func (r *rangeAllocator) updateCIDRAllocation(data nodeAndCIDR) error {
} }
} }
if err != nil { if err != nil {
recordNodeStatusChange(r.recorder, node, "CIDRAssignmentFailed") util.RecordNodeStatusChange(r.recorder, node, "CIDRAssignmentFailed")
// We accept the fact that we may leek CIDRs here. This is safer than releasing // We accept the fact that we may leek CIDRs here. This is safer than releasing
// them in case when we don't know if request went through. // them in case when we don't know if request went through.
// NodeController restart will return all falsely allocated CIDRs to the pool. // NodeController restart will return all falsely allocated CIDRs to the pool.
if !apierrors.IsServerTimeout(err) { if !apierrors.IsServerTimeout(err) {
glog.Errorf("CIDR assignment for node %v failed: %v. Releasing allocated CIDR", data.nodeName, err) glog.Errorf("CIDR assignment for node %v failed: %v. Releasing allocated CIDR", data.nodeName, err)
if releaseErr := r.cidrs.release(data.cidr); releaseErr != nil { if releaseErr := r.cidrs.Release(data.cidr); releaseErr != nil {
glog.Errorf("Error releasing allocated CIDR for node %v: %v", data.nodeName, releaseErr) glog.Errorf("Error releasing allocated CIDR for node %v: %v", data.nodeName, releaseErr)
} }
} }
......
...@@ -39,6 +39,9 @@ import ( ...@@ -39,6 +39,9 @@ import (
"k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/cloudprovider"
fakecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/fake" fakecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/fake"
"k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/controller"
"k8s.io/kubernetes/pkg/controller/node/ipam"
"k8s.io/kubernetes/pkg/controller/node/scheduler"
"k8s.io/kubernetes/pkg/controller/node/util"
"k8s.io/kubernetes/pkg/controller/testutil" "k8s.io/kubernetes/pkg/controller/testutil"
kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
"k8s.io/kubernetes/pkg/util/node" "k8s.io/kubernetes/pkg/util/node"
...@@ -103,7 +106,7 @@ func NewNodeControllerFromClient( ...@@ -103,7 +106,7 @@ func NewNodeControllerFromClient(
serviceCIDR, serviceCIDR,
nodeCIDRMaskSize, nodeCIDRMaskSize,
allocateNodeCIDRs, allocateNodeCIDRs,
RangeAllocatorType, ipam.RangeAllocatorType,
useTaints, useTaints,
useTaints, useTaints,
) )
...@@ -637,9 +640,9 @@ func TestMonitorNodeStatusEvictPods(t *testing.T) { ...@@ -637,9 +640,9 @@ func TestMonitorNodeStatusEvictPods(t *testing.T) {
zones := testutil.GetZones(item.fakeNodeHandler) zones := testutil.GetZones(item.fakeNodeHandler)
for _, zone := range zones { for _, zone := range zones {
if _, ok := nodeController.zonePodEvictor[zone]; ok { if _, ok := nodeController.zonePodEvictor[zone]; ok {
nodeController.zonePodEvictor[zone].Try(func(value TimedValue) (bool, time.Duration) { nodeController.zonePodEvictor[zone].Try(func(value scheduler.TimedValue) (bool, time.Duration) {
nodeUid, _ := value.UID.(string) nodeUid, _ := value.UID.(string)
deletePods(item.fakeNodeHandler, nodeController.recorder, value.Value, nodeUid, nodeController.daemonSetInformer.Lister()) util.DeletePods(item.fakeNodeHandler, nodeController.recorder, value.Value, nodeUid, nodeController.daemonSetInformer.Lister())
return true, 0 return true, 0
}) })
} else { } else {
...@@ -782,9 +785,9 @@ func TestPodStatusChange(t *testing.T) { ...@@ -782,9 +785,9 @@ func TestPodStatusChange(t *testing.T) {
} }
zones := testutil.GetZones(item.fakeNodeHandler) zones := testutil.GetZones(item.fakeNodeHandler)
for _, zone := range zones { for _, zone := range zones {
nodeController.zonePodEvictor[zone].Try(func(value TimedValue) (bool, time.Duration) { nodeController.zonePodEvictor[zone].Try(func(value scheduler.TimedValue) (bool, time.Duration) {
nodeUid, _ := value.UID.(string) nodeUid, _ := value.UID.(string)
deletePods(item.fakeNodeHandler, nodeController.recorder, value.Value, nodeUid, nodeController.daemonSetStore) util.DeletePods(item.fakeNodeHandler, nodeController.recorder, value.Value, nodeUid, nodeController.daemonSetStore)
return true, 0 return true, 0
}) })
} }
...@@ -1337,9 +1340,9 @@ func (nc *nodeController) doEviction(fakeNodeHandler *testutil.FakeNodeHandler) ...@@ -1337,9 +1340,9 @@ func (nc *nodeController) doEviction(fakeNodeHandler *testutil.FakeNodeHandler)
var podEvicted bool var podEvicted bool
zones := testutil.GetZones(fakeNodeHandler) zones := testutil.GetZones(fakeNodeHandler)
for _, zone := range zones { for _, zone := range zones {
nc.zonePodEvictor[zone].Try(func(value TimedValue) (bool, time.Duration) { nc.zonePodEvictor[zone].Try(func(value scheduler.TimedValue) (bool, time.Duration) {
uid, _ := value.UID.(string) uid, _ := value.UID.(string)
deletePods(fakeNodeHandler, nc.recorder, value.Value, uid, nc.daemonSetStore) util.DeletePods(fakeNodeHandler, nc.recorder, value.Value, uid, nc.daemonSetStore)
return true, 0 return true, 0
}) })
} }
...@@ -2310,7 +2313,7 @@ func TestCheckNodeKubeletVersionParsing(t *testing.T) { ...@@ -2310,7 +2313,7 @@ func TestCheckNodeKubeletVersionParsing(t *testing.T) {
}, },
}, },
} }
isOutdated := nodeRunningOutdatedKubelet(n) isOutdated := util.NodeRunningOutdatedKubelet(n)
if ov.outdated != isOutdated { if ov.outdated != isOutdated {
t.Errorf("Version %v doesn't match test expectation. Expected outdated %v got %v", n.Status.NodeInfo.KubeletVersion, ov.outdated, isOutdated) t.Errorf("Version %v doesn't match test expectation. Expected outdated %v got %v", n.Status.NodeInfo.KubeletVersion, ov.outdated, isOutdated)
} else { } else {
......
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_test(
name = "go_default_test",
srcs = [
"rate_limited_queue_test.go",
"taint_controller_test.go",
"timed_workers_test.go",
],
library = ":go_default_library",
tags = ["automanaged"],
deps = [
"//pkg/controller/testutil:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
"//vendor/k8s.io/client-go/testing:go_default_library",
"//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"rate_limited_queue.go",
"taint_controller.go",
"timed_workers.go",
],
tags = ["automanaged"],
deps = [
"//pkg/api/helper:go_default_library",
"//pkg/api/v1/helper:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
"//vendor/k8s.io/client-go/tools/record:go_default_library",
"//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
"//vendor/k8s.io/client-go/util/workqueue:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)
...@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and ...@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package node package scheduler
import ( import (
"container/heap" "container/heap"
...@@ -27,6 +27,18 @@ import ( ...@@ -27,6 +27,18 @@ import (
"github.com/golang/glog" "github.com/golang/glog"
) )
const (
// NodeStatusUpdateRetry controls the number of retries of writing
// NodeStatus update.
NodeStatusUpdateRetry = 5
// NodeEvictionPeriod controls how often NodeController will try to
// evict Pods from non-responsive Nodes.
NodeEvictionPeriod = 100 * time.Millisecond
// EvictionRateLimiterBurst is the burst value for all eviction rate
// limiters
EvictionRateLimiterBurst = 1
)
// TimedValue is a value that should be processed at a designated time. // TimedValue is a value that should be processed at a designated time.
type TimedValue struct { type TimedValue struct {
Value string Value string
...@@ -37,19 +49,26 @@ type TimedValue struct { ...@@ -37,19 +49,26 @@ type TimedValue struct {
} }
// now is used to test time // now is used to test time
var now func() time.Time = time.Now var now = time.Now
// TimedQueue is a priority heap where the lowest ProcessAt is at the front of the queue // TimedQueue is a priority heap where the lowest ProcessAt is at the front of the queue
type TimedQueue []*TimedValue type TimedQueue []*TimedValue
func (h TimedQueue) Len() int { return len(h) } // Len is the length of the queue.
func (h TimedQueue) Len() int { return len(h) }
// Less returns true if queue[i] < queue[j].
func (h TimedQueue) Less(i, j int) bool { return h[i].ProcessAt.Before(h[j].ProcessAt) } func (h TimedQueue) Less(i, j int) bool { return h[i].ProcessAt.Before(h[j].ProcessAt) }
func (h TimedQueue) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
// Swap swaps index i and j.
func (h TimedQueue) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
// Push a new TimedValue on to the queue.
func (h *TimedQueue) Push(x interface{}) { func (h *TimedQueue) Push(x interface{}) {
*h = append(*h, x.(*TimedValue)) *h = append(*h, x.(*TimedValue))
} }
// Pop the lowest ProcessAt item.
func (h *TimedQueue) Pop() interface{} { func (h *TimedQueue) Pop() interface{} {
old := *h old := *h
n := len(old) n := len(old)
...@@ -58,16 +77,17 @@ func (h *TimedQueue) Pop() interface{} { ...@@ -58,16 +77,17 @@ func (h *TimedQueue) Pop() interface{} {
return x return x
} }
// A FIFO queue which additionally guarantees that any element can be added only once until // UniqueQueue is a FIFO queue which additionally guarantees that any
// it is removed. // element can be added only once until it is removed.
type UniqueQueue struct { type UniqueQueue struct {
lock sync.Mutex lock sync.Mutex
queue TimedQueue queue TimedQueue
set sets.String set sets.String
} }
// Adds a new value to the queue if it wasn't added before, or was explicitly removed by the // Add a new value to the queue if it wasn't added before, or was
// Remove call. Returns true if new value was added. // explicitly removed by the Remove call. Returns true if new value
// was added.
func (q *UniqueQueue) Add(value TimedValue) bool { func (q *UniqueQueue) Add(value TimedValue) bool {
q.lock.Lock() q.lock.Lock()
defer q.lock.Unlock() defer q.lock.Unlock()
...@@ -80,8 +100,9 @@ func (q *UniqueQueue) Add(value TimedValue) bool { ...@@ -80,8 +100,9 @@ func (q *UniqueQueue) Add(value TimedValue) bool {
return true return true
} }
// Replace replaces an existing value in the queue if it already exists, otherwise it does nothing. // Replace replaces an existing value in the queue if it already
// Returns true if the item was found. // exists, otherwise it does nothing. Returns true if the item was
// found.
func (q *UniqueQueue) Replace(value TimedValue) bool { func (q *UniqueQueue) Replace(value TimedValue) bool {
q.lock.Lock() q.lock.Lock()
defer q.lock.Unlock() defer q.lock.Unlock()
...@@ -97,8 +118,9 @@ func (q *UniqueQueue) Replace(value TimedValue) bool { ...@@ -97,8 +118,9 @@ func (q *UniqueQueue) Replace(value TimedValue) bool {
return false return false
} }
// Removes the value from the queue, but keeps it in the set, so it won't be added second time. // RemoveFromQueue the value from the queue, but keeps it in the set,
// Returns true if something was removed. // so it won't be added second time. Returns true if something was
// removed.
func (q *UniqueQueue) RemoveFromQueue(value string) bool { func (q *UniqueQueue) RemoveFromQueue(value string) bool {
q.lock.Lock() q.lock.Lock()
defer q.lock.Unlock() defer q.lock.Unlock()
...@@ -115,8 +137,9 @@ func (q *UniqueQueue) RemoveFromQueue(value string) bool { ...@@ -115,8 +137,9 @@ func (q *UniqueQueue) RemoveFromQueue(value string) bool {
return false return false
} }
// Removes the value from the queue, so Get() call won't return it, and allow subsequent addition // Remove the value from the queue, so Get() call won't return it, and
// of the given value. If the value is not present does nothing and returns false. // allow subsequent addition of the given value. If the value is not
// present does nothing and returns false.
func (q *UniqueQueue) Remove(value string) bool { func (q *UniqueQueue) Remove(value string) bool {
q.lock.Lock() q.lock.Lock()
defer q.lock.Unlock() defer q.lock.Unlock()
...@@ -134,7 +157,7 @@ func (q *UniqueQueue) Remove(value string) bool { ...@@ -134,7 +157,7 @@ func (q *UniqueQueue) Remove(value string) bool {
return true return true
} }
// Returns the oldest added value that wasn't returned yet. // Get returns the oldest added value that wasn't returned yet.
func (q *UniqueQueue) Get() (TimedValue, bool) { func (q *UniqueQueue) Get() (TimedValue, bool) {
q.lock.Lock() q.lock.Lock()
defer q.lock.Unlock() defer q.lock.Unlock()
...@@ -146,7 +169,8 @@ func (q *UniqueQueue) Get() (TimedValue, bool) { ...@@ -146,7 +169,8 @@ func (q *UniqueQueue) Get() (TimedValue, bool) {
return *result, true return *result, true
} }
// Head returns the oldest added value that wasn't returned yet without removing it. // Head returns the oldest added value that wasn't returned yet
// without removing it.
func (q *UniqueQueue) Head() (TimedValue, bool) { func (q *UniqueQueue) Head() (TimedValue, bool) {
q.lock.Lock() q.lock.Lock()
defer q.lock.Unlock() defer q.lock.Unlock()
...@@ -157,7 +181,8 @@ func (q *UniqueQueue) Head() (TimedValue, bool) { ...@@ -157,7 +181,8 @@ func (q *UniqueQueue) Head() (TimedValue, bool) {
return *result, true return *result, true
} }
// Clear removes all items from the queue and duplication preventing set. // Clear removes all items from the queue and duplication preventing
// set.
func (q *UniqueQueue) Clear() { func (q *UniqueQueue) Clear() {
q.lock.Lock() q.lock.Lock()
defer q.lock.Unlock() defer q.lock.Unlock()
...@@ -169,15 +194,16 @@ func (q *UniqueQueue) Clear() { ...@@ -169,15 +194,16 @@ func (q *UniqueQueue) Clear() {
} }
} }
// RateLimitedTimedQueue is a unique item priority queue ordered by the expected next time // RateLimitedTimedQueue is a unique item priority queue ordered by
// of execution. It is also rate limited. // the expected next time of execution. It is also rate limited.
type RateLimitedTimedQueue struct { type RateLimitedTimedQueue struct {
queue UniqueQueue queue UniqueQueue
limiterLock sync.Mutex limiterLock sync.Mutex
limiter flowcontrol.RateLimiter limiter flowcontrol.RateLimiter
} }
// Creates new queue which will use given RateLimiter to oversee execution. // NewRateLimitedTimedQueue creates new queue which will use given
// RateLimiter to oversee execution.
func NewRateLimitedTimedQueue(limiter flowcontrol.RateLimiter) *RateLimitedTimedQueue { func NewRateLimitedTimedQueue(limiter flowcontrol.RateLimiter) *RateLimitedTimedQueue {
return &RateLimitedTimedQueue{ return &RateLimitedTimedQueue{
queue: UniqueQueue{ queue: UniqueQueue{
...@@ -188,18 +214,21 @@ func NewRateLimitedTimedQueue(limiter flowcontrol.RateLimiter) *RateLimitedTimed ...@@ -188,18 +214,21 @@ func NewRateLimitedTimedQueue(limiter flowcontrol.RateLimiter) *RateLimitedTimed
} }
} }
// ActionFunc takes a timed value and returns false if the item must be retried, with an optional // ActionFunc takes a timed value and returns false if the item must
// time.Duration if some minimum wait interval should be used. // be retried, with an optional time.Duration if some minimum wait
// interval should be used.
type ActionFunc func(TimedValue) (bool, time.Duration) type ActionFunc func(TimedValue) (bool, time.Duration)
// Try processes the queue. Ends prematurely if RateLimiter forbids an action and leak is true. // Try processes the queue.Ends prematurely if RateLimiter forbids an
// Otherwise, requeues the item to be processed. Each value is processed once if fn returns true, // action and leak is true. Otherwise, requeues the item to be
// otherwise it is added back to the queue. The returned remaining is used to identify the minimum // processed. Each value is processed once if fn returns true,
// time to execute the next item in the queue. The same value is processed only once unless // otherwise it is added back to the queue. The returned remaining is
// Remove is explicitly called on it (it's done by the cancelPodEviction function in NodeController // used to identify the minimum time to execute the next item in the
// when Node becomes Ready again) // queue. The same value is processed only once unless Remove is
// TODO: figure out a good way to do garbage collection for all Nodes that were removed from // explicitly called on it (it's done by the cancelPodEviction
// the cluster. // function in NodeController when Node becomes Ready again) TODO:
// figure out a good way to do garbage collection for all Nodes that
// were removed from the cluster.
func (q *RateLimitedTimedQueue) Try(fn ActionFunc) { func (q *RateLimitedTimedQueue) Try(fn ActionFunc) {
val, ok := q.queue.Head() val, ok := q.queue.Head()
q.limiterLock.Lock() q.limiterLock.Lock()
...@@ -227,8 +256,9 @@ func (q *RateLimitedTimedQueue) Try(fn ActionFunc) { ...@@ -227,8 +256,9 @@ func (q *RateLimitedTimedQueue) Try(fn ActionFunc) {
} }
} }
// Adds value to the queue to be processed. Won't add the same value(comparsion by value) a second time // Add value to the queue to be processed. Won't add the same
// if it was already added and not removed. // value(comparsion by value) a second time if it was already added
// and not removed.
func (q *RateLimitedTimedQueue) Add(value string, uid interface{}) bool { func (q *RateLimitedTimedQueue) Add(value string, uid interface{}) bool {
now := now() now := now()
return q.queue.Add(TimedValue{ return q.queue.Add(TimedValue{
...@@ -239,17 +269,19 @@ func (q *RateLimitedTimedQueue) Add(value string, uid interface{}) bool { ...@@ -239,17 +269,19 @@ func (q *RateLimitedTimedQueue) Add(value string, uid interface{}) bool {
}) })
} }
// Removes Node from the Evictor. The Node won't be processed until added again. // Remove Node from the Evictor. The Node won't be processed until
// added again.
func (q *RateLimitedTimedQueue) Remove(value string) bool { func (q *RateLimitedTimedQueue) Remove(value string) bool {
return q.queue.Remove(value) return q.queue.Remove(value)
} }
// Removes all items from the queue // Clear removes all items from the queue
func (q *RateLimitedTimedQueue) Clear() { func (q *RateLimitedTimedQueue) Clear() {
q.queue.Clear() q.queue.Clear()
} }
// SwapLimiter safely swaps current limiter for this queue with the passed one if capacities or qps's differ. // SwapLimiter safely swaps current limiter for this queue with the
// passed one if capacities or qps's differ.
func (q *RateLimitedTimedQueue) SwapLimiter(newQPS float32) { func (q *RateLimitedTimedQueue) SwapLimiter(newQPS float32) {
q.limiterLock.Lock() q.limiterLock.Lock()
defer q.limiterLock.Unlock() defer q.limiterLock.Unlock()
...@@ -260,7 +292,7 @@ func (q *RateLimitedTimedQueue) SwapLimiter(newQPS float32) { ...@@ -260,7 +292,7 @@ func (q *RateLimitedTimedQueue) SwapLimiter(newQPS float32) {
if newQPS <= 0 { if newQPS <= 0 {
newLimiter = flowcontrol.NewFakeNeverRateLimiter() newLimiter = flowcontrol.NewFakeNeverRateLimiter()
} else { } else {
newLimiter = flowcontrol.NewTokenBucketRateLimiter(newQPS, evictionRateLimiterBurst) newLimiter = flowcontrol.NewTokenBucketRateLimiter(newQPS, EvictionRateLimiterBurst)
} }
// If we're currently waiting on limiter, we drain the new one - this is a good approach when Burst value is 1 // If we're currently waiting on limiter, we drain the new one - this is a good approach when Burst value is 1
// TODO: figure out if we need to support higher Burst values and decide on the drain logic, should we keep: // TODO: figure out if we need to support higher Burst values and decide on the drain logic, should we keep:
......
...@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and ...@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package node package scheduler
import ( import (
"reflect" "reflect"
......
...@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and ...@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package node package scheduler
import ( import (
"fmt" "fmt"
...@@ -122,7 +122,7 @@ func getPodsAssignedToNode(c clientset.Interface, nodeName string) ([]v1.Pod, er ...@@ -122,7 +122,7 @@ func getPodsAssignedToNode(c clientset.Interface, nodeName string) ([]v1.Pod, er
time.Sleep(100 * time.Millisecond) time.Sleep(100 * time.Millisecond)
} }
if err != nil { if err != nil {
return []v1.Pod{}, fmt.Errorf("Failed to get Pods assigned to node %v. Skipping update.", nodeName) return []v1.Pod{}, fmt.Errorf("failed to get Pods assigned to node %v", nodeName)
} }
return pods.Items, nil return pods.Items, nil
} }
...@@ -325,9 +325,8 @@ func (tc *NoExecuteTaintManager) processPodOnNode( ...@@ -325,9 +325,8 @@ func (tc *NoExecuteTaintManager) processPodOnNode(
startTime = scheduledEviction.CreatedAt startTime = scheduledEviction.CreatedAt
if startTime.Add(minTolerationTime).Before(triggerTime) { if startTime.Add(minTolerationTime).Before(triggerTime) {
return return
} else {
tc.cancelWorkWithEvent(podNamespacedName)
} }
tc.cancelWorkWithEvent(podNamespacedName)
} }
tc.taintEvictionQueue.AddWork(NewWorkArgs(podNamespacedName.Name, podNamespacedName.Namespace), startTime, triggerTime) tc.taintEvictionQueue.AddWork(NewWorkArgs(podNamespacedName.Name, podNamespacedName.Namespace), startTime, triggerTime)
} }
......
...@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and ...@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package node package scheduler
import ( import (
"fmt" "fmt"
......
...@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and ...@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package node package scheduler
import ( import (
"sync" "sync"
......
...@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and ...@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package node package scheduler
import ( import (
"sync" "sync"
......
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["controller_utils.go"],
tags = ["automanaged"],
deps = [
"//pkg/api:go_default_library",
"//pkg/cloudprovider:go_default_library",
"//pkg/controller:go_default_library",
"//pkg/kubelet/util/format:go_default_library",
"//pkg/util/node:go_default_library",
"//pkg/util/version:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library",
"//vendor/k8s.io/client-go/listers/extensions/v1beta1:go_default_library",
"//vendor/k8s.io/client-go/tools/cache:go_default_library",
"//vendor/k8s.io/client-go/tools/record:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment