Rate limit events in kubelet

cmd/kubelet/app/server.go

@@ -84,6 +84,8 @@ type KubeletServer struct {
 	HostNetworkSources             string
 	RegistryPullQPS                float64
 	RegistryBurst                  int
+	EventRecordQPS                 float32
+	EventBurst                     int
 	RunOnce                        bool
 	EnableDebuggingHandlers        bool
 	MinimumGCAge                   time.Duration
@@ -220,6 +222,8 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.HostNetworkSources, "host-network-sources", s.HostNetworkSources, "Comma-separated list of sources from which the Kubelet allows pods to use of host network. For all sources use \"*\" [default=\"file\"]")
 	fs.Float64Var(&s.RegistryPullQPS, "registry-qps", s.RegistryPullQPS, "If > 0, limit registry pull QPS to this value.  If 0, unlimited. [default=0.0]")
 	fs.IntVar(&s.RegistryBurst, "registry-burst", s.RegistryBurst, "Maximum size of a bursty pulls, temporarily allows pulls to burst to this number, while still not exceeding registry-qps.  Only used if --registry-qps > 0")
+	fs.Float32Var(&s.EventRecordQPS, "event-qps", s.EventRecordQPS, "If > 0, limit event creations per second to this value. If 0, unlimited. [default=0.0]")
+	fs.IntVar(&s.EventBurst, "event-burst", s.EventBurst, "Maximum size of a bursty event records, temporarily allows event records to burst to this number, while still not exceeding event-qps. Only used if --event-qps > 0")
 	fs.BoolVar(&s.RunOnce, "runonce", s.RunOnce, "If true, exit after spawning pods from local manifests or remote urls. Exclusive with --api-servers, and --enable-server")
 	fs.BoolVar(&s.EnableDebuggingHandlers, "enable-debugging-handlers", s.EnableDebuggingHandlers, "Enables server endpoints for log collection and local running of containers and commands")
 	fs.DurationVar(&s.MinimumGCAge, "minimum-container-ttl-duration", s.MinimumGCAge, "Minimum age for a finished container before it is garbage collected.  Examples: '300ms', '10s' or '2h45m'")
@@ -327,6 +331,8 @@ func (s *KubeletServer) KubeletConfig() (*KubeletConfig, error) {
 		SyncFrequency:                  s.SyncFrequency,
 		RegistryPullQPS:                s.RegistryPullQPS,
 		RegistryBurst:                  s.RegistryBurst,
+		EventRecordQPS:                 s.EventRecordQPS,
+		EventBurst:                     s.EventBurst,
 		MinimumGCAge:                   s.MinimumGCAge,
 		MaxPerPodContainerCount:        s.MaxPerPodContainerCount,
 		MaxContainerCount:              s.MaxContainerCount,
@@ -646,7 +652,13 @@ func RunKubelet(kcfg *KubeletConfig, builder KubeletBuilder) error {
 	eventBroadcaster.StartLogging(glog.V(3).Infof)
 	if kcfg.KubeClient != nil {
 		glog.V(4).Infof("Sending events to api server.")
-		eventBroadcaster.StartRecordingToSink(kcfg.KubeClient.Events(""))
+		if kcfg.EventRecordQPS == 0.0 {
+			eventBroadcaster.StartRecordingToSink(kcfg.KubeClient.Events(""))
+		} else {
+			eventClient := *kcfg.KubeClient
+			eventClient.Throttle = util.NewTokenBucketRateLimiter(kcfg.EventRecordQPS, kcfg.EventBurst)
+			eventBroadcaster.StartRecordingToSink(eventClient.Events(""))
+		}
 	} else {
 		glog.Warning("No api server defined - no events will be sent to API server.")
 	}
@@ -742,6 +754,8 @@ type KubeletConfig struct {
 	SyncFrequency                  time.Duration
 	RegistryPullQPS                float64
 	RegistryBurst                  int
+	EventRecordQPS                 float32
+	EventBurst                     int
 	MinimumGCAge                   time.Duration
 	MaxPerPodContainerCount        int
 	MaxContainerCount              int
@@ -809,6 +823,8 @@ func createAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod
 		kc.SyncFrequency,
 		float32(kc.RegistryPullQPS),
 		kc.RegistryBurst,
+		kc.EventRecordQPS,
+		kc.EventBurst,
 		gcPolicy,
 		pc.SeenAllSources,
 		kc.RegisterNode,

contrib/mesos/pkg/executor/service/service.go

@@ -301,6 +301,8 @@ func (ks *KubeletExecutorServer) createAndInitKubelet(
 		kc.SyncFrequency,
 		float32(kc.RegistryPullQPS),
 		kc.RegistryBurst,
+		kc.EventRecordQPS,
+		kc.EventBurst,
 		gcPolicy,
 		pc.SeenAllSources,
 		kc.RegisterNode,

hack/verify-flags/known-flags.txt

-accept-hosts
-accept-paths
-account-for-pod-resources
-admission-control
-admission-control-config-file
-advertise-address
-advertised-address
-algorithm-provider
-all-namespaces
-allocate-node-cidrs
-allow-privileged
-api-prefix
-api-servers
-api-token
-api-version
-authorization-mode
-authorization-policy-file
-auth-path
-basic-auth-file
-bench-pods
-bench-quiet
-bench-tasks
-bench-workers
-bind-address
-bind-pods-burst
-bind-pods-qps
-cadvisor-port
-cert-dir
-certificate-authority
-cgroup-root
-chaos-chance
-cleanup-iptables
-client-ca-file
-client-certificate
-client-key
-cloud-config
-cloud-provider
-cluster-cidr
-cluster-dns
-cluster-domain
-cluster-name
-cluster-tag
-concurrent-endpoint-syncs
-configure-cbr0
-contain-pod-resources
-container-port
-container-runtime
-cors-allowed-origins
-create-external-load-balancer
-current-release-pr
-current-replicas
-default-container-cpu-limit
-default-container-mem-limit
-delay-shutdown
-deleting-pods-burst
-deleting-pods-qps
-deployment-label-key
-dest-file
-disable-filter
-docker-endpoint
-docker-exec-handler
-dockercfg-path
-driver-port
-dry-run
-duration-sec
-e2e-output-dir
-enable-debugging-handlers
-enable-horizontal-pod-autoscaler
-enable-server
-etcd-config
-etcd-prefix
-etcd-server
-etcd-servers
-event-ttl
-executor-bindall
-executor-logv
-executor-path
-executor-suicide-timeout
-experimental-keystone-url
-experimental-prefix
-external-hostname
-external-ip
-failover-timeout
-file-check-frequency
-file-suffix
-forward-services
-framework-name
-framework-weburi
-func-dest
-fuzz-iters
-gce-project
-gce-zone
-gke-cluster
-google-json-key
-grace-period
-ha-domain
-healthz-bind-address
-healthz-port
-horizontal-pod-autoscaler-sync-period
-hostname-override
-host-network-sources
-http-check-frequency
-http-port
-ignore-not-found
-image-gc-high-threshold
-image-gc-low-threshold
-insecure-bind-address
-insecure-port
-insecure-skip-tls-verify
-iptables-sync-period
-ir-data-source
-ir-dbname
-ir-influxdb-host
-ir-password
-ir-user
-jenkins-host
-jenkins-jobs
-km-path
-kubectl-path
-kubelet-cadvisor-port
-kubelet-certificate-authority
-kubelet-client-certificate
-kubelet-client-key
-kubelet-docker-endpoint
-kubelet-host-network-sources
-kubelet-https
-kubelet-network-plugin
-kubelet-pod-infra-container-image
-kubelet-port
-kubelet-root-dir
-kubelet-sync-frequency
-kubelet-timeout
-kube-master
-label-columns
-last-release-pr
-legacy-userspace-proxy
-log-flush-frequency
-long-running-request-regexp
-low-diskspace-threshold-mb
-manifest-url
-manifest-url-header
-masquerade-all
-master-service-namespace
-max-concurrency
-max-connection-bytes-per-sec
-maximum-dead-containers
-maximum-dead-containers-per-container
-max-log-age
-max-log-backups
-max-log-size
-max-outgoing-burst
-max-outgoing-qps
-max-pods
-max-requests-inflight
-mesos-authentication-principal
-mesos-authentication-provider
-mesos-authentication-secret-file
-mesos-cgroup-prefix
-mesos-executor-cpus
-mesos-executor-mem
-mesos-master
-mesos-role
-mesos-user
-minimum-container-ttl-duration
-minion-max-log-age
-minion-max-log-backups
-minion-max-log-size
-minion-path-override
-min-pr-number
-min-request-timeout
-namespace-sync-period
-network-plugin
-network-plugin-dir
-node-instance-group
-node-monitor-grace-period
-node-monitor-period
-node-startup-grace-period
-node-status-update-frequency
-node-sync-period
-no-headers
-num-nodes
-oidc-ca-file
-oidc-client-id
-oidc-issuer-url
-oidc-username-claim
-oom-score-adj
-output-version
-out-version
-path-override
-pod-cidr
-pod-eviction-timeout
-pod-infra-container-image
-policy-config-file
-poll-interval
-portal-net
-private-mountns
-prom-push-gateway
-proxy-bindall
-proxy-logv
-proxy-port-range
-public-address-override
-pvclaimbinder-sync-period
-read-only-port
-really-crash-for-testing
-reconcile-cooldown
-reconcile-interval
-register-node
-register-retry-count
-registry-burst
-registry-qps
-reject-methods
-reject-paths
-repo-root
-report-dir
-required-contexts
-resolv-conf
-resource-container
-resource-quota-sync-period
-resource-version
-rkt-path
-root-ca-file
-root-dir
-run-proxy
-runtime-config
-scheduler-config
-secure-port
-service-account-key-file
-service-account-lookup
-service-account-private-key-file
-service-address
-service-cluster-ip-range
-service-node-port-range
-service-node-ports
-service-sync-period
-session-affinity
-show-all
-shutdown-fd
-shutdown-fifo
-skip-munges
-sort-by
-source-file
-ssh-keyfile
-ssh-user
-static-pods-config
-stats-port
-storage-version
-streaming-connection-idle-timeout
-suicide-timeout
-sync-frequency
-system-container
-target-port
-tcp-services
-tls-cert-file
-tls-private-key-file
-token-auth-file
-ttl-secs
-type-src
-unix-socket
-update-period
-upgrade-target
-use-kubernetes-cluster-service
-user-whitelist
-watch-cache
-watch-only
-whitelist-override-label
-www-prefix
-retry_time
-file_content_in_loop
-cpu-cfs-quota
+accept-hosts
+accept-paths
+account-for-pod-resources
+admission-control
+admission-control-config-file
+advertise-address
+advertised-address
+algorithm-provider
+all-namespaces
+allocate-node-cidrs
+allow-privileged
+api-burst
+api-prefix
+api-rate
+api-servers
+api-token
+api-version
+authorization-mode
+authorization-policy-file
+auth-path
+basic-auth-file
+bench-pods
+bench-quiet
+bench-tasks
+bench-workers
+bind-address
+bind-pods-burst
+bind-pods-qps
+cadvisor-port
+cert-dir
+certificate-authority
+cgroup-root
+chaos-chance
+cleanup-iptables
+client-ca-file
+client-certificate
+client-key
+cloud-config
+cloud-provider
+cluster-cidr
+cluster-dns
+cluster-domain
+cluster-name
+cluster-tag
+concurrent-endpoint-syncs
+configure-cbr0
+contain-pod-resources
+container-port
+container-runtime
+cors-allowed-origins
+create-external-load-balancer
+current-release-pr
+current-replicas
+default-container-cpu-limit
+default-container-mem-limit
+delay-shutdown
+deleting-pods-burst
+deleting-pods-qps
+deployment-label-key
+dest-file
+disable-filter
+docker-endpoint
+docker-exec-handler
+dockercfg-path
+driver-port
+dry-run
+duration-sec
+e2e-output-dir
+enable-debugging-handlers
+enable-horizontal-pod-autoscaler
+enable-server
+etcd-config
+etcd-prefix
+etcd-server
+etcd-servers
+event-burst
+event-qps
+event-ttl
+executor-bindall
+executor-logv
+executor-path
+executor-suicide-timeout
+experimental-keystone-url
+experimental-prefix
+external-hostname
+external-ip
+failover-timeout
+file-check-frequency
+file-suffix
+forward-services
+framework-name
+framework-weburi
+func-dest
+fuzz-iters
+gce-project
+gce-zone
+gke-cluster
+google-json-key
+grace-period
+ha-domain
+healthz-bind-address
+healthz-port
+horizontal-pod-autoscaler-sync-period
+hostname-override
+host-network-sources
+http-check-frequency
+http-port
+ignore-not-found
+image-gc-high-threshold
+image-gc-low-threshold
+insecure-bind-address
+insecure-port
+insecure-skip-tls-verify
+iptables-sync-period
+ir-data-source
+ir-dbname
+ir-influxdb-host
+ir-password
+ir-user
+jenkins-host
+jenkins-jobs
+km-path
+kubectl-path
+kubelet-cadvisor-port
+kubelet-certificate-authority
+kubelet-client-certificate
+kubelet-client-key
+kubelet-docker-endpoint
+kubelet-host-network-sources
+kubelet-https
+kubelet-network-plugin
+kubelet-pod-infra-container-image
+kubelet-port
+kubelet-root-dir
+kubelet-sync-frequency
+kubelet-timeout
+kube-master
+label-columns
+last-release-pr
+legacy-userspace-proxy
+log-flush-frequency
+long-running-request-regexp
+low-diskspace-threshold-mb
+manifest-url
+manifest-url-header
+masquerade-all
+master-service-namespace
+max-concurrency
+max-connection-bytes-per-sec
+maximum-dead-containers
+maximum-dead-containers-per-container
+max-log-age
+max-log-backups
+max-log-size
+max-outgoing-burst
+max-outgoing-qps
+max-pods
+max-requests-inflight
+mesos-authentication-principal
+mesos-authentication-provider
+mesos-authentication-secret-file
+mesos-cgroup-prefix
+mesos-executor-cpus
+mesos-executor-mem
+mesos-master
+mesos-role
+mesos-user
+minimum-container-ttl-duration
+minion-max-log-age
+minion-max-log-backups
+minion-max-log-size
+minion-path-override
+min-pr-number
+min-request-timeout
+namespace-sync-period
+network-plugin
+network-plugin-dir
+node-instance-group
+node-monitor-grace-period
+node-monitor-period
+node-startup-grace-period
+node-status-update-frequency
+node-sync-period
+no-headers
+num-nodes
+oidc-ca-file
+oidc-client-id
+oidc-issuer-url
+oidc-username-claim
+oom-score-adj
+output-version
+out-version
+path-override
+pod-cidr
+pod-eviction-timeout
+pod-infra-container-image
+policy-config-file
+poll-interval
+portal-net
+private-mountns
+prom-push-gateway
+proxy-bindall
+proxy-logv
+proxy-port-range
+public-address-override
+pvclaimbinder-sync-period
+read-only-port
+really-crash-for-testing
+reconcile-cooldown
+reconcile-interval
+register-node
+register-retry-count
+registry-burst
+registry-qps
+reject-methods
+reject-paths
+repo-root
+report-dir
+required-contexts
+resolv-conf
+resource-container
+resource-quota-sync-period
+resource-version
+rkt-path
+root-ca-file
+root-dir
+run-proxy
+runtime-config
+scheduler-config
+secure-port
+service-account-key-file
+service-account-lookup
+service-account-private-key-file
+service-address
+service-cluster-ip-range
+service-node-port-range
+service-node-ports
+service-sync-period
+session-affinity
+show-all
+shutdown-fd
+shutdown-fifo
+skip-munges
+sort-by
+source-file
+ssh-keyfile
+ssh-user
+static-pods-config
+stats-port
+storage-version
+streaming-connection-idle-timeout
+suicide-timeout
+sync-frequency
+system-container
+target-port
+tcp-services
+tls-cert-file
+tls-private-key-file
+token-auth-file
+ttl-secs
+type-src
+unix-socket
+update-period
+upgrade-target
+use-kubernetes-cluster-service
+user-whitelist
+watch-cache
+watch-only
+whitelist-override-label
+www-prefix
+retry_time
+file_content_in_loop
+cpu-cfs-quota

pkg/kubelet/kubelet.go

@@ -139,6 +139,8 @@ func NewMainKubelet(
 	resyncInterval time.Duration,
 	pullQPS float32,
 	pullBurst int,
+	eventQPS float32,
+	eventBurst int,
 	containerGCPolicy ContainerGCPolicy,
 	sourcesReady SourcesReadyFn,
 	registerNode bool,