Skip to content

K
k3s
Commit 46b0a03b authored by Brad Davidson's avatar Brad Davidson Committed by Brad Davidson
Browse files
Options

Handle conntrack-related sysctls in supervisor agent setup

(cherry picked from commit 7e175e8a) Signed-off-by: 's avatarBrad Davidson <brad.davidson@rancher.com>
parent 20b58fe4
Hide whitespace changes
Inline Side-by-side
Showing with 145 additions and 23 deletions
go.mod
View file @ 46b0a03b
......@@ -33,6 +33,7 @@ replace (
github.com/prometheus/client_model => github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910
github.com/prometheus/common => github.com/prometheus/common v0.0.0-20181126121408-4724e9255275
github.com/prometheus/procfs => github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a
github.com/rancher/k3s/pkg/data => ./pkg/data
k8s.io/api => github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.18.19-k3s1
k8s.io/apiextensions-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.18.19-k3s1
k8s.io/apimachinery => github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.18.19-k3s1
......@@ -86,6 +87,7 @@ require (
github.com/go-bindata/go-bindata v3.1.2+incompatible
github.com/go-sql-driver/mysql v1.4.1
github.com/gogo/googleapis v1.3.0 // indirect
github.com/google/cadvisor v0.35.0
github.com/google/tcpproxy v0.0.0-20180808230851-dfa16c61dad2
github.com/gorilla/mux v1.7.3
github.com/gorilla/websocket v1.4.1
......@@ -126,4 +128,5 @@ require (
k8s.io/cri-api v0.18.19
k8s.io/klog v1.0.0
k8s.io/kubernetes v1.18.19
k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89
)
pkg/agent/run.go
View file @ 46b0a03b
......@@ -2,7 +2,6 @@ package agent
import (
"context"
"errors"
"io/ioutil"
"os"
"path/filepath"
......@@ -12,6 +11,7 @@ import (
"k8s.io/apimachinery/pkg/labels"
systemd "github.com/coreos/go-systemd/daemon"
"github.com/pkg/errors"
"github.com/rancher/k3s/pkg/agent/config"
"github.com/rancher/k3s/pkg/agent/containerd"
"github.com/rancher/k3s/pkg/agent/flannel"
......@@ -31,6 +31,9 @@ import (
"k8s.io/client-go/kubernetes"
v1 "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/tools/clientcmd"
app2 "k8s.io/kubernetes/cmd/kube-proxy/app"
kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config"
utilpointer "k8s.io/utils/pointer"
)
const (
......@@ -42,6 +45,12 @@ const (
func run(ctx context.Context, cfg cmds.Agent, lb *loadbalancer.LoadBalancer) error {
nodeConfig := config.Get(ctx, cfg)
conntrackConfig, err := getConntrackConfig(nodeConfig)
if err != nil {
return errors.Wrap(err, "failed to validate kube-proxy conntrack configuration")
}
syssetup.Configure(conntrackConfig)
if !nodeConfig.NoFlannel {
if err := flannel.Prepare(ctx, nodeConfig); err != nil {
return err
......@@ -86,6 +95,49 @@ func run(ctx context.Context, cfg cmds.Agent, lb *loadbalancer.LoadBalancer) err
return ctx.Err()
}
// getConntrackConfig uses the kube-proxy code to parse the user-provided kube-proxy-arg values, and
// extract the conntrack settings so that K3s can set them itself. This allows us to soft-fail when
// running K3s in Docker, where kube-proxy is no longer allowed to set conntrack sysctls on newer kernels.
// When running rootless, we do not attempt to set conntrack sysctls - this behavior is copied from kubeadm.
func getConntrackConfig(nodeConfig *daemonconfig.Node) (*kubeproxyconfig.KubeProxyConntrackConfiguration, error) {
ctConfig := &kubeproxyconfig.KubeProxyConntrackConfiguration{
MaxPerCore: utilpointer.Int32Ptr(0),
Min: utilpointer.Int32Ptr(0),
TCPEstablishedTimeout: &metav1.Duration{},
TCPCloseWaitTimeout: &metav1.Duration{},
}
if nodeConfig.AgentConfig.Rootless {
return ctConfig, nil
}
cmd := app2.NewProxyCommand()
if err := cmd.ParseFlags(daemonconfig.GetArgsList(map[string]string{}, nodeConfig.AgentConfig.ExtraKubeProxyArgs)); err != nil {
return nil, err
}
maxPerCore, err := cmd.Flags().GetInt32("conntrack-max-per-core")
if err != nil {
return nil, err
}
ctConfig.MaxPerCore = &maxPerCore
min, err := cmd.Flags().GetInt32("conntrack-min")
if err != nil {
return nil, err
}
ctConfig.Min = &min
establishedTimeout, err := cmd.Flags().GetDuration("conntrack-tcp-timeout-established")
if err != nil {
return nil, err
}
ctConfig.TCPEstablishedTimeout.Duration = establishedTimeout
closeWaitTimeout, err := cmd.Flags().GetDuration("conntrack-tcp-timeout-close-wait")
if err != nil {
return nil, err
}
ctConfig.TCPCloseWaitTimeout.Duration = closeWaitTimeout
return ctConfig, nil
}
func coreClient(cfg string) (kubernetes.Interface, error) {
restConfig, err := clientcmd.BuildConfigFromFlags("", cfg)
if err != nil {
......@@ -99,7 +151,6 @@ func Run(ctx context.Context, cfg cmds.Agent) error {
if err := validate(); err != nil {
return err
}
syssetup.Configure()
if cfg.Rootless && !cfg.RootlessAlreadyUnshared {
if err := rootless.Rootless(cfg.DataDir); err != nil {
......
pkg/agent/syssetup/setup.go
View file @ 46b0a03b
......@@ -6,8 +6,14 @@ import (
"io/ioutil"
"os"
"os/exec"
"runtime"
"time"
"github.com/google/cadvisor/machine"
"github.com/google/cadvisor/utils/sysfs"
"github.com/sirupsen/logrus"
kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config"
"k8s.io/kubernetes/pkg/util/sysctl"
)
func loadKernelModule(moduleName string) {
......@@ -16,18 +22,14 @@ func loadKernelModule(moduleName string) {
return
}
if err := exec.Command("modprobe", moduleName).Run(); err != nil {
logrus.Warnf("failed to start %s module", moduleName)
if err := exec.Command("modprobe", "--", moduleName).Run(); err != nil {
logrus.Warnf("Failed to load kernel module %v with modprobe", moduleName)
}
}
func enableSystemControl(file string) {
if err := ioutil.WriteFile(file, []byte("1"), 0640); err != nil {
logrus.Warnf("failed to write value 1 at %s: %v", file, err)
}
}
func Configure() {
// Configure loads required kernel modules and sets sysctls required for other components to
// function properly.
func Configure(config *kubeproxyconfig.KubeProxyConntrackConfiguration) {
loadKernelModule("overlay")
loadKernelModule("nf_conntrack")
loadKernelModule("br_netfilter")
......@@ -36,10 +38,67 @@ func Configure() {
// Kernel is inconsistent about how devconf is configured for
// new network namespaces between ipv4 and ipv6. Make sure to
// enable forwarding on all and default for both ipv4 and ipv8.
enableSystemControl("/proc/sys/net/ipv4/conf/all/forwarding")
enableSystemControl("/proc/sys/net/ipv4/conf/default/forwarding")
enableSystemControl("/proc/sys/net/ipv6/conf/all/forwarding")
enableSystemControl("/proc/sys/net/ipv6/conf/default/forwarding")
enableSystemControl("/proc/sys/net/bridge/bridge-nf-call-iptables")
enableSystemControl("/proc/sys/net/bridge/bridge-nf-call-ip6tables")
sysctls := map[string]int{
"net/ipv4/conf/all/forwarding": 1,
"net/ipv4/conf/default/forwarding": 1,
"net/ipv6/conf/all/forwarding": 1,
"net/ipv6/conf/default/forwarding": 1,
"net/bridge/bridge-nf-call-iptables": 1,
"net/bridge/bridge-nf-call-ip6tables": 1,
}
if conntrackMax := getConntrackMax(config); conntrackMax > 0 {
sysctls["net/netfilter/nf_conntrack_max"] = conntrackMax
}
if config.TCPEstablishedTimeout.Duration > 0 {
sysctls["net/netfilter/nf_conntrack_tcp_timeout_established"] = int(config.TCPEstablishedTimeout.Duration / time.Second)
}
if config.TCPCloseWaitTimeout.Duration > 0 {
sysctls["net/netfilter/nf_conntrack_tcp_timeout_close_wait"] = int(config.TCPCloseWaitTimeout.Duration / time.Second)
}
sys := sysctl.New()
for entry, value := range sysctls {
if val, _ := sys.GetSysctl(entry); val != value {
logrus.Infof("Set sysctl '%v' to %v", entry, value)
if err := sys.SetSysctl(entry, value); err != nil {
logrus.Errorf("Failed to set sysctl: %v", err)
}
}
}
}
// getConntrackMax is cribbed from kube-proxy, as recent kernels no longer allow non-init namespaces
// to set conntrack-related sysctls.
// ref: https://github.com/kubernetes/kubernetes/blob/v1.21.1/cmd/kube-proxy/app/server.go#L780
// ref: https://github.com/kubernetes-sigs/kind/issues/2240
func getConntrackMax(config *kubeproxyconfig.KubeProxyConntrackConfiguration) int {
if config.MaxPerCore != nil && *config.MaxPerCore > 0 {
floor := 0
if config.Min != nil {
floor = int(*config.Min)
}
scaled := int(*config.MaxPerCore) * detectNumCPU()
if scaled > floor {
logrus.Debugf("getConntrackMax: using scaled conntrack-max-per-core")
return scaled
}
logrus.Debugf("getConntrackMax: using conntrack-min")
return floor
}
return 0
}
// detectNumCPU is also cribbed from kube-proxy
func detectNumCPU() int {
// try get numCPU from /sys firstly due to a known issue (https://github.com/kubernetes/kubernetes/issues/99225)
cpuinfo, err := ioutil.ReadFile("/proc/cpuinfo")
if err != nil {
return runtime.NumCPU()
}
_, numCPU, err := machine.GetTopology(sysfs.NewRealSysFs(), string(cpuinfo))
if err != nil || numCPU < 1 {
return runtime.NumCPU()
}
return numCPU
}
pkg/daemons/agent/agent.go
View file @ 46b0a03b
......@@ -36,10 +36,13 @@ func Agent(config *config.Agent) error {
func startKubeProxy(cfg *config.Agent) {
argsMap := map[string]string{
"proxy-mode": "iptables",
"healthz-bind-address": "127.0.0.1",
"kubeconfig": cfg.KubeConfigKubeProxy,
"cluster-cidr": cfg.ClusterCIDR.String(),
"proxy-mode": "iptables",
"healthz-bind-address": "127.0.0.1",
"kubeconfig": cfg.KubeConfigKubeProxy,
"cluster-cidr": cfg.ClusterCIDR.String(),
"conntrack-max-per-core": "0",
"conntrack-tcp-timeout-established": "0s",
"conntrack-tcp-timeout-close-wait": "0s",
}
if cfg.NodeName != "" {
argsMap["hostname-override"] = cfg.NodeName
......
scripts/download
View file @ 46b0a03b
......@@ -8,13 +8,19 @@ RUNC_VERSION=v1.0.0-rc94
ROOT_VERSION=v0.8.1
TRAEFIK_VERSION=1.81.0
CHARTS_DIR=build/static/charts
RUNC_DIR=build/src/github.com/opencontainers/runc
DATA_DIR=build/data
umask 022
rm -rf ${CHARTS_DIR}
rm -rf ${RUNC_DIR}
mkdir -p ${CHARTS_DIR}
mkdir -p ${DATA_DIR}
curl --compressed -sfL https://github.com/rancher/k3s-root/releases/download/${ROOT_VERSION}/k3s-root-${ARCH}.tar | tar xf -
git clone --depth=1 https://github.com/opencontainers/runc build/src/github.com/opencontainers/runc || true
pushd build/src/github.com/opencontainers/runc
git clone --depth=1 https://github.com/opencontainers/runc ${RUNC_DIR} || true
pushd ${RUNC_DIR}
git fetch --all --tags
git checkout ${RUNC_VERSION} -b k3s
popd
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment