Skip to content

K
k3s
Commit a126f40c authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub
Browse files
Options

Merge pull request #45622 from shyamjvs/mock-proxy

Automatic merge from submit-queue (batch tested with PRs 45571, 45657, 45638, 45663, 45622) Use real proxier inside hollow-proxy but with mocked syscalls Fixes https://github.com/kubernetes/kubernetes/issues/43701 This should make hollow-proxy better mimic the real kube-proxy in performance. Maybe next we should have a more realistic implementation even for fake iptables (adding/updating/deleting rules/chains in an table, just not on the real one)? Though I'm not sure how important it is. cc @kubernetes/sig-scalability-misc @kubernetes/sig-network-misc @wojtek-t @gmarek
parents 990cbb31 27fa5239
Hide whitespace changes
Inline Side-by-side
Showing with 68 additions and 32 deletions
cmd/kubemark/BUILD
View file @ a126f40c
...@@ -27,7 +27,9 @@ go_library( ...@@ -27,7 +27,9 @@ go_library(
"//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/cm:go_default_library",
"//pkg/kubelet/dockershim/libdocker:go_default_library", "//pkg/kubelet/dockershim/libdocker:go_default_library",
"//pkg/kubemark:go_default_library", "//pkg/kubemark:go_default_library",
"//pkg/util/exec:go_default_library",
"//pkg/util/iptables/testing:go_default_library", "//pkg/util/iptables/testing:go_default_library",
"//pkg/util/sysctl/testing:go_default_library",
"//pkg/version/prometheus:go_default_library", "//pkg/version/prometheus:go_default_library",
"//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library",
......
cmd/kubemark/hollow-node.go
View file @ a126f40c
...@@ -35,7 +35,9 @@ import ( ...@@ -35,7 +35,9 @@ import (
"k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/pkg/kubelet/cm"
"k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker"
"k8s.io/kubernetes/pkg/kubemark" "k8s.io/kubernetes/pkg/kubemark"
fakeexec "k8s.io/kubernetes/pkg/util/exec"
fakeiptables "k8s.io/kubernetes/pkg/util/iptables/testing" fakeiptables "k8s.io/kubernetes/pkg/util/iptables/testing"
fakesysctl "k8s.io/kubernetes/pkg/util/sysctl/testing"
_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration _ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
"github.com/golang/glog" "github.com/golang/glog"
...@@ -112,7 +114,6 @@ func main() { ...@@ -112,7 +114,6 @@ func main() {
if config.Morph == "kubelet" { if config.Morph == "kubelet" {
cadvisorInterface := new(cadvisortest.Fake) cadvisorInterface := new(cadvisortest.Fake)
containerManager := cm.NewStubContainerManager() containerManager := cm.NewStubContainerManager()
fakeDockerClient := libdocker.NewFakeDockerClient().WithTraceDisabled() fakeDockerClient := libdocker.NewFakeDockerClient().WithTraceDisabled()
fakeDockerClient.EnableSleep = true fakeDockerClient.EnableSleep = true
...@@ -131,24 +132,29 @@ func main() { ...@@ -131,24 +132,29 @@ func main() {
} }
if config.Morph == "proxy" { if config.Morph == "proxy" {
eventBroadcaster := record.NewBroadcaster()
recorder := eventBroadcaster.NewRecorder(api.Scheme, clientv1.EventSource{Component: "kube-proxy", Host: config.NodeName})
iptInterface := fakeiptables.NewFake()
eventClient, err := clientgoclientset.NewForConfig(clientConfig) eventClient, err := clientgoclientset.NewForConfig(clientConfig)
if err != nil { if err != nil {
glog.Fatalf("Failed to create API Server client: %v", err) glog.Fatalf("Failed to create API Server client: %v", err)
} }
iptInterface := fakeiptables.NewFake()
sysctl := fakesysctl.NewFake()
execer := &fakeexec.FakeExec{}
eventBroadcaster := record.NewBroadcaster()
recorder := eventBroadcaster.NewRecorder(api.Scheme, clientv1.EventSource{Component: "kube-proxy", Host: config.NodeName})
hollowProxy := kubemark.NewHollowProxyOrDie( hollowProxy, err := kubemark.NewHollowProxyOrDie(
config.NodeName, config.NodeName,
internalClientset, internalClientset,
eventClient, eventClient,
iptInterface, iptInterface,
sysctl,
execer,
eventBroadcaster, eventBroadcaster,
recorder, recorder,
) )
if err != nil {
glog.Fatalf("Failed to create hollowProxy instance: %v", err)
}
hollowProxy.Run() hollowProxy.Run()
} }
} }
pkg/kubemark/BUILD
View file @ a126f40c
...@@ -29,11 +29,15 @@ go_library( ...@@ -29,11 +29,15 @@ go_library(
"//pkg/kubelet/container/testing:go_default_library", "//pkg/kubelet/container/testing:go_default_library",
"//pkg/kubelet/dockershim/libdocker:go_default_library", "//pkg/kubelet/dockershim/libdocker:go_default_library",
"//pkg/kubelet/types:go_default_library", "//pkg/kubelet/types:go_default_library",
"//pkg/proxy/iptables:go_default_library",
"//pkg/util:go_default_library", "//pkg/util:go_default_library",
"//pkg/util/exec:go_default_library",
"//pkg/util/io:go_default_library", "//pkg/util/io:go_default_library",
"//pkg/util/iptables:go_default_library", "//pkg/util/iptables:go_default_library",
"//pkg/util/mount:go_default_library", "//pkg/util/mount:go_default_library",
"//pkg/util/node:go_default_library",
"//pkg/util/oom:go_default_library", "//pkg/util/oom:go_default_library",
"//pkg/util/sysctl:go_default_library",
"//pkg/volume/empty_dir:go_default_library", "//pkg/volume/empty_dir:go_default_library",
"//pkg/volume/secret:go_default_library", "//pkg/volume/secret:go_default_library",
"//test/utils:go_default_library", "//test/utils:go_default_library",
......
pkg/kubemark/hollow_proxy.go
View file @ a126f40c
...@@ -17,17 +17,23 @@ limitations under the License. ...@@ -17,17 +17,23 @@ limitations under the License.
package kubemark package kubemark
import ( import (
"fmt"
"net"
"time" "time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/types"
v1core "k8s.io/client-go/kubernetes/typed/core/v1" v1core "k8s.io/client-go/kubernetes/typed/core/v1"
clientv1 "k8s.io/client-go/pkg/api/v1" clientv1 "k8s.io/client-go/pkg/api/v1"
"k8s.io/client-go/tools/record" "k8s.io/client-go/tools/record"
proxyapp "k8s.io/kubernetes/cmd/kube-proxy/app" proxyapp "k8s.io/kubernetes/cmd/kube-proxy/app"
"k8s.io/kubernetes/pkg/api"
clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
"k8s.io/kubernetes/pkg/proxy/iptables"
"k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util"
utilexec "k8s.io/kubernetes/pkg/util/exec"
utiliptables "k8s.io/kubernetes/pkg/util/iptables" utiliptables "k8s.io/kubernetes/pkg/util/iptables"
nodeutil "k8s.io/kubernetes/pkg/util/node"
utilsysctl "k8s.io/kubernetes/pkg/util/sysctl"
"github.com/golang/glog" "github.com/golang/glog"
) )
...@@ -36,32 +42,35 @@ type HollowProxy struct { ...@@ -36,32 +42,35 @@ type HollowProxy struct {
ProxyServer *proxyapp.ProxyServer ProxyServer *proxyapp.ProxyServer
} }
type FakeProxyHandler struct{}
func (*FakeProxyHandler) OnServiceAdd(service *api.Service) {}
func (*FakeProxyHandler) OnServiceUpdate(oldService, service *api.Service) {}
func (*FakeProxyHandler) OnServiceDelete(service *api.Service) {}
func (*FakeProxyHandler) OnServiceSynced() {}
func (*FakeProxyHandler) OnEndpointsAdd(endpoints *api.Endpoints) {}
func (*FakeProxyHandler) OnEndpointsUpdate(oldEndpoints, endpoints *api.Endpoints) {}
func (*FakeProxyHandler) OnEndpointsDelete(endpoints *api.Endpoints) {}
func (*FakeProxyHandler) OnEndpointsSynced() {}
type FakeProxier struct{}
func (*FakeProxier) Sync() {}
func (*FakeProxier) SyncLoop() {
select {}
}
func NewHollowProxyOrDie( func NewHollowProxyOrDie(
nodeName string, nodeName string,
client clientset.Interface, client clientset.Interface,
eventClient v1core.EventsGetter, eventClient v1core.EventsGetter,
iptInterface utiliptables.Interface, iptInterface utiliptables.Interface,
sysctl utilsysctl.Interface,
execer utilexec.Interface,
broadcaster record.EventBroadcaster, broadcaster record.EventBroadcaster,
recorder record.EventRecorder, recorder record.EventRecorder,
) *HollowProxy { ) (*HollowProxy, error) {
// Create a proxier with fake iptables underneath it.
proxier, err := iptables.NewProxier(
iptInterface,
sysctl,
execer,
30*time.Second,
5*time.Second,
false,
0,
"10.0.0.0/8",
nodeName,
getNodeIP(client, nodeName),
recorder,
nil,
)
if err != nil {
return nil, fmt.Errorf("unable to create proxier: %v", err)
}
// Create and start Hollow Proxy // Create and start Hollow Proxy
nodeRef := &clientv1.ObjectReference{ nodeRef := &clientv1.ObjectReference{
Kind: "Node", Kind: "Node",
...@@ -75,7 +84,7 @@ func NewHollowProxyOrDie( ...@@ -75,7 +84,7 @@ func NewHollowProxyOrDie(
Client: client, Client: client,
EventClient: eventClient, EventClient: eventClient,
IptInterface: iptInterface, IptInterface: iptInterface,
Proxier: &FakeProxier{}, Proxier: proxier,
Broadcaster: broadcaster, Broadcaster: broadcaster,
Recorder: recorder, Recorder: recorder,
ProxyMode: "fake", ProxyMode: "fake",
...@@ -83,10 +92,10 @@ func NewHollowProxyOrDie( ...@@ -83,10 +92,10 @@ func NewHollowProxyOrDie(
OOMScoreAdj: util.Int32Ptr(0), OOMScoreAdj: util.Int32Ptr(0),
ResourceContainer: "", ResourceContainer: "",
ConfigSyncPeriod: 30 * time.Second, ConfigSyncPeriod: 30 * time.Second,
ServiceEventHandler: &FakeProxyHandler{}, ServiceEventHandler: proxier,
EndpointsEventHandler: &FakeProxyHandler{}, EndpointsEventHandler: proxier,
}, },
} }, nil
} }
func (hp *HollowProxy) Run() { func (hp *HollowProxy) Run() {
...@@ -94,3 +103,18 @@ func (hp *HollowProxy) Run() { ...@@ -94,3 +103,18 @@ func (hp *HollowProxy) Run() {
glog.Fatalf("Error while running proxy: %v\n", err) glog.Fatalf("Error while running proxy: %v\n", err)
} }
} }
func getNodeIP(client clientset.Interface, hostname string) net.IP {
var nodeIP net.IP
node, err := client.Core().Nodes().Get(hostname, metav1.GetOptions{})
if err != nil {
glog.Warningf("Failed to retrieve node info: %v", err)
return nil
}
nodeIP, err = nodeutil.InternalGetNodeHostIP(node)
if err != nil {
glog.Warningf("Failed to retrieve node IP: %v", err)
return nil
}
return nodeIP
}
pkg/proxy/iptables/proxier.go
View file @ a126f40c
...@@ -356,7 +356,7 @@ func NewProxier(ipt utiliptables.Interface, ...@@ -356,7 +356,7 @@ func NewProxier(ipt utiliptables.Interface,
) (*Proxier, error) { ) (*Proxier, error) {
// check valid user input // check valid user input
if minSyncPeriod > syncPeriod { if minSyncPeriod > syncPeriod {
return nil, fmt.Errorf("min-sync (%v) must be < sync(%v)", minSyncPeriod, syncPeriod) return nil, fmt.Errorf("min-sync (%v) must be <= sync(%v)", minSyncPeriod, syncPeriod)
} }
// Set the route_localnet sysctl we need for // Set the route_localnet sysctl we need for
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment