Skip to content

K
k3s
Commit 6c64b4dd authored by Brad Davidson's avatar Brad Davidson Committed by Brad Davidson
Browse files
Options

Fix issues with certs.d template generation

* Fix issue with bare host or IP as endpoint * Fix issue with localhost registries not defaulting to http. * Move the registry template prep to a separate function, and adds tests of that function so that we can ensure we're generating the correct content. Signed-off-by: 's avatarBrad Davidson <brad.davidson@rancher.com>
parent ce6158f8
Hide whitespace changes
Inline Side-by-side
Showing with 1245 additions and 42 deletions
pkg/agent/containerd/config.go
View file @ 6c64b4dd
......@@ -13,9 +13,12 @@ import (
"github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/k3s-io/k3s/pkg/spegel"
"github.com/k3s-io/k3s/pkg/version"
"github.com/rancher/wharfie/pkg/registries"
"github.com/sirupsen/logrus"
)
type HostConfigs map[string]templates.HostConfig
// writeContainerdConfig renders and saves config.toml from the filled template
func writeContainerdConfig(cfg *config.Node, containerdConfig templates.ContainerdConfig) error {
var containerdTemplate string
......@@ -39,79 +42,163 @@ func writeContainerdConfig(cfg *config.Node, containerdConfig templates.Containe
// writeContainerdHosts merges registry mirrors/configs, and renders and saves hosts.toml from the filled template
func writeContainerdHosts(cfg *config.Node, containerdConfig templates.ContainerdConfig) error {
mirrorAddr := net.JoinHostPort(spegel.DefaultRegistry.InternalAddress, spegel.DefaultRegistry.RegistryPort)
registry := containerdConfig.PrivateRegistryConfig
hosts := getHostConfigs(containerdConfig.PrivateRegistryConfig, containerdConfig.NoDefaultEndpoint, mirrorAddr)
// Clean up previous configuration templates
os.RemoveAll(cfg.Containerd.Registry)
// Write out new templates
for host, config := range hosts {
hostDir := filepath.Join(cfg.Containerd.Registry, host)
hostsFile := filepath.Join(hostDir, "hosts.toml")
hostsTemplate, err := templates.ParseHostsTemplateFromConfig(templates.HostsTomlTemplate, config)
if err != nil {
return err
}
if err := os.MkdirAll(hostDir, 0700); err != nil {
return err
}
if err := util2.WriteFile(hostsFile, hostsTemplate); err != nil {
return err
}
}
return nil
}
// getHostConfigs merges the registry mirrors/configs into HostConfig template structs
func getHostConfigs(registry *registries.Registry, noDefaultEndpoint bool, mirrorAddr string) HostConfigs {
hosts := map[string]templates.HostConfig{}
// create endpoints for mirrors
for host, mirror := range registry.Mirrors {
defaultHost, _ := docker.DefaultHost(host)
config := templates.HostConfig{
Host: defaultHost,
Program: version.Program,
}
if host == "*" {
host = "_default"
config.Host = ""
} else if containerdConfig.NoDefaultEndpoint {
config.Host = ""
if uri, _, err := normalizeEndpointAddress(host, mirrorAddr); err == nil {
config.DefaultEndpoint = uri.String()
}
// TODO: rewrites are currently copied from the mirror settings into each endpoint.
// In the future, we should allow for per-endpoint rewrites, instead of expecting
// all mirrors to have the same structure. This will require changes to the registries.yaml
// structure, which is defined in rancher/wharfie.
for _, endpoint := range mirror.Endpoints {
if endpointURL, err := url.Parse(endpoint); err == nil {
re := templates.RegistryEndpoint{
OverridePath: endpointURL.Path != "" && endpointURL.Path != "/" && !strings.HasSuffix(endpointURL.Path, "/v2"),
Config: registry.Configs[endpointURL.Host],
Rewrites: mirror.Rewrites,
URI: endpoint,
}
uri, override, err := normalizeEndpointAddress(endpoint, mirrorAddr)
if err != nil {
logrus.Warnf("Ignoring invalid endpoint URL %s for %s: %v", endpoint, host, err)
} else {
var rewrites map[string]string
// Do not apply rewrites to the embedded registry endpoint
if endpointURL.Host == mirrorAddr {
re.Rewrites = nil
if uri.Host != mirrorAddr {
rewrites = mirror.Rewrites
}
config.Endpoints = append(config.Endpoints, re)
config.Endpoints = append(config.Endpoints, templates.RegistryEndpoint{
Config: registry.Configs[uri.Host],
Rewrites: rewrites,
OverridePath: override,
URI: uri.String(),
})
}
}
if host == "*" {
host = "_default"
}
hosts[host] = config
}
// create endpoints for registries using default endpoints
for host, registry := range registry.Configs {
config, ok := hosts[host]
if !ok {
config = templates.HostConfig{
Program: version.Program,
}
if uri, _, err := normalizeEndpointAddress(host, mirrorAddr); err == nil {
config.DefaultEndpoint = uri.String()
}
}
// If there is config for this host but no endpoints, inject the config for the default endpoint.
if len(config.Endpoints) == 0 {
config.Endpoints = []templates.RegistryEndpoint{
{
uri, _, err := normalizeEndpointAddress(host, mirrorAddr)
if err != nil {
logrus.Warnf("Ignoring invalid endpoint URL %s for %s: %v", host, host, err)
} else {
config.Endpoints = append(config.Endpoints, templates.RegistryEndpoint{
Config: registry,
URI: "https://" + host,
},
URI: uri.String(),
})
}
}
if host == "*" {
host = "_default"
}
hosts[host] = config
}
// Clean up previous configuration templates
os.RemoveAll(cfg.Containerd.Registry)
// Write out new templates
// Clean up hosts and default endpoints where resulting config leaves only defaults
for host, config := range hosts {
hostDir := filepath.Join(cfg.Containerd.Registry, host)
hostsFile := filepath.Join(hostDir, "hosts.toml")
hostsTemplate, err := templates.ParseHostsTemplateFromConfig(templates.HostsTomlTemplate, config)
if err != nil {
return err
// if default endpoint is disabled, or this is the wildcard host, delete the default endpoint
if noDefaultEndpoint || host == "_default" {
config.DefaultEndpoint = ""
hosts[host] = config
}
if err := os.MkdirAll(hostDir, 0700); err != nil {
return err
if l := len(config.Endpoints); l > 0 {
if ep := config.Endpoints[l-1]; ep.URI == config.DefaultEndpoint {
// if the last endpoint is the default endpoint
if ep.Config.Auth == nil && ep.Config.TLS == nil && len(ep.Rewrites) == 0 {
// if has no config, delete this host to use the default config
delete(hosts, host)
} else {
// if it has config, delete the default endpoint
config.DefaultEndpoint = ""
hosts[host] = config
}
}
} else {
// if this host has no endpoints, delete this host to use the default config
delete(hosts, host)
}
if err := util2.WriteFile(hostsFile, hostsTemplate); err != nil {
return err
}
return hosts
}
// normalizeEndpointAddress normalizes the endpoint address.
// If successful, it returns the URL, and a bool indicating if the endpoint path should be overridden.
// If unsuccessful, an error is returned.
// Scheme and hostname logic should match containerd:
// https://github.com/containerd/containerd/blob/v1.7.13/remotes/docker/config/hosts.go#L99-L131
func normalizeEndpointAddress(endpoint, mirrorAddr string) (*url.URL, bool, error) {
// Ensure that the endpoint address has a scheme so that the URL is parsed properly
if !strings.Contains(endpoint, "://") {
endpoint = "//" + endpoint
}
endpointURL, err := url.Parse(endpoint)
if err != nil {
return nil, false, err
}
port := endpointURL.Port()
// set default scheme, if not provided
if endpointURL.Scheme == "" {
// localhost on odd ports defaults to http, unless it's the embedded mirror
if docker.IsLocalhost(endpointURL.Host) && port != "" && port != "443" && endpointURL.Host != mirrorAddr {
endpointURL.Scheme = "http"
} else {
endpointURL.Scheme = "https"
}
}
endpointURL.Host, _ = docker.DefaultHost(endpointURL.Host)
return nil
switch endpointURL.Path {
case "", "/", "/v2":
// If the path is empty, /, or /v2, use the default path.
endpointURL.Path = "/v2"
return endpointURL, false, nil
}
return endpointURL, true, nil
}
pkg/agent/containerd/config_test.go 0 → 100644
View file @ 6c64b4dd
package containerd
import (
"net"
"os"
"path/filepath"
"strings"
"testing"
"github.com/k3s-io/k3s/pkg/agent/templates"
"github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/k3s-io/k3s/pkg/spegel"
"github.com/rancher/wharfie/pkg/registries"
"github.com/stretchr/testify/assert"
)
func Test_UnitGetHostConfigs(t *testing.T) {
type args struct {
registryContent string
noDefaultEndpoint bool
mirrorAddr string
}
tests := []struct {
name string
args args
want HostConfigs
}{
{
name: "no registries",
want: HostConfigs{},
},
{
name: "registry with default endpoint",
args: args{
registryContent: `
mirrors:
docker.io:
`,
},
want: HostConfigs{},
},
{
name: "registry with default endpoint - embedded registry",
args: args{
mirrorAddr: "127.0.0.1:6443",
registryContent: `
mirrors:
docker.io:
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://127.0.0.1:6443/v2",
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
},
},
},
},
"127.0.0.1:6443": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://127.0.0.1:6443/v2",
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
},
},
},
},
},
},
{
name: "registry with default endpoint and creds",
args: args{
registryContent: `
mirrors:
docker.io:
configs:
docker.io:
auth:
username: user
password: pass
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://registry-1.docker.io/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
},
},
},
},
{
name: "registry with only creds",
args: args{
registryContent: `
configs:
docker.io:
auth:
username: user
password: pass
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://registry-1.docker.io/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
},
},
},
},
{
name: "private registry with default endpoint",
args: args{
registryContent: `
mirrors:
registry.example.com:
`,
},
want: HostConfigs{},
},
{
name: "private registry with default endpoint and creds",
args: args{
registryContent: `
mirrors:
registry.example.com:
configs:
registry.example.com:
auth:
username: user
password: pass
`,
},
want: HostConfigs{
"registry.example.com": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://registry.example.com/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
},
},
},
},
{
name: "private registry with only creds",
args: args{
registryContent: `
configs:
registry.example.com:
auth:
username: user
password: pass
`,
},
want: HostConfigs{
"registry.example.com": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://registry.example.com/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
},
},
},
},
{
name: "registry with mirror endpoint - full URL with override path",
args: args{
registryContent: `
mirrors:
docker.io:
endpoint:
- https://registry.example.com/prefix/v2
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
OverridePath: true,
URI: "https://registry.example.com/prefix/v2",
},
},
},
},
},
{
name: "registry with mirror endpoint - hostname only with override path",
args: args{
registryContent: `
mirrors:
docker.io:
endpoint:
- registry.example.com/prefix/v2
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
OverridePath: true,
URI: "https://registry.example.com/prefix/v2",
},
},
},
},
},
{
name: "registry with mirror endpoint - hostname only with default path",
args: args{
registryContent: `
mirrors:
docker.io:
endpoint:
- registry.example.com/v2
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://registry.example.com/v2",
},
},
},
},
},
{
name: "registry with mirror endpoint - full URL",
args: args{
registryContent: `
mirrors:
docker.io:
endpoint:
- https://registry.example.com/v2
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://registry.example.com/v2",
},
},
},
},
},
{
name: "registry with mirror endpoint - URL without path",
args: args{
registryContent: `
mirrors:
docker.io:
endpoint:
- https://registry.example.com
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://registry.example.com/v2",
},
},
},
},
},
{
name: "registry with mirror endpoint - hostname only",
args: args{
registryContent: `
mirrors:
docker.io:
endpoint:
- registry.example.com
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://registry.example.com/v2",
},
},
},
},
},
{
name: "registry with mirror endpoint - hostname and port only",
args: args{
registryContent: `
mirrors:
docker.io:
endpoint:
- registry.example.com:443
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://registry.example.com:443/v2",
},
},
},
},
},
{
name: "registry with mirror endpoint - ip address only",
args: args{
registryContent: `
mirrors:
docker.io:
endpoint:
- 1.2.3.4
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://1.2.3.4/v2",
},
},
},
},
},
{
name: "registry with mirror endpoint - ip and port only",
args: args{
registryContent: `
mirrors:
docker.io:
endpoint:
- 1.2.3.4:443
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://1.2.3.4:443/v2",
},
},
},
},
},
{
name: "registry with mirror endpoint - localhost and port only",
args: args{
registryContent: `
mirrors:
docker.io:
endpoint:
- localhost:5000
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "http://localhost:5000/v2",
},
},
},
},
},
{
name: "registry with mirror endpoint - localhost and port with scheme",
args: args{
registryContent: `
mirrors:
docker.io:
endpoint:
- https://localhost:5000
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://localhost:5000/v2",
},
},
},
},
},
{
name: "registry with mirror endpoint - loopback ip and port only",
args: args{
registryContent: `
mirrors:
docker.io:
endpoint:
- 127.0.0.1:5000
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "http://127.0.0.1:5000/v2",
},
},
},
},
},
{
name: "registry with mirror endpoint - loopback ip and port with scheme",
args: args{
registryContent: `
mirrors:
docker.io:
endpoint:
- https://127.0.0.1:5000
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://127.0.0.1:5000/v2",
},
},
},
},
},
{
name: "registry with mirror endpoint and mirror creds",
args: args{
registryContent: `
mirrors:
docker.io:
endpoint:
- https://registry.example.com/v2
configs:
registry.example.com:
auth:
username: user
password: pass
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://registry.example.com/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
},
},
"registry.example.com": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://registry.example.com/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
},
},
},
},
{
name: "registry with mirror endpoint and mirror creds - override path with v2",
args: args{
registryContent: `
mirrors:
docker.io:
endpoint:
- https://registry.example.com/prefix/v2
registry.example.com:
endpoint:
- https://registry.example.com/prefix/v2
configs:
registry.example.com:
auth:
username: user
password: pass
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
OverridePath: true,
URI: "https://registry.example.com/prefix/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
},
},
"registry.example.com": templates.HostConfig{
DefaultEndpoint: "https://registry.example.com/v2",
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
OverridePath: true,
URI: "https://registry.example.com/prefix/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
},
},
},
},
{
name: "registry with mirror endpoint and mirror creds - override path without v2",
args: args{
registryContent: `
mirrors:
docker.io:
endpoint:
- https://registry.example.com/project/registry
registry.example.com:
endpoint:
- https://registry.example.com/project/registry
configs:
registry.example.com:
auth:
username: user
password: pass
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
OverridePath: true,
URI: "https://registry.example.com/project/registry",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
},
},
"registry.example.com": templates.HostConfig{
DefaultEndpoint: "https://registry.example.com/v2",
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
OverridePath: true,
URI: "https://registry.example.com/project/registry",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
},
},
},
},
{
name: "registry with mirror endpoint and mirror creds - no default endpoint",
args: args{
noDefaultEndpoint: true,
registryContent: `
mirrors:
docker.io:
endpoint:
- https://registry.example.com/v2
configs:
registry.example.com:
auth:
username: user
password: pass
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://registry.example.com/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
},
},
"registry.example.com": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://registry.example.com/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
},
},
},
},
{
name: "registry with mirror endpoint and mirror creds - embedded registry",
args: args{
mirrorAddr: "127.0.0.1:6443",
registryContent: `
mirrors:
docker.io:
endpoint:
- https://registry.example.com/v2
configs:
registry.example.com:
auth:
username: user
password: pass
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://127.0.0.1:6443/v2",
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
},
},
templates.RegistryEndpoint{
URI: "https://registry.example.com/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
},
},
"registry.example.com": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://registry.example.com/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
},
},
"127.0.0.1:6443": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://127.0.0.1:6443/v2",
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
},
},
},
},
},
},
{
name: "registry with mirror endpoint and mirror creds - embedded registry with rewrites",
args: args{
mirrorAddr: "127.0.0.1:6443",
registryContent: `
mirrors:
docker.io:
endpoint:
- https://registry.example.com/v2
rewrite:
"^rancher/(.*)": "docker/rancher-images/$1"
configs:
registry.example.com:
auth:
username: user
password: pass
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://127.0.0.1:6443/v2",
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
},
},
templates.RegistryEndpoint{
URI: "https://registry.example.com/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
Rewrites: map[string]string{
"^rancher/(.*)": "docker/rancher-images/$1",
},
},
},
},
"registry.example.com": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://registry.example.com/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
},
},
"127.0.0.1:6443": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://127.0.0.1:6443/v2",
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
},
},
},
},
},
},
{
name: "registry with mirror endpoint and mirror creds - embedded registry and no default endpoint",
args: args{
mirrorAddr: "127.0.0.1:6443",
noDefaultEndpoint: true,
registryContent: `
mirrors:
docker.io:
endpoint:
- https://registry.example.com/v2
configs:
registry.example.com:
auth:
username: user
password: pass
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://127.0.0.1:6443/v2",
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
},
},
templates.RegistryEndpoint{
URI: "https://registry.example.com/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
},
},
"registry.example.com": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://registry.example.com/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
},
},
"127.0.0.1:6443": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://127.0.0.1:6443/v2",
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
},
},
},
},
},
},
{
name: "wildcard mirror endpoint - full URL",
args: args{
registryContent: `
mirrors:
"*":
endpoint:
- https://registry.example.com/v2
`,
},
want: HostConfigs{
"_default": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://registry.example.com/v2",
},
},
},
},
},
{
name: "wildcard mirror endpoint - full URL, embedded registry",
args: args{
mirrorAddr: "127.0.0.1:6443",
registryContent: `
mirrors:
"*":
endpoint:
- https://registry.example.com/v2
`,
},
want: HostConfigs{
"_default": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
// note that the embedded registry mirror is NOT listed as an endpoint.
// individual registries must be enabled for mirroring by name.
templates.RegistryEndpoint{
URI: "https://registry.example.com/v2",
},
},
},
"127.0.0.1:6443": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://127.0.0.1:6443/v2",
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
},
},
},
},
},
},
{
name: "localhost registry - default https endpoint on unspecified port",
args: args{
registryContent: `
mirrors:
"localhost":
`,
},
want: HostConfigs{},
},
{
name: "localhost registry - default https endpoint on https port",
args: args{
registryContent: `
mirrors:
"localhost:443":
`,
},
want: HostConfigs{},
},
{
name: "localhost registry - default http endpoint on odd port",
args: args{
registryContent: `
mirrors:
"localhost:5000":
`,
},
want: HostConfigs{},
},
{
name: "localhost registry - default http endpoint on http port",
args: args{
registryContent: `
mirrors:
"localhost:80":
`,
},
want: HostConfigs{},
},
{
name: "localhost registry - default http endpoint on odd port, embedded registry",
args: args{
mirrorAddr: "127.0.0.1:6443",
registryContent: `
mirrors:
"localhost:5000":
`,
},
want: HostConfigs{
// localhost registries are not handled by the embedded registry mirror.
"127.0.0.1:6443": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://127.0.0.1:6443/v2",
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
},
},
},
},
},
},
{
name: "localhost registry - https endpoint on odd port with tls verification disabled",
args: args{
registryContent: `
mirrors:
localhost:5000:
endpoint:
- https://localhost:5000
configs:
localhost:5000:
tls:
insecure_skip_verify: true
`,
},
want: HostConfigs{
"localhost:5000": templates.HostConfig{
DefaultEndpoint: "http://localhost:5000/v2",
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
templates.RegistryEndpoint{
URI: "https://localhost:5000/v2",
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
InsecureSkipVerify: true,
},
},
},
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// replace tabs from the inline yaml with spaces; yaml doesn't support tabs for indentation.
tt.args.registryContent = strings.ReplaceAll(tt.args.registryContent, "\t", " ")
registriesFile := filepath.Join(t.TempDir(), "registries.yaml")
os.WriteFile(registriesFile, []byte(tt.args.registryContent), 0644)
registry, err := registries.GetPrivateRegistries(registriesFile)
if err != nil {
t.Fatalf("failed to parse %s: %v\n", registriesFile, err)
}
// set up embedded registry, if enabled for the test
if tt.args.mirrorAddr != "" {
conf := spegel.DefaultRegistry
conf.ServerCAFile = "server-ca"
conf.ClientKeyFile = "client-key"
conf.ClientCertFile = "client-cert"
conf.InternalAddress, conf.RegistryPort, _ = net.SplitHostPort(tt.args.mirrorAddr)
conf.InjectMirror(&config.Node{AgentConfig: config.Agent{Registry: registry.Registry}})
}
got := getHostConfigs(registry.Registry, tt.args.noDefaultEndpoint, tt.args.mirrorAddr)
assert.Equal(t, tt.want, got, "getHostConfigs()")
})
}
}
pkg/agent/templates/templates.go
View file @ 6c64b4dd
......@@ -34,15 +34,15 @@ type RegistryEndpoint struct {
}
type HostConfig struct {
Host string
Program string
Endpoints []RegistryEndpoint
DefaultEndpoint string
Program string
Endpoints []RegistryEndpoint
}
const HostsTomlTemplate = `
{{- /* */ -}}
# File generated by {{ .Program }}. DO NOT EDIT.
{{ if .Host }}server = "https://{{ .Host }}"{{ end }}
{{ if .DefaultEndpoint }}server = "{{ .DefaultEndpoint }}"{{ end }}
{{ range $e := .Endpoints -}}
[host."{{ $e.URI }}"]
......
pkg/spegel/registry.go
View file @ 6c64b4dd
......@@ -3,6 +3,7 @@ package spegel
import (
"net"
"github.com/containerd/containerd/remotes/docker"
"github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/rancher/wharfie/pkg/registries"
)
......@@ -11,6 +12,7 @@ import (
// to all configured registries.
func (c *Config) InjectMirror(nodeConfig *config.Node) error {
mirrorAddr := net.JoinHostPort(c.InternalAddress, c.RegistryPort)
mirrorURL := "https://" + mirrorAddr + "/v2"
registry := nodeConfig.AgentConfig.Registry
if registry.Configs == nil {
......@@ -28,11 +30,15 @@ func (c *Config) InjectMirror(nodeConfig *config.Node) error {
registry.Mirrors = map[string]registries.Mirror{}
}
for host, mirror := range registry.Mirrors {
if host != "*" {
mirror.Endpoints = append([]string{"https://" + mirrorAddr}, mirror.Endpoints...)
// Don't handle wildcard or local registry entries
if host != "*" && !docker.IsLocalhost(host) {
mirror.Endpoints = append([]string{mirrorURL}, mirror.Endpoints...)
registry.Mirrors[host] = mirror
}
}
registry.Mirrors[mirrorAddr] = registries.Mirror{
Endpoints: []string{mirrorURL},
}
return nil
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment