Commit 8e1b2340 authored by Brad Davidson's avatar Brad Davidson Committed by Brad Davidson

Bump wharfie to v0.5.1 and use shared decompression code

Signed-off-by: 's avatarBrad Davidson <brad.davidson@rancher.com> (cherry picked from commit 7d3447ce)
parent f468e10f
......@@ -22,6 +22,7 @@ replace (
github.com/docker/docker => github.com/docker/docker v20.10.2+incompatible
github.com/docker/libnetwork => github.com/docker/libnetwork v0.8.0-dev.2.0.20190624125649-f0e46a78ea34
github.com/golang/protobuf => github.com/k3s-io/protobuf v1.4.3-k3s1
github.com/googleapis/gax-go/v2 => github.com/googleapis/gax-go/v2 v2.0.5
github.com/juju/errors => github.com/k3s-io/nocode v0.0.0-20200630202308-cb097102c09f
github.com/kubernetes-sigs/cri-tools => github.com/k3s-io/cri-tools v1.21.0-k3s1
github.com/matryer/moq => github.com/rancher/moq v0.0.0-20190404221404-ee5226d43009
......@@ -70,14 +71,14 @@ replace (
require (
github.com/containerd/cgroups v1.0.1
github.com/containerd/containerd v1.5.1
github.com/containerd/containerd v1.5.7
github.com/containerd/cri v1.11.1-0.20200820101445-b0cc07999aa5
github.com/containerd/fuse-overlayfs-snapshotter v1.0.2
github.com/containerd/go-cni v1.0.2 // indirect
github.com/containerd/imgcrypt v1.1.1 // indirect
github.com/coreos/go-iptables v0.4.5
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f
github.com/docker/docker v20.10.5+incompatible
github.com/docker/docker v20.10.10+incompatible
github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83
github.com/flannel-io/flannel v0.15.1
github.com/go-bindata/go-bindata v3.1.2+incompatible
......@@ -101,11 +102,10 @@ require (
github.com/opencontainers/runc v1.0.2
github.com/opencontainers/selinux v1.8.2
github.com/otiai10/copy v1.6.0
github.com/pierrec/lz4 v2.6.0+incompatible
github.com/pkg/errors v0.9.1
github.com/rancher/dynamiclistener v0.2.3-k3s1
github.com/rancher/remotedialer v0.2.0
github.com/rancher/wharfie v0.3.4
github.com/rancher/wharfie v0.5.1
github.com/rancher/wrangler v0.8.3
github.com/rancher/wrangler-api v0.6.0
github.com/robfig/cron/v3 v3.0.1
......@@ -114,12 +114,12 @@ require (
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.7.0
github.com/tchap/go-patricia v2.3.0+incompatible // indirect
github.com/urfave/cli v1.22.2
github.com/urfave/cli v1.22.4
go.etcd.io/etcd v0.5.0-alpha.5.0.20201208200253-50621aee4aea
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
golang.org/x/sys v0.0.0-20210426230700-d19ff857e887
google.golang.org/grpc v1.37.0
golang.org/x/net v0.0.0-20211005215030-d2e5035098b3
golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef
google.golang.org/grpc v1.41.0
gopkg.in/yaml.v2 v2.4.0
k8s.io/api v0.21.7
k8s.io/apimachinery v0.21.7
......
......@@ -2,8 +2,6 @@ package containerd
import (
"bufio"
"compress/bzip2"
"compress/gzip"
"context"
"fmt"
"io"
......@@ -19,18 +17,16 @@ import (
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/namespaces"
"github.com/containerd/containerd/reference/docker"
"github.com/klauspost/compress/zstd"
"github.com/natefinch/lumberjack"
"github.com/opencontainers/runc/libcontainer/userns"
"github.com/pierrec/lz4"
"github.com/pkg/errors"
"github.com/rancher/k3s/pkg/agent/templates"
util2 "github.com/rancher/k3s/pkg/agent/util"
"github.com/rancher/k3s/pkg/daemons/agent"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/rancher/k3s/pkg/untar"
"github.com/rancher/k3s/pkg/version"
"github.com/rancher/wharfie/pkg/registries"
"github.com/rancher/wharfie/pkg/tarfile"
"github.com/rancher/wrangler/pkg/merr"
"github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
......@@ -199,7 +195,7 @@ func preloadImages(ctx context.Context, cfg *config.Node) error {
logrus.Errorf("Error encountered while importing %s: %v", filePath, err)
continue
}
logrus.Debugf("Imported images from %s in %s", filePath, time.Since(start))
logrus.Infof("Imported images from %s in %s", filePath, time.Since(start))
}
return nil
}
......@@ -208,39 +204,26 @@ func preloadImages(ctx context.Context, cfg *config.Node) error {
// This is in its own function so that we can ensure that the various readers are properly closed, as some
// decompressing readers need to be explicitly closed and others do not.
func preloadFile(ctx context.Context, cfg *config.Node, client *containerd.Client, criConn *grpc.ClientConn, filePath string) error {
file, err := os.Open(filePath)
if util2.HasSuffixI(filePath, ".txt") {
file, err := os.Open(filePath)
if err != nil {
return err
}
defer file.Close()
logrus.Infof("Pulling images from %s", filePath)
return prePullImages(ctx, criConn, file)
}
opener, err := tarfile.GetOpener(filePath)
if err != nil {
return err
}
defer file.Close()
var imageReader io.Reader
switch {
case util2.HasSuffixI(filePath, ".txt"):
return prePullImages(ctx, criConn, file)
case util2.HasSuffixI(filePath, ".tar"):
imageReader = file
case util2.HasSuffixI(filePath, ".tar.lz4"):
imageReader = lz4.NewReader(file)
case util2.HasSuffixI(filePath, ".tar.bz2", ".tbz"):
imageReader = bzip2.NewReader(file)
case util2.HasSuffixI(filePath, ".tar.gz", ".tgz"):
zr, err := gzip.NewReader(file)
if err != nil {
return err
}
defer zr.Close()
imageReader = zr
case util2.HasSuffixI(filePath, "tar.zst", ".tzst"):
zr, err := zstd.NewReader(file, zstd.WithDecoderMaxMemory(untar.MaxDecoderMemory))
if err != nil {
return err
}
defer zr.Close()
imageReader = zr
default:
return errors.New("unhandled file type")
imageReader, err := opener()
if err != nil {
return err
}
defer imageReader.Close()
logrus.Infof("Importing images from %s", filePath)
......@@ -353,7 +336,7 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
NodeConfig: cfg,
DisableCgroup: disableCgroup,
IsRunningInUserNS: isRunningInUserNS,
PrivateRegistryConfig: privRegistries.Registry(),
PrivateRegistryConfig: privRegistries.Registry,
}
selEnabled, selConfigured, err := selinuxStatus()
......
......@@ -15,20 +15,10 @@ import (
"time"
"github.com/klauspost/compress/zstd"
"github.com/rancher/wharfie/pkg/tarfile"
"github.com/sirupsen/logrus"
)
const (
// The zstd decoder will attempt to use up to 1GB memory for streaming operations by default,
// which is excessive and will OOM low-memory devices.
// NOTE: This must be at least as large as the window size used when compressing tarballs, or you
// will see a "window size exceeded" error when decompressing. The zstd CLI tool uses 4MB by
// default; the --long option defaults to 27 or 128M, which is still too much for a Pi3. 32MB
// (--long=25) has been tested to work acceptably while still compressing by an additional 3-6% on
// our datasets.
MaxDecoderMemory = 1 << 25
)
// TODO(bradfitz): this was copied from x/build/cmd/buildlet/buildlet.go
// but there were some buildlet-specific bits in there, so the code is
// forked for now. Unfork and add some opts arguments here, so the
......@@ -49,7 +39,7 @@ func untar(r io.Reader, dir string) (err error) {
logrus.Printf("error extracting tarball into %s after %d files, %d dirs, %v: %v", dir, nFiles, len(madeDir), td, err)
}
}()
zr, err := zstd.NewReader(r, zstd.WithDecoderMaxMemory(MaxDecoderMemory))
zr, err := zstd.NewReader(r, zstd.WithDecoderMaxMemory(tarfile.MaxDecoderMemory))
if err != nil {
return fmt.Errorf("error extracting zstd-compressed body: %v", err)
}
......
......@@ -140,7 +140,7 @@ func testOnGCE() bool {
}()
go func() {
addrs, err := net.LookupHost("metadata.google.internal")
addrs, err := net.DefaultResolver.LookupHost(ctx, "metadata.google.internal")
if err != nil || len(addrs) == 0 {
resc <- false
return
......@@ -282,6 +282,7 @@ func NewClient(c *http.Client) *Client {
// getETag returns a value from the metadata service as well as the associated ETag.
// This func is otherwise equivalent to Get.
func (c *Client) getETag(suffix string) (value, etag string, err error) {
ctx := context.TODO()
// Using a fixed IP makes it very difficult to spoof the metadata service in
// a container, which is an important use-case for local testing of cloud
// deployments. To enable spoofing of the metadata service, the environment
......@@ -296,6 +297,7 @@ func (c *Client) getETag(suffix string) (value, etag string, err error) {
// being stable anyway.
host = metadataIP
}
suffix = strings.TrimLeft(suffix, "/")
u := "http://" + host + "/computeMetadata/v1/" + suffix
req, err := http.NewRequest("GET", u, nil)
if err != nil {
......@@ -303,9 +305,25 @@ func (c *Client) getETag(suffix string) (value, etag string, err error) {
}
req.Header.Set("Metadata-Flavor", "Google")
req.Header.Set("User-Agent", userAgent)
res, err := c.hc.Do(req)
if err != nil {
return "", "", err
var res *http.Response
var reqErr error
retryer := newRetryer()
for {
res, reqErr = c.hc.Do(req)
var code int
if res != nil {
code = res.StatusCode
}
if delay, shouldRetry := retryer.Retry(code, reqErr); shouldRetry {
if err := sleep(ctx, delay); err != nil {
return "", "", err
}
continue
}
break
}
if reqErr != nil {
return "", "", nil
}
defer res.Body.Close()
if res.StatusCode == http.StatusNotFound {
......
// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package metadata
import (
"context"
"io"
"math/rand"
"net/http"
"time"
)
const (
maxRetryAttempts = 5
)
var (
syscallRetryable = func(err error) bool { return false }
)
// defaultBackoff is basically equivalent to gax.Backoff without the need for
// the dependency.
type defaultBackoff struct {
max time.Duration
mul float64
cur time.Duration
}
func (b *defaultBackoff) Pause() time.Duration {
d := time.Duration(1 + rand.Int63n(int64(b.cur)))
b.cur = time.Duration(float64(b.cur) * b.mul)
if b.cur > b.max {
b.cur = b.max
}
return d
}
// sleep is the equivalent of gax.Sleep without the need for the dependency.
func sleep(ctx context.Context, d time.Duration) error {
t := time.NewTimer(d)
select {
case <-ctx.Done():
t.Stop()
return ctx.Err()
case <-t.C:
return nil
}
}
func newRetryer() *metadataRetryer {
return &metadataRetryer{bo: &defaultBackoff{
cur: 100 * time.Millisecond,
max: 30 * time.Second,
mul: 2,
}}
}
type backoff interface {
Pause() time.Duration
}
type metadataRetryer struct {
bo backoff
attempts int
}
func (r *metadataRetryer) Retry(status int, err error) (time.Duration, bool) {
if status == http.StatusOK {
return 0, false
}
retryOk := shouldRetry(status, err)
if !retryOk {
return 0, false
}
if r.attempts == maxRetryAttempts {
return 0, false
}
r.attempts++
return r.bo.Pause(), true
}
func shouldRetry(status int, err error) bool {
if 500 <= status && status <= 599 {
return true
}
if err == io.ErrUnexpectedEOF {
return true
}
// Transient network errors should be retried.
if syscallRetryable(err) {
return true
}
if err, ok := err.(interface{ Temporary() bool }); ok {
if err.Temporary() {
return true
}
}
if err, ok := err.(interface{ Unwrap() error }); ok {
return shouldRetry(status, err.Unwrap())
}
return false
}
// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build linux
// +build linux
package metadata
import "syscall"
func init() {
// Initialize syscallRetryable to return true on transient socket-level
// errors. These errors are specific to Linux.
syscallRetryable = func(err error) bool { return err == syscall.ECONNRESET || err == syscall.ECONNREFUSED }
}
......@@ -5,21 +5,14 @@ package winio
import (
"os"
"runtime"
"syscall"
"unsafe"
)
//sys getFileInformationByHandleEx(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) = GetFileInformationByHandleEx
//sys setFileInformationByHandle(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) = SetFileInformationByHandle
const (
fileBasicInfo = 0
fileIDInfo = 0x12
"golang.org/x/sys/windows"
)
// FileBasicInfo contains file access time and file attributes information.
type FileBasicInfo struct {
CreationTime, LastAccessTime, LastWriteTime, ChangeTime syscall.Filetime
CreationTime, LastAccessTime, LastWriteTime, ChangeTime windows.Filetime
FileAttributes uint32
pad uint32 // padding
}
......@@ -27,7 +20,7 @@ type FileBasicInfo struct {
// GetFileBasicInfo retrieves times and attributes for a file.
func GetFileBasicInfo(f *os.File) (*FileBasicInfo, error) {
bi := &FileBasicInfo{}
if err := getFileInformationByHandleEx(syscall.Handle(f.Fd()), fileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), windows.FileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
......@@ -36,13 +29,32 @@ func GetFileBasicInfo(f *os.File) (*FileBasicInfo, error) {
// SetFileBasicInfo sets times and attributes for a file.
func SetFileBasicInfo(f *os.File, bi *FileBasicInfo) error {
if err := setFileInformationByHandle(syscall.Handle(f.Fd()), fileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
if err := windows.SetFileInformationByHandle(windows.Handle(f.Fd()), windows.FileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
return &os.PathError{Op: "SetFileInformationByHandle", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
return nil
}
// FileStandardInfo contains extended information for the file.
// FILE_STANDARD_INFO in WinBase.h
// https://docs.microsoft.com/en-us/windows/win32/api/winbase/ns-winbase-file_standard_info
type FileStandardInfo struct {
AllocationSize, EndOfFile int64
NumberOfLinks uint32
DeletePending, Directory bool
}
// GetFileStandardInfo retrieves ended information for the file.
func GetFileStandardInfo(f *os.File) (*FileStandardInfo, error) {
si := &FileStandardInfo{}
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), windows.FileStandardInfo, (*byte)(unsafe.Pointer(si)), uint32(unsafe.Sizeof(*si))); err != nil {
return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
return si, nil
}
// FileIDInfo contains the volume serial number and file ID for a file. This pair should be
// unique on a system.
type FileIDInfo struct {
......@@ -53,7 +65,7 @@ type FileIDInfo struct {
// GetFileID retrieves the unique (volume, file ID) pair for a file.
func GetFileID(f *os.File) (*FileIDInfo, error) {
fileID := &FileIDInfo{}
if err := getFileInformationByHandleEx(syscall.Handle(f.Fd()), fileIDInfo, (*byte)(unsafe.Pointer(fileID)), uint32(unsafe.Sizeof(*fileID))); err != nil {
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), windows.FileIdInfo, (*byte)(unsafe.Pointer(fileID)), uint32(unsafe.Sizeof(*fileID))); err != nil {
return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
......
......@@ -63,14 +63,12 @@ var (
procCreateIoCompletionPort = modkernel32.NewProc("CreateIoCompletionPort")
procCreateNamedPipeW = modkernel32.NewProc("CreateNamedPipeW")
procGetCurrentThread = modkernel32.NewProc("GetCurrentThread")
procGetFileInformationByHandleEx = modkernel32.NewProc("GetFileInformationByHandleEx")
procGetNamedPipeHandleStateW = modkernel32.NewProc("GetNamedPipeHandleStateW")
procGetNamedPipeInfo = modkernel32.NewProc("GetNamedPipeInfo")
procGetQueuedCompletionStatus = modkernel32.NewProc("GetQueuedCompletionStatus")
procLocalAlloc = modkernel32.NewProc("LocalAlloc")
procLocalFree = modkernel32.NewProc("LocalFree")
procSetFileCompletionNotificationModes = modkernel32.NewProc("SetFileCompletionNotificationModes")
procSetFileInformationByHandle = modkernel32.NewProc("SetFileInformationByHandle")
procNtCreateNamedPipeFile = modntdll.NewProc("NtCreateNamedPipeFile")
procRtlDefaultNpAcl = modntdll.NewProc("RtlDefaultNpAcl")
procRtlDosPathNameToNtPathName_U = modntdll.NewProc("RtlDosPathNameToNtPathName_U")
......@@ -339,14 +337,6 @@ func getCurrentThread() (h syscall.Handle) {
return
}
func getFileInformationByHandleEx(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) {
r1, _, e1 := syscall.Syscall6(procGetFileInformationByHandleEx.Addr(), 4, uintptr(h), uintptr(class), uintptr(unsafe.Pointer(buffer)), uintptr(size), 0, 0)
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func getNamedPipeHandleState(pipe syscall.Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) {
r1, _, e1 := syscall.Syscall9(procGetNamedPipeHandleStateW.Addr(), 7, uintptr(pipe), uintptr(unsafe.Pointer(state)), uintptr(unsafe.Pointer(curInstances)), uintptr(unsafe.Pointer(maxCollectionCount)), uintptr(unsafe.Pointer(collectDataTimeout)), uintptr(unsafe.Pointer(userName)), uintptr(maxUserNameSize), 0, 0)
if r1 == 0 {
......@@ -390,14 +380,6 @@ func setFileCompletionNotificationModes(h syscall.Handle, flags uint8) (err erro
return
}
func setFileInformationByHandle(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) {
r1, _, e1 := syscall.Syscall6(procSetFileInformationByHandle.Addr(), 4, uintptr(h), uintptr(class), uintptr(unsafe.Pointer(buffer)), uintptr(size), 0, 0)
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func ntCreateNamedPipeFile(pipe *syscall.Handle, access uint32, oa *objectAttributes, iosb *ioStatusBlock, share uint32, disposition uint32, options uint32, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntstatus) {
r0, _, _ := syscall.Syscall15(procNtCreateNamedPipeFile.Addr(), 14, uintptr(unsafe.Pointer(pipe)), uintptr(access), uintptr(unsafe.Pointer(oa)), uintptr(unsafe.Pointer(iosb)), uintptr(share), uintptr(disposition), uintptr(options), uintptr(typ), uintptr(readMode), uintptr(completionMode), uintptr(maxInstances), uintptr(inboundQuota), uintptr(outputQuota), uintptr(unsafe.Pointer(timeout)), 0)
status = ntstatus(r0)
......
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package errorutil
import (
"errors"
"fmt"
"strings"
)
// Aggregate combines a list of errors into a single new error.
func Aggregate(errs []error) error {
switch len(errs) {
case 0:
return nil
case 1:
return errs[0]
default:
points := make([]string, len(errs)+1)
points[0] = fmt.Sprintf("%d error(s) occurred:", len(errs))
for i, err := range errs {
points[i+1] = fmt.Sprintf("* %s", err)
}
return errors.New(strings.Join(points, "\n\t"))
}
}
module github.com/containerd/stargz-snapshotter/estargz
go 1.16
require (
github.com/klauspost/compress v1.13.6
github.com/opencontainers/go-digest v1.0.0
github.com/pkg/errors v0.9.1
github.com/vbatts/tar-split v0.11.2
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
)
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME=
github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
/*
Copyright 2019 The Go Authors. All rights reserved.
Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file.
*/
package estargz
import (
"archive/tar"
"bytes"
"compress/gzip"
"encoding/binary"
"encoding/json"
"fmt"
"hash"
"io"
"strconv"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
)
type gzipCompression struct {
*gzipCompressor
*GzipDecompressor
}
func newGzipCompressionWithLevel(level int) Compression {
return &gzipCompression{
&gzipCompressor{level},
&GzipDecompressor{},
}
}
func NewGzipCompressorWithLevel(level int) Compressor {
return &gzipCompressor{level}
}
type gzipCompressor struct {
compressionLevel int
}
func (gc *gzipCompressor) Writer(w io.Writer) (io.WriteCloser, error) {
return gzip.NewWriterLevel(w, gc.compressionLevel)
}
func (gc *gzipCompressor) WriteTOCAndFooter(w io.Writer, off int64, toc *JTOC, diffHash hash.Hash) (digest.Digest, error) {
tocJSON, err := json.MarshalIndent(toc, "", "\t")
if err != nil {
return "", err
}
gz, _ := gzip.NewWriterLevel(w, gc.compressionLevel)
gw := io.Writer(gz)
if diffHash != nil {
gw = io.MultiWriter(gz, diffHash)
}
tw := tar.NewWriter(gw)
if err := tw.WriteHeader(&tar.Header{
Typeflag: tar.TypeReg,
Name: TOCTarName,
Size: int64(len(tocJSON)),
}); err != nil {
return "", err
}
if _, err := tw.Write(tocJSON); err != nil {
return "", err
}
if err := tw.Close(); err != nil {
return "", err
}
if err := gz.Close(); err != nil {
return "", err
}
if _, err := w.Write(gzipFooterBytes(off)); err != nil {
return "", err
}
return digest.FromBytes(tocJSON), nil
}
// gzipFooterBytes returns the 51 bytes footer.
func gzipFooterBytes(tocOff int64) []byte {
buf := bytes.NewBuffer(make([]byte, 0, FooterSize))
gz, _ := gzip.NewWriterLevel(buf, gzip.NoCompression) // MUST be NoCompression to keep 51 bytes
// Extra header indicating the offset of TOCJSON
// https://tools.ietf.org/html/rfc1952#section-2.3.1.1
header := make([]byte, 4)
header[0], header[1] = 'S', 'G'
subfield := fmt.Sprintf("%016xSTARGZ", tocOff)
binary.LittleEndian.PutUint16(header[2:4], uint16(len(subfield))) // little-endian per RFC1952
gz.Header.Extra = append(header, []byte(subfield)...)
gz.Close()
if buf.Len() != FooterSize {
panic(fmt.Sprintf("footer buffer = %d, not %d", buf.Len(), FooterSize))
}
return buf.Bytes()
}
type GzipDecompressor struct{}
func (gz *GzipDecompressor) Reader(r io.Reader) (io.ReadCloser, error) {
return gzip.NewReader(r)
}
func (gz *GzipDecompressor) ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) {
return parseTOCEStargz(r)
}
func (gz *GzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) {
if len(p) != FooterSize {
return 0, 0, 0, fmt.Errorf("invalid length %d cannot be parsed", len(p))
}
zr, err := gzip.NewReader(bytes.NewReader(p))
if err != nil {
return 0, 0, 0, err
}
defer zr.Close()
extra := zr.Header.Extra
si1, si2, subfieldlen, subfield := extra[0], extra[1], extra[2:4], extra[4:]
if si1 != 'S' || si2 != 'G' {
return 0, 0, 0, fmt.Errorf("invalid subfield IDs: %q, %q; want E, S", si1, si2)
}
if slen := binary.LittleEndian.Uint16(subfieldlen); slen != uint16(16+len("STARGZ")) {
return 0, 0, 0, fmt.Errorf("invalid length of subfield %d; want %d", slen, 16+len("STARGZ"))
}
if string(subfield[16:]) != "STARGZ" {
return 0, 0, 0, fmt.Errorf("STARGZ magic string must be included in the footer subfield")
}
tocOffset, err = strconv.ParseInt(string(subfield[:16]), 16, 64)
if err != nil {
return 0, 0, 0, errors.Wrapf(err, "legacy: failed to parse toc offset")
}
return tocOffset, tocOffset, 0, nil
}
func (gz *GzipDecompressor) FooterSize() int64 {
return FooterSize
}
type legacyGzipDecompressor struct{}
func (gz *legacyGzipDecompressor) Reader(r io.Reader) (io.ReadCloser, error) {
return gzip.NewReader(r)
}
func (gz *legacyGzipDecompressor) ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) {
return parseTOCEStargz(r)
}
func (gz *legacyGzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) {
if len(p) != legacyFooterSize {
return 0, 0, 0, fmt.Errorf("legacy: invalid length %d cannot be parsed", len(p))
}
zr, err := gzip.NewReader(bytes.NewReader(p))
if err != nil {
return 0, 0, 0, errors.Wrapf(err, "legacy: failed to get footer gzip reader")
}
defer zr.Close()
extra := zr.Header.Extra
if len(extra) != 16+len("STARGZ") {
return 0, 0, 0, fmt.Errorf("legacy: invalid stargz's extra field size")
}
if string(extra[16:]) != "STARGZ" {
return 0, 0, 0, fmt.Errorf("legacy: magic string STARGZ not found")
}
tocOffset, err = strconv.ParseInt(string(extra[:16]), 16, 64)
if err != nil {
return 0, 0, 0, errors.Wrapf(err, "legacy: failed to parse toc offset")
}
return tocOffset, tocOffset, 0, nil
}
func (gz *legacyGzipDecompressor) FooterSize() int64 {
return legacyFooterSize
}
func parseTOCEStargz(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) {
zr, err := gzip.NewReader(r)
if err != nil {
return nil, "", fmt.Errorf("malformed TOC gzip header: %v", err)
}
defer zr.Close()
zr.Multistream(false)
tr := tar.NewReader(zr)
h, err := tr.Next()
if err != nil {
return nil, "", fmt.Errorf("failed to find tar header in TOC gzip stream: %v", err)
}
if h.Name != TOCTarName {
return nil, "", fmt.Errorf("TOC tar entry had name %q; expected %q", h.Name, TOCTarName)
}
dgstr := digest.Canonical.Digester()
toc = new(JTOC)
if err := json.NewDecoder(io.TeeReader(tr, dgstr.Hash())).Decode(&toc); err != nil {
return nil, "", fmt.Errorf("error decoding TOC JSON: %v", err)
}
return toc, dgstr.Digest(), nil
}
......@@ -15,7 +15,7 @@ type roffRenderer struct {
extensions blackfriday.Extensions
listCounters []int
firstHeader bool
defineTerm bool
firstDD bool
listDepth int
}
......@@ -42,7 +42,8 @@ const (
quoteCloseTag = "\n.RE\n"
listTag = "\n.RS\n"
listCloseTag = "\n.RE\n"
arglistTag = "\n.TP\n"
dtTag = "\n.TP\n"
dd2Tag = "\n"
tableStart = "\n.TS\nallbox;\n"
tableEnd = ".TE\n"
tableCellStart = "T{\n"
......@@ -90,7 +91,7 @@ func (r *roffRenderer) RenderNode(w io.Writer, node *blackfriday.Node, entering
switch node.Type {
case blackfriday.Text:
r.handleText(w, node, entering)
escapeSpecialChars(w, node.Literal)
case blackfriday.Softbreak:
out(w, crTag)
case blackfriday.Hardbreak:
......@@ -150,40 +151,21 @@ func (r *roffRenderer) RenderNode(w io.Writer, node *blackfriday.Node, entering
out(w, codeCloseTag)
case blackfriday.Table:
r.handleTable(w, node, entering)
case blackfriday.TableCell:
r.handleTableCell(w, node, entering)
case blackfriday.TableHead:
case blackfriday.TableBody:
case blackfriday.TableRow:
// no action as cell entries do all the nroff formatting
return blackfriday.GoToNext
case blackfriday.TableCell:
r.handleTableCell(w, node, entering)
case blackfriday.HTMLSpan:
// ignore other HTML tags
default:
fmt.Fprintln(os.Stderr, "WARNING: go-md2man does not handle node type "+node.Type.String())
}
return walkAction
}
func (r *roffRenderer) handleText(w io.Writer, node *blackfriday.Node, entering bool) {
var (
start, end string
)
// handle special roff table cell text encapsulation
if node.Parent.Type == blackfriday.TableCell {
if len(node.Literal) > 30 {
start = tableCellStart
end = tableCellEnd
} else {
// end rows that aren't terminated by "tableCellEnd" with a cr if end of row
if node.Parent.Next == nil && !node.Parent.IsHeader {
end = crTag
}
}
}
out(w, start)
escapeSpecialChars(w, node.Literal)
out(w, end)
}
func (r *roffRenderer) handleHeading(w io.Writer, node *blackfriday.Node, entering bool) {
if entering {
switch node.Level {
......@@ -230,15 +212,20 @@ func (r *roffRenderer) handleItem(w io.Writer, node *blackfriday.Node, entering
if node.ListFlags&blackfriday.ListTypeOrdered != 0 {
out(w, fmt.Sprintf(".IP \"%3d.\" 5\n", r.listCounters[len(r.listCounters)-1]))
r.listCounters[len(r.listCounters)-1]++
} else if node.ListFlags&blackfriday.ListTypeTerm != 0 {
// DT (definition term): line just before DD (see below).
out(w, dtTag)
r.firstDD = true
} else if node.ListFlags&blackfriday.ListTypeDefinition != 0 {
// state machine for handling terms and following definitions
// since blackfriday does not distinguish them properly, nor
// does it seperate them into separate lists as it should
if !r.defineTerm {
out(w, arglistTag)
r.defineTerm = true
// DD (definition description): line that starts with ": ".
//
// We have to distinguish between the first DD and the
// subsequent ones, as there should be no vertical
// whitespace between the DT and the first DD.
if r.firstDD {
r.firstDD = false
} else {
r.defineTerm = false
out(w, dd2Tag)
}
} else {
out(w, ".IP \\(bu 2\n")
......@@ -251,7 +238,7 @@ func (r *roffRenderer) handleItem(w io.Writer, node *blackfriday.Node, entering
func (r *roffRenderer) handleTable(w io.Writer, node *blackfriday.Node, entering bool) {
if entering {
out(w, tableStart)
//call walker to count cells (and rows?) so format section can be produced
// call walker to count cells (and rows?) so format section can be produced
columns := countColumns(node)
out(w, strings.Repeat("l ", columns)+"\n")
out(w, strings.Repeat("l ", columns)+".\n")
......@@ -261,28 +248,41 @@ func (r *roffRenderer) handleTable(w io.Writer, node *blackfriday.Node, entering
}
func (r *roffRenderer) handleTableCell(w io.Writer, node *blackfriday.Node, entering bool) {
var (
start, end string
)
if node.IsHeader {
start = codespanTag
end = codespanCloseTag
}
if entering {
var start string
if node.Prev != nil && node.Prev.Type == blackfriday.TableCell {
out(w, "\t"+start)
} else {
out(w, start)
start = "\t"
}
if node.IsHeader {
start += codespanTag
} else if nodeLiteralSize(node) > 30 {
start += tableCellStart
}
out(w, start)
} else {
// need to carriage return if we are at the end of the header row
if node.IsHeader && node.Next == nil {
end = end + crTag
var end string
if node.IsHeader {
end = codespanCloseTag
} else if nodeLiteralSize(node) > 30 {
end = tableCellEnd
}
if node.Next == nil && end != tableCellEnd {
// Last cell: need to carriage return if we are at the end of the
// header row and content isn't wrapped in a "tablecell"
end += crTag
}
out(w, end)
}
}
func nodeLiteralSize(node *blackfriday.Node) int {
total := 0
for n := node.FirstChild; n != nil; n = n.FirstChild {
total += len(n.Literal)
}
return total
}
// because roff format requires knowing the column count before outputting any table
// data we need to walk a table tree and count the columns
func countColumns(node *blackfriday.Node) int {
......@@ -309,15 +309,6 @@ func out(w io.Writer, output string) {
io.WriteString(w, output) // nolint: errcheck
}
func needsBackslash(c byte) bool {
for _, r := range []byte("-_&\\~") {
if c == r {
return true
}
}
return false
}
func escapeSpecialChars(w io.Writer, text []byte) {
for i := 0; i < len(text); i++ {
// escape initial apostrophe or period
......@@ -328,7 +319,7 @@ func escapeSpecialChars(w io.Writer, text []byte) {
// directly copy normal characters
org := i
for i < len(text) && !needsBackslash(text[i]) {
for i < len(text) && text[i] != '\\' {
i++
}
if i > org {
......
......@@ -6,6 +6,7 @@ import (
"os"
"path/filepath"
"strings"
"sync"
"github.com/docker/cli/cli/config/configfile"
"github.com/docker/cli/cli/config/credentials"
......@@ -23,17 +24,44 @@ const (
)
var (
configDir = os.Getenv("DOCKER_CONFIG")
initConfigDir = new(sync.Once)
configDir string
homeDir string
)
func init() {
// resetHomeDir is used in testing to reset the "homeDir" package variable to
// force re-lookup of the home directory between tests.
func resetHomeDir() {
homeDir = ""
}
func getHomeDir() string {
if homeDir == "" {
homeDir = homedir.Get()
}
return homeDir
}
// resetConfigDir is used in testing to reset the "configDir" package variable
// and its sync.Once to force re-lookup between tests.
func resetConfigDir() {
configDir = ""
initConfigDir = new(sync.Once)
}
func setConfigDir() {
if configDir != "" {
return
}
configDir = os.Getenv("DOCKER_CONFIG")
if configDir == "" {
configDir = filepath.Join(homedir.Get(), configFileDir)
configDir = filepath.Join(getHomeDir(), configFileDir)
}
}
// Dir returns the directory the configuration file is stored in
func Dir() string {
initConfigDir.Do(setConfigDir)
return configDir
}
......@@ -76,10 +104,15 @@ func LoadFromReader(configData io.Reader) (*configfile.ConfigFile, error) {
return &configFile, err
}
// TODO remove this temporary hack, which is used to warn about the deprecated ~/.dockercfg file
var printLegacyFileWarning bool
// Load reads the configuration files in the given directory, and sets up
// the auth config information and returns values.
// FIXME: use the internal golang config parser
func Load(configDir string) (*configfile.ConfigFile, error) {
printLegacyFileWarning = false
if configDir == "" {
configDir = Dir()
}
......@@ -88,11 +121,7 @@ func Load(configDir string) (*configfile.ConfigFile, error) {
configFile := configfile.New(filename)
// Try happy path first - latest config file
if _, err := os.Stat(filename); err == nil {
file, err := os.Open(filename)
if err != nil {
return configFile, errors.Wrap(err, filename)
}
if file, err := os.Open(filename); err == nil {
defer file.Close()
err = configFile.LoadFromReader(file)
if err != nil {
......@@ -106,22 +135,13 @@ func Load(configDir string) (*configfile.ConfigFile, error) {
}
// Can't find latest config file so check for the old one
homedir, err := os.UserHomeDir()
if err != nil {
return configFile, errors.Wrap(err, oldConfigfile)
}
confFile := filepath.Join(homedir, oldConfigfile)
if _, err := os.Stat(confFile); err != nil {
return configFile, nil //missing file is not an error
}
file, err := os.Open(confFile)
if err != nil {
return configFile, errors.Wrap(err, filename)
}
defer file.Close()
err = configFile.LegacyLoadFromReader(file)
if err != nil {
return configFile, errors.Wrap(err, filename)
filename = filepath.Join(getHomeDir(), oldConfigfile)
if file, err := os.Open(filename); err == nil {
printLegacyFileWarning = true
defer file.Close()
if err := configFile.LegacyLoadFromReader(file); err != nil {
return configFile, errors.Wrap(err, filename)
}
}
return configFile, nil
}
......@@ -133,6 +153,9 @@ func LoadDefaultConfigFile(stderr io.Writer) *configfile.ConfigFile {
if err != nil {
fmt.Fprintf(stderr, "WARNING: Error loading config file: %v\n", err)
}
if printLegacyFileWarning {
_, _ = fmt.Fprintln(stderr, "WARNING: Support for the legacy ~/.dockercfg configuration file and file-format is deprecated and will be removed in an upcoming release")
}
if !configFile.ContainsAuth() {
configFile.CredentialsStore = credentials.DetectDefaultStore(configFile.CredentialsStore)
}
......
......@@ -13,6 +13,7 @@ import (
"github.com/docker/cli/cli/config/credentials"
"github.com/docker/cli/cli/config/types"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
const (
......@@ -118,7 +119,7 @@ func (configFile *ConfigFile) LegacyLoadFromReader(configData io.Reader) error {
// LoadFromReader reads the configuration data given and sets up the auth config
// information with given directory and populates the receiver object
func (configFile *ConfigFile) LoadFromReader(configData io.Reader) error {
if err := json.NewDecoder(configData).Decode(&configFile); err != nil {
if err := json.NewDecoder(configData).Decode(&configFile); err != nil && !errors.Is(err, io.EOF) {
return err
}
var err error
......@@ -168,6 +169,13 @@ func (configFile *ConfigFile) SaveToWriter(writer io.Writer) error {
configFile.AuthConfigs = tmpAuthConfigs
defer func() { configFile.AuthConfigs = saveAuthConfigs }()
// User-Agent header is automatically set, and should not be stored in the configuration
for v := range configFile.HTTPHeaders {
if strings.EqualFold(v, "User-Agent") {
delete(configFile.HTTPHeaders, v)
}
}
data, err := json.MarshalIndent(configFile, "", "\t")
if err != nil {
return err
......@@ -177,7 +185,7 @@ func (configFile *ConfigFile) SaveToWriter(writer io.Writer) error {
}
// Save encodes and writes out all the authorization information
func (configFile *ConfigFile) Save() error {
func (configFile *ConfigFile) Save() (retErr error) {
if configFile.Filename == "" {
return errors.Errorf("Can't save config with empty filename")
}
......@@ -190,13 +198,33 @@ func (configFile *ConfigFile) Save() error {
if err != nil {
return err
}
defer func() {
temp.Close()
if retErr != nil {
if err := os.Remove(temp.Name()); err != nil {
logrus.WithError(err).WithField("file", temp.Name()).Debug("Error cleaning up temp file")
}
}
}()
err = configFile.SaveToWriter(temp)
temp.Close()
if err != nil {
os.Remove(temp.Name())
return err
}
return os.Rename(temp.Name(), configFile.Filename)
if err := temp.Close(); err != nil {
return errors.Wrap(err, "error closing temp file")
}
// Handle situation where the configfile is a symlink
cfgFile := configFile.Filename
if f, err := os.Readlink(cfgFile); err == nil {
cfgFile = f
}
// Try copying the current config file (if any) ownership and permissions
copyFilePermissions(cfgFile, temp.Name())
return os.Rename(temp.Name(), cfgFile)
}
// ParseProxyConfig computes proxy configuration by retrieving the config for the provided host and
......
// +build !windows
package configfile
import (
"os"
"syscall"
)
// copyFilePermissions copies file ownership and permissions from "src" to "dst",
// ignoring any error during the process.
func copyFilePermissions(src, dst string) {
var (
mode os.FileMode = 0600
uid, gid int
)
fi, err := os.Stat(src)
if err != nil {
return
}
if fi.Mode().IsRegular() {
mode = fi.Mode()
}
if err := os.Chmod(dst, mode); err != nil {
return
}
uid = int(fi.Sys().(*syscall.Stat_t).Uid)
gid = int(fi.Sys().(*syscall.Stat_t).Gid)
if uid > 0 && gid > 0 {
_ = os.Chown(dst, uid, gid)
}
}
package configfile
func copyFilePermissions(src, dst string) {
// TODO implement for Windows
}
package credentials
import (
"os/exec"
exec "golang.org/x/sys/execabs"
)
// DetectDefaultStore return the default credentials store for the platform if
......
package challenge
import (
"net/url"
"strings"
)
// FROM: https://golang.org/src/net/http/http.go
// Given a string of the form "host", "host:port", or "[ipv6::address]:port",
// return true if the string includes a port.
func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") }
// FROM: http://golang.org/src/net/http/transport.go
var portMap = map[string]string{
"http": "80",
"https": "443",
}
// canonicalAddr returns url.Host but always with a ":port" suffix
// FROM: http://golang.org/src/net/http/transport.go
func canonicalAddr(url *url.URL) string {
addr := url.Host
if !hasPort(addr) {
return addr + ":" + portMap[url.Scheme]
}
return addr
}
package challenge
import (
"fmt"
"net/http"
"net/url"
"strings"
"sync"
)
// Challenge carries information from a WWW-Authenticate response header.
// See RFC 2617.
type Challenge struct {
// Scheme is the auth-scheme according to RFC 2617
Scheme string
// Parameters are the auth-params according to RFC 2617
Parameters map[string]string
}
// Manager manages the challenges for endpoints.
// The challenges are pulled out of HTTP responses. Only
// responses which expect challenges should be added to
// the manager, since a non-unauthorized request will be
// viewed as not requiring challenges.
type Manager interface {
// GetChallenges returns the challenges for the given
// endpoint URL.
GetChallenges(endpoint url.URL) ([]Challenge, error)
// AddResponse adds the response to the challenge
// manager. The challenges will be parsed out of
// the WWW-Authenicate headers and added to the
// URL which was produced the response. If the
// response was authorized, any challenges for the
// endpoint will be cleared.
AddResponse(resp *http.Response) error
}
// NewSimpleManager returns an instance of
// Manger which only maps endpoints to challenges
// based on the responses which have been added the
// manager. The simple manager will make no attempt to
// perform requests on the endpoints or cache the responses
// to a backend.
func NewSimpleManager() Manager {
return &simpleManager{
Challenges: make(map[string][]Challenge),
}
}
type simpleManager struct {
sync.RWMutex
Challenges map[string][]Challenge
}
func normalizeURL(endpoint *url.URL) {
endpoint.Host = strings.ToLower(endpoint.Host)
endpoint.Host = canonicalAddr(endpoint)
}
func (m *simpleManager) GetChallenges(endpoint url.URL) ([]Challenge, error) {
normalizeURL(&endpoint)
m.RLock()
defer m.RUnlock()
challenges := m.Challenges[endpoint.String()]
return challenges, nil
}
func (m *simpleManager) AddResponse(resp *http.Response) error {
challenges := ResponseChallenges(resp)
if resp.Request == nil {
return fmt.Errorf("missing request reference")
}
urlCopy := url.URL{
Path: resp.Request.URL.Path,
Host: resp.Request.URL.Host,
Scheme: resp.Request.URL.Scheme,
}
normalizeURL(&urlCopy)
m.Lock()
defer m.Unlock()
m.Challenges[urlCopy.String()] = challenges
return nil
}
// Octet types from RFC 2616.
type octetType byte
var octetTypes [256]octetType
const (
isToken octetType = 1 << iota
isSpace
)
func init() {
// OCTET = <any 8-bit sequence of data>
// CHAR = <any US-ASCII character (octets 0 - 127)>
// CTL = <any US-ASCII control character (octets 0 - 31) and DEL (127)>
// CR = <US-ASCII CR, carriage return (13)>
// LF = <US-ASCII LF, linefeed (10)>
// SP = <US-ASCII SP, space (32)>
// HT = <US-ASCII HT, horizontal-tab (9)>
// <"> = <US-ASCII double-quote mark (34)>
// CRLF = CR LF
// LWS = [CRLF] 1*( SP | HT )
// TEXT = <any OCTET except CTLs, but including LWS>
// separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <">
// | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT
// token = 1*<any CHAR except CTLs or separators>
// qdtext = <any TEXT except <">>
for c := 0; c < 256; c++ {
var t octetType
isCtl := c <= 31 || c == 127
isChar := 0 <= c && c <= 127
isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0
if strings.IndexRune(" \t\r\n", rune(c)) >= 0 {
t |= isSpace
}
if isChar && !isCtl && !isSeparator {
t |= isToken
}
octetTypes[c] = t
}
}
// ResponseChallenges returns a list of authorization challenges
// for the given http Response. Challenges are only checked if
// the response status code was a 401.
func ResponseChallenges(resp *http.Response) []Challenge {
if resp.StatusCode == http.StatusUnauthorized {
// Parse the WWW-Authenticate Header and store the challenges
// on this endpoint object.
return parseAuthHeader(resp.Header)
}
return nil
}
func parseAuthHeader(header http.Header) []Challenge {
challenges := []Challenge{}
for _, h := range header[http.CanonicalHeaderKey("WWW-Authenticate")] {
v, p := parseValueAndParams(h)
if v != "" {
challenges = append(challenges, Challenge{Scheme: v, Parameters: p})
}
}
return challenges
}
func parseValueAndParams(header string) (value string, params map[string]string) {
params = make(map[string]string)
value, s := expectToken(header)
if value == "" {
return
}
value = strings.ToLower(value)
s = "," + skipSpace(s)
for strings.HasPrefix(s, ",") {
var pkey string
pkey, s = expectToken(skipSpace(s[1:]))
if pkey == "" {
return
}
if !strings.HasPrefix(s, "=") {
return
}
var pvalue string
pvalue, s = expectTokenOrQuoted(s[1:])
if pvalue == "" {
return
}
pkey = strings.ToLower(pkey)
params[pkey] = pvalue
s = skipSpace(s)
}
return
}
func skipSpace(s string) (rest string) {
i := 0
for ; i < len(s); i++ {
if octetTypes[s[i]]&isSpace == 0 {
break
}
}
return s[i:]
}
func expectToken(s string) (token, rest string) {
i := 0
for ; i < len(s); i++ {
if octetTypes[s[i]]&isToken == 0 {
break
}
}
return s[:i], s[i:]
}
func expectTokenOrQuoted(s string) (value string, rest string) {
if !strings.HasPrefix(s, "\"") {
return expectToken(s)
}
s = s[1:]
for i := 0; i < len(s); i++ {
switch s[i] {
case '"':
return s[:i], s[i+1:]
case '\\':
p := make([]byte, len(s)-1)
j := copy(p, s[:i])
escape := true
for i = i + 1; i < len(s); i++ {
b := s[i]
switch {
case escape:
escape = false
p[j] = b
j++
case b == '\\':
escape = true
case b == '"':
return string(p[:j]), s[i+1:]
default:
p[j] = b
j++
}
}
return "", ""
}
}
return "", ""
}
......@@ -4,7 +4,8 @@ import (
"fmt"
"io"
"os"
"os/exec"
exec "golang.org/x/sys/execabs"
)
// Program is an interface to execute external programs.
......
package credentials
// Version holds a string describing the current version
const Version = "0.6.3"
const Version = "0.6.4"
......@@ -50,19 +50,22 @@ func newCall(t TestHelper, receiver interface{}, method string, methodType refle
t.Helper()
// TODO: check arity, types.
margs := make([]Matcher, len(args))
mArgs := make([]Matcher, len(args))
for i, arg := range args {
if m, ok := arg.(Matcher); ok {
margs[i] = m
mArgs[i] = m
} else if arg == nil {
// Handle nil specially so that passing a nil interface value
// will match the typed nils of concrete args.
margs[i] = Nil()
mArgs[i] = Nil()
} else {
margs[i] = Eq(arg)
mArgs[i] = Eq(arg)
}
}
// callerInfo's skip should be updated if the number of calls between the user's test
// and this line changes, i.e. this code is wrapped in another anonymous function.
// 0 is us, 1 is RecordCallWithMethodType(), 2 is the generated recorder, and 3 is the user's test.
origin := callerInfo(3)
actions := []func([]interface{}) []interface{}{func([]interface{}) []interface{} {
// Synthesize the zero value for each of the return args' types.
......@@ -73,7 +76,7 @@ func newCall(t TestHelper, receiver interface{}, method string, methodType refle
return rets
}}
return &Call{t: t, receiver: receiver, method: method, methodType: methodType,
args: margs, origin: origin, minCalls: 1, maxCalls: 1, actions: actions}
args: mArgs, origin: origin, minCalls: 1, maxCalls: 1, actions: actions}
}
// AnyTimes allows the expectation to be called 0 or more times
......@@ -110,19 +113,25 @@ func (c *Call) DoAndReturn(f interface{}) *Call {
v := reflect.ValueOf(f)
c.addAction(func(args []interface{}) []interface{} {
vargs := make([]reflect.Value, len(args))
c.t.Helper()
vArgs := make([]reflect.Value, len(args))
ft := v.Type()
if c.methodType.NumIn() != ft.NumIn() {
c.t.Fatalf("wrong number of arguments in DoAndReturn func for %T.%v: got %d, want %d [%s]",
c.receiver, c.method, ft.NumIn(), c.methodType.NumIn(), c.origin)
return nil
}
for i := 0; i < len(args); i++ {
if args[i] != nil {
vargs[i] = reflect.ValueOf(args[i])
vArgs[i] = reflect.ValueOf(args[i])
} else {
// Use the zero value for the arg.
vargs[i] = reflect.Zero(ft.In(i))
vArgs[i] = reflect.Zero(ft.In(i))
}
}
vrets := v.Call(vargs)
rets := make([]interface{}, len(vrets))
for i, ret := range vrets {
vRets := v.Call(vArgs)
rets := make([]interface{}, len(vRets))
for i, ret := range vRets {
rets[i] = ret.Interface()
}
return rets
......@@ -139,17 +148,23 @@ func (c *Call) Do(f interface{}) *Call {
v := reflect.ValueOf(f)
c.addAction(func(args []interface{}) []interface{} {
vargs := make([]reflect.Value, len(args))
c.t.Helper()
if c.methodType.NumIn() != v.Type().NumIn() {
c.t.Fatalf("wrong number of arguments in Do func for %T.%v: got %d, want %d [%s]",
c.receiver, c.method, v.Type().NumIn(), c.methodType.NumIn(), c.origin)
return nil
}
vArgs := make([]reflect.Value, len(args))
ft := v.Type()
for i := 0; i < len(args); i++ {
if args[i] != nil {
vargs[i] = reflect.ValueOf(args[i])
vArgs[i] = reflect.ValueOf(args[i])
} else {
// Use the zero value for the arg.
vargs[i] = reflect.Zero(ft.In(i))
vArgs[i] = reflect.Zero(ft.In(i))
}
}
v.Call(vargs)
v.Call(vArgs)
return nil
})
return c
......@@ -301,14 +316,9 @@ func (c *Call) matches(args []interface{}) error {
for i, m := range c.args {
if !m.Matches(args[i]) {
got := fmt.Sprintf("%v", args[i])
if gs, ok := m.(GotFormatter); ok {
got = gs.Got(args[i])
}
return fmt.Errorf(
"expected call at %s doesn't match the argument at index %d.\nGot: %v\nWant: %v",
c.origin, i, got, m,
c.origin, i, formatGottenArg(m, args[i]), m,
)
}
}
......@@ -331,7 +341,7 @@ func (c *Call) matches(args []interface{}) error {
// Non-variadic args
if !m.Matches(args[i]) {
return fmt.Errorf("expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v",
c.origin, strconv.Itoa(i), args[i], m)
c.origin, strconv.Itoa(i), formatGottenArg(m, args[i]), m)
}
continue
}
......@@ -355,12 +365,12 @@ func (c *Call) matches(args []interface{}) error {
// matches all the remaining arguments or the lack of any.
// Convert the remaining arguments, if any, into a slice of the
// expected type.
vargsType := c.methodType.In(c.methodType.NumIn() - 1)
vargs := reflect.MakeSlice(vargsType, 0, len(args)-i)
vArgsType := c.methodType.In(c.methodType.NumIn() - 1)
vArgs := reflect.MakeSlice(vArgsType, 0, len(args)-i)
for _, arg := range args[i:] {
vargs = reflect.Append(vargs, reflect.ValueOf(arg))
vArgs = reflect.Append(vArgs, reflect.ValueOf(arg))
}
if m.Matches(vargs.Interface()) {
if m.Matches(vArgs.Interface()) {
// Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, gomock.Any())
// Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, someSliceMatcher)
// Got Foo(a, b) want Foo(matcherA, matcherB, gomock.Any())
......@@ -373,16 +383,16 @@ func (c *Call) matches(args []interface{}) error {
// Got Foo(a, b, c, d) want Foo(matcherA, matcherB, matcherC, matcherD, matcherE)
// Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, matcherC, matcherD)
// Got Foo(a, b, c) want Foo(matcherA, matcherB)
return fmt.Errorf("Expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v",
c.origin, strconv.Itoa(i), args[i:], c.args[i])
return fmt.Errorf("expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v",
c.origin, strconv.Itoa(i), formatGottenArg(m, args[i:]), c.args[i])
}
}
// Check that all prerequisite calls have been satisfied.
for _, preReqCall := range c.preReqs {
if !preReqCall.satisfied() {
return fmt.Errorf("Expected call at %s doesn't have a prerequisite call satisfied:\n%v\nshould be called before:\n%v",
return fmt.Errorf("expected call at %s doesn't have a prerequisite call satisfied:\n%v\nshould be called before:\n%v",
c.origin, preReqCall, c)
}
}
......@@ -425,3 +435,11 @@ func setSlice(arg interface{}, v reflect.Value) {
func (c *Call) addAction(action func([]interface{}) []interface{}) {
c.actions = append(c.actions, action)
}
func formatGottenArg(m Matcher, arg interface{}) string {
got := fmt.Sprintf("%v (%T)", arg, arg)
if gs, ok := m.(GotFormatter); ok {
got = gs.Got(arg)
}
return got
}
......@@ -16,6 +16,7 @@ package gomock
import (
"bytes"
"errors"
"fmt"
)
......@@ -84,14 +85,18 @@ func (cs callSet) FindMatch(receiver interface{}, method string, args []interfac
for _, call := range exhausted {
if err := call.matches(args); err != nil {
_, _ = fmt.Fprintf(&callsErrors, "\n%v", err)
continue
}
_, _ = fmt.Fprintf(
&callsErrors, "all expected calls for method %q have been exhausted", method,
)
}
if len(expected)+len(exhausted) == 0 {
_, _ = fmt.Fprintf(&callsErrors, "there are no expected calls of the method %q for that receiver", method)
}
return nil, fmt.Errorf(callsErrors.String())
return nil, errors.New(callsErrors.String())
}
// Failures returns the calls that are not satisfied.
......
......@@ -50,9 +50,6 @@
// mockObj.EXPECT().SomeMethod(2, "second"),
// mockObj.EXPECT().SomeMethod(3, "third"),
// )
//
// TODO:
// - Handle different argument/return types (e.g. ..., chan, map, interface).
package gomock
import (
......@@ -77,6 +74,15 @@ type TestHelper interface {
Helper()
}
// cleanuper is used to check if TestHelper also has the `Cleanup` method. A
// common pattern is to pass in a `*testing.T` to
// `NewController(t TestReporter)`. In Go 1.14+, `*testing.T` has a cleanup
// method. This can be utilized to call `Finish()` so the caller of this library
// does not have to.
type cleanuper interface {
Cleanup(func())
}
// A Controller represents the top-level control of a mock ecosystem. It
// defines the scope and lifetime of mock objects, as well as their
// expectations. It is safe to call Controller's methods from multiple
......@@ -115,29 +121,43 @@ type Controller struct {
// NewController returns a new Controller. It is the preferred way to create a
// Controller.
//
// New in go1.14+, if you are passing a *testing.T into this function you no
// longer need to call ctrl.Finish() in your test methods.
func NewController(t TestReporter) *Controller {
h, ok := t.(TestHelper)
if !ok {
h = nopTestHelper{t}
h = &nopTestHelper{t}
}
return &Controller{
ctrl := &Controller{
T: h,
expectedCalls: newCallSet(),
}
if c, ok := isCleanuper(ctrl.T); ok {
c.Cleanup(func() {
ctrl.T.Helper()
ctrl.finish(true, nil)
})
}
return ctrl
}
type cancelReporter struct {
TestHelper
t TestHelper
cancel func()
}
func (r *cancelReporter) Errorf(format string, args ...interface{}) {
r.TestHelper.Errorf(format, args...)
r.t.Errorf(format, args...)
}
func (r *cancelReporter) Fatalf(format string, args ...interface{}) {
defer r.cancel()
r.TestHelper.Fatalf(format, args...)
r.t.Fatalf(format, args...)
}
func (r *cancelReporter) Helper() {
r.t.Helper()
}
// WithContext returns a new Controller and a Context, which is cancelled on any
......@@ -145,15 +165,22 @@ func (r *cancelReporter) Fatalf(format string, args ...interface{}) {
func WithContext(ctx context.Context, t TestReporter) (*Controller, context.Context) {
h, ok := t.(TestHelper)
if !ok {
h = nopTestHelper{t}
h = &nopTestHelper{t: t}
}
ctx, cancel := context.WithCancel(ctx)
return NewController(&cancelReporter{h, cancel}), ctx
return NewController(&cancelReporter{t: h, cancel: cancel}), ctx
}
type nopTestHelper struct {
TestReporter
t TestReporter
}
func (h *nopTestHelper) Errorf(format string, args ...interface{}) {
h.t.Errorf(format, args...)
}
func (h *nopTestHelper) Fatalf(format string, args ...interface{}) {
h.t.Fatalf(format, args...)
}
func (h nopTestHelper) Helper() {}
......@@ -197,7 +224,10 @@ func (ctrl *Controller) Call(receiver interface{}, method string, args ...interf
expected, err := ctrl.expectedCalls.FindMatch(receiver, method, args)
if err != nil {
origin := callerInfo(2)
// callerInfo's skip should be updated if the number of calls between the user's test
// and this line changes, i.e. this code is wrapped in another anonymous function.
// 0 is us, 1 is controller.Call(), 2 is the generated mock, and 3 is the user's test.
origin := callerInfo(3)
ctrl.T.Fatalf("Unexpected call to %T.%v(%v) at %s because: %s", receiver, method, args, origin, err)
}
......@@ -229,21 +259,33 @@ func (ctrl *Controller) Call(receiver interface{}, method string, args ...interf
// Finish checks to see if all the methods that were expected to be called
// were called. It should be invoked for each Controller. It is not idempotent
// and therefore can only be invoked once.
//
// New in go1.14+, if you are passing a *testing.T into NewController function you no
// longer need to call ctrl.Finish() in your test methods.
func (ctrl *Controller) Finish() {
// If we're currently panicking, probably because this is a deferred call.
// This must be recovered in the deferred function.
err := recover()
ctrl.finish(false, err)
}
func (ctrl *Controller) finish(cleanup bool, panicErr interface{}) {
ctrl.T.Helper()
ctrl.mu.Lock()
defer ctrl.mu.Unlock()
if ctrl.finished {
ctrl.T.Fatalf("Controller.Finish was called more than once. It has to be called exactly once.")
if _, ok := isCleanuper(ctrl.T); !ok {
ctrl.T.Fatalf("Controller.Finish was called more than once. It has to be called exactly once.")
}
return
}
ctrl.finished = true
// If we're currently panicking, probably because this is a deferred call,
// pass through the panic.
if err := recover(); err != nil {
panic(err)
// Short-circuit, pass through the panic.
if panicErr != nil {
panic(panicErr)
}
// Check that all remaining expected calls are satisfied.
......@@ -252,13 +294,43 @@ func (ctrl *Controller) Finish() {
ctrl.T.Errorf("missing call(s) to %v", call)
}
if len(failures) != 0 {
ctrl.T.Fatalf("aborting test due to missing call(s)")
if !cleanup {
ctrl.T.Fatalf("aborting test due to missing call(s)")
return
}
ctrl.T.Errorf("aborting test due to missing call(s)")
}
}
// callerInfo returns the file:line of the call site. skip is the number
// of stack frames to skip when reporting. 0 is callerInfo's call site.
func callerInfo(skip int) string {
if _, file, line, ok := runtime.Caller(skip + 1); ok {
return fmt.Sprintf("%s:%d", file, line)
}
return "unknown file"
}
// isCleanuper checks it if t's base TestReporter has a Cleanup method.
func isCleanuper(t TestReporter) (cleanuper, bool) {
tr := unwrapTestReporter(t)
c, ok := tr.(cleanuper)
return c, ok
}
// unwrapTestReporter unwraps TestReporter to the base implementation.
func unwrapTestReporter(t TestReporter) TestReporter {
tr := t
switch nt := t.(type) {
case *cancelReporter:
tr = nt.t
if h, check := tr.(*nopTestHelper); check {
tr = h.t
}
case *nopTestHelper:
tr = nt.t
default:
// not wrapped
}
return tr
}
......@@ -102,11 +102,25 @@ type eqMatcher struct {
}
func (e eqMatcher) Matches(x interface{}) bool {
return reflect.DeepEqual(e.x, x)
// In case, some value is nil
if e.x == nil || x == nil {
return reflect.DeepEqual(e.x, x)
}
// Check if types assignable and convert them to common type
x1Val := reflect.ValueOf(e.x)
x2Val := reflect.ValueOf(x)
if x1Val.Type().AssignableTo(x2Val.Type()) {
x1ValConverted := x1Val.Convert(x2Val.Type())
return reflect.DeepEqual(x1ValConverted.Interface(), x2Val.Interface())
}
return false
}
func (e eqMatcher) String() string {
return fmt.Sprintf("is equal to %v", e.x)
return fmt.Sprintf("is equal to %v (%T)", e.x, e.x)
}
type nilMatcher struct{}
......@@ -139,7 +153,6 @@ func (n notMatcher) Matches(x interface{}) bool {
}
func (n notMatcher) String() string {
// TODO: Improve this if we add a NotString method to the Matcher interface.
return "not(" + n.m.String() + ")"
}
......@@ -194,6 +207,70 @@ func (m lenMatcher) String() string {
return fmt.Sprintf("has length %d", m.i)
}
type inAnyOrderMatcher struct {
x interface{}
}
func (m inAnyOrderMatcher) Matches(x interface{}) bool {
given, ok := m.prepareValue(x)
if !ok {
return false
}
wanted, ok := m.prepareValue(m.x)
if !ok {
return false
}
if given.Len() != wanted.Len() {
return false
}
usedFromGiven := make([]bool, given.Len())
foundFromWanted := make([]bool, wanted.Len())
for i := 0; i < wanted.Len(); i++ {
wantedMatcher := Eq(wanted.Index(i).Interface())
for j := 0; j < given.Len(); j++ {
if usedFromGiven[j] {
continue
}
if wantedMatcher.Matches(given.Index(j).Interface()) {
foundFromWanted[i] = true
usedFromGiven[j] = true
break
}
}
}
missingFromWanted := 0
for _, found := range foundFromWanted {
if !found {
missingFromWanted++
}
}
extraInGiven := 0
for _, used := range usedFromGiven {
if !used {
extraInGiven++
}
}
return extraInGiven == 0 && missingFromWanted == 0
}
func (m inAnyOrderMatcher) prepareValue(x interface{}) (reflect.Value, bool) {
xValue := reflect.ValueOf(x)
switch xValue.Kind() {
case reflect.Slice, reflect.Array:
return xValue, true
default:
return reflect.Value{}, false
}
}
func (m inAnyOrderMatcher) String() string {
return fmt.Sprintf("has the same elements as %v", m.x)
}
// Constructors
// All returns a composite Matcher that returns true if and only all of the
......@@ -245,7 +322,7 @@ func Not(x interface{}) Matcher {
// AssignableToTypeOf(s).Matches(time.Second) // returns true
// AssignableToTypeOf(s).Matches(99) // returns false
//
// var ctx = reflect.TypeOf((*context.Context)).Elem()
// var ctx = reflect.TypeOf((*context.Context)(nil)).Elem()
// AssignableToTypeOf(ctx).Matches(context.Background()) // returns true
func AssignableToTypeOf(x interface{}) Matcher {
if xt, ok := x.(reflect.Type); ok {
......@@ -253,3 +330,12 @@ func AssignableToTypeOf(x interface{}) Matcher {
}
return assignableToTypeOfMatcher{reflect.TypeOf(x)}
}
// InAnyOrder is a Matcher that returns true for collections of the same elements ignoring the order.
//
// Example usage:
// InAnyOrder([]int{1, 2, 3}).Matches([]int{1, 3, 2}) // returns true
// InAnyOrder([]int{1, 2, 3}).Matches([]int{1, 2}) // returns false
func InAnyOrder(x interface{}) Matcher {
return inAnyOrderMatcher{x}
}
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package descriptor provides functions for obtaining the protocol buffer
// descriptors of generated Go types.
//
// Deprecated: See the "google.golang.org/protobuf/reflect/protoreflect" package
// for how to obtain an EnumDescriptor or MessageDescriptor in order to
// programatically interact with the protobuf type system.
package descriptor
import (
"bytes"
"compress/gzip"
"io/ioutil"
"sync"
"github.com/golang/protobuf/proto"
"google.golang.org/protobuf/reflect/protodesc"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/runtime/protoimpl"
descriptorpb "github.com/golang/protobuf/protoc-gen-go/descriptor"
)
// Message is proto.Message with a method to return its descriptor.
//
// Deprecated: The Descriptor method may not be generated by future
// versions of protoc-gen-go, meaning that this interface may not
// be implemented by many concrete message types.
type Message interface {
proto.Message
Descriptor() ([]byte, []int)
}
// ForMessage returns the file descriptor proto containing
// the message and the message descriptor proto for the message itself.
// The returned proto messages must not be mutated.
//
// Deprecated: Not all concrete message types satisfy the Message interface.
// Use MessageDescriptorProto instead. If possible, the calling code should
// be rewritten to use protobuf reflection instead.
// See package "google.golang.org/protobuf/reflect/protoreflect" for details.
func ForMessage(m Message) (*descriptorpb.FileDescriptorProto, *descriptorpb.DescriptorProto) {
return MessageDescriptorProto(m)
}
type rawDesc struct {
fileDesc []byte
indexes []int
}
var rawDescCache sync.Map // map[protoreflect.Descriptor]*rawDesc
func deriveRawDescriptor(d protoreflect.Descriptor) ([]byte, []int) {
// Fast-path: check whether raw descriptors are already cached.
origDesc := d
if v, ok := rawDescCache.Load(origDesc); ok {
return v.(*rawDesc).fileDesc, v.(*rawDesc).indexes
}
// Slow-path: derive the raw descriptor from the v2 descriptor.
// Start with the leaf (a given enum or message declaration) and
// ascend upwards until we hit the parent file descriptor.
var idxs []int
for {
idxs = append(idxs, d.Index())
d = d.Parent()
if d == nil {
// TODO: We could construct a FileDescriptor stub for standalone
// descriptors to satisfy the API.
return nil, nil
}
if _, ok := d.(protoreflect.FileDescriptor); ok {
break
}
}
// Obtain the raw file descriptor.
var raw []byte
switch fd := d.(type) {
case interface{ ProtoLegacyRawDesc() []byte }:
raw = fd.ProtoLegacyRawDesc()
case protoreflect.FileDescriptor:
raw, _ = proto.Marshal(protodesc.ToFileDescriptorProto(fd))
}
file := protoimpl.X.CompressGZIP(raw)
// Reverse the indexes, since we populated it in reverse.
for i, j := 0, len(idxs)-1; i < j; i, j = i+1, j-1 {
idxs[i], idxs[j] = idxs[j], idxs[i]
}
if v, ok := rawDescCache.LoadOrStore(origDesc, &rawDesc{file, idxs}); ok {
return v.(*rawDesc).fileDesc, v.(*rawDesc).indexes
}
return file, idxs
}
// EnumRawDescriptor returns the GZIP'd raw file descriptor representing
// the enum and the index path to reach the enum declaration.
// The returned slices must not be mutated.
func EnumRawDescriptor(e proto.GeneratedEnum) ([]byte, []int) {
if ev, ok := e.(interface{ EnumDescriptor() ([]byte, []int) }); ok {
return ev.EnumDescriptor()
}
ed := protoimpl.X.EnumTypeOf(e)
return deriveRawDescriptor(ed.Descriptor())
}
// MessageRawDescriptor returns the GZIP'd raw file descriptor representing
// the message and the index path to reach the message declaration.
// The returned slices must not be mutated.
func MessageRawDescriptor(m proto.GeneratedMessage) ([]byte, []int) {
if mv, ok := m.(interface{ Descriptor() ([]byte, []int) }); ok {
return mv.Descriptor()
}
md := protoimpl.X.MessageTypeOf(m)
return deriveRawDescriptor(md.Descriptor())
}
var fileDescCache sync.Map // map[*byte]*descriptorpb.FileDescriptorProto
func deriveFileDescriptor(rawDesc []byte) *descriptorpb.FileDescriptorProto {
// Fast-path: check whether descriptor protos are already cached.
if v, ok := fileDescCache.Load(&rawDesc[0]); ok {
return v.(*descriptorpb.FileDescriptorProto)
}
// Slow-path: derive the descriptor proto from the GZIP'd message.
zr, err := gzip.NewReader(bytes.NewReader(rawDesc))
if err != nil {
panic(err)
}
b, err := ioutil.ReadAll(zr)
if err != nil {
panic(err)
}
fd := new(descriptorpb.FileDescriptorProto)
if err := proto.Unmarshal(b, fd); err != nil {
panic(err)
}
if v, ok := fileDescCache.LoadOrStore(&rawDesc[0], fd); ok {
return v.(*descriptorpb.FileDescriptorProto)
}
return fd
}
// EnumDescriptorProto returns the file descriptor proto representing
// the enum and the enum descriptor proto for the enum itself.
// The returned proto messages must not be mutated.
func EnumDescriptorProto(e proto.GeneratedEnum) (*descriptorpb.FileDescriptorProto, *descriptorpb.EnumDescriptorProto) {
rawDesc, idxs := EnumRawDescriptor(e)
if rawDesc == nil || idxs == nil {
return nil, nil
}
fd := deriveFileDescriptor(rawDesc)
if len(idxs) == 1 {
return fd, fd.EnumType[idxs[0]]
}
md := fd.MessageType[idxs[0]]
for _, i := range idxs[1 : len(idxs)-1] {
md = md.NestedType[i]
}
ed := md.EnumType[idxs[len(idxs)-1]]
return fd, ed
}
// MessageDescriptorProto returns the file descriptor proto representing
// the message and the message descriptor proto for the message itself.
// The returned proto messages must not be mutated.
func MessageDescriptorProto(m proto.GeneratedMessage) (*descriptorpb.FileDescriptorProto, *descriptorpb.DescriptorProto) {
rawDesc, idxs := MessageRawDescriptor(m)
if rawDesc == nil || idxs == nil {
return nil, nil
}
fd := deriveFileDescriptor(rawDesc)
md := fd.MessageType[idxs[0]]
for _, i := range idxs[1:] {
md = md.NestedType[i]
}
return fd, md
}
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package remap handles tracking the locations of Go tokens in a source text
// across a rewrite by the Go formatter.
package remap
import (
"fmt"
"go/scanner"
"go/token"
)
// A Location represents a span of byte offsets in the source text.
type Location struct {
Pos, End int // End is exclusive
}
// A Map represents a mapping between token locations in an input source text
// and locations in the correspnding output text.
type Map map[Location]Location
// Find reports whether the specified span is recorded by m, and if so returns
// the new location it was mapped to. If the input span was not found, the
// returned location is the same as the input.
func (m Map) Find(pos, end int) (Location, bool) {
key := Location{
Pos: pos,
End: end,
}
if loc, ok := m[key]; ok {
return loc, true
}
return key, false
}
func (m Map) add(opos, oend, npos, nend int) {
m[Location{Pos: opos, End: oend}] = Location{Pos: npos, End: nend}
}
// Compute constructs a location mapping from input to output. An error is
// reported if any of the tokens of output cannot be mapped.
func Compute(input, output []byte) (Map, error) {
itok := tokenize(input)
otok := tokenize(output)
if len(itok) != len(otok) {
return nil, fmt.Errorf("wrong number of tokens, %d ≠ %d", len(itok), len(otok))
}
m := make(Map)
for i, ti := range itok {
to := otok[i]
if ti.Token != to.Token {
return nil, fmt.Errorf("token %d type mismatch: %s ≠ %s", i+1, ti, to)
}
m.add(ti.pos, ti.end, to.pos, to.end)
}
return m, nil
}
// tokinfo records the span and type of a source token.
type tokinfo struct {
pos, end int
token.Token
}
func tokenize(src []byte) []tokinfo {
fs := token.NewFileSet()
var s scanner.Scanner
s.Init(fs.AddFile("src", fs.Base(), len(src)), src, nil, scanner.ScanComments)
var info []tokinfo
for {
pos, next, lit := s.Scan()
switch next {
case token.SEMICOLON:
continue
}
info = append(info, tokinfo{
pos: int(pos - 1),
end: int(pos + token.Pos(len(lit)) - 1),
Token: next,
})
if next == token.EOF {
break
}
}
return info
}
// Code generated by protoc-gen-go. DO NOT EDIT.
// source: github.com/golang/protobuf/protoc-gen-go/plugin/plugin.proto
package plugin_go
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
pluginpb "google.golang.org/protobuf/types/pluginpb"
reflect "reflect"
)
// Symbols defined in public import of google/protobuf/compiler/plugin.proto.
type CodeGeneratorResponse_Feature = pluginpb.CodeGeneratorResponse_Feature
const CodeGeneratorResponse_FEATURE_NONE = pluginpb.CodeGeneratorResponse_FEATURE_NONE
const CodeGeneratorResponse_FEATURE_PROTO3_OPTIONAL = pluginpb.CodeGeneratorResponse_FEATURE_PROTO3_OPTIONAL
var CodeGeneratorResponse_Feature_name = pluginpb.CodeGeneratorResponse_Feature_name
var CodeGeneratorResponse_Feature_value = pluginpb.CodeGeneratorResponse_Feature_value
type Version = pluginpb.Version
type CodeGeneratorRequest = pluginpb.CodeGeneratorRequest
type CodeGeneratorResponse = pluginpb.CodeGeneratorResponse
type CodeGeneratorResponse_File = pluginpb.CodeGeneratorResponse_File
var File_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto protoreflect.FileDescriptor
var file_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto_rawDesc = []byte{
0x0a, 0x3c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6c,
0x61, 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x63, 0x2d, 0x67, 0x65, 0x6e, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69,
0x6e, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x25,
0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f,
0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x72, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2e,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x62, 0x75, 0x66, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x2d, 0x67, 0x65, 0x6e, 0x2d, 0x67,
0x6f, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x3b, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x5f,
0x67, 0x6f, 0x50, 0x00, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32,
}
var file_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto_goTypes = []interface{}{}
var file_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto_depIdxs = []int32{
0, // [0:0] is the sub-list for method output_type
0, // [0:0] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
}
func init() { file_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto_init() }
func file_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto_init() {
if File_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto != nil {
return
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto_rawDesc,
NumEnums: 0,
NumMessages: 0,
NumExtensions: 0,
NumServices: 0,
},
GoTypes: file_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto_goTypes,
DependencyIndexes: file_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto_depIdxs,
}.Build()
File_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto = out.File
file_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto_rawDesc = nil
file_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto_goTypes = nil
file_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto_depIdxs = nil
}
......@@ -315,7 +315,7 @@ func (tf Transform) Option() Option { return tf.trans }
// pops the address from the stack. Thus, when traversing into a pointer from
// reflect.Ptr, reflect.Slice element, or reflect.Map, we can detect cycles
// by checking whether the pointer has already been visited. The cycle detection
// uses a seperate stack for the x and y values.
// uses a separate stack for the x and y values.
//
// If a cycle is detected we need to determine whether the two pointers
// should be considered equal. The definition of equality chosen by Equal
......
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package and provides helpers for adding Close to io.{Reader|Writer}.
package and
import (
"io"
)
// ReadCloser implements io.ReadCloser by reading from a particular io.Reader
// and then calling the provided "Close()" method.
type ReadCloser struct {
io.Reader
CloseFunc func() error
}
var _ io.ReadCloser = (*ReadCloser)(nil)
// Close implements io.ReadCloser
func (rac *ReadCloser) Close() error {
return rac.CloseFunc()
}
// WriteCloser implements io.WriteCloser by reading from a particular io.Writer
// and then calling the provided "Close()" method.
type WriteCloser struct {
io.Writer
CloseFunc func() error
}
var _ io.WriteCloser = (*WriteCloser)(nil)
// Close implements io.WriteCloser
func (wac *WriteCloser) Close() error {
return wac.CloseFunc()
}
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package estargz adapts the containerd estargz package to our abstractions.
package estargz
import (
"bytes"
"io"
"io/ioutil"
"github.com/containerd/stargz-snapshotter/estargz"
v1 "github.com/google/go-containerregistry/pkg/v1"
)
// Assert that what we're returning is an io.ReadCloser
var _ io.ReadCloser = (*estargz.Blob)(nil)
// ReadCloser reads uncompressed tarball input from the io.ReadCloser and
// returns:
// * An io.ReadCloser from which compressed data may be read, and
// * A v1.Hash with the hash of the estargz table of contents, or
// * An error if the estargz processing encountered a problem.
//
// Refer to estargz for the options:
// https://pkg.go.dev/github.com/containerd/stargz-snapshotter/estargz@v0.4.1#Option
func ReadCloser(r io.ReadCloser, opts ...estargz.Option) (*estargz.Blob, v1.Hash, error) {
defer r.Close()
// TODO(#876): Avoid buffering into memory.
bs, err := ioutil.ReadAll(r)
if err != nil {
return nil, v1.Hash{}, err
}
br := bytes.NewReader(bs)
rc, err := estargz.Build(io.NewSectionReader(br, 0, int64(len(bs))), opts...)
if err != nil {
return nil, v1.Hash{}, err
}
h, err := v1.NewHash(rc.TOCDigest().String())
return rc, h, err
}
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package gzip provides helper functions for interacting with gzipped streams.
package gzip
import (
"bufio"
"bytes"
"compress/gzip"
"io"
"github.com/google/go-containerregistry/internal/and"
)
var gzipMagicHeader = []byte{'\x1f', '\x8b'}
// ReadCloser reads uncompressed input data from the io.ReadCloser and
// returns an io.ReadCloser from which compressed data may be read.
// This uses gzip.BestSpeed for the compression level.
func ReadCloser(r io.ReadCloser) io.ReadCloser {
return ReadCloserLevel(r, gzip.BestSpeed)
}
// ReadCloserLevel reads uncompressed input data from the io.ReadCloser and
// returns an io.ReadCloser from which compressed data may be read.
// Refer to compress/gzip for the level:
// https://golang.org/pkg/compress/gzip/#pkg-constants
func ReadCloserLevel(r io.ReadCloser, level int) io.ReadCloser {
pr, pw := io.Pipe()
// For highly compressible layers, gzip.Writer will output a very small
// number of bytes per Write(). This is normally fine, but when pushing
// to a registry, we want to ensure that we're taking full advantage of
// the available bandwidth instead of sending tons of tiny writes over
// the wire.
// 64K ought to be small enough for anybody.
bw := bufio.NewWriterSize(pw, 2<<16)
// Returns err so we can pw.CloseWithError(err)
go func() error {
// TODO(go1.14): Just defer {pw,gw,r}.Close like you'd expect.
// Context: https://golang.org/issue/24283
gw, err := gzip.NewWriterLevel(bw, level)
if err != nil {
return pw.CloseWithError(err)
}
if _, err := io.Copy(gw, r); err != nil {
defer r.Close()
defer gw.Close()
return pw.CloseWithError(err)
}
// Close gzip writer to Flush it and write gzip trailers.
if err := gw.Close(); err != nil {
return pw.CloseWithError(err)
}
// Flush bufio writer to ensure we write out everything.
if err := bw.Flush(); err != nil {
return pw.CloseWithError(err)
}
// We don't really care if these fail.
defer pw.Close()
defer r.Close()
return nil
}()
return pr
}
// UnzipReadCloser reads compressed input data from the io.ReadCloser and
// returns an io.ReadCloser from which uncompessed data may be read.
func UnzipReadCloser(r io.ReadCloser) (io.ReadCloser, error) {
gr, err := gzip.NewReader(r)
if err != nil {
return nil, err
}
return &and.ReadCloser{
Reader: gr,
CloseFunc: func() error {
// If the unzip fails, then this seems to return the same
// error as the read. We don't want this to interfere with
// us closing the main ReadCloser, since this could leave
// an open file descriptor (fails on Windows).
gr.Close()
return r.Close()
},
}, nil
}
// Is detects whether the input stream is compressed.
func Is(r io.Reader) (bool, error) {
magicHeader := make([]byte, 2)
n, err := r.Read(magicHeader)
if n == 0 && err == io.EOF {
return false, nil
}
if err != nil {
return false, err
}
return bytes.Equal(magicHeader, gzipMagicHeader), nil
}
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package redact contains a simple context signal for redacting requests.
package redact
import (
"context"
)
type contextKey string
var redactKey = contextKey("redact")
// NewContext creates a new ctx with the reason for redaction.
func NewContext(ctx context.Context, reason string) context.Context {
return context.WithValue(ctx, redactKey, reason)
}
// FromContext returns the redaction reason, if any.
func FromContext(ctx context.Context) (bool, string) {
reason, ok := ctx.Value(redactKey).(string)
return ok, reason
}
// Copyright 2019 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package retry provides methods for retrying operations. It is a thin wrapper
// around k8s.io/apimachinery/pkg/util/wait to make certain operations easier.
package retry
import (
"context"
"fmt"
"github.com/google/go-containerregistry/internal/retry/wait"
)
// Backoff is an alias of our own wait.Backoff to avoid name conflicts with
// the kubernetes wait package. Typing retry.Backoff is aesier than fixing
// the wrong import every time you use wait.Backoff.
type Backoff = wait.Backoff
// This is implemented by several errors in the net package as well as our
// transport.Error.
type temporary interface {
Temporary() bool
}
// IsTemporary returns true if err implements Temporary() and it returns true.
func IsTemporary(err error) bool {
if err == context.DeadlineExceeded {
return false
}
if te, ok := err.(temporary); ok && te.Temporary() {
return true
}
return false
}
// IsNotNil returns true if err is not nil.
func IsNotNil(err error) bool {
return err != nil
}
// Predicate determines whether an error should be retried.
type Predicate func(error) (retry bool)
// Retry retries a given function, f, until a predicate is satisfied, using
// exponential backoff. If the predicate is never satisfied, it will return the
// last error returned by f.
func Retry(f func() error, p Predicate, backoff wait.Backoff) (err error) {
if f == nil {
return fmt.Errorf("nil f passed to retry")
}
if p == nil {
return fmt.Errorf("nil p passed to retry")
}
condition := func() (bool, error) {
err = f()
if p(err) {
return false, nil
}
return true, err
}
wait.ExponentialBackoff(backoff, condition)
return
}
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package wait is a subset of k8s.io/apimachinery to avoid conflicts
// in dependencies (specifically, logging).
package wait
import (
"errors"
"math/rand"
"time"
)
// Jitter returns a time.Duration between duration and duration + maxFactor *
// duration.
//
// This allows clients to avoid converging on periodic behavior. If maxFactor
// is 0.0, a suggested default value will be chosen.
func Jitter(duration time.Duration, maxFactor float64) time.Duration {
if maxFactor <= 0.0 {
maxFactor = 1.0
}
wait := duration + time.Duration(rand.Float64()*maxFactor*float64(duration))
return wait
}
// ErrWaitTimeout is returned when the condition exited without success.
var ErrWaitTimeout = errors.New("timed out waiting for the condition")
// ConditionFunc returns true if the condition is satisfied, or an error
// if the loop should be aborted.
type ConditionFunc func() (done bool, err error)
// Backoff holds parameters applied to a Backoff function.
type Backoff struct {
// The initial duration.
Duration time.Duration
// Duration is multiplied by factor each iteration, if factor is not zero
// and the limits imposed by Steps and Cap have not been reached.
// Should not be negative.
// The jitter does not contribute to the updates to the duration parameter.
Factor float64
// The sleep at each iteration is the duration plus an additional
// amount chosen uniformly at random from the interval between
// zero and `jitter*duration`.
Jitter float64
// The remaining number of iterations in which the duration
// parameter may change (but progress can be stopped earlier by
// hitting the cap). If not positive, the duration is not
// changed. Used for exponential backoff in combination with
// Factor and Cap.
Steps int
// A limit on revised values of the duration parameter. If a
// multiplication by the factor parameter would make the duration
// exceed the cap then the duration is set to the cap and the
// steps parameter is set to zero.
Cap time.Duration
}
// Step (1) returns an amount of time to sleep determined by the
// original Duration and Jitter and (2) mutates the provided Backoff
// to update its Steps and Duration.
func (b *Backoff) Step() time.Duration {
if b.Steps < 1 {
if b.Jitter > 0 {
return Jitter(b.Duration, b.Jitter)
}
return b.Duration
}
b.Steps--
duration := b.Duration
// calculate the next step
if b.Factor != 0 {
b.Duration = time.Duration(float64(b.Duration) * b.Factor)
if b.Cap > 0 && b.Duration > b.Cap {
b.Duration = b.Cap
b.Steps = 0
}
}
if b.Jitter > 0 {
duration = Jitter(duration, b.Jitter)
}
return duration
}
// ExponentialBackoff repeats a condition check with exponential backoff.
//
// It repeatedly checks the condition and then sleeps, using `backoff.Step()`
// to determine the length of the sleep and adjust Duration and Steps.
// Stops and returns as soon as:
// 1. the condition check returns true or an error,
// 2. `backoff.Steps` checks of the condition have been done, or
// 3. a sleep truncated by the cap on duration has been completed.
// In case (1) the returned error is what the condition function returned.
// In all other cases, ErrWaitTimeout is returned.
func ExponentialBackoff(backoff Backoff, condition ConditionFunc) error {
for backoff.Steps > 0 {
if ok, err := condition(); err != nil || ok {
return err
}
if backoff.Steps == 1 {
break
}
time.Sleep(backoff.Step())
}
return ErrWaitTimeout
}
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package verify provides a ReadCloser that verifies content matches the
// expected hash values.
package verify
import (
"bytes"
"encoding/hex"
"errors"
"fmt"
"hash"
"io"
"github.com/google/go-containerregistry/internal/and"
v1 "github.com/google/go-containerregistry/pkg/v1"
)
// SizeUnknown is a sentinel value to indicate that the expected size is not known.
const SizeUnknown = -1
type verifyReader struct {
inner io.Reader
hasher hash.Hash
expected v1.Hash
gotSize, wantSize int64
}
// Read implements io.Reader
func (vc *verifyReader) Read(b []byte) (int, error) {
n, err := vc.inner.Read(b)
vc.gotSize += int64(n)
if err == io.EOF {
if vc.wantSize != SizeUnknown && vc.gotSize != vc.wantSize {
return n, fmt.Errorf("error verifying size; got %d, want %d", vc.gotSize, vc.wantSize)
}
got := hex.EncodeToString(vc.hasher.Sum(make([]byte, 0, vc.hasher.Size())))
if want := vc.expected.Hex; got != want {
return n, fmt.Errorf("error verifying %s checksum after reading %d bytes; got %q, want %q",
vc.expected.Algorithm, vc.gotSize, got, want)
}
}
return n, err
}
// ReadCloser wraps the given io.ReadCloser to verify that its contents match
// the provided v1.Hash before io.EOF is returned.
//
// The reader will only be read up to size bytes, to prevent resource
// exhaustion. If EOF is returned before size bytes are read, an error is
// returned.
//
// A size of SizeUnknown (-1) indicates disables size verification when the size
// is unknown ahead of time.
func ReadCloser(r io.ReadCloser, size int64, h v1.Hash) (io.ReadCloser, error) {
w, err := v1.Hasher(h.Algorithm)
if err != nil {
return nil, err
}
var r2 io.Reader = r
if size != SizeUnknown {
r2 = io.LimitReader(io.TeeReader(r, w), size)
}
return &and.ReadCloser{
Reader: &verifyReader{
inner: r2,
hasher: w,
expected: h,
wantSize: size,
},
CloseFunc: r.Close,
}, nil
}
// Descriptor verifies that the embedded Data field matches the Size and Digest
// fields of the given v1.Descriptor, returning an error if the Data field is
// missing or if it contains incorrect data.
func Descriptor(d v1.Descriptor) error {
if d.Data == nil {
return errors.New("error verifying descriptor; Data == nil")
}
h, sz, err := v1.SHA256(bytes.NewReader(d.Data))
if err != nil {
return err
}
if h != d.Digest {
return fmt.Errorf("error verifying Digest; got %q, want %q", h, d.Digest)
}
if sz != d.Size {
return fmt.Errorf("error verifying Size; got %d, want %d", sz, d.Size)
}
return nil
}
......@@ -16,6 +16,7 @@ package authn
import (
"os"
"sync"
"github.com/docker/cli/cli/config"
"github.com/docker/cli/cli/config/types"
......@@ -42,7 +43,9 @@ type Keychain interface {
// defaultKeychain implements Keychain with the semantics of the standard Docker
// credential keychain.
type defaultKeychain struct{}
type defaultKeychain struct {
mu sync.Mutex
}
var (
// DefaultKeychain implements Keychain by interpreting the docker config file.
......@@ -57,6 +60,8 @@ const (
// Resolve implements Keychain.
func (dk *defaultKeychain) Resolve(target Resource) (Authenticator, error) {
dk.mu.Lock()
defer dk.mu.Unlock()
cf, err := config.Load(os.Getenv("DOCKER_CONFIG"))
if err != nil {
return nil, err
......
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package logs exposes the loggers used by this library.
package logs
import (
"io/ioutil"
"log"
)
var (
// Warn is used to log non-fatal errors.
Warn = log.New(ioutil.Discard, "", log.LstdFlags)
// Progress is used to log notable, successful events.
Progress = log.New(ioutil.Discard, "", log.LstdFlags)
// Debug is used to log information that is useful for debugging.
Debug = log.New(ioutil.Discard, "", log.LstdFlags)
)
// Enabled checks to see if the logger's writer is set to something other
// than ioutil.Discard. This allows callers to avoid expensive operations
// that will end up in /dev/null anyway.
func Enabled(l *log.Logger) bool {
return l.Writer() != ioutil.Discard
}
......@@ -35,9 +35,9 @@ func stripRunesFn(runes string) func(rune) rune {
func checkElement(name, element, allowedRunes string, minRunes, maxRunes int) error {
numRunes := utf8.RuneCountInString(element)
if (numRunes < minRunes) || (maxRunes < numRunes) {
return NewErrBadName("%s must be between %d and %d runes in length: %s", name, minRunes, maxRunes, element)
return newErrBadName("%s must be between %d and %d runes in length: %s", name, minRunes, maxRunes, element)
} else if len(strings.Map(stripRunesFn(allowedRunes), element)) != 0 {
return NewErrBadName("%s can only contain the runes `%s`: %s", name, allowedRunes, element)
return newErrBadName("%s can only contain the runes `%s`: %s", name, allowedRunes, element)
}
return nil
}
......@@ -69,7 +69,7 @@ func NewDigest(name string, opts ...Option) (Digest, error) {
// Split on "@"
parts := strings.Split(name, digestDelim)
if len(parts) != 2 {
return Digest{}, NewErrBadName("a digest must contain exactly one '@' separator (e.g. registry/repository@digest) saw: %s", name)
return Digest{}, newErrBadName("a digest must contain exactly one '@' separator (e.g. registry/repository@digest) saw: %s", name)
}
base := parts[0]
digest := parts[1]
......
......@@ -14,7 +14,10 @@
package name
import "fmt"
import (
"errors"
"fmt"
)
// ErrBadName is an error for when a bad docker name is supplied.
type ErrBadName struct {
......@@ -25,13 +28,15 @@ func (e *ErrBadName) Error() string {
return e.info
}
// NewErrBadName returns a ErrBadName which returns the given formatted string from Error().
func NewErrBadName(fmtStr string, args ...interface{}) *ErrBadName {
// newErrBadName returns a ErrBadName which returns the given formatted string from Error().
func newErrBadName(fmtStr string, args ...interface{}) *ErrBadName {
return &ErrBadName{fmt.Sprintf(fmtStr, args...)}
}
// IsErrBadName returns true if the given error is an ErrBadName.
//
// Deprecated: Use errors.Is.
func IsErrBadName(err error) bool {
_, ok := err.(*ErrBadName)
return ok
var berr *ErrBadName
return errors.As(err, &berr)
}
......@@ -44,8 +44,7 @@ func ParseReference(s string, opts ...Option) (Reference, error) {
if d, err := NewDigest(s, opts...); err == nil {
return d, nil
}
return nil, NewErrBadName("could not parse reference: " + s)
return nil, newErrBadName("could not parse reference: " + s)
}
type stringConst string
......
......@@ -98,7 +98,7 @@ func checkRegistry(name string) error {
// Per RFC 3986, registries (authorities) are required to be prefixed with "//"
// url.Host == hostname[:port] == authority
if url, err := url.Parse("//" + name); err != nil || url.Host != name {
return NewErrBadName("registries must be valid RFC 3986 URI authorities: %s", name)
return newErrBadName("registries must be valid RFC 3986 URI authorities: %s", name)
}
return nil
}
......@@ -108,7 +108,7 @@ func checkRegistry(name string) error {
func NewRegistry(name string, opts ...Option) (Registry, error) {
opt := makeOptions(opts...)
if opt.strict && len(name) == 0 {
return Registry{}, NewErrBadName("strict validation requires the registry to be explicitly defined")
return Registry{}, newErrBadName("strict validation requires the registry to be explicitly defined")
}
if err := checkRegistry(name); err != nil {
......
......@@ -72,7 +72,7 @@ func checkRepository(repository string) error {
func NewRepository(name string, opts ...Option) (Repository, error) {
opt := makeOptions(opts...)
if len(name) == 0 {
return Repository{}, NewErrBadName("a repository name must be specified")
return Repository{}, newErrBadName("a repository name must be specified")
}
var registry string
......@@ -95,7 +95,7 @@ func NewRepository(name string, opts ...Option) (Repository, error) {
return Repository{}, err
}
if hasImplicitNamespace(repo, reg) && opt.strict {
return Repository{}, NewErrBadName("strict validation requires the full repository path (missing 'library')")
return Repository{}, newErrBadName("strict validation requires the full repository path (missing 'library')")
}
return Repository{reg, repo}, nil
}
......
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"encoding/json"
"io"
"time"
)
// ConfigFile is the configuration file that holds the metadata describing
// how to launch a container. See:
// https://github.com/opencontainers/image-spec/blob/master/config.md
//
// docker_version and os.version are not part of the spec but included
// for backwards compatibility.
type ConfigFile struct {
Architecture string `json:"architecture"`
Author string `json:"author,omitempty"`
Container string `json:"container,omitempty"`
Created Time `json:"created,omitempty"`
DockerVersion string `json:"docker_version,omitempty"`
History []History `json:"history,omitempty"`
OS string `json:"os"`
RootFS RootFS `json:"rootfs"`
Config Config `json:"config"`
OSVersion string `json:"os.version,omitempty"`
}
// History is one entry of a list recording how this container image was built.
type History struct {
Author string `json:"author,omitempty"`
Created Time `json:"created,omitempty"`
CreatedBy string `json:"created_by,omitempty"`
Comment string `json:"comment,omitempty"`
EmptyLayer bool `json:"empty_layer,omitempty"`
}
// Time is a wrapper around time.Time to help with deep copying
type Time struct {
time.Time
}
// DeepCopyInto creates a deep-copy of the Time value. The underlying time.Time
// type is effectively immutable in the time API, so it is safe to
// copy-by-assign, despite the presence of (unexported) Pointer fields.
func (t *Time) DeepCopyInto(out *Time) {
*out = *t
}
// RootFS holds the ordered list of file system deltas that comprise the
// container image's root filesystem.
type RootFS struct {
Type string `json:"type"`
DiffIDs []Hash `json:"diff_ids"`
}
// HealthConfig holds configuration settings for the HEALTHCHECK feature.
type HealthConfig struct {
// Test is the test to perform to check that the container is healthy.
// An empty slice means to inherit the default.
// The options are:
// {} : inherit healthcheck
// {"NONE"} : disable healthcheck
// {"CMD", args...} : exec arguments directly
// {"CMD-SHELL", command} : run command with system's default shell
Test []string `json:",omitempty"`
// Zero means to inherit. Durations are expressed as integer nanoseconds.
Interval time.Duration `json:",omitempty"` // Interval is the time to wait between checks.
Timeout time.Duration `json:",omitempty"` // Timeout is the time to wait before considering the check to have hung.
StartPeriod time.Duration `json:",omitempty"` // The start period for the container to initialize before the retries starts to count down.
// Retries is the number of consecutive failures needed to consider a container as unhealthy.
// Zero means inherit.
Retries int `json:",omitempty"`
}
// Config is a submessage of the config file described as:
// The execution parameters which SHOULD be used as a base when running
// a container using the image.
// The names of the fields in this message are chosen to reflect the JSON
// payload of the Config as defined here:
// https://git.io/vrAET
// and
// https://github.com/opencontainers/image-spec/blob/master/config.md
type Config struct {
AttachStderr bool `json:"AttachStderr,omitempty"`
AttachStdin bool `json:"AttachStdin,omitempty"`
AttachStdout bool `json:"AttachStdout,omitempty"`
Cmd []string `json:"Cmd,omitempty"`
Healthcheck *HealthConfig `json:"Healthcheck,omitempty"`
Domainname string `json:"Domainname,omitempty"`
Entrypoint []string `json:"Entrypoint,omitempty"`
Env []string `json:"Env,omitempty"`
Hostname string `json:"Hostname,omitempty"`
Image string `json:"Image,omitempty"`
Labels map[string]string `json:"Labels,omitempty"`
OnBuild []string `json:"OnBuild,omitempty"`
OpenStdin bool `json:"OpenStdin,omitempty"`
StdinOnce bool `json:"StdinOnce,omitempty"`
Tty bool `json:"Tty,omitempty"`
User string `json:"User,omitempty"`
Volumes map[string]struct{} `json:"Volumes,omitempty"`
WorkingDir string `json:"WorkingDir,omitempty"`
ExposedPorts map[string]struct{} `json:"ExposedPorts,omitempty"`
ArgsEscaped bool `json:"ArgsEscaped,omitempty"`
NetworkDisabled bool `json:"NetworkDisabled,omitempty"`
MacAddress string `json:"MacAddress,omitempty"`
StopSignal string `json:"StopSignal,omitempty"`
Shell []string `json:"Shell,omitempty"`
}
// ParseConfigFile parses the io.Reader's contents into a ConfigFile.
func ParseConfigFile(r io.Reader) (*ConfigFile, error) {
cf := ConfigFile{}
if err := json.NewDecoder(r).Decode(&cf); err != nil {
return nil, err
}
return &cf, nil
}
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +k8s:deepcopy-gen=package
// Package v1 defines structured types for OCI v1 images
package v1
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"hash"
"io"
"strconv"
"strings"
)
// Hash is an unqualified digest of some content, e.g. sha256:deadbeef
type Hash struct {
// Algorithm holds the algorithm used to compute the hash.
Algorithm string
// Hex holds the hex portion of the content hash.
Hex string
}
// String reverses NewHash returning the string-form of the hash.
func (h Hash) String() string {
return fmt.Sprintf("%s:%s", h.Algorithm, h.Hex)
}
// NewHash validates the input string is a hash and returns a strongly type Hash object.
func NewHash(s string) (Hash, error) {
h := Hash{}
if err := h.parse(s); err != nil {
return Hash{}, err
}
return h, nil
}
// MarshalJSON implements json.Marshaler
func (h Hash) MarshalJSON() ([]byte, error) {
return json.Marshal(h.String())
}
// UnmarshalJSON implements json.Unmarshaler
func (h *Hash) UnmarshalJSON(data []byte) error {
s, err := strconv.Unquote(string(data))
if err != nil {
return err
}
return h.parse(s)
}
// MarshalText implements encoding.TextMarshaler. This is required to use
// v1.Hash as a key in a map when marshalling JSON.
func (h Hash) MarshalText() (text []byte, err error) {
return []byte(h.String()), nil
}
// UnmarshalText implements encoding.TextUnmarshaler. This is required to use
// v1.Hash as a key in a map when unmarshalling JSON.
func (h *Hash) UnmarshalText(text []byte) error {
return h.parse(string(text))
}
// Hasher returns a hash.Hash for the named algorithm (e.g. "sha256")
func Hasher(name string) (hash.Hash, error) {
switch name {
case "sha256":
return sha256.New(), nil
default:
return nil, fmt.Errorf("unsupported hash: %q", name)
}
}
func (h *Hash) parse(unquoted string) error {
parts := strings.Split(unquoted, ":")
if len(parts) != 2 {
return fmt.Errorf("cannot parse hash: %q", unquoted)
}
rest := strings.TrimLeft(parts[1], "0123456789abcdef")
if len(rest) != 0 {
return fmt.Errorf("found non-hex character in hash: %c", rest[0])
}
hasher, err := Hasher(parts[0])
if err != nil {
return err
}
// Compare the hex to the expected size (2 hex characters per byte)
if len(parts[1]) != hasher.Size()*2 {
return fmt.Errorf("wrong number of hex digits for %s: %s", parts[0], parts[1])
}
h.Algorithm = parts[0]
h.Hex = parts[1]
return nil
}
// SHA256 computes the Hash of the provided io.Reader's content.
func SHA256(r io.Reader) (Hash, int64, error) {
hasher := sha256.New()
n, err := io.Copy(hasher, r)
if err != nil {
return Hash{}, 0, err
}
return Hash{
Algorithm: "sha256",
Hex: hex.EncodeToString(hasher.Sum(make([]byte, 0, hasher.Size()))),
}, n, nil
}
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"github.com/google/go-containerregistry/pkg/v1/types"
)
// Image defines the interface for interacting with an OCI v1 image.
type Image interface {
// Layers returns the ordered collection of filesystem layers that comprise this image.
// The order of the list is oldest/base layer first, and most-recent/top layer last.
Layers() ([]Layer, error)
// MediaType of this image's manifest.
MediaType() (types.MediaType, error)
// Size returns the size of the manifest.
Size() (int64, error)
// ConfigName returns the hash of the image's config file, also known as
// the Image ID.
ConfigName() (Hash, error)
// ConfigFile returns this image's config file.
ConfigFile() (*ConfigFile, error)
// RawConfigFile returns the serialized bytes of ConfigFile().
RawConfigFile() ([]byte, error)
// Digest returns the sha256 of this image's manifest.
Digest() (Hash, error)
// Manifest returns this image's Manifest object.
Manifest() (*Manifest, error)
// RawManifest returns the serialized bytes of Manifest()
RawManifest() ([]byte, error)
// LayerByDigest returns a Layer for interacting with a particular layer of
// the image, looking it up by "digest" (the compressed hash).
LayerByDigest(Hash) (Layer, error)
// LayerByDiffID is an analog to LayerByDigest, looking up by "diff id"
// (the uncompressed hash).
LayerByDiffID(Hash) (Layer, error)
}
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"github.com/google/go-containerregistry/pkg/v1/types"
)
// ImageIndex defines the interface for interacting with an OCI image index.
type ImageIndex interface {
// MediaType of this image's manifest.
MediaType() (types.MediaType, error)
// Digest returns the sha256 of this index's manifest.
Digest() (Hash, error)
// Size returns the size of the manifest.
Size() (int64, error)
// IndexManifest returns this image index's manifest object.
IndexManifest() (*IndexManifest, error)
// RawManifest returns the serialized bytes of IndexManifest().
RawManifest() ([]byte, error)
// Image returns a v1.Image that this ImageIndex references.
Image(Hash) (Image, error)
// ImageIndex returns a v1.ImageIndex that this ImageIndex references.
ImageIndex(Hash) (ImageIndex, error)
}
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"io"
"github.com/google/go-containerregistry/pkg/v1/types"
)
// Layer is an interface for accessing the properties of a particular layer of a v1.Image
type Layer interface {
// Digest returns the Hash of the compressed layer.
Digest() (Hash, error)
// DiffID returns the Hash of the uncompressed layer.
DiffID() (Hash, error)
// Compressed returns an io.ReadCloser for the compressed layer contents.
Compressed() (io.ReadCloser, error)
// Uncompressed returns an io.ReadCloser for the uncompressed layer contents.
Uncompressed() (io.ReadCloser, error)
// Size returns the compressed size of the Layer.
Size() (int64, error)
// MediaType returns the media type of the Layer.
MediaType() (types.MediaType, error)
}
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"encoding/json"
"io"
"github.com/google/go-containerregistry/pkg/v1/types"
)
// Manifest represents the OCI image manifest in a structured way.
type Manifest struct {
SchemaVersion int64 `json:"schemaVersion"`
MediaType types.MediaType `json:"mediaType,omitempty"`
Config Descriptor `json:"config"`
Layers []Descriptor `json:"layers"`
Annotations map[string]string `json:"annotations,omitempty"`
}
// IndexManifest represents an OCI image index in a structured way.
type IndexManifest struct {
SchemaVersion int64 `json:"schemaVersion"`
MediaType types.MediaType `json:"mediaType,omitempty"`
Manifests []Descriptor `json:"manifests"`
Annotations map[string]string `json:"annotations,omitempty"`
}
// Descriptor holds a reference from the manifest to one of its constituent elements.
type Descriptor struct {
MediaType types.MediaType `json:"mediaType"`
Size int64 `json:"size"`
Digest Hash `json:"digest"`
Data []byte `json:"data,omitempty"`
URLs []string `json:"urls,omitempty"`
Annotations map[string]string `json:"annotations,omitempty"`
Platform *Platform `json:"platform,omitempty"`
}
// ParseManifest parses the io.Reader's contents into a Manifest.
func ParseManifest(r io.Reader) (*Manifest, error) {
m := Manifest{}
if err := json.NewDecoder(r).Decode(&m); err != nil {
return nil, err
}
return &m, nil
}
// ParseIndexManifest parses the io.Reader's contents into an IndexManifest.
func ParseIndexManifest(r io.Reader) (*IndexManifest, error) {
im := IndexManifest{}
if err := json.NewDecoder(r).Decode(&im); err != nil {
return nil, err
}
return &im, nil
}
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package match provides functionality for conveniently matching a v1.Descriptor.
package match
import (
v1 "github.com/google/go-containerregistry/pkg/v1"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
)
// Matcher function that is given a v1.Descriptor, and returns whether or
// not it matches a given rule. Can match on anything it wants in the Descriptor.
type Matcher func(desc v1.Descriptor) bool
// Name returns a match.Matcher that matches based on the value of the
// "org.opencontainers.image.ref.name" annotation:
// github.com/opencontainers/image-spec/blob/v1.0.1/annotations.md#pre-defined-annotation-keys
func Name(name string) Matcher {
return Annotation(imagespec.AnnotationRefName, name)
}
// Annotation returns a match.Matcher that matches based on the provided annotation.
func Annotation(key, value string) Matcher {
return func(desc v1.Descriptor) bool {
if desc.Annotations == nil {
return false
}
if aValue, ok := desc.Annotations[key]; ok && aValue == value {
return true
}
return false
}
}
// Platforms returns a match.Matcher that matches on any one of the provided platforms.
// Ignores any descriptors that do not have a platform.
func Platforms(platforms ...v1.Platform) Matcher {
return func(desc v1.Descriptor) bool {
if desc.Platform == nil {
return false
}
for _, platform := range platforms {
if desc.Platform.Equals(platform) {
return true
}
}
return false
}
}
// MediaTypes returns a match.Matcher that matches at least one of the provided media types.
func MediaTypes(mediaTypes ...string) Matcher {
mts := map[string]bool{}
for _, media := range mediaTypes {
mts[media] = true
}
return func(desc v1.Descriptor) bool {
if desc.MediaType == "" {
return false
}
if _, ok := mts[string(desc.MediaType)]; ok {
return true
}
return false
}
}
// Digests returns a match.Matcher that matches at least one of the provided Digests
func Digests(digests ...v1.Hash) Matcher {
digs := map[v1.Hash]bool{}
for _, digest := range digests {
digs[digest] = true
}
return func(desc v1.Descriptor) bool {
_, ok := digs[desc.Digest]
return ok
}
}
# `partial`
[![GoDoc](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/partial?status.svg)](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/partial)
## Partial Implementations
There are roughly two kinds of image representations: compressed and uncompressed.
The implementations for these kinds of images are almost identical, with the only
major difference being how blobs (config and layers) are fetched. This common
code lives in this package, where you provide a _partial_ implementation of a
compressed or uncompressed image, and you get back a full `v1.Image` implementation.
### Examples
In a registry, blobs are compressed, so it's easiest to implement a `v1.Image` in terms
of compressed layers. `remote.remoteImage` does this by implementing `CompressedImageCore`:
```go
type CompressedImageCore interface {
RawConfigFile() ([]byte, error)
MediaType() (types.MediaType, error)
RawManifest() ([]byte, error)
LayerByDigest(v1.Hash) (CompressedLayer, error)
}
```
In a tarball, blobs are (often) uncompressed, so it's easiest to implement a `v1.Image` in terms
of uncompressed layers. `tarball.uncompressedImage` does this by implementing `UncompressedImageCore`:
```go
type UncompressedImageCore interface {
RawConfigFile() ([]byte, error)
MediaType() (types.MediaType, error)
LayerByDiffID(v1.Hash) (UncompressedLayer, error)
}
```
## Optional Methods
Where possible, we access some information via optional methods as an optimization.
### [`partial.Descriptor`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/partial#Descriptor)
There are some properties of a [`Descriptor`](https://github.com/opencontainers/image-spec/blob/master/descriptor.md#properties) that aren't derivable from just image data:
* `MediaType`
* `Platform`
* `URLs`
* `Annotations`
For example, in a `tarball.Image`, there is a `LayerSources` field that contains
an entire layer descriptor with `URLs` information for foreign layers. This
information can be passed through to callers by implementing this optional
`Descriptor` method.
See [`#654`](https://github.com/google/go-containerregistry/pull/654).
### [`partial.UncompressedSize`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/partial#UncompressedSize)
Usually, you don't need to know the uncompressed size of a layer, since that
information isn't stored in a config file (just he sha256 is needed); however,
there are cases where it is very helpful to know the layer size, e.g. when
writing the uncompressed layer into a tarball.
See [`#655`](https://github.com/google/go-containerregistry/pull/655).
### [`partial.Exists`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/partial#Exists)
We generally don't care about the existence of something as granular as a
layer, and would rather ensure all the invariants of an image are upheld via
the `validate` package. However, there are situations where we want to do a
quick smoke test to ensure that the underlying storage engine hasn't been
corrupted by something e.g. deleting files or blobs. Thus, we've exposed an
optional `Exists` method that does an existence check without actually reading
any bytes.
The `remote` package implements this via `HEAD` requests.
The `layout` package implements this via `os.Stat`.
See [`#838`](https://github.com/google/go-containerregistry/pull/838).
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package partial
import (
"io"
"github.com/google/go-containerregistry/internal/gzip"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
)
// CompressedLayer represents the bare minimum interface a natively
// compressed layer must implement for us to produce a v1.Layer
type CompressedLayer interface {
// Digest returns the Hash of the compressed layer.
Digest() (v1.Hash, error)
// Compressed returns an io.ReadCloser for the compressed layer contents.
Compressed() (io.ReadCloser, error)
// Size returns the compressed size of the Layer.
Size() (int64, error)
// Returns the mediaType for the compressed Layer
MediaType() (types.MediaType, error)
}
// compressedLayerExtender implements v1.Image using the compressed base properties.
type compressedLayerExtender struct {
CompressedLayer
}
// Uncompressed implements v1.Layer
func (cle *compressedLayerExtender) Uncompressed() (io.ReadCloser, error) {
r, err := cle.Compressed()
if err != nil {
return nil, err
}
return gzip.UnzipReadCloser(r)
}
// DiffID implements v1.Layer
func (cle *compressedLayerExtender) DiffID() (v1.Hash, error) {
// If our nested CompressedLayer implements DiffID,
// then delegate to it instead.
if wdi, ok := cle.CompressedLayer.(WithDiffID); ok {
return wdi.DiffID()
}
r, err := cle.Uncompressed()
if err != nil {
return v1.Hash{}, err
}
defer r.Close()
h, _, err := v1.SHA256(r)
return h, err
}
// CompressedToLayer fills in the missing methods from a CompressedLayer so that it implements v1.Layer
func CompressedToLayer(ul CompressedLayer) (v1.Layer, error) {
return &compressedLayerExtender{ul}, nil
}
// CompressedImageCore represents the base minimum interface a natively
// compressed image must implement for us to produce a v1.Image.
type CompressedImageCore interface {
ImageCore
// RawManifest returns the serialized bytes of the manifest.
RawManifest() ([]byte, error)
// LayerByDigest is a variation on the v1.Image method, which returns
// a CompressedLayer instead.
LayerByDigest(v1.Hash) (CompressedLayer, error)
}
// compressedImageExtender implements v1.Image by extending CompressedImageCore with the
// appropriate methods computed from the minimal core.
type compressedImageExtender struct {
CompressedImageCore
}
// Assert that our extender type completes the v1.Image interface
var _ v1.Image = (*compressedImageExtender)(nil)
// Digest implements v1.Image
func (i *compressedImageExtender) Digest() (v1.Hash, error) {
return Digest(i)
}
// ConfigName implements v1.Image
func (i *compressedImageExtender) ConfigName() (v1.Hash, error) {
return ConfigName(i)
}
// Layers implements v1.Image
func (i *compressedImageExtender) Layers() ([]v1.Layer, error) {
hs, err := FSLayers(i)
if err != nil {
return nil, err
}
ls := make([]v1.Layer, 0, len(hs))
for _, h := range hs {
l, err := i.LayerByDigest(h)
if err != nil {
return nil, err
}
ls = append(ls, l)
}
return ls, nil
}
// LayerByDigest implements v1.Image
func (i *compressedImageExtender) LayerByDigest(h v1.Hash) (v1.Layer, error) {
cl, err := i.CompressedImageCore.LayerByDigest(h)
if err != nil {
return nil, err
}
return CompressedToLayer(cl)
}
// LayerByDiffID implements v1.Image
func (i *compressedImageExtender) LayerByDiffID(h v1.Hash) (v1.Layer, error) {
h, err := DiffIDToBlob(i, h)
if err != nil {
return nil, err
}
return i.LayerByDigest(h)
}
// ConfigFile implements v1.Image
func (i *compressedImageExtender) ConfigFile() (*v1.ConfigFile, error) {
return ConfigFile(i)
}
// Manifest implements v1.Image
func (i *compressedImageExtender) Manifest() (*v1.Manifest, error) {
return Manifest(i)
}
// Size implements v1.Image
func (i *compressedImageExtender) Size() (int64, error) {
return Size(i)
}
// CompressedToImage fills in the missing methods from a CompressedImageCore so that it implements v1.Image
func CompressedToImage(cic CompressedImageCore) (v1.Image, error) {
return &compressedImageExtender{
CompressedImageCore: cic,
}, nil
}
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package partial defines methods for building up a v1.Image from
// minimal subsets that are sufficient for defining a v1.Image.
package partial
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package partial
import (
"github.com/google/go-containerregistry/pkg/v1/types"
)
// ImageCore is the core set of properties without which we cannot build a v1.Image
type ImageCore interface {
// RawConfigFile returns the serialized bytes of this image's config file.
RawConfigFile() ([]byte, error)
// MediaType of this image's manifest.
MediaType() (types.MediaType, error)
}
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package partial
import (
"fmt"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/match"
)
// FindManifests given a v1.ImageIndex, find the manifests that fit the matcher.
func FindManifests(index v1.ImageIndex, matcher match.Matcher) ([]v1.Descriptor, error) {
// get the actual manifest list
indexManifest, err := index.IndexManifest()
if err != nil {
return nil, fmt.Errorf("unable to get raw index: %w", err)
}
manifests := []v1.Descriptor{}
// try to get the root of our image
for _, manifest := range indexManifest.Manifests {
if matcher(manifest) {
manifests = append(manifests, manifest)
}
}
return manifests, nil
}
// FindImages given a v1.ImageIndex, find the images that fit the matcher. If a Descriptor
// matches the provider Matcher, but the referenced item is not an Image, ignores it.
// Only returns those that match the Matcher and are images.
func FindImages(index v1.ImageIndex, matcher match.Matcher) ([]v1.Image, error) {
matches := []v1.Image{}
manifests, err := FindManifests(index, matcher)
if err != nil {
return nil, err
}
for _, desc := range manifests {
// if it is not an image, ignore it
if !desc.MediaType.IsImage() {
continue
}
img, err := index.Image(desc.Digest)
if err != nil {
return nil, err
}
matches = append(matches, img)
}
return matches, nil
}
// FindIndexes given a v1.ImageIndex, find the indexes that fit the matcher. If a Descriptor
// matches the provider Matcher, but the referenced item is not an Index, ignores it.
// Only returns those that match the Matcher and are indexes.
func FindIndexes(index v1.ImageIndex, matcher match.Matcher) ([]v1.ImageIndex, error) {
matches := []v1.ImageIndex{}
manifests, err := FindManifests(index, matcher)
if err != nil {
return nil, err
}
for _, desc := range manifests {
if !desc.MediaType.IsIndex() {
continue
}
// if it is not an index, ignore it
idx, err := index.ImageIndex(desc.Digest)
if err != nil {
return nil, err
}
matches = append(matches, idx)
}
return matches, nil
}
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package partial
import (
"bytes"
"io"
"sync"
"github.com/google/go-containerregistry/internal/gzip"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
)
// UncompressedLayer represents the bare minimum interface a natively
// uncompressed layer must implement for us to produce a v1.Layer
type UncompressedLayer interface {
// DiffID returns the Hash of the uncompressed layer.
DiffID() (v1.Hash, error)
// Uncompressed returns an io.ReadCloser for the uncompressed layer contents.
Uncompressed() (io.ReadCloser, error)
// Returns the mediaType for the compressed Layer
MediaType() (types.MediaType, error)
}
// uncompressedLayerExtender implements v1.Image using the uncompressed base properties.
type uncompressedLayerExtender struct {
UncompressedLayer
// Memoize size/hash so that the methods aren't twice as
// expensive as doing this manually.
hash v1.Hash
size int64
hashSizeError error
once sync.Once
}
// Compressed implements v1.Layer
func (ule *uncompressedLayerExtender) Compressed() (io.ReadCloser, error) {
u, err := ule.Uncompressed()
if err != nil {
return nil, err
}
return gzip.ReadCloser(u), nil
}
// Digest implements v1.Layer
func (ule *uncompressedLayerExtender) Digest() (v1.Hash, error) {
ule.calcSizeHash()
return ule.hash, ule.hashSizeError
}
// Size implements v1.Layer
func (ule *uncompressedLayerExtender) Size() (int64, error) {
ule.calcSizeHash()
return ule.size, ule.hashSizeError
}
func (ule *uncompressedLayerExtender) calcSizeHash() {
ule.once.Do(func() {
var r io.ReadCloser
r, ule.hashSizeError = ule.Compressed()
if ule.hashSizeError != nil {
return
}
defer r.Close()
ule.hash, ule.size, ule.hashSizeError = v1.SHA256(r)
})
}
// UncompressedToLayer fills in the missing methods from an UncompressedLayer so that it implements v1.Layer
func UncompressedToLayer(ul UncompressedLayer) (v1.Layer, error) {
return &uncompressedLayerExtender{UncompressedLayer: ul}, nil
}
// UncompressedImageCore represents the bare minimum interface a natively
// uncompressed image must implement for us to produce a v1.Image
type UncompressedImageCore interface {
ImageCore
// LayerByDiffID is a variation on the v1.Image method, which returns
// an UncompressedLayer instead.
LayerByDiffID(v1.Hash) (UncompressedLayer, error)
}
// UncompressedToImage fills in the missing methods from an UncompressedImageCore so that it implements v1.Image.
func UncompressedToImage(uic UncompressedImageCore) (v1.Image, error) {
return &uncompressedImageExtender{
UncompressedImageCore: uic,
}, nil
}
// uncompressedImageExtender implements v1.Image by extending UncompressedImageCore with the
// appropriate methods computed from the minimal core.
type uncompressedImageExtender struct {
UncompressedImageCore
lock sync.Mutex
manifest *v1.Manifest
}
// Assert that our extender type completes the v1.Image interface
var _ v1.Image = (*uncompressedImageExtender)(nil)
// Digest implements v1.Image
func (i *uncompressedImageExtender) Digest() (v1.Hash, error) {
return Digest(i)
}
// Manifest implements v1.Image
func (i *uncompressedImageExtender) Manifest() (*v1.Manifest, error) {
i.lock.Lock()
defer i.lock.Unlock()
if i.manifest != nil {
return i.manifest, nil
}
b, err := i.RawConfigFile()
if err != nil {
return nil, err
}
cfgHash, cfgSize, err := v1.SHA256(bytes.NewReader(b))
if err != nil {
return nil, err
}
m := &v1.Manifest{
SchemaVersion: 2,
MediaType: types.DockerManifestSchema2,
Config: v1.Descriptor{
MediaType: types.DockerConfigJSON,
Size: cfgSize,
Digest: cfgHash,
},
}
ls, err := i.Layers()
if err != nil {
return nil, err
}
m.Layers = make([]v1.Descriptor, len(ls))
for i, l := range ls {
desc, err := Descriptor(l)
if err != nil {
return nil, err
}
m.Layers[i] = *desc
}
i.manifest = m
return i.manifest, nil
}
// RawManifest implements v1.Image
func (i *uncompressedImageExtender) RawManifest() ([]byte, error) {
return RawManifest(i)
}
// Size implements v1.Image
func (i *uncompressedImageExtender) Size() (int64, error) {
return Size(i)
}
// ConfigName implements v1.Image
func (i *uncompressedImageExtender) ConfigName() (v1.Hash, error) {
return ConfigName(i)
}
// ConfigFile implements v1.Image
func (i *uncompressedImageExtender) ConfigFile() (*v1.ConfigFile, error) {
return ConfigFile(i)
}
// Layers implements v1.Image
func (i *uncompressedImageExtender) Layers() ([]v1.Layer, error) {
diffIDs, err := DiffIDs(i)
if err != nil {
return nil, err
}
ls := make([]v1.Layer, 0, len(diffIDs))
for _, h := range diffIDs {
l, err := i.LayerByDiffID(h)
if err != nil {
return nil, err
}
ls = append(ls, l)
}
return ls, nil
}
// LayerByDiffID implements v1.Image
func (i *uncompressedImageExtender) LayerByDiffID(diffID v1.Hash) (v1.Layer, error) {
ul, err := i.UncompressedImageCore.LayerByDiffID(diffID)
if err != nil {
return nil, err
}
return UncompressedToLayer(ul)
}
// LayerByDigest implements v1.Image
func (i *uncompressedImageExtender) LayerByDigest(h v1.Hash) (v1.Layer, error) {
diffID, err := BlobToDiffID(i, h)
if err != nil {
return nil, err
}
return i.LayerByDiffID(diffID)
}
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"sort"
)
// Platform represents the target os/arch for an image.
type Platform struct {
Architecture string `json:"architecture"`
OS string `json:"os"`
OSVersion string `json:"os.version,omitempty"`
OSFeatures []string `json:"os.features,omitempty"`
Variant string `json:"variant,omitempty"`
Features []string `json:"features,omitempty"`
}
// Equals returns true if the given platform is semantically equivalent to this one.
// The order of Features and OSFeatures is not important.
func (p Platform) Equals(o Platform) bool {
return p.OS == o.OS && p.Architecture == o.Architecture && p.Variant == o.Variant && p.OSVersion == o.OSVersion &&
stringSliceEqualIgnoreOrder(p.OSFeatures, o.OSFeatures) && stringSliceEqualIgnoreOrder(p.Features, o.Features)
}
// stringSliceEqual compares 2 string slices and returns if their contents are identical.
func stringSliceEqual(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i, elm := range a {
if elm != b[i] {
return false
}
}
return true
}
// stringSliceEqualIgnoreOrder compares 2 string slices and returns if their contents are identical, ignoring order
func stringSliceEqualIgnoreOrder(a, b []string) bool {
if a != nil && b != nil {
sort.Strings(a)
sort.Strings(b)
}
return stringSliceEqual(a, b)
}
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
// Update representation of an update of transfer progress. Some functions
// in this module can take a channel to which updates will be sent while a
// transfer is in progress.
// +k8s:deepcopy-gen=false
type Update struct {
Total int64
Complete int64
Error error
}
# `remote`
[![GoDoc](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/remote?status.svg)](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/remote)
The `remote` package implements a client for accessing a registry,
per the [OCI distribution spec](https://github.com/opencontainers/distribution-spec/blob/master/spec.md).
It leans heavily on the lower level [`transport`](/pkg/v1/remote/transport) package, which handles the
authentication handshake and structured errors.
## Usage
```go
package main
import (
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/remote"
)
func main() {
ref, err := name.ParseReference("gcr.io/google-containers/pause")
if err != nil {
panic(err)
}
img, err := remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain))
if err != nil {
panic(err)
}
// do stuff with img
}
```
## Structure
<p align="center">
<img src="/images/remote.dot.svg" />
</p>
## Background
There are a lot of confusingly similar terms that come up when talking about images in registries.
### Anatomy of an image
In general...
* A tag refers to an image manifest.
* An image manifest references a config file and an orderered list of _compressed_ layers by sha256 digest.
* A config file references an ordered list of _uncompressed_ layers by sha256 digest and contains runtime configuration.
* The sha256 digest of the config file is the [image id](https://github.com/opencontainers/image-spec/blob/master/config.md#imageid) for the image.
For example, an image with two layers would look something like this:
![image anatomy](/images/image-anatomy.dot.svg)
### Anatomy of an index
In the normal case, an [index](https://github.com/opencontainers/image-spec/blob/master/image-index.md) is used to represent a multi-platform image.
This was the original use case for a [manifest
list](https://docs.docker.com/registry/spec/manifest-v2-2/#manifest-list).
![image index anatomy](/images/index-anatomy.dot.svg)
It is possible for an index to reference another index, per the OCI
[image-spec](https://github.com/opencontainers/image-spec/blob/master/media-types.md#compatibility-matrix).
In theory, both an image and image index can reference arbitrary things via
[descriptors](https://github.com/opencontainers/image-spec/blob/master/descriptor.md),
e.g. see the [image layout
example](https://github.com/opencontainers/image-spec/blob/master/image-layout.md#index-example),
which references an application/xml file from an image index.
That could look something like this:
![strange image index anatomy](/images/index-anatomy-strange.dot.svg)
Using a recursive index like this might not be possible with all registries,
but this flexibility allows for some interesting applications, e.g. the
[OCI Artifacts](https://github.com/opencontainers/artifacts) effort.
### Anatomy of an image upload
The structure of an image requires a delicate ordering when uploading an image to a registry.
Below is a (slightly simplified) figure that describes how an image is prepared for upload
to a registry and how the data flows between various artifacts:
![upload](/images/upload.dot.svg)
Note that:
* A config file references the uncompressed layer contents by sha256.
* A manifest references the compressed layer contents by sha256 and the size of the layer.
* A manifest references the config file contents by sha256 and the size of the file.
It follows that during an upload, we need to upload layers before the config file,
and we need to upload the config file before the manifest.
Sometimes, we know all of this information ahead of time, (e.g. when copying from remote.Image),
so the ordering is less important.
In other cases, e.g. when using a [`stream.Layer`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/stream#Layer),
we can't compute anything until we have already uploaded the layer, so we need to be careful about ordering.
## Caveats
### schema 1
This package does not support schema 1 images, see [`#377`](https://github.com/google/go-containerregistry/issues/377),
however, it's possible to do _something_ useful with them via [`remote.Get`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/remote#Get),
which doesn't try to interpret what is returned by the registry.
[`crane.Copy`](https://godoc.org/github.com/google/go-containerregistry/pkg/crane#Copy) takes advantage of this to implement support for copying schema 1 images,
see [here](https://github.com/google/go-containerregistry/blob/main/pkg/internal/legacy/copy.go).
// Copyright 2019 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package remote
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/remote/transport"
)
type catalog struct {
Repos []string `json:"repositories"`
}
// CatalogPage calls /_catalog, returning the list of repositories on the registry.
func CatalogPage(target name.Registry, last string, n int, options ...Option) ([]string, error) {
o, err := makeOptions(target, options...)
if err != nil {
return nil, err
}
scopes := []string{target.Scope(transport.PullScope)}
tr, err := transport.NewWithContext(o.context, target, o.auth, o.transport, scopes)
if err != nil {
return nil, err
}
query := fmt.Sprintf("last=%s&n=%d", url.QueryEscape(last), n)
uri := url.URL{
Scheme: target.Scheme(),
Host: target.RegistryStr(),
Path: "/v2/_catalog",
RawQuery: query,
}
client := http.Client{Transport: tr}
req, err := http.NewRequest(http.MethodGet, uri.String(), nil)
if err != nil {
return nil, err
}
resp, err := client.Do(req.WithContext(o.context))
if err != nil {
return nil, err
}
defer resp.Body.Close()
if err := transport.CheckError(resp, http.StatusOK); err != nil {
return nil, err
}
var parsed catalog
if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
return nil, err
}
return parsed.Repos, nil
}
// Catalog calls /_catalog, returning the list of repositories on the registry.
func Catalog(ctx context.Context, target name.Registry, options ...Option) ([]string, error) {
o, err := makeOptions(target, options...)
if err != nil {
return nil, err
}
scopes := []string{target.Scope(transport.PullScope)}
tr, err := transport.NewWithContext(o.context, target, o.auth, o.transport, scopes)
if err != nil {
return nil, err
}
uri := &url.URL{
Scheme: target.Scheme(),
Host: target.RegistryStr(),
Path: "/v2/_catalog",
}
if o.pageSize > 0 {
uri.RawQuery = fmt.Sprintf("n=%d", o.pageSize)
}
client := http.Client{Transport: tr}
// WithContext overrides the ctx passed directly.
if o.context != context.Background() {
ctx = o.context
}
var (
parsed catalog
repoList []string
)
// get responses until there is no next page
for {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
req, err := http.NewRequest("GET", uri.String(), nil)
if err != nil {
return nil, err
}
req = req.WithContext(ctx)
resp, err := client.Do(req)
if err != nil {
return nil, err
}
if err := transport.CheckError(resp, http.StatusOK); err != nil {
return nil, err
}
if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
return nil, err
}
if err := resp.Body.Close(); err != nil {
return nil, err
}
repoList = append(repoList, parsed.Repos...)
uri, err = getNextPageURL(resp)
if err != nil {
return nil, err
}
// no next page
if uri == nil {
break
}
}
return repoList, nil
}
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment