Skip to content

K
k3s
Commit 6f4217a3 authored by Brad Davidson's avatar Brad Davidson Committed by Brad Davidson
Browse files
Options

Build standalone containerd

Signed-off-by: 's avatarBrad Davidson <brad.davidson@rancher.com>
parent 8737e2e1
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 2413 additions and 474 deletions
cmd/k3s/main.go
View file @ 6f4217a3
......@@ -182,13 +182,13 @@ func getAssetAndDir(dataDir string) (string, string) {
func extract(dataDir string) (string, error) {
// first look for global asset folder so we don't create a HOME version if not needed
_, dir := getAssetAndDir(datadir.DefaultDataDir)
if _, err := os.Stat(filepath.Join(dir, "bin", "containerd")); err == nil {
if _, err := os.Stat(filepath.Join(dir, "bin", "k3s")); err == nil {
return dir, nil
}
asset, dir := getAssetAndDir(dataDir)
// check if target content already exists
if _, err := os.Stat(filepath.Join(dir, "bin", "containerd")); err == nil {
if _, err := os.Stat(filepath.Join(dir, "bin", "k3s")); err == nil {
return dir, nil
}
......
go.mod
View file @ 6f4217a3
......@@ -5,17 +5,7 @@ go 1.16
replace (
github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.8.20
github.com/benmoss/go-powershell => github.com/k3s-io/go-powershell v0.0.0-20201118222746-51f4c451fbd7
github.com/containerd/aufs => github.com/containerd/aufs v1.0.0
github.com/containerd/btrfs => github.com/containerd/btrfs v1.0.0
github.com/containerd/cgroups => github.com/containerd/cgroups v1.0.1
github.com/containerd/console => github.com/containerd/console v1.0.2
github.com/containerd/containerd => github.com/k3s-io/containerd v1.5.8-k3s1 // k3s-release/1.5
github.com/containerd/continuity => github.com/k3s-io/continuity v0.0.0-20210309170710-f93269e0d5c1
github.com/containerd/fifo => github.com/containerd/fifo v1.0.0
github.com/containerd/go-runc => github.com/containerd/go-runc v1.0.0
github.com/containerd/ttrpc => github.com/containerd/ttrpc v1.0.2
github.com/containerd/typeurl => github.com/containerd/typeurl v1.0.2
github.com/containerd/zfs => github.com/containerd/zfs v1.0.0
github.com/containerd/containerd => github.com/k3s-io/containerd v1.5.8-k3s2 // k3s-release/1.5
github.com/coreos/go-systemd => github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e
github.com/docker/distribution => github.com/docker/distribution v2.7.1+incompatible
github.com/docker/docker => github.com/docker/docker v20.10.2+incompatible
......@@ -25,7 +15,6 @@ replace (
github.com/juju/errors => github.com/k3s-io/nocode v0.0.0-20200630202308-cb097102c09f
github.com/kubernetes-sigs/cri-tools => github.com/k3s-io/cri-tools v1.21.0-k3s1
github.com/matryer/moq => github.com/rancher/moq v0.0.0-20190404221404-ee5226d43009
// LOOK TO scripts/download FOR THE VERSION OF runc THAT WE ARE BUILDING/SHIPPING
github.com/opencontainers/runc => github.com/opencontainers/runc v1.0.3
github.com/opencontainers/runtime-spec => github.com/opencontainers/runtime-spec v1.0.3-0.20210316141917-a8c4a9ee0f6b
github.com/rancher/k3s/pkg/data => ./pkg/data
......@@ -77,9 +66,9 @@ replace (
require (
github.com/Microsoft/hcsshim v0.8.23
github.com/containerd/cgroups v1.0.1
github.com/containerd/containerd v1.5.8
github.com/containerd/fuse-overlayfs-snapshotter v1.0.3
github.com/containerd/stargz-snapshotter v0.8.0
github.com/containerd/containerd v1.6.0-beta.2.0.20211117185425-a776a27af54a
github.com/containerd/fuse-overlayfs-snapshotter v1.0.4
github.com/containerd/stargz-snapshotter v0.10.1
github.com/coreos/go-iptables v0.5.0
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f
github.com/docker/docker v20.10.10+incompatible
......@@ -102,7 +91,6 @@ require (
github.com/natefinch/lumberjack v2.0.0+incompatible
github.com/onsi/ginkgo v1.16.4
github.com/onsi/gomega v1.11.0
// LOOK TO scripts/download FOR THE VERSION OF runc THAT WE ARE BUILDING/SHIPPING
github.com/opencontainers/runc v1.0.3
github.com/opencontainers/selinux v1.8.2
github.com/otiai10/copy v1.6.0
......@@ -125,8 +113,8 @@ require (
go.etcd.io/etcd/server/v3 v3.5.0
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
golang.org/x/net v0.0.0-20211005215030-d2e5035098b3
golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef
google.golang.org/grpc v1.41.0
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359
google.golang.org/grpc v1.42.0
gopkg.in/yaml.v2 v2.4.0
inet.af/tcpproxy v0.0.0-20200125044825-b6bb9b5b8252
k8s.io/api v0.22.4
......@@ -136,7 +124,7 @@ require (
k8s.io/cloud-provider v0.22.4
k8s.io/component-base v0.22.4
k8s.io/controller-manager v0.22.4 // indirect
k8s.io/cri-api v0.22.4
k8s.io/cri-api v0.23.0-alpha.4
k8s.io/klog v1.0.0
k8s.io/klog/v2 v2.9.0
k8s.io/kubectl v0.22.4
......
package/Dockerfile
View file @ 6f4217a3
......@@ -4,9 +4,6 @@ COPY build/out/data.tar.zst /
RUN mkdir -p /image/etc/ssl/certs /image/run /image/var/run /image/tmp /image/lib/modules /image/lib/firmware && \
tar -xa -C /image -f /data.tar.zst && \
cp /etc/ssl/certs/ca-certificates.crt /image/etc/ssl/certs/ca-certificates.crt
RUN cd image/bin && \
rm -f k3s && \
ln -s k3s-server k3s
FROM scratch
ARG VERSION="dev"
......
pkg/containerd/builtins_linux.go
View file @ 6f4217a3
......@@ -22,9 +22,9 @@ import (
_ "github.com/containerd/containerd/metrics/cgroups"
_ "github.com/containerd/containerd/runtime/v2"
_ "github.com/containerd/containerd/runtime/v2/runc/options"
_ "github.com/containerd/containerd/snapshots/btrfs/plugin"
_ "github.com/containerd/containerd/snapshots/native/plugin"
_ "github.com/containerd/containerd/snapshots/overlay/plugin"
_ "github.com/containerd/containerd/snapshots/btrfs/plugin"
_ "github.com/containerd/fuse-overlayfs-snapshotter/plugin"
_ "github.com/containerd/stargz-snapshotter/service/plugin"
)
scripts/binary_size_check.sh
View file @ 6f4217a3
......@@ -8,7 +8,10 @@ fi
. ./scripts/version.sh
MAX_BINARY_SIZE=61000000
# Try to keep the K3s binary under 64 megabytes.
# "64M ought to be enough for anybody"
MAX_BINARY_MB=64
MAX_BINARY_SIZE=$((MAX_BINARY_MB * 1024 * 1024))
BIN_SUFFIX="-${ARCH}"
if [ ${ARCH} = amd64 ]; then
BIN_SUFFIX=""
......@@ -20,9 +23,9 @@ CMD_NAME="dist/artifacts/k3s${BIN_SUFFIX}"
SIZE=$(stat -c '%s' ${CMD_NAME})
if [ ${SIZE} -gt ${MAX_BINARY_SIZE} ]; then
echo "k3s binary ${CMD_NAME} size ${SIZE} exceeds max acceptable size of ${MAX_BINARY_SIZE} bytes"
exit 1
echo "k3s binary ${CMD_NAME} size ${SIZE} exceeds max acceptable size of ${MAX_BINARY_SIZE} bytes (${MAX_BINARY_MB} MiB)"
exit 1
fi
echo "k3s binary ${CMD_NAME} size ${SIZE} is less than max acceptable size of ${MAX_BINARY_SIZE} bytes"
echo "k3s binary ${CMD_NAME} size ${SIZE} is less than max acceptable size of ${MAX_BINARY_SIZE} bytes (${MAX_BINARY_MB} MiB)"
exit 0
scripts/build
View file @ 6f4217a3
......@@ -10,7 +10,7 @@ GO=${GO-go}
PKG="github.com/rancher/k3s"
PKG_CONTAINERD="github.com/containerd/containerd"
PKG_K3S_CONTAINERD="github.com/k3s-io/containerd"
PKG_CRICTL="github.com/kubernetes-sigs/cri-tools"
PKG_CRICTL="github.com/kubernetes-sigs/cri-tools/pkg"
PKG_K8S_BASE="k8s.io/component-base"
PKG_K8S_CLIENT="k8s.io/client-go/pkg"
......@@ -35,7 +35,9 @@ VERSIONFLAGS="
-X ${VENDOR_PREFIX}${PKG_CONTAINERD}/version.Version=${VERSION_CONTAINERD}
-X ${VENDOR_PREFIX}${PKG_CONTAINERD}/version.Package=${PKG_K3S_CONTAINERD}
-X ${VENDOR_PREFIX}${PKG_CRICTL}/pkg/version.Version=${VERSION_CRICTL}
-X ${PKG_CONTAINERD}/version.Version=${VERSION_CONTAINERD}
-X ${PKG_CONTAINERD}/version.Package=${PKG_K3S_CONTAINERD}
"
LDFLAGS="
-w -s"
......@@ -43,7 +45,7 @@ LDFLAGS="
STATIC="
-extldflags '-static -lm -ldl -lz -lpthread'
"
TAGS="ctrd apparmor seccomp netcgo osusergo providerless"
TAGS="apparmor seccomp netcgo osusergo providerless"
RUNC_TAGS="apparmor seccomp"
RUNC_STATIC="static"
......@@ -101,22 +103,29 @@ if [ ! -x ${INSTALLBIN}/cni ]; then
fi
echo Building k3s
CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd ./cmd/server/main.go
ln -s containerd ./bin/k3s-agent
ln -s containerd ./bin/k3s-server
ln -s containerd ./bin/k3s-etcd-snapshot
ln -s containerd ./bin/k3s-secrets-encrypt
ln -s containerd ./bin/k3s-certificate
ln -s containerd ./bin/kubectl
ln -s containerd ./bin/crictl
ln -s containerd ./bin/ctr
CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/k3s ./cmd/server/main.go
ln -s k3s ./bin/k3s-agent
ln -s k3s ./bin/k3s-server
ln -s k3s ./bin/k3s-etcd-snapshot
ln -s k3s ./bin/k3s-secrets-encrypt
ln -s k3s ./bin/k3s-certificate
ln -s k3s ./bin/kubectl
ln -s k3s ./bin/crictl
ln -s k3s ./bin/ctr
export GOPATH=$(pwd)/build
echo Building containerd
pushd ./build/src/github.com/containerd/containerd
TAGS="${TAGS/netcgo/netgo}"
CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd ./cmd/containerd
CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd-shim-runc-v2 ./cmd/containerd-shim-runc-v2
popd
cp -vf ./build/src/github.com/containerd/containerd/bin/* ./bin/
echo Building runc
rm -f ./build/src/github.com/opencontainers/runc/runc
make GOPATH=$(pwd)/build EXTRA_LDFLAGS="-w -s" BUILDTAGS="$RUNC_TAGS" -C ./build/src/github.com/opencontainers/runc $RUNC_STATIC
cp -f ./build/src/github.com/opencontainers/runc/runc ./bin/runc
echo Building containerd-shim-runc-v2
rm -f ./vendor/github.com/containerd/containerd/bin/containerd-shim-runc-v2
make -C ./vendor/github.com/containerd/containerd bin/containerd-shim-runc-v2
cp -f ./vendor/github.com/containerd/containerd/bin/containerd-shim-runc-v2 ./bin/containerd-shim-runc-v2
pushd ./build/src/github.com/opencontainers/runc
rm -f runc
make EXTRA_LDFLAGS="-w -s" BUILDTAGS="$RUNC_TAGS" $RUNC_STATIC
popd
cp -vf ./build/src/github.com/opencontainers/runc/runc ./bin/
scripts/download
View file @ 6f4217a3
......@@ -13,12 +13,14 @@ TRAEFIK_CRD_FILE=traefik-crd-${TRAEFIK_CHART_VERSION}${TRAEFIK_PACKAGE_VERSION}.
TRAEFIK_URL=https://helm.traefik.io/traefik/traefik-${TRAEFIK_CHART_VERSION}.tgz
CHARTS_DIR=build/static/charts
RUNC_DIR=build/src/github.com/opencontainers/runc
CONTAINERD_DIR=build/src/github.com/containerd/containerd
DATA_DIR=build/data
export TZ=UTC
umask 022
rm -rf ${CHARTS_DIR}
rm -rf ${RUNC_DIR}
rm -rf ${CONTAINERD_DIR}
mkdir -p ${CHARTS_DIR}
mkdir -p ${DATA_DIR}
......@@ -26,6 +28,8 @@ curl --compressed -sfL https://github.com/k3s-io/k3s-root/releases/download/${VE
git clone --single-branch --branch=${VERSION_RUNC} --depth=1 https://github.com/opencontainers/runc ${RUNC_DIR}
git clone --single-branch --branch=${VERSION_CONTAINERD} --depth=1 https://github.com/k3s-io/containerd ${CONTAINERD_DIR}
setup_tmp() {
TMP_DIR=$(mktemp -d --tmpdir=${CHARTS_DIR})
cleanup() {
......
scripts/package-cli
View file @ 6f4217a3
......@@ -7,9 +7,9 @@ cd $(dirname $0)/..
GO=${GO-go}
for i in crictl kubectl k3s-agent k3s-server k3s-etcd-snapshot k3s-secrets-encrypt k3s-certificate k3s ; do
for i in crictl kubectl k3s-agent k3s-server k3s-etcd-snapshot k3s-secrets-encrypt k3s-certificate; do
rm -f bin/$i
ln -s containerd bin/$i
ln -s k3s bin/$i
done
for i in bridge flannel host-local loopback portmap; do
......
vendor/github.com/containerd/console/console_windows.go
View file @ 6f4217a3
......@@ -17,10 +17,10 @@
package console
import (
"errors"
"fmt"
"os"
"github.com/pkg/errors"
"golang.org/x/sys/windows"
)
......@@ -103,7 +103,7 @@ func (m *master) Reset() error {
{m.err, m.errMode},
} {
if err := windows.SetConsoleMode(s.fd, s.mode); err != nil {
return errors.Wrap(err, "unable to restore console mode")
return fmt.Errorf("unable to restore console mode: %w", err)
}
}
......@@ -114,7 +114,7 @@ func (m *master) Size() (WinSize, error) {
var info windows.ConsoleScreenBufferInfo
err := windows.GetConsoleScreenBufferInfo(m.out, &info)
if err != nil {
return WinSize{}, errors.Wrap(err, "unable to get console info")
return WinSize{}, fmt.Errorf("unable to get console info: %w", err)
}
winsize := WinSize{
......@@ -139,7 +139,7 @@ func (m *master) DisableEcho() error {
mode |= windows.ENABLE_LINE_INPUT
if err := windows.SetConsoleMode(m.in, mode); err != nil {
return errors.Wrap(err, "unable to set console to disable echo")
return fmt.Errorf("unable to set console to disable echo: %w", err)
}
return nil
......@@ -192,7 +192,7 @@ func makeInputRaw(fd windows.Handle, mode uint32) error {
}
if err := windows.SetConsoleMode(fd, mode); err != nil {
return errors.Wrap(err, "unable to set console to raw mode")
return fmt.Errorf("unable to set console to raw mode: %w", err)
}
return nil
......
vendor/github.com/containerd/console/console_zos.go 0 → 100644
View file @ 6f4217a3
// +build zos
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package console
import (
"fmt"
"os"
"golang.org/x/sys/unix"
)
// NewPty creates a new pty pair
// The master is returned as the first console and a string
// with the path to the pty slave is returned as the second
func NewPty() (Console, string, error) {
var f File
var err error
var slave string
for i := 0;; i++ {
ptyp := fmt.Sprintf("/dev/ptyp%04d", i)
f, err = os.OpenFile(ptyp, os.O_RDWR, 0600)
if err == nil {
slave = fmt.Sprintf("/dev/ttyp%04d", i)
break
}
if os.IsNotExist(err) {
return nil, "", err
}
// else probably Resource Busy
}
m, err := newMaster(f)
if err != nil {
return nil, "", err
}
return m, slave, nil
}
type master struct {
f File
original *unix.Termios
}
func (m *master) Read(b []byte) (int, error) {
return m.f.Read(b)
}
func (m *master) Write(b []byte) (int, error) {
return m.f.Write(b)
}
func (m *master) Close() error {
return m.f.Close()
}
func (m *master) Resize(ws WinSize) error {
return tcswinsz(m.f.Fd(), ws)
}
func (m *master) ResizeFrom(c Console) error {
ws, err := c.Size()
if err != nil {
return err
}
return m.Resize(ws)
}
func (m *master) Reset() error {
if m.original == nil {
return nil
}
return tcset(m.f.Fd(), m.original)
}
func (m *master) getCurrent() (unix.Termios, error) {
var termios unix.Termios
if err := tcget(m.f.Fd(), &termios); err != nil {
return unix.Termios{}, err
}
return termios, nil
}
func (m *master) SetRaw() error {
rawState, err := m.getCurrent()
if err != nil {
return err
}
rawState = cfmakeraw(rawState)
rawState.Oflag = rawState.Oflag | unix.OPOST
return tcset(m.f.Fd(), &rawState)
}
func (m *master) DisableEcho() error {
rawState, err := m.getCurrent()
if err != nil {
return err
}
rawState.Lflag = rawState.Lflag &^ unix.ECHO
return tcset(m.f.Fd(), &rawState)
}
func (m *master) Size() (WinSize, error) {
return tcgwinsz(m.f.Fd())
}
func (m *master) Fd() uintptr {
return m.f.Fd()
}
func (m *master) Name() string {
return m.f.Name()
}
// checkConsole checks if the provided file is a console
func checkConsole(f File) error {
var termios unix.Termios
if tcget(f.Fd(), &termios) != nil {
return ErrNotAConsole
}
return nil
}
func newMaster(f File) (Console, error) {
m := &master{
f: f,
}
t, err := m.getCurrent()
if err != nil {
return nil, err
}
m.original = &t
return m, nil
}
// ClearONLCR sets the necessary tty_ioctl(4)s to ensure that a pty pair
// created by us acts normally. In particular, a not-very-well-known default of
// Linux unix98 ptys is that they have +onlcr by default. While this isn't a
// problem for terminal emulators, because we relay data from the terminal we
// also relay that funky line discipline.
func ClearONLCR(fd uintptr) error {
return setONLCR(fd, false)
}
// SetONLCR sets the necessary tty_ioctl(4)s to ensure that a pty pair
// created by us acts as intended for a terminal emulator.
func SetONLCR(fd uintptr) error {
return setONLCR(fd, true)
}
vendor/github.com/containerd/console/go.mod
View file @ 6f4217a3
......@@ -2,7 +2,4 @@ module github.com/containerd/console
go 1.13
require (
github.com/pkg/errors v0.9.1
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c
)
require golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c
vendor/github.com/containerd/console/go.sum
View file @ 6f4217a3
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
vendor/github.com/containerd/console/tc_unix.go
View file @ 6f4217a3
// +build darwin freebsd linux netbsd openbsd solaris
// +build darwin freebsd linux netbsd openbsd solaris zos
/*
Copyright The containerd Authors.
......
vendor/github.com/containerd/console/tc_zos.go 0 → 100644
View file @ 6f4217a3
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package console
import (
"golang.org/x/sys/unix"
)
const (
cmdTcGet = unix.TCGETS
cmdTcSet = unix.TCSETS
)
vendor/github.com/containerd/containerd/go.mod
View file @ 6f4217a3
......@@ -8,13 +8,15 @@ require (
github.com/containerd/aufs v1.0.0
github.com/containerd/btrfs v1.0.0
github.com/containerd/cgroups v1.0.1
github.com/containerd/console v1.0.2
github.com/containerd/continuity v0.1.0
github.com/containerd/console v1.0.3
github.com/containerd/continuity v0.2.1
github.com/containerd/fifo v1.0.0
github.com/containerd/fuse-overlayfs-snapshotter v1.0.4
github.com/containerd/go-cni v1.0.2
github.com/containerd/go-runc v1.0.0
github.com/containerd/imgcrypt v1.1.1
github.com/containerd/nri v0.1.0
github.com/containerd/stargz-snapshotter v0.10.1
github.com/containerd/ttrpc v1.1.0
github.com/containerd/typeurl v1.0.2
github.com/containerd/zfs v1.0.0
......@@ -31,41 +33,40 @@ require (
github.com/google/go-cmp v0.5.6
github.com/google/uuid v1.2.0
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/hashicorp/go-multierror v1.0.0
github.com/hashicorp/go-multierror v1.1.1
github.com/imdario/mergo v0.3.12
github.com/klauspost/compress v1.11.13
github.com/klauspost/compress v1.13.6
github.com/moby/locker v1.0.1
github.com/moby/sys/mountinfo v0.4.1
github.com/moby/sys/mountinfo v0.5.0
github.com/moby/sys/symlink v0.1.0
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.1
github.com/opencontainers/image-spec v1.0.2-0.20210819154149-5ad6f50d6283
github.com/opencontainers/runc v1.0.2
github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417
github.com/opencontainers/selinux v1.8.2
github.com/pelletier/go-toml v1.8.1
github.com/pelletier/go-toml v1.9.4
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.11.0
github.com/prometheus/procfs v0.6.0 // indirect; temporarily force v0.6.0, which was previously defined in imgcrypt as explicit version
github.com/satori/go.uuid v1.2.0 // indirect
github.com/sirupsen/logrus v1.8.1
github.com/stretchr/testify v1.7.0
github.com/tchap/go-patricia v2.2.6+incompatible
github.com/urfave/cli v1.22.2
github.com/urfave/cli v1.22.4
go.etcd.io/bbolt v1.3.6
golang.org/x/net v0.0.0-20210520170846-37e1c6afe023
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c
google.golang.org/grpc v1.38.0
google.golang.org/protobuf v1.26.0
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359
google.golang.org/grpc v1.42.0
google.golang.org/protobuf v1.27.1
gotest.tools/v3 v3.0.3
k8s.io/api v0.22.0
k8s.io/apimachinery v0.22.0
k8s.io/apiserver v0.22.0
k8s.io/client-go v0.22.0
k8s.io/component-base v0.22.0
k8s.io/cri-api v0.22.0
k8s.io/api v0.22.4
k8s.io/apimachinery v0.22.4
k8s.io/apiserver v0.22.4
k8s.io/client-go v0.22.4
k8s.io/component-base v0.22.4
k8s.io/cri-api v0.23.0-alpha.4
k8s.io/klog/v2 v2.9.0
k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9
k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a
)
// When updating replace rules, make sure to also update the rules in integration/client/go.mod
......@@ -77,4 +78,5 @@ replace (
// urfave/cli must be <= v1.22.1 due to a regression: https://github.com/urfave/cli/issues/1092
github.com/urfave/cli => github.com/urfave/cli v1.22.1
google.golang.org/genproto => google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63
k8s.io/cri-api => k8s.io/cri-api v0.22.4
)
vendor/github.com/containerd/continuity/AUTHORS
View file @ 6f4217a3
......@@ -12,8 +12,11 @@ Darren Stahl <darst@microsoft.com>
Derek McGowan <derek@mcg.dev>
Derek McGowan <derek@mcgstyle.net>
Edward Pilatowicz <edward.pilatowicz@oracle.com>
Fu Wei <fuweid89@gmail.com>
Hajime Tazaki <thehajime@gmail.com>
Ian Campbell <ijc@docker.com>
Ivan Markin <sw@nogoegst.net>
Jacob Blain Christen <jacob@rancher.com>
Justin Cormack <justin.cormack@docker.com>
Justin Cummins <sul3n3t@gmail.com>
Kasper Fabæch Brandt <poizan@poizan.dk>
......@@ -23,10 +26,11 @@ Michael Crosby <michael@thepasture.io>
Michael Wan <zirenwan@gmail.com>
Mike Brown <brownwm@us.ibm.com>
Niels de Vos <ndevos@redhat.com>
Phil Estes <estesp@amazon.com>
Phil Estes <estesp@gmail.com>
Phil Estes <estesp@linux.vnet.ibm.com>
Samuel Karp <me@samuelkarp.com>
Sam Whited <sam@samwhited.com>
Samuel Karp <me@samuelkarp.com>
Sebastiaan van Stijn <github@gone.nl>
Shengjing Zhu <zhsj@debian.org>
Stephen J Day <stephen.day@docker.com>
......
vendor/github.com/containerd/continuity/README.md
View file @ 6f4217a3
# continuity
[![GoDoc](https://godoc.org/github.com/containerd/continuity?status.svg)](https://godoc.org/github.com/containerd/continuity)
[![Build Status](https://travis-ci.org/containerd/continuity.svg?branch=master)](https://travis-ci.org/containerd/continuity)
[![Build Status](https://travis-ci.org/containerd/continuity.svg?branch=main)](https://travis-ci.org/containerd/continuity)
A transport-agnostic, filesystem metadata manifest system
......@@ -81,8 +81,8 @@ $ go generate ./proto
continuity is a containerd sub-project, licensed under the [Apache 2.0 license](./LICENSE).
As a containerd sub-project, you will find the:
* [Project governance](https://github.com/containerd/project/blob/master/GOVERNANCE.md),
* [Maintainers](https://github.com/containerd/project/blob/master/MAINTAINERS),
* and [Contributing guidelines](https://github.com/containerd/project/blob/master/CONTRIBUTING.md)
* [Project governance](https://github.com/containerd/project/blob/main/GOVERNANCE.md),
* [Maintainers](https://github.com/containerd/project/blob/main/MAINTAINERS),
* and [Contributing guidelines](https://github.com/containerd/project/blob/main/CONTRIBUTING.md)
information in our [`containerd/project`](https://github.com/containerd/project) repository.
vendor/github.com/containerd/continuity/devices/devices_unix.go
View file @ 6f4217a3
//go:build linux || darwin || freebsd || solaris
// +build linux darwin freebsd solaris
/*
......
vendor/github.com/containerd/continuity/devices/devices_windows.go
View file @ 6f4217a3
......@@ -17,11 +17,10 @@
package devices
import (
"fmt"
"os"
"github.com/pkg/errors"
)
func DeviceInfo(fi os.FileInfo) (uint64, uint64, error) {
return 0, 0, errors.Wrap(ErrNotSupported, "cannot get device info on windows")
return 0, 0, fmt.Errorf("cannot get device info on windows: %w", ErrNotSupported)
}
vendor/github.com/containerd/continuity/devices/mknod_unix.go
View file @ 6f4217a3
//go:build linux || darwin || solaris
// +build linux darwin solaris
/*
......
vendor/github.com/containerd/continuity/driver/driver_unix.go
View file @ 6f4217a3
//go:build linux || darwin || freebsd || solaris
// +build linux darwin freebsd solaris
/*
......
vendor/github.com/containerd/continuity/driver/lchmod_unix.go
View file @ 6f4217a3
//go:build darwin || freebsd || solaris
// +build darwin freebsd solaris
/*
......
vendor/github.com/containerd/continuity/fs/copy.go
View file @ 6f4217a3
......@@ -17,12 +17,11 @@
package fs
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"sync"
"github.com/pkg/errors"
)
var bufferPool = &sync.Pool{
......@@ -92,35 +91,35 @@ func CopyDir(dst, src string, opts ...CopyDirOpt) error {
func copyDirectory(dst, src string, inodes map[uint64]string, o *copyDirOpts) error {
stat, err := os.Stat(src)
if err != nil {
return errors.Wrapf(err, "failed to stat %s", src)
return fmt.Errorf("failed to stat %s: %w", src, err)
}
if !stat.IsDir() {
return errors.Errorf("source %s is not directory", src)
return fmt.Errorf("source %s is not directory", src)
}
if st, err := os.Stat(dst); err != nil {
if err := os.Mkdir(dst, stat.Mode()); err != nil {
return errors.Wrapf(err, "failed to mkdir %s", dst)
return fmt.Errorf("failed to mkdir %s: %w", dst, err)
}
} else if !st.IsDir() {
return errors.Errorf("cannot copy to non-directory: %s", dst)
return fmt.Errorf("cannot copy to non-directory: %s", dst)
} else {
if err := os.Chmod(dst, stat.Mode()); err != nil {
return errors.Wrapf(err, "failed to chmod on %s", dst)
return fmt.Errorf("failed to chmod on %s: %w", dst, err)
}
}
fis, err := ioutil.ReadDir(src)
if err != nil {
return errors.Wrapf(err, "failed to read %s", src)
return fmt.Errorf("failed to read %s: %w", src, err)
}
if err := copyFileInfo(stat, dst); err != nil {
return errors.Wrapf(err, "failed to copy file info for %s", dst)
return fmt.Errorf("failed to copy file info for %s: %w", dst, err)
}
if err := copyXAttrs(dst, src, o.xex, o.xeh); err != nil {
return errors.Wrap(err, "failed to copy xattrs")
return fmt.Errorf("failed to copy xattrs: %w", err)
}
for _, fi := range fis {
......@@ -136,37 +135,37 @@ func copyDirectory(dst, src string, inodes map[uint64]string, o *copyDirOpts) er
case (fi.Mode() & os.ModeType) == 0:
link, err := getLinkSource(target, fi, inodes)
if err != nil {
return errors.Wrap(err, "failed to get hardlink")
return fmt.Errorf("failed to get hardlink: %w", err)
}
if link != "" {
if err := os.Link(link, target); err != nil {
return errors.Wrap(err, "failed to create hard link")
return fmt.Errorf("failed to create hard link: %w", err)
}
} else if err := CopyFile(target, source); err != nil {
return errors.Wrap(err, "failed to copy files")
return fmt.Errorf("failed to copy files: %w", err)
}
case (fi.Mode() & os.ModeSymlink) == os.ModeSymlink:
link, err := os.Readlink(source)
if err != nil {
return errors.Wrapf(err, "failed to read link: %s", source)
return fmt.Errorf("failed to read link: %s: %w", source, err)
}
if err := os.Symlink(link, target); err != nil {
return errors.Wrapf(err, "failed to create symlink: %s", target)
return fmt.Errorf("failed to create symlink: %s: %w", target, err)
}
case (fi.Mode() & os.ModeDevice) == os.ModeDevice:
if err := copyDevice(target, fi); err != nil {
return errors.Wrapf(err, "failed to create device")
return fmt.Errorf("failed to create device: %w", err)
}
default:
// TODO: Support pipes and sockets
return errors.Wrapf(err, "unsupported mode %s", fi.Mode())
return fmt.Errorf("unsupported mode %s: %w", fi.Mode(), err)
}
if err := copyFileInfo(fi, target); err != nil {
return errors.Wrap(err, "failed to copy file info")
return fmt.Errorf("failed to copy file info: %w", err)
}
if err := copyXAttrs(target, source, o.xex, o.xeh); err != nil {
return errors.Wrap(err, "failed to copy xattrs")
return fmt.Errorf("failed to copy xattrs: %w", err)
}
}
......@@ -178,12 +177,12 @@ func copyDirectory(dst, src string, inodes map[uint64]string, o *copyDirOpts) er
func CopyFile(target, source string) error {
src, err := os.Open(source)
if err != nil {
return errors.Wrapf(err, "failed to open source %s", source)
return fmt.Errorf("failed to open source %s: %w", source, err)
}
defer src.Close()
tgt, err := os.Create(target)
if err != nil {
return errors.Wrapf(err, "failed to open target %s", target)
return fmt.Errorf("failed to open target %s: %w", target, err)
}
defer tgt.Close()
......
vendor/github.com/containerd/continuity/fs/copy_darwin.go 0 → 100644
View file @ 6f4217a3
//go:build darwin
// +build darwin
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"errors"
"os"
"syscall"
"golang.org/x/sys/unix"
)
func copyDevice(dst string, fi os.FileInfo) error {
st, ok := fi.Sys().(*syscall.Stat_t)
if !ok {
return errors.New("unsupported stat type")
}
return unix.Mknod(dst, uint32(fi.Mode()), int(st.Rdev))
}
func utimesNano(name string, atime, mtime syscall.Timespec) error {
at := unix.NsecToTimespec(atime.Nano())
mt := unix.NsecToTimespec(mtime.Nano())
utimes := [2]unix.Timespec{at, mt}
return unix.UtimesNanoAt(unix.AT_FDCWD, name, utimes[0:], unix.AT_SYMLINK_NOFOLLOW)
}
vendor/github.com/containerd/continuity/fs/copy_freebsd.go
View file @ 6f4217a3
//go:build freebsd
// +build freebsd
/*
......@@ -19,10 +20,10 @@
package fs
import (
"errors"
"os"
"syscall"
"github.com/pkg/errors"
"golang.org/x/sys/unix"
)
......
vendor/github.com/containerd/continuity/fs/copy_linux.go
View file @ 6f4217a3
......@@ -17,12 +17,13 @@
package fs
import (
"errors"
"fmt"
"io"
"os"
"syscall"
"github.com/containerd/continuity/sysx"
"github.com/pkg/errors"
"golang.org/x/sys/unix"
)
......@@ -41,13 +42,13 @@ func copyFileInfo(fi os.FileInfo, name string) error {
}
}
if err != nil {
return errors.Wrapf(err, "failed to chown %s", name)
return fmt.Errorf("failed to chown %s: %w", name, err)
}
}
if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink {
if err := os.Chmod(name, fi.Mode()); err != nil {
return errors.Wrapf(err, "failed to chmod %s", name)
return fmt.Errorf("failed to chmod %s: %w", name, err)
}
}
......@@ -56,7 +57,7 @@ func copyFileInfo(fi os.FileInfo, name string) error {
unix.NsecToTimespec(syscall.TimespecToNsec(StatMtime(st))),
}
if err := unix.UtimesNanoAt(unix.AT_FDCWD, name, timespec, unix.AT_SYMLINK_NOFOLLOW); err != nil {
return errors.Wrapf(err, "failed to utime %s", name)
return fmt.Errorf("failed to utime %s: %w", name, err)
}
return nil
......@@ -67,7 +68,7 @@ const maxSSizeT = int64(^uint(0) >> 1)
func copyFileContent(dst, src *os.File) error {
st, err := src.Stat()
if err != nil {
return errors.Wrap(err, "unable to stat source")
return fmt.Errorf("unable to stat source: %w", err)
}
size := st.Size()
......@@ -88,13 +89,16 @@ func copyFileContent(dst, src *os.File) error {
n, err := unix.CopyFileRange(srcFd, nil, dstFd, nil, copySize, 0)
if err != nil {
if (err != unix.ENOSYS && err != unix.EXDEV) || !first {
return errors.Wrap(err, "copy file range failed")
return fmt.Errorf("copy file range failed: %w", err)
}
buf := bufferPool.Get().(*[]byte)
_, err = io.CopyBuffer(dst, src, *buf)
bufferPool.Put(buf)
return errors.Wrap(err, "userspace copy failed")
if err != nil {
return fmt.Errorf("userspace copy failed: %w", err)
}
return nil
}
first = false
......@@ -107,7 +111,7 @@ func copyFileContent(dst, src *os.File) error {
func copyXAttrs(dst, src string, excludes map[string]struct{}, errorHandler XAttrErrorHandler) error {
xattrKeys, err := sysx.LListxattr(src)
if err != nil {
e := errors.Wrapf(err, "failed to list xattrs on %s", src)
e := fmt.Errorf("failed to list xattrs on %s: %w", src, err)
if errorHandler != nil {
e = errorHandler(dst, src, "", e)
}
......@@ -119,7 +123,7 @@ func copyXAttrs(dst, src string, excludes map[string]struct{}, errorHandler XAtt
}
data, err := sysx.LGetxattr(src, xattr)
if err != nil {
e := errors.Wrapf(err, "failed to get xattr %q on %s", xattr, src)
e := fmt.Errorf("failed to get xattr %q on %s: %w", xattr, src, err)
if errorHandler != nil {
if e = errorHandler(dst, src, xattr, e); e == nil {
continue
......@@ -128,7 +132,7 @@ func copyXAttrs(dst, src string, excludes map[string]struct{}, errorHandler XAtt
return e
}
if err := sysx.LSetxattr(dst, xattr, data, 0); err != nil {
e := errors.Wrapf(err, "failed to set xattr %q on %s", xattr, dst)
e := fmt.Errorf("failed to set xattr %q on %s: %w", xattr, dst, err)
if errorHandler != nil {
if e = errorHandler(dst, src, xattr, e); e == nil {
continue
......
vendor/github.com/containerd/continuity/fs/copy_darwinopenbsdsolaris.go → vendor/github.com/containerd/continuity/fs/copy_openbsdsolaris.go
View file @ 6f4217a3
// +build darwin openbsd solaris
//go:build openbsd || solaris
// +build openbsd solaris
/*
Copyright The containerd Authors.
......@@ -19,10 +20,10 @@
package fs
import (
"errors"
"os"
"syscall"
"github.com/pkg/errors"
"golang.org/x/sys/unix"
)
......
vendor/github.com/containerd/continuity/fs/copy_unix.go
View file @ 6f4217a3
//go:build darwin || freebsd || openbsd || solaris
// +build darwin freebsd openbsd solaris
/*
......@@ -19,12 +20,12 @@
package fs
import (
"fmt"
"io"
"os"
"syscall"
"github.com/containerd/continuity/sysx"
"github.com/pkg/errors"
)
func copyFileInfo(fi os.FileInfo, name string) error {
......@@ -42,18 +43,18 @@ func copyFileInfo(fi os.FileInfo, name string) error {
}
}
if err != nil {
return errors.Wrapf(err, "failed to chown %s", name)
return fmt.Errorf("failed to chown %s: %w", name, err)
}
}
if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink {
if err := os.Chmod(name, fi.Mode()); err != nil {
return errors.Wrapf(err, "failed to chmod %s", name)
return fmt.Errorf("failed to chmod %s: %w", name, err)
}
}
if err := utimesNano(name, StatAtime(st), StatMtime(st)); err != nil {
return errors.Wrapf(err, "failed to utime %s", name)
return fmt.Errorf("failed to utime %s: %w", name, err)
}
return nil
......@@ -70,7 +71,7 @@ func copyFileContent(dst, src *os.File) error {
func copyXAttrs(dst, src string, excludes map[string]struct{}, errorHandler XAttrErrorHandler) error {
xattrKeys, err := sysx.LListxattr(src)
if err != nil {
e := errors.Wrapf(err, "failed to list xattrs on %s", src)
e := fmt.Errorf("failed to list xattrs on %s: %w", src, err)
if errorHandler != nil {
e = errorHandler(dst, src, "", e)
}
......@@ -82,7 +83,7 @@ func copyXAttrs(dst, src string, excludes map[string]struct{}, errorHandler XAtt
}
data, err := sysx.LGetxattr(src, xattr)
if err != nil {
e := errors.Wrapf(err, "failed to get xattr %q on %s", xattr, src)
e := fmt.Errorf("failed to get xattr %q on %s: %w", xattr, src, err)
if errorHandler != nil {
if e = errorHandler(dst, src, xattr, e); e == nil {
continue
......@@ -91,7 +92,7 @@ func copyXAttrs(dst, src string, excludes map[string]struct{}, errorHandler XAtt
return e
}
if err := sysx.LSetxattr(dst, xattr, data, 0); err != nil {
e := errors.Wrapf(err, "failed to set xattr %q on %s", xattr, dst)
e := fmt.Errorf("failed to set xattr %q on %s: %w", xattr, dst, err)
if errorHandler != nil {
if e = errorHandler(dst, src, xattr, e); e == nil {
continue
......
vendor/github.com/containerd/continuity/fs/copy_windows.go
View file @ 6f4217a3
......@@ -17,15 +17,15 @@
package fs
import (
"errors"
"fmt"
"io"
"os"
"github.com/pkg/errors"
)
func copyFileInfo(fi os.FileInfo, name string) error {
if err := os.Chmod(name, fi.Mode()); err != nil {
return errors.Wrapf(err, "failed to chmod %s", name)
return fmt.Errorf("failed to chmod %s: %w", name, err)
}
// TODO: copy windows specific metadata
......
vendor/github.com/containerd/continuity/fs/diff_unix.go
View file @ 6f4217a3
//go:build !windows
// +build !windows
/*
......@@ -20,11 +21,11 @@ package fs
import (
"bytes"
"fmt"
"os"
"syscall"
"github.com/containerd/continuity/sysx"
"github.com/pkg/errors"
)
// detectDirDiff returns diff dir options if a directory could
......@@ -56,11 +57,11 @@ func compareSysStat(s1, s2 interface{}) (bool, error) {
func compareCapabilities(p1, p2 string) (bool, error) {
c1, err := sysx.LGetxattr(p1, "security.capability")
if err != nil && err != sysx.ENODATA {
return false, errors.Wrapf(err, "failed to get xattr for %s", p1)
return false, fmt.Errorf("failed to get xattr for %s: %w", p1, err)
}
c2, err := sysx.LGetxattr(p2, "security.capability")
if err != nil && err != sysx.ENODATA {
return false, errors.Wrapf(err, "failed to get xattr for %s", p2)
return false, fmt.Errorf("failed to get xattr for %s: %w", p2, err)
}
return bytes.Equal(c1, c2), nil
}
......
vendor/github.com/containerd/continuity/fs/du_unix.go
View file @ 6f4217a3
//go:build !windows
// +build !windows
/*
......@@ -41,10 +42,8 @@ type inode struct {
func newInode(stat *syscall.Stat_t) inode {
return inode{
// Dev is uint32 on darwin/bsd, uint64 on linux/solaris/freebsd
dev: uint64(stat.Dev), // nolint: unconvert
// Ino is uint32 on bsd, uint64 on darwin/linux/solaris/freebsd
ino: uint64(stat.Ino), // nolint: unconvert
dev: uint64(stat.Dev), //nolint: unconvert // dev is uint32 on darwin/bsd, uint64 on linux/solaris/freebsd
ino: uint64(stat.Ino), //nolint: unconvert // ino is uint32 on bsd, uint64 on darwin/linux/solaris/freebsd
}
}
......
vendor/github.com/containerd/continuity/fs/hardlink_unix.go
View file @ 6f4217a3
//go:build !windows
// +build !windows
/*
......@@ -29,6 +30,5 @@ func getLinkInfo(fi os.FileInfo) (uint64, bool) {
return 0, false
}
// Ino is uint32 on bsd, uint64 on darwin/linux/solaris
return uint64(s.Ino), !fi.IsDir() && s.Nlink > 1 // nolint: unconvert
return uint64(s.Ino), !fi.IsDir() && s.Nlink > 1 //nolint: unconvert // ino is uint32 on bsd, uint64 on darwin/linux/solaris
}
vendor/github.com/containerd/continuity/fs/path.go
View file @ 6f4217a3
......@@ -19,11 +19,10 @@ package fs
import (
"bytes"
"context"
"errors"
"io"
"os"
"path/filepath"
"github.com/pkg/errors"
)
var (
......
vendor/github.com/containerd/continuity/fs/stat_darwinfreebsd.go → vendor/github.com/containerd/continuity/fs/stat_darwinbsd.go
View file @ 6f4217a3
// +build darwin freebsd
//go:build darwin || freebsd || netbsd
// +build darwin freebsd netbsd
/*
Copyright The containerd Authors.
......@@ -40,5 +41,5 @@ func StatMtime(st *syscall.Stat_t) syscall.Timespec {
// StatATimeAsTime returns the access time as a time.Time
func StatATimeAsTime(st *syscall.Stat_t) time.Time {
return time.Unix(int64(st.Atimespec.Sec), int64(st.Atimespec.Nsec)) // nolint: unconvert
return time.Unix(int64(st.Atimespec.Sec), int64(st.Atimespec.Nsec)) //nolint: unconvert // int64 conversions ensure the line compiles for 32-bit systems as well.
}
vendor/github.com/containerd/continuity/fs/stat_linuxopenbsd.go
View file @ 6f4217a3
//go:build linux || openbsd
// +build linux openbsd
/*
......@@ -40,6 +41,5 @@ func StatMtime(st *syscall.Stat_t) syscall.Timespec {
// StatATimeAsTime returns st.Atim as a time.Time
func StatATimeAsTime(st *syscall.Stat_t) time.Time {
// The int64 conversions ensure the line compiles for 32-bit systems as well.
return time.Unix(int64(st.Atim.Sec), int64(st.Atim.Nsec)) // nolint: unconvert
return time.Unix(int64(st.Atim.Sec), int64(st.Atim.Nsec)) //nolint: unconvert // int64 conversions ensure the line compiles for 32-bit systems as well.
}
vendor/github.com/containerd/continuity/go.mod
View file @ 6f4217a3
......@@ -3,17 +3,14 @@ module github.com/containerd/continuity
go 1.13
require (
bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898
// 5883e5a4b512fe2e32f915b1c66a1ddfef81cb3f is the last version to support macOS
// see https://github.com/bazil/fuse/commit/60eaf8f021ce00e5c52529cdcba1067e13c1c2c2
bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512
github.com/dustin/go-humanize v1.0.0
github.com/golang/protobuf v1.2.0
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/golang/protobuf v1.3.5
github.com/opencontainers/go-digest v1.0.0
github.com/pkg/errors v0.9.1
github.com/sirupsen/logrus v1.7.0
github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee
github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95 // indirect
github.com/stretchr/testify v1.4.0 // indirect
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd // indirect
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3
github.com/spf13/cobra v1.0.0
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c
)
vendor/github.com/containerd/continuity/hardlinks_unix.go
View file @ 6f4217a3
//go:build linux || darwin || freebsd || solaris
// +build linux darwin freebsd solaris
/*
......
vendor/github.com/containerd/continuity/manifest.go
View file @ 6f4217a3
......@@ -75,7 +75,7 @@ func MarshalText(w io.Writer, m *Manifest) error {
// BuildManifest creates the manifest for the given context
func BuildManifest(ctx Context) (*Manifest, error) {
resourcesByPath := map[string]Resource{}
hardlinks := newHardlinkManager()
hardLinks := newHardlinkManager()
if err := ctx.Walk(func(p string, fi os.FileInfo, err error) error {
if err != nil {
......@@ -97,7 +97,7 @@ func BuildManifest(ctx Context) (*Manifest, error) {
}
// add to the hardlink manager
if err := hardlinks.Add(fi, resource); err == nil {
if err := hardLinks.Add(fi, resource); err == nil {
// Resource has been accepted by hardlink manager so we don't add
// it to the resourcesByPath until we merge at the end.
return nil
......@@ -114,14 +114,12 @@ func BuildManifest(ctx Context) (*Manifest, error) {
}
// merge and post-process the hardlinks.
// nolint:misspell
hardlinked, err := hardlinks.Merge()
hardLinked, err := hardLinks.Merge()
if err != nil {
return nil, err
}
// nolint:misspell
for _, resource := range hardlinked {
for _, resource := range hardLinked {
resourcesByPath[resource.Path()] = resource
}
......
vendor/github.com/containerd/continuity/proto/manifest.pb.go
View file @ 6f4217a3
// Code generated by protoc-gen-go.
// Code generated by protoc-gen-go. DO NOT EDIT.
// source: manifest.proto
// DO NOT EDIT!
/*
Package proto is a generated protocol buffer package.
......@@ -97,6 +96,83 @@ func (m *Resource) String() string { return proto1.CompactTextString(
func (*Resource) ProtoMessage() {}
func (*Resource) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
func (m *Resource) GetPath() []string {
if m != nil {
return m.Path
}
return nil
}
func (m *Resource) GetUid() int64 {
if m != nil {
return m.Uid
}
return 0
}
func (m *Resource) GetGid() int64 {
if m != nil {
return m.Gid
}
return 0
}
func (m *Resource) GetUser() string {
if m != nil {
return m.User
}
return ""
}
func (m *Resource) GetGroup() string {
if m != nil {
return m.Group
}
return ""
}
func (m *Resource) GetMode() uint32 {
if m != nil {
return m.Mode
}
return 0
}
func (m *Resource) GetSize() uint64 {
if m != nil {
return m.Size
}
return 0
}
func (m *Resource) GetDigest() []string {
if m != nil {
return m.Digest
}
return nil
}
func (m *Resource) GetTarget() string {
if m != nil {
return m.Target
}
return ""
}
func (m *Resource) GetMajor() uint64 {
if m != nil {
return m.Major
}
return 0
}
func (m *Resource) GetMinor() uint64 {
if m != nil {
return m.Minor
}
return 0
}
func (m *Resource) GetXattr() []*XAttr {
if m != nil {
return m.Xattr
......@@ -124,6 +200,20 @@ func (m *XAttr) String() string { return proto1.CompactTextString(m)
func (*XAttr) ProtoMessage() {}
func (*XAttr) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
func (m *XAttr) GetName() string {
if m != nil {
return m.Name
}
return ""
}
func (m *XAttr) GetData() []byte {
if m != nil {
return m.Data
}
return nil
}
// ADSEntry encodes information for a Windows Alternate Data Stream.
type ADSEntry struct {
// Name specifices the stream name.
......@@ -147,6 +237,27 @@ func (m *ADSEntry) String() string { return proto1.CompactTextString(
func (*ADSEntry) ProtoMessage() {}
func (*ADSEntry) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
func (m *ADSEntry) GetName() string {
if m != nil {
return m.Name
}
return ""
}
func (m *ADSEntry) GetData() []byte {
if m != nil {
return m.Data
}
return nil
}
func (m *ADSEntry) GetDigest() string {
if m != nil {
return m.Digest
}
return ""
}
func init() {
proto1.RegisterType((*Manifest)(nil), "proto.Manifest")
proto1.RegisterType((*Resource)(nil), "proto.Resource")
......@@ -158,7 +269,7 @@ func init() { proto1.RegisterFile("manifest.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
// 317 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x90, 0x4f, 0x4b, 0xf3, 0x40,
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x90, 0x4f, 0x4b, 0xf3, 0x40,
0x10, 0xc6, 0x49, 0x93, 0xf4, 0x4d, 0xa7, 0xed, 0xab, 0x2c, 0x52, 0xe6, 0x18, 0x73, 0x0a, 0x08,
0x15, 0xf4, 0xe0, 0xb9, 0xa2, 0x17, 0xc1, 0xcb, 0x7a, 0xf1, 0xba, 0xba, 0x6b, 0x5c, 0x21, 0xd9,
0xb0, 0xd9, 0x80, 0xfa, 0xe5, 0xfc, 0x6a, 0x32, 0xb3, 0x69, 0xd1, 0x9b, 0xa7, 0x3c, 0xcf, 0x6f,
......
vendor/github.com/containerd/continuity/resource_unix.go
View file @ 6f4217a3
//go:build linux || darwin || freebsd || solaris
// +build linux darwin freebsd solaris
/*
......
vendor/github.com/containerd/continuity/sysx/nodata_unix.go
View file @ 6f4217a3
//go:build darwin || freebsd || openbsd
// +build darwin freebsd openbsd
/*
......
vendor/github.com/containerd/continuity/sysx/xattr.go
View file @ 6f4217a3
//go:build linux || darwin
// +build linux darwin
/*
......
vendor/github.com/containerd/continuity/sysx/xattr_unsupported.go
View file @ 6f4217a3
//go:build !linux && !darwin
// +build !linux,!darwin
/*
......
vendor/github.com/containerd/fuse-overlayfs-snapshotter/Dockerfile
View file @ 6f4217a3
......@@ -13,12 +13,12 @@
# limitations under the License.
ARG FUSEOVERLAYFS_COMMIT=master
ARG ROOTLESSKIT_COMMIT=v0.14.0
ARG SHADOW_COMMIT=4.8.1
ARG ROOTLESSKIT_COMMIT=v0.14.6
ARG SHADOW_COMMIT=4.9
ARG GO_VERSION=1.16
ARG DEBIAN_VERSION=10
ARG ALPINE_VERSION=3.13
ARG GO_VERSION=1.17
ARG DEBIAN_VERSION=11
ARG ALPINE_VERSION=3.14
FROM golang:${GO_VERSION}-alpine AS containerd-fuse-overlayfs-test
COPY . /go/src/github.com/containerd/fuse-overlayfs-snapshotter
......
vendor/github.com/containerd/fuse-overlayfs-snapshotter/README.md
View file @ 6f4217a3
......@@ -5,8 +5,6 @@ Unlike `overlayfs`, `fuse-overlayfs` can be used as a non-root user on almost al
You do NOT need this `fuse-overlayfs` plugin on the following environments, because they support the real `overlayfs` for non-root users:
- [kernel >= 5.11](https://github.com/torvalds/linux/commit/459c7c565ac36ba09ffbf24231147f408fde4203)
- [Ubuntu kernel, since circa 2015](https://kernel.ubuntu.com/git/ubuntu/ubuntu-bionic.git/commit/fs/overlayfs?id=3b7da90f28fe1ed4b79ef2d994c81efbc58f1144)
- [Debian 10 kernel](https://salsa.debian.org/kernel-team/linux/blob/283390e7feb21b47779b48e0c8eb0cc409d2c815/debian/patches/debian/overlayfs-permit-mounts-in-userns.patch)
- Debian 10 needs `sudo modprobe overlay permit_mounts_in_userns=1`. Future release of Debian with kernel >= 5.11 will not need this `modprobe` hack.
fuse-overlayfs-snapshotter is a **non-core** sub-project of containerd.
......
vendor/github.com/containerd/fuse-overlayfs-snapshotter/check.go
View file @ 6f4217a3
//go:build linux
// +build linux
/*
......@@ -27,7 +28,6 @@ import (
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/mount"
"github.com/pkg/errors"
)
// supportsReadonlyMultipleLowerDir checks if read-only multiple lowerdirs can be mounted with fuse-overlayfs.
......@@ -57,7 +57,7 @@ func supportsReadonlyMultipleLowerDir(d string) error {
}
dest := filepath.Join(td, "merged")
if err := m.Mount(dest); err != nil {
return errors.Wrapf(err, "failed to mount fuse-overlayfs (%+v) on %s", m, dest)
return fmt.Errorf("failed to mount fuse-overlayfs (%+v) on %s: %w", m, dest, err)
}
if err := mount.UnmountAll(dest, 0); err != nil {
log.L.WithError(err).Warnf("Failed to unmount check directory %v", dest)
......@@ -70,13 +70,13 @@ func supportsReadonlyMultipleLowerDir(d string) error {
// this snapshotter as a library.
func Supported(root string) error {
if _, err := exec.LookPath(fuseoverlayfsBinary); err != nil {
return errors.Wrapf(err, "%s not installed", fuseoverlayfsBinary)
return fmt.Errorf("%s not installed: %w", fuseoverlayfsBinary, err)
}
if err := os.MkdirAll(root, 0700); err != nil {
return err
}
if err := supportsReadonlyMultipleLowerDir(root); err != nil {
return errors.Wrap(err, "fuse-overlayfs not functional, make sure running with kernel >= 4.18")
return fmt.Errorf("fuse-overlayfs not functional, make sure running with kernel >= 4.18: %w", err)
}
return nil
}
vendor/github.com/containerd/fuse-overlayfs-snapshotter/fuseoverlayfs.go
View file @ 6f4217a3
//go:build linux
// +build linux
/*
......@@ -32,7 +33,6 @@ import (
"github.com/containerd/containerd/snapshots"
"github.com/containerd/containerd/snapshots/storage"
"github.com/containerd/continuity/fs"
"github.com/pkg/errors"
)
const (
......@@ -186,7 +186,7 @@ func (o *snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, er
_, info, _, err := storage.GetInfo(ctx, key)
t.Rollback()
if err != nil {
return nil, errors.Wrap(err, "failed to get active mount")
return nil, fmt.Errorf("failed to get active mount: %w", err)
}
return o.mounts(s, info), nil
}
......@@ -217,7 +217,7 @@ func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snap
}
if _, err = storage.CommitActive(ctx, key, name, snapshots.Usage(usage), opts...); err != nil {
return errors.Wrap(err, "failed to commit snapshot")
return fmt.Errorf("failed to commit snapshot: %w", err)
}
return t.Commit()
}
......@@ -240,14 +240,14 @@ func (o *snapshotter) Remove(ctx context.Context, key string) (err error) {
_, _, err = storage.Remove(ctx, key)
if err != nil {
return errors.Wrap(err, "failed to remove")
return fmt.Errorf("failed to remove: %w", err)
}
if !o.asyncRemove {
var removals []string
removals, err = o.getCleanupDirectories(ctx, t)
if err != nil {
return errors.Wrap(err, "unable to get directories for removal")
return fmt.Errorf("unable to get directories for removal: %w", err)
}
// Remove directories after the transaction is closed, failures must not
......@@ -353,7 +353,7 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k
if path != "" {
if err1 := os.RemoveAll(path); err1 != nil {
log.G(ctx).WithError(err1).WithField("path", path).Error("failed to reclaim snapshot directory, directory may need removal")
err = errors.Wrapf(err, "failed to remove path: %v", err1)
err = fmt.Errorf("failed to remove path: %v: %w", err1, err)
}
}
}
......@@ -365,7 +365,7 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k
if rerr := t.Rollback(); rerr != nil {
log.G(ctx).WithError(rerr).Warn("failed to rollback transaction")
}
return nil, errors.Wrap(err, "failed to create prepare snapshot dir")
return nil, fmt.Errorf("failed to create prepare snapshot dir: %w", err)
}
rollback := true
defer func() {
......@@ -378,13 +378,13 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k
s, err := storage.CreateSnapshot(ctx, kind, key, parent, opts...)
if err != nil {
return nil, errors.Wrap(err, "failed to create snapshot")
return nil, fmt.Errorf("failed to create snapshot: %w", err)
}
if len(s.ParentIDs) > 0 {
st, err := os.Stat(o.upperPath(s.ParentIDs[0]))
if err != nil {
return nil, errors.Wrap(err, "failed to stat parent")
return nil, fmt.Errorf("failed to stat parent: %w", err)
}
stat := st.Sys().(*syscall.Stat_t)
......@@ -393,13 +393,13 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k
if rerr := t.Rollback(); rerr != nil {
log.G(ctx).WithError(rerr).Warn("failed to rollback transaction")
}
return nil, errors.Wrap(err, "failed to chown")
return nil, fmt.Errorf("failed to chown: %w", err)
}
}
path = filepath.Join(snapshotDir, s.ID)
if err = os.Rename(td, path); err != nil {
return nil, errors.Wrap(err, "failed to rename")
return nil, fmt.Errorf("failed to rename: %w", err)
}
td = ""
......@@ -407,7 +407,7 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k
rollback = false
if err = t.Commit(); err != nil {
return nil, errors.Wrap(err, "commit failed")
return nil, fmt.Errorf("commit failed: %w", err)
}
return o.mounts(s, info), nil
......@@ -416,7 +416,7 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k
func (o *snapshotter) prepareDirectory(ctx context.Context, snapshotDir string, kind snapshots.Kind) (string, error) {
td, err := ioutil.TempDir(snapshotDir, "new-")
if err != nil {
return "", errors.Wrap(err, "failed to create temp dir")
return "", fmt.Errorf("failed to create temp dir: %w", err)
}
if err := os.Mkdir(filepath.Join(td, "fs"), 0755); err != nil {
......
vendor/github.com/containerd/fuse-overlayfs-snapshotter/go.mod
View file @ 6f4217a3
......@@ -3,13 +3,9 @@ module github.com/containerd/fuse-overlayfs-snapshotter
go 1.16
require (
github.com/containerd/containerd v1.5.2
github.com/containerd/containerd v1.5.8
github.com/containerd/continuity v0.1.0
github.com/coreos/go-systemd/v22 v22.1.0
github.com/pkg/errors v0.9.1
github.com/sirupsen/logrus v1.7.0
github.com/coreos/go-systemd/v22 v22.3.2
github.com/sirupsen/logrus v1.8.1
google.golang.org/grpc v1.33.2
)
// replace grpc to match https://github.com/containerd/containerd/blob/v1.5.2/go.mod
replace google.golang.org/grpc => google.golang.org/grpc v1.27.1
vendor/github.com/containerd/stargz-snapshotter/cache/cache.go
View file @ 6f4217a3
......@@ -132,6 +132,7 @@ func NewDirectoryCache(directory string, config DirectoryCacheConfig) (BlobCache
}
dataCache = lrucache.New(maxEntry)
dataCache.OnEvicted = func(key string, value interface{}) {
value.(*bytes.Buffer).Reset()
bufPool.Put(value)
}
}
......@@ -296,7 +297,6 @@ func (dc *directoryCache) Add(key string, opts ...Option) (Writer, error) {
}
b := dc.bufPool.Get().(*bytes.Buffer)
b.Reset()
memW := &writer{
WriteCloser: nopWriteCloser(io.Writer(b)),
commitFunc: func() error {
......@@ -306,7 +306,7 @@ func (dc *directoryCache) Add(key string, opts ...Option) (Writer, error) {
}
cached, done, added := dc.cache.Add(key, b)
if !added {
dc.bufPool.Put(b) // already exists in the cache. abort it.
dc.putBuffer(b) // already exists in the cache. abort it.
}
commit := func() error {
defer done()
......@@ -331,7 +331,7 @@ func (dc *directoryCache) Add(key string, opts ...Option) (Writer, error) {
abortFunc: func() error {
defer w.Close()
defer w.Abort()
dc.bufPool.Put(b) // abort it.
dc.putBuffer(b) // abort it.
return nil
},
}
......@@ -339,6 +339,11 @@ func (dc *directoryCache) Add(key string, opts ...Option) (Writer, error) {
return memW, nil
}
func (dc *directoryCache) putBuffer(b *bytes.Buffer) {
b.Reset()
dc.bufPool.Put(b)
}
func (dc *directoryCache) Close() error {
dc.closedMu.Lock()
defer dc.closedMu.Unlock()
......
vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go
View file @ 6f4217a3
......@@ -118,7 +118,7 @@ func Open(sr *io.SectionReader, opt ...OpenOption) (*Reader, error) {
}
}
gzipCompressors := []Decompressor{new(GzipDecompressor), new(legacyGzipDecompressor)}
gzipCompressors := []Decompressor{new(GzipDecompressor), new(LegacyGzipDecompressor)}
decompressors := append(gzipCompressors, opts.decompressors...)
// Determine the size to fetch. Try to fetch as many bytes as possible.
......@@ -184,7 +184,7 @@ func OpenFooter(sr *io.SectionReader) (tocOffset int64, footerSize int64, rErr e
return 0, 0, fmt.Errorf("error reading footer: %v", err)
}
var allErr []error
for _, d := range []Decompressor{new(GzipDecompressor), new(legacyGzipDecompressor)} {
for _, d := range []Decompressor{new(GzipDecompressor), new(LegacyGzipDecompressor)} {
fSize := d.FooterSize()
fOffset := positive(int64(len(footer)) - fSize)
_, tocOffset, _, err := d.ParseFooter(footer[fOffset:])
......@@ -279,12 +279,12 @@ func (r *Reader) initFields() error {
pdir := r.getOrCreateDir(pdirName)
ent.NumLink++ // at least one name(ent.Name) references this entry.
if ent.Type == "hardlink" {
if org, ok := r.m[cleanEntryName(ent.LinkName)]; ok {
org.NumLink++ // original entry is referenced by this ent.Name.
ent = org
} else {
return fmt.Errorf("%q is a hardlink but the linkname %q isn't found", ent.Name, ent.LinkName)
org, err := r.getSource(ent)
if err != nil {
return err
}
org.NumLink++ // original entry is referenced by this ent.Name.
ent = org
}
pdir.addChild(path.Base(name), ent)
}
......@@ -303,6 +303,20 @@ func (r *Reader) initFields() error {
return nil
}
func (r *Reader) getSource(ent *TOCEntry) (_ *TOCEntry, err error) {
if ent.Type == "hardlink" {
org, ok := r.m[cleanEntryName(ent.LinkName)]
if !ok {
return nil, fmt.Errorf("%q is a hardlink but the linkname %q isn't found", ent.Name, ent.LinkName)
}
ent, err = r.getSource(org)
if err != nil {
return nil, err
}
}
return ent, nil
}
func parentDir(p string) string {
dir, _ := path.Split(p)
return strings.TrimSuffix(dir, "/")
......@@ -464,7 +478,11 @@ func (r *Reader) Lookup(path string) (e *TOCEntry, ok bool) {
}
e, ok = r.m[path]
if ok && e.Type == "hardlink" {
e, ok = r.m[e.LinkName]
var err error
e, err = r.getSource(e)
if err != nil {
return nil, false
}
}
return
}
......
vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go
View file @ 6f4217a3
......@@ -38,30 +38,34 @@ import (
)
type gzipCompression struct {
*gzipCompressor
*GzipCompressor
*GzipDecompressor
}
func newGzipCompressionWithLevel(level int) Compression {
return &gzipCompression{
&gzipCompressor{level},
&GzipCompressor{level},
&GzipDecompressor{},
}
}
func NewGzipCompressorWithLevel(level int) Compressor {
return &gzipCompressor{level}
func NewGzipCompressor() *GzipCompressor {
return &GzipCompressor{gzip.BestCompression}
}
type gzipCompressor struct {
func NewGzipCompressorWithLevel(level int) *GzipCompressor {
return &GzipCompressor{level}
}
type GzipCompressor struct {
compressionLevel int
}
func (gc *gzipCompressor) Writer(w io.Writer) (io.WriteCloser, error) {
func (gc *GzipCompressor) Writer(w io.Writer) (io.WriteCloser, error) {
return gzip.NewWriterLevel(w, gc.compressionLevel)
}
func (gc *gzipCompressor) WriteTOCAndFooter(w io.Writer, off int64, toc *JTOC, diffHash hash.Hash) (digest.Digest, error) {
func (gc *GzipCompressor) WriteTOCAndFooter(w io.Writer, off int64, toc *JTOC, diffHash hash.Hash) (digest.Digest, error) {
tocJSON, err := json.MarshalIndent(toc, "", "\t")
if err != nil {
return "", err
......@@ -155,17 +159,21 @@ func (gz *GzipDecompressor) FooterSize() int64 {
return FooterSize
}
type legacyGzipDecompressor struct{}
func (gz *GzipDecompressor) DecompressTOC(r io.Reader) (tocJSON io.ReadCloser, err error) {
return decompressTOCEStargz(r)
}
type LegacyGzipDecompressor struct{}
func (gz *legacyGzipDecompressor) Reader(r io.Reader) (io.ReadCloser, error) {
func (gz *LegacyGzipDecompressor) Reader(r io.Reader) (io.ReadCloser, error) {
return gzip.NewReader(r)
}
func (gz *legacyGzipDecompressor) ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) {
func (gz *LegacyGzipDecompressor) ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) {
return parseTOCEStargz(r)
}
func (gz *legacyGzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) {
func (gz *LegacyGzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) {
if len(p) != legacyFooterSize {
return 0, 0, 0, fmt.Errorf("legacy: invalid length %d cannot be parsed", len(p))
}
......@@ -188,29 +196,43 @@ func (gz *legacyGzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOff
return tocOffset, tocOffset, 0, nil
}
func (gz *legacyGzipDecompressor) FooterSize() int64 {
func (gz *LegacyGzipDecompressor) FooterSize() int64 {
return legacyFooterSize
}
func (gz *LegacyGzipDecompressor) DecompressTOC(r io.Reader) (tocJSON io.ReadCloser, err error) {
return decompressTOCEStargz(r)
}
func parseTOCEStargz(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) {
tr, err := decompressTOCEStargz(r)
if err != nil {
return nil, "", err
}
dgstr := digest.Canonical.Digester()
toc = new(JTOC)
if err := json.NewDecoder(io.TeeReader(tr, dgstr.Hash())).Decode(&toc); err != nil {
return nil, "", fmt.Errorf("error decoding TOC JSON: %v", err)
}
if err := tr.Close(); err != nil {
return nil, "", err
}
return toc, dgstr.Digest(), nil
}
func decompressTOCEStargz(r io.Reader) (tocJSON io.ReadCloser, err error) {
zr, err := gzip.NewReader(r)
if err != nil {
return nil, "", fmt.Errorf("malformed TOC gzip header: %v", err)
return nil, fmt.Errorf("malformed TOC gzip header: %v", err)
}
defer zr.Close()
zr.Multistream(false)
tr := tar.NewReader(zr)
h, err := tr.Next()
if err != nil {
return nil, "", fmt.Errorf("failed to find tar header in TOC gzip stream: %v", err)
return nil, fmt.Errorf("failed to find tar header in TOC gzip stream: %v", err)
}
if h.Name != TOCTarName {
return nil, "", fmt.Errorf("TOC tar entry had name %q; expected %q", h.Name, TOCTarName)
return nil, fmt.Errorf("TOC tar entry had name %q; expected %q", h.Name, TOCTarName)
}
dgstr := digest.Canonical.Digester()
toc = new(JTOC)
if err := json.NewDecoder(io.TeeReader(tr, dgstr.Hash())).Decode(&toc); err != nil {
return nil, "", fmt.Errorf("error decoding TOC JSON: %v", err)
}
return toc, dgstr.Digest(), nil
return readCloser{tr, zr.Close}, nil
}
vendor/github.com/containerd/stargz-snapshotter/estargz/zstdchunked/zstdchunked.go
View file @ 6f4217a3
......@@ -17,6 +17,7 @@
package zstdchunked
import (
"bufio"
"bytes"
"encoding/binary"
"encoding/json"
......@@ -88,6 +89,25 @@ func (zz *Decompressor) FooterSize() int64 {
return FooterSize
}
func (zz *Decompressor) DecompressTOC(r io.Reader) (tocJSON io.ReadCloser, err error) {
decoder, err := zstd.NewReader(r)
if err != nil {
return nil, err
}
br := bufio.NewReader(decoder)
if _, err := br.Peek(1); err != nil {
return nil, err
}
return &reader{br, decoder.Close}, nil
}
type reader struct {
io.Reader
closeFunc func()
}
func (r *reader) Close() error { r.closeFunc(); return nil }
type zstdReadCloser struct{ *zstd.Decoder }
func (z *zstdReadCloser) Close() error {
......
vendor/github.com/containerd/stargz-snapshotter/fs/config/config.go
View file @ 6f4217a3
......@@ -46,6 +46,7 @@ type Config struct {
DisableVerification bool `toml:"disable_verification"`
MaxConcurrency int64 `toml:"max_concurrency"`
NoPrometheus bool `toml:"no_prometheus"`
MetadataStore string `toml:"metadata_store" default:"memory"`
// BlobConfig is config for layer blob management.
BlobConfig `toml:"blob"`
......@@ -69,13 +70,17 @@ type BlobConfig struct {
// If PrefetchChunkSize < ChunkSize prefetch bytes will be fetched as a single http GET,
// else total GET requests for prefetch = ceil(PrefetchSize / PrefetchChunkSize).
PrefetchChunkSize int64 `toml:"prefetch_chunk_size"`
MaxRetries int `toml:"max_retries"`
MinWaitMSec int `toml:"min_wait_msec"`
MaxWaitMSec int `toml:"max_wait_msec"`
}
type DirectoryCacheConfig struct {
MaxLRUCacheEntry int `toml:"max_lru_cache_entry"`
MaxCacheFds int `toml:"max_cache_fds"`
SyncAdd bool `toml:"sync_add"`
Direct bool `toml:"direct"`
Direct bool `toml:"direct" default:"true"`
}
type FuseConfig struct {
......
vendor/github.com/containerd/stargz-snapshotter/fs/fs.go
View file @ 6f4217a3
......@@ -53,6 +53,7 @@ import (
"github.com/containerd/stargz-snapshotter/fs/layer"
commonmetrics "github.com/containerd/stargz-snapshotter/fs/metrics/common"
layermetrics "github.com/containerd/stargz-snapshotter/fs/metrics/layer"
"github.com/containerd/stargz-snapshotter/fs/remote"
"github.com/containerd/stargz-snapshotter/fs/source"
"github.com/containerd/stargz-snapshotter/snapshot"
"github.com/containerd/stargz-snapshotter/task"
......@@ -73,7 +74,8 @@ const (
type Option func(*options)
type options struct {
getSources source.GetSources
getSources source.GetSources
resolveHandlers map[string]remote.Handler
}
func WithGetSources(s source.GetSources) Option {
......@@ -82,6 +84,15 @@ func WithGetSources(s source.GetSources) Option {
}
}
func WithResolveHandler(name string, handler remote.Handler) Option {
return func(opts *options) {
if opts.resolveHandlers == nil {
opts.resolveHandlers = make(map[string]remote.Handler)
}
opts.resolveHandlers[name] = handler
}
}
func NewFilesystem(root string, cfg config.Config, opts ...Option) (_ snapshot.FileSystem, err error) {
var fsOpts options
for _, o := range opts {
......@@ -109,7 +120,7 @@ func NewFilesystem(root string, cfg config.Config, opts ...Option) (_ snapshot.F
})
}
tm := task.NewBackgroundTaskManager(maxConcurrency, 5*time.Second)
r, err := layer.NewResolver(root, tm, cfg)
r, err := layer.NewResolver(root, tm, cfg, fsOpts.resolveHandlers)
if err != nil {
return nil, errors.Wrapf(err, "failed to setup resolver")
}
......@@ -177,6 +188,13 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s
return fmt.Errorf("source must be passed")
}
defaultPrefetchSize := fs.prefetchSize
if psStr, ok := labels[config.TargetPrefetchSizeLabel]; ok {
if ps, err := strconv.ParseInt(psStr, 10, 64); err == nil {
defaultPrefetchSize = ps
}
}
// Resolve the target layer
var (
resultChan = make(chan layer.Layer)
......@@ -188,10 +206,10 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s
l, err := fs.resolver.Resolve(ctx, s.Hosts, s.Name, s.Target)
if err == nil {
resultChan <- l
fs.prefetch(ctx, l, defaultPrefetchSize, start)
return
}
rErr = errors.Wrapf(rErr, "failed to resolve layer %q from %q: %v",
s.Target.Digest, s.Name, err)
rErr = errors.Wrapf(rErr, "failed to resolve layer %q from %q: %v", s.Target.Digest, s.Name, err)
}
errChan <- rErr
}()
......@@ -202,12 +220,17 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s
desc := desc
go func() {
// Avoids to get canceled by client.
ctx := log.WithLogger(context.Background(),
log.G(ctx).WithField("mountpoint", mountpoint))
err := fs.resolver.Cache(ctx, preResolve.Hosts, preResolve.Name, desc)
ctx := log.WithLogger(context.Background(), log.G(ctx).WithField("mountpoint", mountpoint))
l, err := fs.resolver.Resolve(ctx, preResolve.Hosts, preResolve.Name, desc)
if err != nil {
log.G(ctx).WithError(err).Debug("failed to pre-resolve")
return
}
fs.prefetch(ctx, l, defaultPrefetchSize, start)
// Release this layer because this isn't target and we don't use it anymore here.
// However, this will remain on the resolver cache until eviction.
l.Done()
}()
}
......@@ -255,7 +278,7 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s
// Verification must be done. Don't mount this layer.
return fmt.Errorf("digest of TOC JSON must be passed")
}
node, err := l.RootNode()
node, err := l.RootNode(0)
if err != nil {
log.G(ctx).WithError(err).Warnf("Failed to get root node")
return errors.Wrapf(err, "failed to get root node")
......@@ -263,7 +286,7 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s
// Measuring duration of Mount operation for resolved layer.
digest := l.Info().Digest // get layer sha
defer commonmetrics.MeasureLatency(commonmetrics.Mount, digest, start)
defer commonmetrics.MeasureLatencyInMilliseconds(commonmetrics.Mount, digest, start)
// Register the mountpoint layer
fs.layerMu.Lock()
......@@ -271,41 +294,6 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s
fs.layerMu.Unlock()
fs.metricsController.Add(mountpoint, l)
// Prefetch this layer. We prefetch several layers in parallel. The first
// Check() for this layer waits for the prefetch completion.
if !fs.noprefetch {
prefetchSize := fs.prefetchSize
if psStr, ok := labels[config.TargetPrefetchSizeLabel]; ok {
if ps, err := strconv.ParseInt(psStr, 10, 64); err == nil {
prefetchSize = ps
}
}
go func() {
fs.backgroundTaskManager.DoPrioritizedTask()
defer fs.backgroundTaskManager.DonePrioritizedTask()
if err := l.Prefetch(prefetchSize); err != nil {
log.G(ctx).WithError(err).Warnf("failed to prefetch layer=%v", digest)
return
}
log.G(ctx).Debug("completed to prefetch")
}()
}
// Fetch whole layer aggressively in background. We use background
// reader for this so prioritized tasks(Mount, Check, etc...) can
// interrupt the reading. This can avoid disturbing prioritized tasks
// about NW traffic.
if !fs.noBackgroundFetch {
go func() {
if err := l.BackgroundFetch(); err != nil {
log.G(ctx).WithError(err).Warnf("failed to fetch whole layer=%v", digest)
return
}
commonmetrics.LogLatencyForLastOnDemandFetch(ctx, digest, start, l.Info().ReadTime) // write log record for the latency between mount start and last on demand fetch
log.G(ctx).Debug("completed to fetch all layer data in background")
}()
}
// mount the node to the specified mountpoint
// TODO: bind mount the state directory as a read-only fs on snapshotter's side
rawFS := fusefs.NewNodeFS(node, &fusefs.Options{
......@@ -341,7 +329,7 @@ func (fs *filesystem) Check(ctx context.Context, mountpoint string, labels map[s
fs.backgroundTaskManager.DoPrioritizedTask()
defer fs.backgroundTaskManager.DonePrioritizedTask()
defer commonmetrics.MeasureLatency(commonmetrics.PrefetchesCompleted, digest.FromString(""), time.Now()) // measuring the time the container launch is blocked on prefetch to complete
defer commonmetrics.MeasureLatencyInMilliseconds(commonmetrics.PrefetchesCompleted, digest.FromString(""), time.Now()) // measuring the time the container launch is blocked on prefetch to complete
ctx = log.WithLogger(ctx, log.G(ctx).WithField("mountpoint", mountpoint))
......@@ -422,6 +410,23 @@ func (fs *filesystem) Unmount(ctx context.Context, mountpoint string) error {
return syscall.Unmount(mountpoint, syscall.MNT_FORCE)
}
func (fs *filesystem) prefetch(ctx context.Context, l layer.Layer, defaultPrefetchSize int64, start time.Time) {
// Prefetch a layer. The first Check() for this layer waits for the prefetch completion.
if !fs.noprefetch {
go l.Prefetch(defaultPrefetchSize)
}
// Fetch whole layer aggressively in background.
if !fs.noBackgroundFetch {
go func() {
if err := l.BackgroundFetch(); err == nil {
// write log record for the latency between mount start and last on demand fetch
commonmetrics.LogLatencyForLastOnDemandFetch(ctx, l.Info().Digest, start, l.Info().ReadTime)
}
}()
}
}
// neighboringLayers returns layer descriptors except the `target` layer in the specified manifest.
func neighboringLayers(manifest ocispec.Manifest, target ocispec.Descriptor) (descs []ocispec.Descriptor) {
for _, desc := range manifest.Layers {
......
vendor/github.com/containerd/stargz-snapshotter/fs/metrics/common/metrics.go
View file @ 6f4217a3
......@@ -27,8 +27,11 @@ import (
)
const (
// OperationLatencyKey is the key for stargz operation latency metrics.
OperationLatencyKey = "operation_duration"
// OperationLatencyKeyMilliseconds is the key for stargz operation latency metrics in milliseconds.
OperationLatencyKeyMilliseconds = "operation_duration_milliseconds"
// OperationLatencyKeyMicroseconds is the key for stargz operation latency metrics in microseconds.
OperationLatencyKeyMicroseconds = "operation_duration_microseconds"
// OperationCountKey is the key for stargz operation count metrics.
OperationCountKey = "operation_count"
......@@ -71,18 +74,32 @@ const (
)
var (
// Buckets for OperationLatency metric in milliseconds.
latencyBuckets = []float64{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384} // in milliseconds
// Buckets for OperationLatency metrics.
latencyBucketsMilliseconds = []float64{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384} // in milliseconds
latencyBucketsMicroseconds = []float64{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024} // in microseconds
// operationLatency collects operation latency numbers by operation
// type and layer digest.
operationLatency = prometheus.NewHistogramVec(
// operationLatencyMilliseconds collects operation latency numbers in milliseconds grouped by
// operation, type and layer digest.
operationLatencyMilliseconds = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: OperationLatencyKey,
Name: OperationLatencyKeyMilliseconds,
Help: "Latency in milliseconds of stargz snapshotter operations. Broken down by operation type and layer sha.",
Buckets: latencyBuckets,
Buckets: latencyBucketsMilliseconds,
},
[]string{"operation_type", "layer"},
)
// operationLatencyMicroseconds collects operation latency numbers in microseconds grouped by
// operation, type and layer digest.
operationLatencyMicroseconds = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: OperationLatencyKeyMicroseconds,
Help: "Latency in microseconds of stargz snapshotter operations. Broken down by operation type and layer sha.",
Buckets: latencyBucketsMicroseconds,
},
[]string{"operation_type", "layer"},
)
......@@ -120,22 +137,39 @@ func sinceInMilliseconds(start time.Time) float64 {
return float64(time.Since(start).Nanoseconds()) / 1e6
}
// sinceInMicroseconds gets the time since the specified start in microseconds.
// The division by 1e3 is made to have the microseconds value as floating point number, since the native method
// .Microseconds() returns an integer value and you can lost a precision for sub-microsecond values.
func sinceInMicroseconds(start time.Time) float64 {
return float64(time.Since(start).Nanoseconds()) / 1e3
}
// Register registers metrics. This is always called only once.
func Register() {
register.Do(func() {
prometheus.MustRegister(operationLatency)
prometheus.MustRegister(operationLatencyMilliseconds)
prometheus.MustRegister(operationLatencyMicroseconds)
prometheus.MustRegister(operationCount)
prometheus.MustRegister(bytesCount)
})
}
// MeasureLatency wraps the labels attachment as well as calling Observe into a single method.
// MeasureLatencyInMilliseconds wraps the labels attachment as well as calling Observe into a single method.
// Right now we attach the operation and layer digest, so it's possible to see the breakdown for latency
// by operation and individual layers.
// If you want this to be layer agnostic, just pass the digest from empty string, e.g.
// layerDigest := digest.FromString("")
func MeasureLatencyInMilliseconds(operation string, layer digest.Digest, start time.Time) {
operationLatencyMilliseconds.WithLabelValues(operation, layer.String()).Observe(sinceInMilliseconds(start))
}
// MeasureLatencyInMicroseconds wraps the labels attachment as well as calling Observe into a single method.
// Right now we attach the operation and layer digest, so it's possible to see the breakdown for latency
// by operation and individual layers.
// If you want this to be layer agnostic, just pass the digest from empty string, e.g.
// layerDigest := digest.FromString("")
func MeasureLatency(operation string, layer digest.Digest, start time.Time) {
operationLatency.WithLabelValues(operation, layer.String()).Observe(sinceInMilliseconds(start))
func MeasureLatencyInMicroseconds(operation string, layer digest.Digest, start time.Time) {
operationLatencyMicroseconds.WithLabelValues(operation, layer.String()).Observe(sinceInMicroseconds(start))
}
// IncOperationCount wraps the labels attachment as well as calling Inc into a single method.
......
vendor/github.com/containerd/stargz-snapshotter/fs/remote/blob.go
View file @ 6f4217a3
......@@ -28,6 +28,8 @@ import (
"io"
"io/ioutil"
"regexp"
"sort"
"strings"
"sync"
"time"
......@@ -37,6 +39,7 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
"golang.org/x/sync/singleflight"
)
var contentRangeRegexp = regexp.MustCompile(`bytes ([0-9]+)-([0-9]+)/([0-9]+|\\*)`)
......@@ -52,7 +55,7 @@ type Blob interface {
}
type blob struct {
fetcher *fetcher
fetcher fetcher
fetcherMu sync.Mutex
size int64
......@@ -64,8 +67,10 @@ type blob struct {
checkInterval time.Duration
fetchTimeout time.Duration
fetchedRegionSet regionSet
fetchedRegionSetMu sync.Mutex
fetchedRegionSet regionSet
fetchedRegionSetMu sync.Mutex
fetchedRegionGroup singleflight.Group
fetchedRegionCopyMu sync.Mutex
resolver *Resolver
......@@ -73,6 +78,22 @@ type blob struct {
closedMu sync.Mutex
}
func makeBlob(fetcher fetcher, size int64, chunkSize int64, prefetchChunkSize int64,
blobCache cache.BlobCache, lastCheck time.Time, checkInterval time.Duration,
r *Resolver, fetchTimeout time.Duration) *blob {
return &blob{
fetcher: fetcher,
size: size,
chunkSize: chunkSize,
prefetchChunkSize: prefetchChunkSize,
cache: blobCache,
lastCheck: lastCheck,
checkInterval: checkInterval,
resolver: r,
fetchTimeout: fetchTimeout,
}
}
func (b *blob) Close() error {
b.closedMu.Lock()
defer b.closedMu.Unlock()
......@@ -96,16 +117,17 @@ func (b *blob) Refresh(ctx context.Context, hosts source.RegistryHosts, refspec
}
// refresh the fetcher
new, newSize, err := newFetcher(ctx, hosts, refspec, desc)
f, newSize, err := b.resolver.resolveFetcher(ctx, hosts, refspec, desc)
if err != nil {
return err
} else if newSize != b.size {
}
if newSize != b.size {
return fmt.Errorf("Invalid size of new blob %d; want %d", newSize, b.size)
}
// update the blob's fetcher with new one
b.fetcherMu.Lock()
b.fetcher = new
b.fetcher = f
b.fetcherMu.Unlock()
b.lastCheckMu.Lock()
b.lastCheck = time.Now()
......@@ -153,7 +175,18 @@ func (b *blob) FetchedSize() int64 {
return sz
}
func (b *blob) cacheAt(offset int64, size int64, fr *fetcher, cacheOpts *options) error {
func makeSyncKey(allData map[region]io.Writer) string {
keys := make([]string, len(allData))
keysIndex := 0
for key := range allData {
keys[keysIndex] = fmt.Sprintf("[%d,%d]", key.b, key.e)
keysIndex++
}
sort.Strings(keys)
return strings.Join(keys, ",")
}
func (b *blob) cacheAt(offset int64, size int64, fr fetcher, cacheOpts *options) error {
fetchReg := region{floor(offset, b.chunkSize), ceil(offset+size-1, b.chunkSize) - 1}
discard := make(map[region]io.Writer)
......@@ -275,8 +308,9 @@ func (b *blob) ReadAt(p []byte, offset int64, opts ...Option) (int, error) {
return len(p), nil
}
// fetchRange fetches all specified chunks from local cache and remote blob.
func (b *blob) fetchRange(allData map[region]io.Writer, opts *options) error {
// fetchRegions fetches all specified chunks from remote blob and puts it in the local cache.
// It must be called from within fetchRange and need to ensure that it is inside the singleflight `Do` operation.
func (b *blob) fetchRegions(allData map[region]io.Writer, fetched map[region]bool, opts *options) error {
if len(allData) == 0 {
return nil
}
......@@ -289,14 +323,18 @@ func (b *blob) fetchRange(allData map[region]io.Writer, opts *options) error {
// request missed regions
var req []region
fetched := make(map[region]bool)
for reg := range allData {
req = append(req, reg)
fetched[reg] = false
}
ctx, cancel := context.WithTimeout(context.Background(), b.fetchTimeout)
fetchCtx, cancel := context.WithTimeout(context.Background(), b.fetchTimeout)
defer cancel()
mr, err := fr.fetch(ctx, req, true, opts)
if opts.ctx != nil {
fetchCtx = opts.ctx
}
mr, err := fr.fetch(fetchCtx, req, true)
if err != nil {
return err
}
......@@ -366,6 +404,64 @@ func (b *blob) fetchRange(allData map[region]io.Writer, opts *options) error {
return nil
}
// fetchRange fetches all specified chunks from local cache and remote blob.
func (b *blob) fetchRange(allData map[region]io.Writer, opts *options) error {
if len(allData) == 0 {
return nil
}
// We build a key based on regions we need to fetch and pass it to singleflightGroup.Do(...)
// to block simultaneous same requests. Once the request is finished and the data is ready,
// all blocked callers will be unblocked and that same data will be returned by all blocked callers.
key := makeSyncKey(allData)
fetched := make(map[region]bool)
_, err, shared := b.fetchedRegionGroup.Do(key, func() (interface{}, error) {
return nil, b.fetchRegions(allData, fetched, opts)
})
// When unblocked try to read from cache in case if there were no errors
// If we fail reading from cache, fetch from remote registry again
if err == nil && shared {
for reg := range allData {
if _, ok := fetched[reg]; ok {
continue
}
err = b.walkChunks(reg, func(chunk region) error {
b.fetcherMu.Lock()
fr := b.fetcher
b.fetcherMu.Unlock()
// Check if the content exists in the cache
// And if exists, read from cache
r, err := b.cache.Get(fr.genID(chunk), opts.cacheOpts...)
if err != nil {
return err
}
defer r.Close()
rr := io.NewSectionReader(r, 0, chunk.size())
// Copy the target chunk
b.fetchedRegionCopyMu.Lock()
defer b.fetchedRegionCopyMu.Unlock()
if _, err := io.CopyN(allData[chunk], rr, chunk.size()); err != nil {
return err
}
return nil
})
if err != nil {
break
}
}
// if we cannot read the data from cache, do fetch again
if err != nil {
return b.fetchRange(allData, opts)
}
}
return err
}
type walkFunc func(reg region) error
// walkChunks walks chunks from begin to end in order in the specified region.
......
vendor/github.com/containerd/stargz-snapshotter/fs/source/source.go
View file @ 6f4217a3
......@@ -70,6 +70,14 @@ const (
// targetImageLayersLabel is a label which contains layer digests contained in
// the target image.
targetImageLayersLabel = "containerd.io/snapshot/remote/stargz.layers"
// targetImageURLsLabelPrefix is a label prefix which constructs a map from the layer index to
// urls of the layer descriptor.
targetImageURLsLabelPrefix = "containerd.io/snapshot/remote/urls."
// targetURsLLabel is a label which contains layer URL. This is only used to pass URL from containerd
// to snapshotter.
targetURLsLabel = "containerd.io/snapshot/remote/urls"
)
// FromDefaultLabels returns a function for converting snapshot labels to
......@@ -94,31 +102,38 @@ func FromDefaultLabels(hosts RegistryHosts) GetSources {
return nil, err
}
var layersDgst []digest.Digest
var neighboringLayers []ocispec.Descriptor
if l, ok := labels[targetImageLayersLabel]; ok {
layersStr := strings.Split(l, ",")
for _, l := range layersStr {
for i, l := range layersStr {
d, err := digest.Parse(l)
if err != nil {
return nil, err
}
if d.String() != target.String() {
layersDgst = append(layersDgst, d)
desc := ocispec.Descriptor{Digest: d}
if urls, ok := labels[targetImageURLsLabelPrefix+fmt.Sprintf("%d", i)]; ok {
desc.URLs = strings.Split(urls, ",")
}
neighboringLayers = append(neighboringLayers, desc)
}
}
}
var layers []ocispec.Descriptor
for _, dgst := range append([]digest.Digest{target}, layersDgst...) {
layers = append(layers, ocispec.Descriptor{Digest: dgst})
targetDesc := ocispec.Descriptor{
Digest: target,
Annotations: labels,
}
if targetURLs, ok := labels[targetURLsLabel]; ok {
targetDesc.URLs = append(targetDesc.URLs, strings.Split(targetURLs, ",")...)
}
return []Source{
{
Hosts: hosts,
Name: refspec,
Target: ocispec.Descriptor{Digest: target},
Manifest: ocispec.Manifest{Layers: layers},
Target: targetDesc,
Manifest: ocispec.Manifest{Layers: append([]ocispec.Descriptor{targetDesc}, neighboringLayers...)},
},
}, nil
}
......@@ -146,7 +161,7 @@ func AppendDefaultLabelsHandlerWrapper(ref string, prefetchSize int64) func(f im
c.Annotations[targetRefLabel] = ref
c.Annotations[targetDigestLabel] = c.Digest.String()
var layers string
for _, l := range children[i:] {
for i, l := range children[i:] {
if images.IsLayerType(l.MediaType) {
ls := fmt.Sprintf("%s,", l.Digest.String())
// This avoids the label hits the size limitation.
......@@ -155,10 +170,17 @@ func AppendDefaultLabelsHandlerWrapper(ref string, prefetchSize int64) func(f im
break
}
layers += ls
// Store URLs of the neighbouring layer as well.
urlsKey := targetImageURLsLabelPrefix + fmt.Sprintf("%d", i)
c.Annotations[urlsKey] = appendWithValidation(urlsKey, l.URLs)
}
}
c.Annotations[targetImageLayersLabel] = strings.TrimSuffix(layers, ",")
c.Annotations[config.TargetPrefetchSizeLabel] = fmt.Sprintf("%d", prefetchSize)
// store URL in annotation to let containerd to pass it to the snapshotter
c.Annotations[targetURLsLabel] = appendWithValidation(targetURLsLabel, c.URLs)
}
}
}
......@@ -166,3 +188,15 @@ func AppendDefaultLabelsHandlerWrapper(ref string, prefetchSize int64) func(f im
})
}
}
func appendWithValidation(key string, values []string) string {
var v string
for _, u := range values {
s := fmt.Sprintf("%s,", u)
if err := labels.Validate(key, v+s); err != nil {
break
}
v += s
}
return strings.TrimSuffix(v, ",")
}
vendor/github.com/containerd/stargz-snapshotter/metadata/memory/reader.go 0 → 100644
View file @ 6f4217a3
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package memory
import (
"fmt"
"io"
"math"
"os"
"sync"
"time"
"github.com/containerd/stargz-snapshotter/estargz"
"github.com/containerd/stargz-snapshotter/metadata"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
)
type reader struct {
r *estargz.Reader
rootID uint32
idMap map[uint32]*estargz.TOCEntry
idOfEntry map[*estargz.TOCEntry]uint32
mu sync.Mutex
curID uint32
curIDMu sync.Mutex
opts *metadata.Options
}
func (r *reader) nextID() (uint32, error) {
r.curIDMu.Lock()
defer r.curIDMu.Unlock()
if r.curID == math.MaxUint32 {
return 0, fmt.Errorf("sequence id too large")
}
r.curID++
return r.curID, nil
}
func NewReader(sr *io.SectionReader, opts ...metadata.Option) (metadata.Reader, error) {
var rOpts metadata.Options
for _, o := range opts {
if err := o(&rOpts); err != nil {
return nil, errors.Wrapf(err, "failed to apply option")
}
}
telemetry := &estargz.Telemetry{}
if rOpts.Telemetry != nil {
telemetry.GetFooterLatency = estargz.MeasureLatencyHook(rOpts.Telemetry.GetFooterLatency)
telemetry.GetTocLatency = estargz.MeasureLatencyHook(rOpts.Telemetry.GetTocLatency)
telemetry.DeserializeTocLatency = estargz.MeasureLatencyHook(rOpts.Telemetry.DeserializeTocLatency)
}
var decompressors []estargz.Decompressor
for _, d := range rOpts.Decompressors {
decompressors = append(decompressors, d)
}
er, err := estargz.Open(sr,
estargz.WithTOCOffset(rOpts.TOCOffset),
estargz.WithTelemetry(telemetry),
estargz.WithDecompressors(decompressors...),
)
if err != nil {
return nil, err
}
root, ok := er.Lookup("")
if !ok {
return nil, fmt.Errorf("failed to get root node")
}
r := &reader{r: er, idMap: make(map[uint32]*estargz.TOCEntry), idOfEntry: make(map[*estargz.TOCEntry]uint32), opts: &rOpts}
rootID, err := r.initID(root)
if err != nil {
return nil, err
}
r.rootID = rootID
return r, nil
}
func (r *reader) initID(e *estargz.TOCEntry) (id uint32, err error) {
var ok bool
r.mu.Lock()
id, ok = r.idOfEntry[e]
if !ok {
id, err = r.nextID()
if err != nil {
return 0, err
}
r.idMap[id] = e
r.idOfEntry[e] = id
}
r.mu.Unlock()
e.ForeachChild(func(_ string, ent *estargz.TOCEntry) bool {
if ent.Type == "hardlink" {
var ok bool
ent, ok = r.r.Lookup(ent.Name)
if !ok {
return false
}
}
_, err = r.initID(ent)
return err == nil
})
return id, err
}
func (r *reader) RootID() uint32 {
return r.rootID
}
func (r *reader) TOCDigest() digest.Digest {
return r.r.TOCDigest()
}
func (r *reader) GetOffset(id uint32) (offset int64, err error) {
r.mu.Lock()
defer r.mu.Unlock()
e, ok := r.idMap[id]
if !ok {
return 0, fmt.Errorf("entry %d not found", id)
}
return e.Offset, nil
}
func (r *reader) GetAttr(id uint32) (attr metadata.Attr, err error) {
r.mu.Lock()
e, ok := r.idMap[id]
r.mu.Unlock()
if !ok {
err = fmt.Errorf("entry %d not found", id)
return
}
// TODO: zero copy
attrFromTOCEntry(e, &attr)
return
}
func (r *reader) GetChild(pid uint32, base string) (id uint32, attr metadata.Attr, err error) {
r.mu.Lock()
e, ok := r.idMap[pid]
r.mu.Unlock()
if !ok {
err = fmt.Errorf("parent entry %d not found", pid)
return
}
child, ok := e.LookupChild(base)
if !ok {
err = fmt.Errorf("child %q of entry %d not found", base, pid)
return
}
if child.Type == "hardlink" {
child, ok = r.r.Lookup(child.Name)
if !ok {
err = fmt.Errorf("child %q ()hardlink of entry %d not found", base, pid)
return
}
}
cid, ok := r.idOfEntry[child]
if !ok {
err = fmt.Errorf("id of entry %q not found", base)
return
}
// TODO: zero copy
attrFromTOCEntry(child, &attr)
return cid, attr, nil
}
func (r *reader) ForeachChild(id uint32, f func(name string, id uint32, mode os.FileMode) bool) error {
r.mu.Lock()
e, ok := r.idMap[id]
r.mu.Unlock()
if !ok {
return fmt.Errorf("parent entry %d not found", id)
}
var err error
e.ForeachChild(func(baseName string, ent *estargz.TOCEntry) bool {
if ent.Type == "hardlink" {
var ok bool
ent, ok = r.r.Lookup(ent.Name)
if !ok {
return false
}
}
r.mu.Lock()
id, ok := r.idOfEntry[ent]
r.mu.Unlock()
if !ok {
err = fmt.Errorf("id of child entry %q not found", baseName)
return false
}
return f(baseName, id, ent.Stat().Mode())
})
return err
}
func (r *reader) OpenFile(id uint32) (metadata.File, error) {
r.mu.Lock()
e, ok := r.idMap[id]
r.mu.Unlock()
if !ok {
return nil, fmt.Errorf("entry %d not found", id)
}
sr, err := r.r.OpenFile(e.Name)
if err != nil {
return nil, err
}
return &file{r, e, sr}, nil
}
func (r *reader) Clone(sr *io.SectionReader) (metadata.Reader, error) {
return NewReader(sr,
metadata.WithTOCOffset(r.opts.TOCOffset),
metadata.WithTelemetry(r.opts.Telemetry),
metadata.WithDecompressors(r.opts.Decompressors...),
)
}
func (r *reader) Close() error {
return nil
}
type file struct {
r *reader
e *estargz.TOCEntry
sr *io.SectionReader
}
func (r *file) ChunkEntryForOffset(offset int64) (off int64, size int64, dgst string, ok bool) {
e, ok := r.r.r.ChunkEntryForOffset(r.e.Name, offset)
if !ok {
return 0, 0, "", false
}
dgst = e.Digest
if e.ChunkDigest != "" {
// NOTE* "reg" also can contain ChunkDigest (e.g. when "reg" is the first entry of
// chunked file)
dgst = e.ChunkDigest
}
return e.ChunkOffset, e.ChunkSize, dgst, true
}
func (r *file) ReadAt(p []byte, off int64) (n int, err error) {
return r.sr.ReadAt(p, off)
}
func (r *reader) NumOfNodes() (i int, _ error) {
return len(r.idMap), nil
}
// TODO: share it with db pkg
func attrFromTOCEntry(src *estargz.TOCEntry, dst *metadata.Attr) *metadata.Attr {
dst.Size = src.Size
dst.ModTime, _ = time.Parse(time.RFC3339, src.ModTime3339)
dst.LinkName = src.LinkName
dst.Mode = src.Stat().Mode()
dst.UID = src.UID
dst.GID = src.GID
dst.DevMajor = src.DevMajor
dst.DevMinor = src.DevMinor
dst.Xattrs = src.Xattrs
dst.NumLink = src.NumLink
return dst
}
vendor/github.com/containerd/stargz-snapshotter/metadata/metadata.go 0 → 100644
View file @ 6f4217a3
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package metadata
import (
"io"
"os"
"time"
"github.com/containerd/stargz-snapshotter/estargz"
digest "github.com/opencontainers/go-digest"
)
// Attr reprensents the attributes of a node.
type Attr struct {
// Size, for regular files, is the logical size of the file.
Size int64
// ModTime is the modification time of the node.
ModTime time.Time
// LinkName, for symlinks, is the link target.
LinkName string
// Mode is the permission and mode bits.
Mode os.FileMode
// UID is the user ID of the owner.
UID int
// GID is the group ID of the owner.
GID int
// DevMajor is the major device number for device.
DevMajor int
// DevMinor is the major device number for device.
DevMinor int
// Xattrs are the extended attribute for the node.
Xattrs map[string][]byte
// NumLink is the number of names pointing to this node.
NumLink int
}
// TODO: docs
type Reader interface {
RootID() uint32
TOCDigest() digest.Digest
GetOffset(id uint32) (offset int64, err error)
GetAttr(id uint32) (attr Attr, err error)
GetChild(pid uint32, base string) (id uint32, attr Attr, err error)
ForeachChild(id uint32, f func(name string, id uint32, mode os.FileMode) bool) error
OpenFile(id uint32) (File, error)
Clone(sr *io.SectionReader) (Reader, error)
Close() error
}
type File interface {
ChunkEntryForOffset(offset int64) (off int64, size int64, dgst string, ok bool)
ReadAt(p []byte, off int64) (n int, err error)
}
type Decompressor interface {
estargz.Decompressor
// DecompressTOC decompresses the passed blob and returns a reader of TOC JSON.
DecompressTOC(io.Reader) (tocJSON io.ReadCloser, err error)
}
type Options struct {
TOCOffset int64
Telemetry *Telemetry
Decompressors []Decompressor
}
// Option is an option to configure the behaviour of reader.
type Option func(o *Options) error
// WithTOCOffset option specifies the offset of TOC
func WithTOCOffset(tocOffset int64) Option {
return func(o *Options) error {
o.TOCOffset = tocOffset
return nil
}
}
// WithTelemetry option specifies the telemetry hooks
func WithTelemetry(telemetry *Telemetry) Option {
return func(o *Options) error {
o.Telemetry = telemetry
return nil
}
}
// WithDecompressors option specifies decompressors to use.
// Default is gzip-based decompressor.
func WithDecompressors(decompressors ...Decompressor) Option {
return func(o *Options) error {
o.Decompressors = decompressors
return nil
}
}
// A func which takes start time and records the diff
type MeasureLatencyHook func(time.Time)
// A struct which defines telemetry hooks. By implementing these hooks you should be able to record
// the latency metrics of the respective steps of estargz open operation.
type Telemetry struct {
GetFooterLatency MeasureLatencyHook // measure time to get stargz footer (in milliseconds)
GetTocLatency MeasureLatencyHook // measure time to GET TOC JSON (in milliseconds)
DeserializeTocLatency MeasureLatencyHook // measure time to deserialize TOC JSON (in milliseconds)
}
vendor/github.com/containerd/stargz-snapshotter/service/resolver/cri.go
View file @ 6f4217a3
......@@ -40,6 +40,7 @@ import (
"github.com/containerd/containerd/remotes/docker"
dconfig "github.com/containerd/containerd/remotes/docker/config"
"github.com/containerd/stargz-snapshotter/fs/source"
rhttp "github.com/hashicorp/go-retryablehttp"
"github.com/pkg/errors"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
)
......@@ -136,18 +137,24 @@ func RegistryHostsFromCRIConfig(ctx context.Context, config Registry, credsFuncs
}
var (
transport = http.DefaultTransport.(*http.Transport).Clone()
client = &http.Client{Transport: transport}
config = config.Configs[u.Host]
rclient = rhttp.NewClient()
config = config.Configs[u.Host]
)
rclient.Logger = nil // disable logging every request
if config.TLS != nil {
transport.TLSClientConfig, err = getTLSConfig(*config.TLS)
if err != nil {
return nil, errors.Wrapf(err, "get TLSConfig for registry %q", e)
if tr, ok := rclient.HTTPClient.Transport.(*http.Transport); ok {
tr.TLSClientConfig, err = getTLSConfig(*config.TLS)
if err != nil {
return nil, errors.Wrapf(err, "get TLSConfig for registry %q", e)
}
} else {
return nil, errors.New("TLS config cannot be applied; Client.Transport is not *http.Transport")
}
}
client := rclient.StandardClient()
authorizer := docker.NewDockerAuthorizer(
docker.WithAuthClient(client),
docker.WithAuthCreds(multiCredsFuncs(ref, credsFuncs...)))
......
vendor/github.com/containerd/stargz-snapshotter/service/resolver/registry.go
View file @ 6f4217a3
......@@ -17,12 +17,12 @@
package resolver
import (
"net/http"
"time"
"github.com/containerd/containerd/reference"
"github.com/containerd/containerd/remotes/docker"
"github.com/containerd/stargz-snapshotter/fs/source"
rhttp "github.com/hashicorp/go-retryablehttp"
)
const defaultRequestTimeoutSec = 30
......@@ -59,7 +59,9 @@ func RegistryHostsFromConfig(cfg Config, credsFuncs ...Credential) source.Regist
for _, h := range append(cfg.Host[host].Mirrors, MirrorConfig{
Host: host,
}) {
tr := &http.Client{Transport: http.DefaultTransport.(*http.Transport).Clone()}
client := rhttp.NewClient()
client.Logger = nil // disable logging every request
tr := client.StandardClient()
if h.RequestTimeoutSec >= 0 {
if h.RequestTimeoutSec == 0 {
tr.Timeout = defaultRequestTimeoutSec * time.Second
......
vendor/github.com/containerd/stargz-snapshotter/service/service.go
View file @ 6f4217a3
......@@ -35,6 +35,7 @@ type Option func(*options)
type options struct {
credsFuncs []resolver.Credential
registryHosts source.RegistryHosts
fsOpts []stargzfs.Option
}
// WithCredsFuncs specifies credsFuncs to be used for connecting to the registries.
......@@ -51,6 +52,13 @@ func WithCustomRegistryHosts(hosts source.RegistryHosts) Option {
}
}
// WithFilesystemOptions allowes to pass filesystem-related configuration.
func WithFilesystemOptions(opts ...stargzfs.Option) Option {
return func(o *options) {
o.fsOpts = opts
}
}
// NewStargzSnapshotterService returns stargz snapshotter.
func NewStargzSnapshotterService(ctx context.Context, root string, config *Config, opts ...Option) (snapshots.Snapshotter, error) {
var sOpts options
......@@ -65,13 +73,11 @@ func NewStargzSnapshotterService(ctx context.Context, root string, config *Confi
}
// Configure filesystem and snapshotter
fs, err := stargzfs.NewFilesystem(fsRoot(root),
config.Config,
stargzfs.WithGetSources(sources(
sourceFromCRILabels(hosts), // provides source info based on CRI labels
source.FromDefaultLabels(hosts), // provides source info based on default labels
)),
)
fsOpts := append(sOpts.fsOpts, stargzfs.WithGetSources(sources(
sourceFromCRILabels(hosts), // provides source info based on CRI labels
source.FromDefaultLabels(hosts), // provides source info based on default labels
)))
fs, err := stargzfs.NewFilesystem(fsRoot(root), config.Config, fsOpts...)
if err != nil {
log.G(ctx).WithError(err).Fatalf("failed to configure filesystem")
}
......
vendor/github.com/containerd/stargz-snapshotter/snapshot/snapshot.go
View file @ 6f4217a3
......@@ -251,7 +251,7 @@ func (o *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...s
WithError(err).Warn("failed to prepare remote snapshot")
} else {
base.Labels[remoteLabel] = remoteLabelVal // Mark this snapshot as remote
err := o.Commit(ctx, target, key, append(opts, snapshots.WithLabels(base.Labels))...)
err := o.commit(ctx, true, target, key, append(opts, snapshots.WithLabels(base.Labels))...)
if err == nil || errdefs.IsAlreadyExists(err) {
// count also AlreadyExists as "success"
log.G(lCtx).WithField(remoteSnapshotLogKey, prepareSucceeded).Debug("prepared remote snapshot")
......@@ -293,6 +293,10 @@ func (o *snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, er
}
func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshots.Opt) error {
return o.commit(ctx, false, name, key, opts...)
}
func (o *snapshotter) commit(ctx context.Context, isRemote bool, name, key string, opts ...snapshots.Opt) error {
ctx, t, err := o.ms.TransactionContext(ctx, true)
if err != nil {
return err
......@@ -307,17 +311,20 @@ func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snap
}()
// grab the existing id
id, _, _, err := storage.GetInfo(ctx, key)
id, _, usage, err := storage.GetInfo(ctx, key)
if err != nil {
return err
}
usage, err := fs.DiskUsage(ctx, o.upperPath(id))
if err != nil {
return err
if !isRemote { // skip diskusage for remote snapshots for allowing lazy preparation of nodes
du, err := fs.DiskUsage(ctx, o.upperPath(id))
if err != nil {
return err
}
usage = snapshots.Usage(du)
}
if _, err = storage.CommitActive(ctx, key, name, snapshots.Usage(usage), opts...); err != nil {
if _, err = storage.CommitActive(ctx, key, name, usage, opts...); err != nil {
return errors.Wrap(err, "failed to commit snapshot")
}
......
vendor/github.com/containerd/stargz-snapshotter/util/testutil/ensurehello.go 0 → 100644
View file @ 6f4217a3
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package testutil
import (
"compress/gzip"
"context"
"io"
"io/ioutil"
"net/http"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/content/local"
"github.com/containerd/containerd/images/archive"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
const (
// HelloArchiveURL points to an OCI archive of `hello-world`.
// Exported from `docker.io/library/hello-world@sha256:1a523af650137b8accdaed439c17d684df61ee4d74feac151b5b337bd29e7eec` .
// See https://github.com/AkihiroSuda/test-oci-archives/releases/tag/v20210101
HelloArchiveURL = "https://github.com/AkihiroSuda/test-oci-archives/releases/download/v20210101/hello-world.tar.gz"
// HelloArchiveDigest is the digest of the archive.
HelloArchiveDigest = "sha256:5aa022621c4de0e941ab2a30d4569c403e156b4ba2de2ec32e382ae8679f40e1"
)
// EnsureHello creates a temp content store and ensures `hello-world` image from HelloArchiveURL into the store.
func EnsureHello(ctx context.Context) (*ocispec.Descriptor, content.Store, error) {
// Pulling an image without the daemon is a mess, so we use OCI archive here.
resp, err := http.Get(HelloArchiveURL)
if err != nil {
return nil, nil, err
}
defer resp.Body.Close()
sha256Digester := digest.SHA256.Digester()
sha256Hasher := sha256Digester.Hash()
tr := io.TeeReader(resp.Body, sha256Hasher)
gzReader, err := gzip.NewReader(tr)
if err != nil {
return nil, nil, err
}
tempDir, err := ioutil.TempDir("", "test-estargz")
if err != nil {
return nil, nil, err
}
cs, err := local.NewStore(tempDir)
if err != nil {
return nil, nil, err
}
desc, err := archive.ImportIndex(ctx, cs, gzReader)
if err != nil {
return nil, nil, err
}
resp.Body.Close()
if d := sha256Digester.Digest().String(); d != HelloArchiveDigest {
err = errors.Errorf("expected digest of %q to be %q, got %q", HelloArchiveURL, HelloArchiveDigest, d)
return nil, nil, err
}
return &desc, cs, nil
}
vendor/github.com/containerd/stargz-snapshotter/util/testutil/estargz.go 0 → 100644
View file @ 6f4217a3
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package testutil
import (
"bytes"
"io"
"github.com/containerd/stargz-snapshotter/estargz"
digest "github.com/opencontainers/go-digest"
)
type buildEStargzOptions struct {
estargzOptions []estargz.Option
buildTarOptions []BuildTarOption
}
type BuildEStargzOption func(o *buildEStargzOptions) error
// WithEStargzOptions specifies options for estargz lib
func WithEStargzOptions(eo ...estargz.Option) BuildEStargzOption {
return func(o *buildEStargzOptions) error {
o.estargzOptions = eo
return nil
}
}
// WithBuildTarOptions option specifies the options for tar creation
func WithBuildTarOptions(to ...BuildTarOption) BuildEStargzOption {
return func(o *buildEStargzOptions) error {
o.buildTarOptions = to
return nil
}
}
func BuildEStargz(ents []TarEntry, opts ...BuildEStargzOption) (*io.SectionReader, digest.Digest, error) {
var beOpts buildEStargzOptions
for _, o := range opts {
o(&beOpts)
}
tarBuf := new(bytes.Buffer)
if _, err := io.Copy(tarBuf, BuildTar(ents, beOpts.buildTarOptions...)); err != nil {
return nil, "", err
}
tarData := tarBuf.Bytes()
rc, err := estargz.Build(
io.NewSectionReader(bytes.NewReader(tarData), 0, int64(len(tarData))),
beOpts.estargzOptions...)
if err != nil {
return nil, "", err
}
defer rc.Close()
vsb := new(bytes.Buffer)
if _, err := io.Copy(vsb, rc); err != nil {
return nil, "", err
}
vsbb := vsb.Bytes()
return io.NewSectionReader(bytes.NewReader(vsbb), 0, int64(len(vsbb))), rc.TOCDigest(), nil
}
vendor/github.com/containerd/stargz-snapshotter/util/testutil/tar.go 0 → 100644
View file @ 6f4217a3
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package testutil
// This utility helps test codes to generate sample tar blobs.
import (
"archive/tar"
"fmt"
"io"
"os"
"strings"
"time"
)
// TarEntry is an entry of tar.
type TarEntry interface {
AppendTar(tw *tar.Writer, opts BuildTarOptions) error
}
// BuildTarOptions is a set of options used during building blob.
type BuildTarOptions struct {
// Prefix is the prefix string need to be added to each file name (e.g. "./", "/", etc.)
Prefix string
}
// BuildTarOption is an option used during building blob.
type BuildTarOption func(o *BuildTarOptions)
// WithPrefix is an option to add a prefix string to each file name (e.g. "./", "/", etc.)
func WithPrefix(prefix string) BuildTarOption {
return func(o *BuildTarOptions) {
o.Prefix = prefix
}
}
// BuildTar builds a tar blob
func BuildTar(ents []TarEntry, opts ...BuildTarOption) io.Reader {
var bo BuildTarOptions
for _, o := range opts {
o(&bo)
}
pr, pw := io.Pipe()
go func() {
tw := tar.NewWriter(pw)
for _, ent := range ents {
if err := ent.AppendTar(tw, bo); err != nil {
pw.CloseWithError(err)
return
}
}
if err := tw.Close(); err != nil {
pw.CloseWithError(err)
return
}
pw.Close()
}()
return pr
}
type tarEntryFunc func(*tar.Writer, BuildTarOptions) error
func (f tarEntryFunc) AppendTar(tw *tar.Writer, opts BuildTarOptions) error { return f(tw, opts) }
// DirectoryBuildTarOption is an option for a directory entry.
type DirectoryBuildTarOption func(o *dirOpts)
type dirOpts struct {
uid int
gid int
xattrs map[string]string
mode *os.FileMode
modTime time.Time
}
// WithFileModTime specifies the modtime of the dir.
func WithDirModTime(modTime time.Time) DirectoryBuildTarOption {
return func(o *dirOpts) {
o.modTime = modTime
}
}
// WithDirOwner specifies the owner of the directory.
func WithDirOwner(uid, gid int) DirectoryBuildTarOption {
return func(o *dirOpts) {
o.uid = uid
o.gid = gid
}
}
// WithDirXattrs specifies the extended attributes of the directory.
func WithDirXattrs(xattrs map[string]string) DirectoryBuildTarOption {
return func(o *dirOpts) {
o.xattrs = xattrs
}
}
// WithDirMode specifies the mode of the directory.
func WithDirMode(mode os.FileMode) DirectoryBuildTarOption {
return func(o *dirOpts) {
o.mode = &mode
}
}
// Dir is a directory entry
func Dir(name string, opts ...DirectoryBuildTarOption) TarEntry {
return tarEntryFunc(func(tw *tar.Writer, buildOpts BuildTarOptions) error {
var dOpts dirOpts
for _, o := range opts {
o(&dOpts)
}
if !strings.HasSuffix(name, "/") {
panic(fmt.Sprintf("missing trailing slash in dir %q ", name))
}
var mode int64 = 0755
if dOpts.mode != nil {
mode = permAndExtraMode2TarMode(*dOpts.mode)
}
return tw.WriteHeader(&tar.Header{
Typeflag: tar.TypeDir,
Name: buildOpts.Prefix + name,
Mode: mode,
ModTime: dOpts.modTime,
Xattrs: dOpts.xattrs,
Uid: dOpts.uid,
Gid: dOpts.gid,
})
})
}
// FileBuildTarOption is an option for a file entry.
type FileBuildTarOption func(o *fileOpts)
type fileOpts struct {
uid int
gid int
xattrs map[string]string
mode *os.FileMode
modTime time.Time
}
// WithFileOwner specifies the owner of the file.
func WithFileOwner(uid, gid int) FileBuildTarOption {
return func(o *fileOpts) {
o.uid = uid
o.gid = gid
}
}
// WithFileXattrs specifies the extended attributes of the file.
func WithFileXattrs(xattrs map[string]string) FileBuildTarOption {
return func(o *fileOpts) {
o.xattrs = xattrs
}
}
// WithFileModTime specifies the modtime of the file.
func WithFileModTime(modTime time.Time) FileBuildTarOption {
return func(o *fileOpts) {
o.modTime = modTime
}
}
// WithFileMode specifies the mode of the file.
func WithFileMode(mode os.FileMode) FileBuildTarOption {
return func(o *fileOpts) {
o.mode = &mode
}
}
// File is a regilar file entry
func File(name, contents string, opts ...FileBuildTarOption) TarEntry {
return tarEntryFunc(func(tw *tar.Writer, buildOpts BuildTarOptions) error {
var fOpts fileOpts
for _, o := range opts {
o(&fOpts)
}
if strings.HasSuffix(name, "/") {
return fmt.Errorf("bogus trailing slash in file %q", name)
}
var mode int64 = 0644
if fOpts.mode != nil {
mode = permAndExtraMode2TarMode(*fOpts.mode)
}
if err := tw.WriteHeader(&tar.Header{
Typeflag: tar.TypeReg,
Name: buildOpts.Prefix + name,
Mode: mode,
ModTime: fOpts.modTime,
Xattrs: fOpts.xattrs,
Size: int64(len(contents)),
Uid: fOpts.uid,
Gid: fOpts.gid,
}); err != nil {
return err
}
_, err := io.WriteString(tw, contents)
return err
})
}
// Symlink is a symlink entry
func Symlink(name, target string) TarEntry {
return tarEntryFunc(func(tw *tar.Writer, buildOpts BuildTarOptions) error {
return tw.WriteHeader(&tar.Header{
Typeflag: tar.TypeSymlink,
Name: buildOpts.Prefix + name,
Linkname: target,
Mode: 0644,
})
})
}
// Link is a hard-link entry
func Link(name, linkname string) TarEntry {
now := time.Now()
return tarEntryFunc(func(w *tar.Writer, buildOpts BuildTarOptions) error {
return w.WriteHeader(&tar.Header{
Typeflag: tar.TypeLink,
Name: buildOpts.Prefix + name,
Linkname: linkname,
ModTime: now,
AccessTime: now,
ChangeTime: now,
})
})
}
// Chardev is a character device entry
func Chardev(name string, major, minor int64) TarEntry {
now := time.Now()
return tarEntryFunc(func(w *tar.Writer, buildOpts BuildTarOptions) error {
return w.WriteHeader(&tar.Header{
Typeflag: tar.TypeChar,
Name: buildOpts.Prefix + name,
Devmajor: major,
Devminor: minor,
ModTime: now,
AccessTime: now,
ChangeTime: now,
})
})
}
// Blockdev is a block device entry
func Blockdev(name string, major, minor int64) TarEntry {
now := time.Now()
return tarEntryFunc(func(w *tar.Writer, buildOpts BuildTarOptions) error {
return w.WriteHeader(&tar.Header{
Typeflag: tar.TypeBlock,
Name: buildOpts.Prefix + name,
Devmajor: major,
Devminor: minor,
ModTime: now,
AccessTime: now,
ChangeTime: now,
})
})
}
// Fifo is a fifo entry
func Fifo(name string) TarEntry {
now := time.Now()
return tarEntryFunc(func(w *tar.Writer, buildOpts BuildTarOptions) error {
return w.WriteHeader(&tar.Header{
Typeflag: tar.TypeFifo,
Name: buildOpts.Prefix + name,
ModTime: now,
AccessTime: now,
ChangeTime: now,
})
})
}
// suid, guid, sticky bits for archive/tar
// https://github.com/golang/go/blob/release-branch.go1.13/src/archive/tar/common.go#L607-L609
const (
cISUID = 04000 // Set uid
cISGID = 02000 // Set gid
cISVTX = 01000 // Save text (sticky bit)
)
func permAndExtraMode2TarMode(fm os.FileMode) (tm int64) {
tm = int64(fm & os.ModePerm)
if fm&os.ModeSetuid != 0 {
tm |= cISUID
}
if fm&os.ModeSetgid != 0 {
tm |= cISGID
}
if fm&os.ModeSticky != 0 {
tm |= cISVTX
}
return
}
vendor/github.com/containerd/ttrpc/.gitignore
View file @ 6f4217a3
......@@ -9,6 +9,3 @@
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
.glide/
vendor/github.com/containerd/ttrpc/.travis.yml deleted 100644 → 0
View file @ 8737e2e1
dist: bionic
language: go
go:
- "1.13.x"
- "1.15.x"
install:
# Don't change local go.{mod, sum} by go get tools.
#
# ref: https://github.com/golang/go/issues/27643
- pushd ..; go get -u github.com/vbatts/git-validation; popd
- pushd ..; go get -u github.com/kunalkushwaha/ltag; popd
before_script:
- pushd ..; git clone https://github.com/containerd/project; popd
script:
- DCO_VERBOSITY=-q ../project/script/validate/dco
- ../project/script/validate/fileheader ../project/
- go test -v -race -covermode=atomic -coverprofile=coverage.txt ./...
after_success:
- bash <(curl -s https://codecov.io/bash)
vendor/github.com/containerd/ttrpc/README.md
View file @ 6f4217a3
# ttrpc
[![Build Status](https://travis-ci.org/containerd/ttrpc.svg?branch=master)](https://travis-ci.org/containerd/ttrpc)
[![Build Status](https://github.com/containerd/ttrpc/workflows/CI/badge.svg)](https://github.com/containerd/ttrpc/actions?query=workflow%3ACI)
[![codecov](https://codecov.io/gh/containerd/ttrpc/branch/main/graph/badge.svg)](https://codecov.io/gh/containerd/ttrpc)
GRPC for low-memory environments.
......@@ -40,13 +41,8 @@ directly, if required.
# Status
Very new. YMMV.
TODO:
- [X] Plumb error codes and GRPC status
- [X] Remove use of any type and dependency on typeurl package
- [X] Ensure that protocol can support streaming in the future
- [ ] Document protocol layout
- [ ] Add testing under concurrent load to ensure
- [ ] Verify connection error handling
......@@ -55,8 +51,8 @@ TODO:
ttrpc is a containerd sub-project, licensed under the [Apache 2.0 license](./LICENSE).
As a containerd sub-project, you will find the:
* [Project governance](https://github.com/containerd/project/blob/master/GOVERNANCE.md),
* [Maintainers](https://github.com/containerd/project/blob/master/MAINTAINERS),
* and [Contributing guidelines](https://github.com/containerd/project/blob/master/CONTRIBUTING.md)
* [Project governance](https://github.com/containerd/project/blob/main/GOVERNANCE.md),
* [Maintainers](https://github.com/containerd/project/blob/main/MAINTAINERS),
* and [Contributing guidelines](https://github.com/containerd/project/blob/main/CONTRIBUTING.md)
information in our [`containerd/project`](https://github.com/containerd/project) repository.
vendor/github.com/containerd/ttrpc/channel.go
View file @ 6f4217a3
......@@ -19,11 +19,11 @@ package ttrpc
import (
"bufio"
"encoding/binary"
"fmt"
"io"
"net"
"sync"
"github.com/pkg/errors"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
......@@ -105,7 +105,7 @@ func (ch *channel) recv() (messageHeader, []byte, error) {
if mh.Length > uint32(messageLengthMax) {
if _, err := ch.br.Discard(int(mh.Length)); err != nil {
return mh, nil, errors.Wrapf(err, "failed to discard after receiving oversized message")
return mh, nil, fmt.Errorf("failed to discard after receiving oversized message: %w", err)
}
return mh, nil, status.Errorf(codes.ResourceExhausted, "message length %v exceed maximum message size of %v", mh.Length, messageLengthMax)
......@@ -113,7 +113,7 @@ func (ch *channel) recv() (messageHeader, []byte, error) {
p := ch.getmbuf(int(mh.Length))
if _, err := io.ReadFull(ch.br, p); err != nil {
return messageHeader{}, nil, errors.Wrapf(err, "failed reading message")
return messageHeader{}, nil, fmt.Errorf("failed reading message: %w", err)
}
return mh, p, nil
......
vendor/github.com/containerd/ttrpc/client.go
View file @ 6f4217a3
......@@ -18,6 +18,7 @@ package ttrpc
import (
"context"
"errors"
"io"
"net"
"os"
......@@ -27,7 +28,6 @@ import (
"time"
"github.com/gogo/protobuf/proto"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
......@@ -194,72 +194,131 @@ type message struct {
err error
}
type receiver struct {
wg *sync.WaitGroup
messages chan *message
err error
// callMap provides access to a map of active calls, guarded by a mutex.
type callMap struct {
m sync.Mutex
activeCalls map[uint32]*callRequest
closeErr error
}
func (r *receiver) run(ctx context.Context, c *channel) {
defer r.wg.Done()
// newCallMap returns a new callMap with an empty set of active calls.
func newCallMap() *callMap {
return &callMap{
activeCalls: make(map[uint32]*callRequest),
}
}
for {
select {
case <-ctx.Done():
r.err = ctx.Err()
return
default:
mh, p, err := c.recv()
if err != nil {
_, ok := status.FromError(err)
if !ok {
// treat all errors that are not an rpc status as terminal.
// all others poison the connection.
r.err = filterCloseErr(err)
return
}
}
select {
case r.messages <- &message{
messageHeader: mh,
p: p[:mh.Length],
err: err,
}:
case <-ctx.Done():
r.err = ctx.Err()
return
}
}
// set adds a call entry to the map with the given streamID key.
func (cm *callMap) set(streamID uint32, cr *callRequest) error {
cm.m.Lock()
defer cm.m.Unlock()
if cm.closeErr != nil {
return cm.closeErr
}
cm.activeCalls[streamID] = cr
return nil
}
// get looks up the call entry for the given streamID key, then removes it
// from the map and returns it.
func (cm *callMap) get(streamID uint32) (cr *callRequest, ok bool, err error) {
cm.m.Lock()
defer cm.m.Unlock()
if cm.closeErr != nil {
return nil, false, cm.closeErr
}
cr, ok = cm.activeCalls[streamID]
if ok {
delete(cm.activeCalls, streamID)
}
return
}
// abort sends the given error to each active call, and clears the map.
// Once abort has been called, any subsequent calls to the callMap will return the error passed to abort.
func (cm *callMap) abort(err error) error {
cm.m.Lock()
defer cm.m.Unlock()
if cm.closeErr != nil {
return cm.closeErr
}
for streamID, call := range cm.activeCalls {
call.errs <- err
delete(cm.activeCalls, streamID)
}
cm.closeErr = err
return nil
}
func (c *Client) run() {
var (
streamID uint32 = 1
waiters = make(map[uint32]*callRequest)
calls = c.calls
incoming = make(chan *message)
receiversDone = make(chan struct{})
wg sync.WaitGroup
waiters = newCallMap()
receiverDone = make(chan struct{})
)
// broadcast the shutdown error to the remaining waiters.
abortWaiters := func(wErr error) {
for _, waiter := range waiters {
waiter.errs <- wErr
// Sender goroutine
// Receives calls from dispatch, adds them to the set of active calls, and sends them
// to the server.
go func() {
var streamID uint32 = 1
for {
select {
case <-c.ctx.Done():
return
case call := <-c.calls:
id := streamID
streamID += 2 // enforce odd client initiated request ids
if err := waiters.set(id, call); err != nil {
call.errs <- err // errs is buffered so should not block.
continue
}
if err := c.send(id, messageTypeRequest, call.req); err != nil {
call.errs <- err // errs is buffered so should not block.
waiters.get(id) // remove from waiters set
}
}
}
}
recv := &receiver{
wg: &wg,
messages: incoming,
}
wg.Add(1)
}()
// Receiver goroutine
// Receives responses from the server, looks up the call info in the set of active calls,
// and notifies the caller of the response.
go func() {
wg.Wait()
close(receiversDone)
defer close(receiverDone)
for {
select {
case <-c.ctx.Done():
c.setError(c.ctx.Err())
return
default:
mh, p, err := c.channel.recv()
if err != nil {
_, ok := status.FromError(err)
if !ok {
// treat all errors that are not an rpc status as terminal.
// all others poison the connection.
c.setError(filterCloseErr(err))
return
}
}
msg := &message{
messageHeader: mh,
p: p[:mh.Length],
err: err,
}
call, ok, err := waiters.get(mh.StreamID)
if err != nil {
logrus.Errorf("ttrpc: failed to look up active call: %s", err)
continue
}
if !ok {
logrus.Errorf("ttrpc: received message for unknown channel %v", mh.StreamID)
continue
}
call.errs <- c.recv(call.resp, msg)
}
}
}()
go recv.run(c.ctx, c.channel)
defer func() {
c.conn.Close()
......@@ -269,32 +328,14 @@ func (c *Client) run() {
for {
select {
case call := <-calls:
if err := c.send(streamID, messageTypeRequest, call.req); err != nil {
call.errs <- err
continue
}
waiters[streamID] = call
streamID += 2 // enforce odd client initiated request ids
case msg := <-incoming:
call, ok := waiters[msg.StreamID]
if !ok {
logrus.Errorf("ttrpc: received message for unknown channel %v", msg.StreamID)
continue
}
call.errs <- c.recv(call.resp, msg)
delete(waiters, msg.StreamID)
case <-receiversDone:
// all the receivers have exited
if recv.err != nil {
c.setError(recv.err)
}
case <-receiverDone:
// The receiver has exited.
// don't return out, let the close of the context trigger the abort of waiters
c.Close()
case <-c.ctx.Done():
abortWaiters(c.error())
// Abort all active calls. This will also prevent any new calls from being added
// to waiters.
waiters.abort(c.error())
return
}
}
......@@ -347,7 +388,7 @@ func filterCloseErr(err error) error {
return nil
case err == io.EOF:
return ErrClosed
case errors.Cause(err) == io.EOF:
case errors.Is(err, io.EOF):
return ErrClosed
case strings.Contains(err.Error(), "use of closed network connection"):
return ErrClosed
......
vendor/github.com/containerd/ttrpc/codec.go
View file @ 6f4217a3
......@@ -17,8 +17,9 @@
package ttrpc
import (
"fmt"
"github.com/gogo/protobuf/proto"
"github.com/pkg/errors"
)
type codec struct{}
......@@ -28,7 +29,7 @@ func (c codec) Marshal(msg interface{}) ([]byte, error) {
case proto.Message:
return proto.Marshal(v)
default:
return nil, errors.Errorf("ttrpc: cannot marshal unknown type: %T", msg)
return nil, fmt.Errorf("ttrpc: cannot marshal unknown type: %T", msg)
}
}
......@@ -37,6 +38,6 @@ func (c codec) Unmarshal(p []byte, msg interface{}) error {
case proto.Message:
return proto.Unmarshal(p, v)
default:
return errors.Errorf("ttrpc: cannot unmarshal into unknown type: %T", msg)
return fmt.Errorf("ttrpc: cannot unmarshal into unknown type: %T", msg)
}
}
vendor/github.com/containerd/ttrpc/config.go
View file @ 6f4217a3
......@@ -16,7 +16,7 @@
package ttrpc
import "github.com/pkg/errors"
import "errors"
type serverConfig struct {
handshaker Handshaker
......
vendor/github.com/containerd/ttrpc/go.mod
View file @ 6f4217a3
......@@ -3,12 +3,11 @@ module github.com/containerd/ttrpc
go 1.13
require (
github.com/gogo/protobuf v1.3.1
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/pkg/errors v0.9.1
github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1
github.com/sirupsen/logrus v1.4.2
golang.org/x/sys v0.0.0-20200120151820-655fe14d7479
google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24
google.golang.org/grpc v1.26.0
github.com/gogo/protobuf v1.3.2
github.com/prometheus/procfs v0.6.0
github.com/sirupsen/logrus v1.8.1
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c
google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63
google.golang.org/grpc v1.27.1
google.golang.org/protobuf v1.27.1
)
vendor/github.com/containerd/ttrpc/server.go
View file @ 6f4217a3
......@@ -18,6 +18,7 @@ package ttrpc
import (
"context"
"errors"
"io"
"math/rand"
"net"
......@@ -25,7 +26,6 @@ import (
"sync/atomic"
"time"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
......
vendor/github.com/containerd/ttrpc/services.go
View file @ 6f4217a3
......@@ -18,13 +18,14 @@ package ttrpc
import (
"context"
"errors"
"fmt"
"io"
"os"
"path"
"unsafe"
"github.com/gogo/protobuf/proto"
"github.com/pkg/errors"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
......@@ -51,7 +52,7 @@ func newServiceSet(interceptor UnaryServerInterceptor) *serviceSet {
func (s *serviceSet) register(name string, methods map[string]Method) {
if _, ok := s.services[name]; ok {
panic(errors.Errorf("duplicate service %v registered", name))
panic(fmt.Errorf("duplicate service %v registered", name))
}
s.services[name] = ServiceDesc{
......@@ -116,12 +117,12 @@ func (s *serviceSet) dispatch(ctx context.Context, serviceName, methodName strin
func (s *serviceSet) resolve(service, method string) (Method, error) {
srv, ok := s.services[service]
if !ok {
return nil, status.Errorf(codes.NotFound, "service %v", service)
return nil, status.Errorf(codes.Unimplemented, "service %v", service)
}
mthd, ok := srv.Methods[method]
if !ok {
return nil, status.Errorf(codes.NotFound, "method %v", method)
return nil, status.Errorf(codes.Unimplemented, "method %v", method)
}
return mthd, nil
......
vendor/github.com/containerd/ttrpc/unixcreds_linux.go
View file @ 6f4217a3
......@@ -18,11 +18,12 @@ package ttrpc
import (
"context"
"errors"
"fmt"
"net"
"os"
"syscall"
"github.com/pkg/errors"
"golang.org/x/sys/unix"
)
......@@ -31,12 +32,12 @@ type UnixCredentialsFunc func(*unix.Ucred) error
func (fn UnixCredentialsFunc) Handshake(ctx context.Context, conn net.Conn) (net.Conn, interface{}, error) {
uc, err := requireUnixSocket(conn)
if err != nil {
return nil, nil, errors.Wrap(err, "ttrpc.UnixCredentialsFunc: require unix socket")
return nil, nil, fmt.Errorf("ttrpc.UnixCredentialsFunc: require unix socket: %w", err)
}
rs, err := uc.SyscallConn()
if err != nil {
return nil, nil, errors.Wrap(err, "ttrpc.UnixCredentialsFunc: (net.UnixConn).SyscallConn failed")
return nil, nil, fmt.Errorf("ttrpc.UnixCredentialsFunc: (net.UnixConn).SyscallConn failed: %w", err)
}
var (
ucred *unix.Ucred
......@@ -45,15 +46,15 @@ func (fn UnixCredentialsFunc) Handshake(ctx context.Context, conn net.Conn) (net
if err := rs.Control(func(fd uintptr) {
ucred, ucredErr = unix.GetsockoptUcred(int(fd), unix.SOL_SOCKET, unix.SO_PEERCRED)
}); err != nil {
return nil, nil, errors.Wrapf(err, "ttrpc.UnixCredentialsFunc: (*syscall.RawConn).Control failed")
return nil, nil, fmt.Errorf("ttrpc.UnixCredentialsFunc: (*syscall.RawConn).Control failed: %w", err)
}
if ucredErr != nil {
return nil, nil, errors.Wrapf(err, "ttrpc.UnixCredentialsFunc: failed to retrieve socket peer credentials")
return nil, nil, fmt.Errorf("ttrpc.UnixCredentialsFunc: failed to retrieve socket peer credentials: %w", err)
}
if err := fn(ucred); err != nil {
return nil, nil, errors.Wrapf(err, "ttrpc.UnixCredentialsFunc: credential check failed")
return nil, nil, fmt.Errorf("ttrpc.UnixCredentialsFunc: credential check failed: %w", err)
}
return uc, ucred, nil
......@@ -93,7 +94,7 @@ func requireRoot(ucred *unix.Ucred) error {
func requireUidGid(ucred *unix.Ucred, uid, gid int) error {
if (uid != -1 && uint32(uid) != ucred.Uid) || (gid != -1 && uint32(gid) != ucred.Gid) {
return errors.Wrap(syscall.EPERM, "ttrpc: invalid credentials")
return fmt.Errorf("ttrpc: invalid credentials: %v", syscall.EPERM)
}
return nil
}
......
vendor/github.com/goccy/go-json/.codecov.yml 0 → 100644
View file @ 6f4217a3
codecov:
require_ci_to_pass: yes
coverage:
precision: 2
round: down
range: "70...100"
status:
project:
default:
target: 70%
threshold: 2%
patch: off
changes: no
parsers:
gcov:
branch_detection:
conditional: yes
loop: yes
method: no
macro: no
comment:
layout: "header,diff"
behavior: default
require_changes: no
ignore:
- internal/encoder/vm_color
- internal/encoder/vm_color_indent
vendor/github.com/goccy/go-json/.golangci.yml 0 → 100644
View file @ 6f4217a3
run:
skip-files:
- encode_optype.go
- ".*_test\\.go$"
linters-settings:
govet:
enable-all: true
disable:
- shadow
linters:
enable-all: true
disable:
- dogsled
- dupl
- exhaustive
- exhaustivestruct
- errorlint
- forbidigo
- funlen
- gci
- gochecknoglobals
- gochecknoinits
- gocognit
- gocritic
- gocyclo
- godot
- godox
- goerr113
- gofumpt
- gomnd
- gosec
- ifshort
- lll
- makezero
- nakedret
- nestif
- nlreturn
- paralleltest
- testpackage
- thelper
- wrapcheck
- interfacer
- lll
- nakedret
- nestif
- nlreturn
- testpackage
- wsl
issues:
exclude-rules:
# not needed
- path: /*.go
text: "ST1003: should not use underscores in package names"
linters:
- stylecheck
- path: /*.go
text: "don't use an underscore in package name"
linters:
- golint
- path: rtype.go
linters:
- golint
- stylecheck
- path: error.go
linters:
- staticcheck
# Maximum issues count per one linter. Set to 0 to disable. Default is 50.
max-issues-per-linter: 0
# Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
max-same-issues: 0
vendor/github.com/goccy/go-json/CHANGELOG.md 0 → 100644
View file @ 6f4217a3
# v0.7.10 - 2021/10/16
* Fix conversion from pointer to uint64 ( #294 )
# v0.7.9 - 2021/09/28
* Fix encoding of nil value about interface type that has method ( #291 )
# v0.7.8 - 2021/09/01
* Fix mapassign_faststr for indirect struct type ( #283 )
* Fix encoding of not empty interface type ( #284 )
* Fix encoding of empty struct interface type ( #286 )
# v0.7.7 - 2021/08/25
* Fix invalid utf8 on stream decoder ( #279 )
* Fix buffer length bug on string stream decoder ( #280 )
Thank you @orisano !!
# v0.7.6 - 2021/08/13
* Fix nil slice assignment ( #276 )
* Improve error message ( #277 )
# v0.7.5 - 2021/08/12
* Fix encoding of embedded struct with tags ( #265 )
* Fix encoding of embedded struct that isn't first field ( #272 )
* Fix decoding of binary type with escaped char ( #273 )
# v0.7.4 - 2021/07/06
* Fix encoding of indirect layout structure ( #264 )
# v0.7.3 - 2021/06/29
* Fix encoding of pointer type in empty interface ( #262 )
# v0.7.2 - 2021/06/26
### Fix decoder
* Add decoder for func type to fix decoding of nil function value ( #257 )
* Fix stream decoding of []byte type ( #258 )
### Performance
* Improve decoding performance of map[string]interface{} type ( use `mapassign_faststr` ) ( #256 )
* Improve encoding performance of empty interface type ( remove recursive calling of `vm.Run` ) ( #259 )
### Benchmark
* Add bytedance/sonic as benchmark target ( #254 )
# v0.7.1 - 2021/06/18
### Fix decoder
* Fix error when unmarshal empty array ( #253 )
# v0.7.0 - 2021/06/12
### Support context for MarshalJSON and UnmarshalJSON ( #248 )
* json.MarshalContext(context.Context, interface{}, ...json.EncodeOption) ([]byte, error)
* json.NewEncoder(io.Writer).EncodeContext(context.Context, interface{}, ...json.EncodeOption) error
* json.UnmarshalContext(context.Context, []byte, interface{}, ...json.DecodeOption) error
* json.NewDecoder(io.Reader).DecodeContext(context.Context, interface{}) error
```go
type MarshalerContext interface {
MarshalJSON(context.Context) ([]byte, error)
}
type UnmarshalerContext interface {
UnmarshalJSON(context.Context, []byte) error
}
```
### Add DecodeFieldPriorityFirstWin option ( #242 )
In the default behavior, go-json, like encoding/json, will reflect the result of the last evaluation when a field with the same name exists. I've added new options to allow you to change this behavior. `json.DecodeFieldPriorityFirstWin` option reflects the result of the first evaluation if a field with the same name exists. This behavior has a performance advantage as it allows the subsequent strings to be skipped if all fields have been evaluated.
### Fix encoder
* Fix indent number contains recursive type ( #249 )
* Fix encoding of using empty interface as map key ( #244 )
### Fix decoder
* Fix decoding fields containing escaped characters ( #237 )
### Refactor
* Move some tests to subdirectory ( #243 )
* Refactor package layout for decoder ( #238 )
# v0.6.1 - 2021/06/02
### Fix encoder
* Fix value of totalLength for encoding ( #236 )
# v0.6.0 - 2021/06/01
### Support Colorize option for encoding (#233)
```go
b, err := json.MarshalWithOption(v, json.Colorize(json.DefaultColorScheme))
if err != nil {
...
}
fmt.Println(string(b)) // print colored json
```
### Refactor
* Fix opcode layout - Adjust memory layout of the opcode to 128 bytes in a 64-bit environment ( #230 )
* Refactor encode option ( #231 )
* Refactor escape string ( #232 )
# v0.5.1 - 2021/5/20
### Optimization
* Add type addrShift to enable bigger encoder/decoder cache ( #213 )
### Fix decoder
* Keep original reference of slice element ( #229 )
### Refactor
* Refactor Debug mode for encoding ( #226 )
* Generate VM sources for encoding ( #227 )
* Refactor validator for null/true/false for decoding ( #221 )
# v0.5.0 - 2021/5/9
### Supports using omitempty and string tags at the same time ( #216 )
### Fix decoder
* Fix stream decoder for unicode char ( #215 )
* Fix decoding of slice element ( #219 )
* Fix calculating of buffer length for stream decoder ( #220 )
### Refactor
* replace skipWhiteSpace goto by loop ( #212 )
# v0.4.14 - 2021/5/4
### Benchmark
* Add valyala/fastjson to benchmark ( #193 )
* Add benchmark task for CI ( #211 )
### Fix decoder
* Fix decoding of slice with unmarshal json type ( #198 )
* Fix decoding of null value for interface type that does not implement Unmarshaler ( #205 )
* Fix decoding of null value to []byte by json.Unmarshal ( #206 )
* Fix decoding of backslash char at the end of string ( #207 )
* Fix stream decoder for null/true/false value ( #208 )
* Fix stream decoder for slow reader ( #211 )
### Performance
* If cap of slice is enough, reuse slice data for compatibility with encoding/json ( #200 )
# v0.4.13 - 2021/4/20
### Fix json.Compact and json.Indent
* Support validation the input buffer for json.Compact and json.Indent ( #189 )
* Optimize json.Compact and json.Indent ( improve memory footprint ) ( #190 )
# v0.4.12 - 2021/4/15
### Fix encoder
* Fix unnecessary indent for empty slice type ( #181 )
* Fix encoding of omitempty feature for the slice or interface type ( #183 )
* Fix encoding custom types zero values with omitempty when marshaller exists ( #187 )
### Fix decoder
* Fix decoder for invalid top level value ( #184 )
* Fix decoder for invalid number value ( #185 )
# v0.4.11 - 2021/4/3
* Improve decoder performance for interface type
# v0.4.10 - 2021/4/2
### Fix encoder
* Fixed a bug when encoding slice and map containing recursive structures
* Fixed a logic to determine if indirect reference
# v0.4.9 - 2021/3/29
### Add debug mode
If you use `json.MarshalWithOption(v, json.Debug())` and `panic` occurred in `go-json`, produces debug information to console.
### Support a new feature to compatible with encoding/json
- invalid UTF-8 is coerced to valid UTF-8 ( without performance down )
### Fix encoder
- Fixed handling of MarshalJSON of function type
### Fix decoding of slice of pointer type
If there is a pointer value, go-json will use it. (This behavior is necessary to achieve the ability to prioritize pre-filled values). However, since slices are reused internally, there was a bug that referred to the previous pointer value. Therefore, it is not necessary to refer to the pointer value in advance for the slice element, so we explicitly initialize slice element by `nil`.
# v0.4.8 - 2021/3/21
### Reduce memory usage at compile time
* go-json have used about 2GB of memory at compile time, but now it can compile with about less than 550MB.
### Fix any encoder's bug
* Add many test cases for encoder
* Fix composite type ( slice/array/map )
* Fix pointer types
* Fix encoding of MarshalJSON or MarshalText or json.Number type
### Refactor encoder
* Change package layout for reducing memory usage at compile
* Remove anonymous and only operation
* Remove root property from encodeCompileContext and opcode
### Fix CI
* Add Go 1.16
* Remove Go 1.13
* Fix `make cover` task
### Number/Delim/Token/RawMessage use the types defined in encoding/json by type alias
# v0.4.7 - 2021/02/22
### Fix decoder
* Fix decoding of deep recursive structure
* Fix decoding of embedded unexported pointer field
* Fix invalid test case
* Fix decoding of invalid value
* Fix decoding of prefilled value
* Fix not being able to return UnmarshalTypeError when it should be returned
* Fix decoding of null value
* Fix decoding of type of null string
* Use pre allocated pointer if exists it at decoding
### Reduce memory usage at compile
* Integrate int/int8/int16/int32/int64 and uint/uint8/uint16/uint32/uint64 operation to reduce memory usage at compile
### Remove unnecessary optype
vendor/github.com/goccy/go-json/LICENSE 0 → 100644
View file @ 6f4217a3
MIT License
Copyright (c) 2020 Masaaki Goshima
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
vendor/github.com/goccy/go-json/Makefile 0 → 100644
View file @ 6f4217a3
PKG := github.com/goccy/go-json
BIN_DIR := $(CURDIR)/bin
PKGS := $(shell go list ./... | grep -v internal/cmd|grep -v test)
COVER_PKGS := $(foreach pkg,$(PKGS),$(subst $(PKG),.,$(pkg)))
COMMA := ,
EMPTY :=
SPACE := $(EMPTY) $(EMPTY)
COVERPKG_OPT := $(subst $(SPACE),$(COMMA),$(COVER_PKGS))
$(BIN_DIR):
@mkdir -p $(BIN_DIR)
.PHONY: cover
cover:
go test -coverpkg=$(COVERPKG_OPT) -coverprofile=cover.out ./...
.PHONY: cover-html
cover-html: cover
go tool cover -html=cover.out
.PHONY: lint
lint: golangci-lint
golangci-lint run
golangci-lint: | $(BIN_DIR)
@{ \
set -e; \
GOLANGCI_LINT_TMP_DIR=$$(mktemp -d); \
cd $$GOLANGCI_LINT_TMP_DIR; \
go mod init tmp; \
GOBIN=$(BIN_DIR) go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.36.0; \
rm -rf $$GOLANGCI_LINT_TMP_DIR; \
}
.PHONY: generate
generate:
go generate ./internal/...
vendor/github.com/goccy/go-json/internal/encoder/vm/vm.go 0 → 100644
View file @ 6f4217a3
This source diff could not be displayed because it is too large. You can view the blob instead.
vendor/github.com/goccy/go-json/internal/encoder/vm_color/vm.go 0 → 100644
View file @ 6f4217a3
This source diff could not be displayed because it is too large. You can view the blob instead.
vendor/github.com/goccy/go-json/internal/encoder/vm_color_indent/vm.go 0 → 100644
View file @ 6f4217a3
This source diff could not be displayed because it is too large. You can view the blob instead.
vendor/github.com/goccy/go-json/internal/encoder/vm_indent/vm.go 0 → 100644
View file @ 6f4217a3
This source diff could not be displayed because it is too large. You can view the blob instead.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment