Merge pull request #35132 from dashpole/per_volume_inode

pkg/kubelet/server/stats/volume_stat_calculator.go

@@ -114,9 +114,13 @@ func (s *volumeStatCalculator) calcAndStoreStats() {
 func (s *volumeStatCalculator) parsePodVolumeStats(podName string, metric *volume.Metrics) stats.VolumeStats {
 	available := uint64(metric.Available.Value())
 	capacity := uint64(metric.Capacity.Value())
-	used := uint64((metric.Used.Value()))
+	used := uint64(metric.Used.Value())
+	inodes := uint64(metric.Inodes.Value())
+	inodesFree := uint64(metric.InodesFree.Value())
+	inodesUsed := uint64(metric.InodesUsed.Value())
 	return stats.VolumeStats{
-		Name:    podName,
-		FsStats: stats.FsStats{AvailableBytes: &available, CapacityBytes: &capacity, UsedBytes: &used},
+		Name: podName,
+		FsStats: stats.FsStats{AvailableBytes: &available, CapacityBytes: &capacity, UsedBytes: &used,
+			Inodes: &inodes, InodesFree: &inodesFree, InodesUsed: &inodesUsed},
 	}
 }

pkg/volume/metrics_du.go

@@ -50,6 +50,11 @@ func (md *metricsDu) GetMetrics() (*Metrics, error) {
 		return metrics, err
 	}
 
+	err = md.runFind(metrics)
+	if err != nil {
+		return metrics, err
+	}
+
 	err = md.getFsInfo(metrics)
 	if err != nil {
 		return metrics, err
@@ -68,14 +73,26 @@ func (md *metricsDu) runDu(metrics *Metrics) error {
 	return nil
 }
 
+// runFind executes the "find" command and writes the results to metrics.InodesUsed
+func (md *metricsDu) runFind(metrics *Metrics) error {
+	inodesUsed, err := util.Find(md.path)
+	if err != nil {
+		return err
+	}
+	metrics.InodesUsed = resource.NewQuantity(inodesUsed, resource.BinarySI)
+	return nil
+}
+
 // getFsInfo writes metrics.Capacity and metrics.Available from the filesystem
 // info
 func (md *metricsDu) getFsInfo(metrics *Metrics) error {
-	available, capacity, _, err := util.FsInfo(md.path)
+	available, capacity, _, inodes, inodesFree, _, err := util.FsInfo(md.path)
 	if err != nil {
 		return NewFsInfoFailedError(err)
 	}
 	metrics.Available = resource.NewQuantity(available, resource.BinarySI)
 	metrics.Capacity = resource.NewQuantity(capacity, resource.BinarySI)
+	metrics.Inodes = resource.NewQuantity(inodes, resource.BinarySI)
+	metrics.InodesFree = resource.NewQuantity(inodesFree, resource.BinarySI)
 	return nil
 }

pkg/volume/metrics_statfs.go

@@ -54,12 +54,15 @@ func (md *metricsStatFS) GetMetrics() (*Metrics, error) {
 
 // getFsInfo writes metrics.Capacity, metrics.Used and metrics.Available from the filesystem info
 func (md *metricsStatFS) getFsInfo(metrics *Metrics) error {
-	available, capacity, usage, err := util.FsInfo(md.path)
+	available, capacity, usage, inodes, inodesFree, inodesUsed, err := util.FsInfo(md.path)
 	if err != nil {
 		return NewFsInfoFailedError(err)
 	}
 	metrics.Available = resource.NewQuantity(available, resource.BinarySI)
 	metrics.Capacity = resource.NewQuantity(capacity, resource.BinarySI)
 	metrics.Used = resource.NewQuantity(usage, resource.BinarySI)
+	metrics.Inodes = resource.NewQuantity(inodes, resource.BinarySI)
+	metrics.InodesFree = resource.NewQuantity(inodesFree, resource.BinarySI)
+	metrics.InodesUsed = resource.NewQuantity(inodesUsed, resource.BinarySI)
 	return nil
 }

pkg/volume/util/fs.go

@@ -19,23 +19,25 @@ limitations under the License.
 package util
 
 import (
+	"bytes"
 	"fmt"
 	"os/exec"
+	"strconv"
 	"strings"
 	"syscall"
 
 	"k8s.io/kubernetes/pkg/api/resource"
 )
 
-// FSInfo linux returns (available bytes, byte capacity, byte usage, error) for the filesystem that
-// path resides upon.
-func FsInfo(path string) (int64, int64, int64, error) {
+// FSInfo linux returns (available bytes, byte capacity, byte usage, total inodes, inodes free, inode usage, error)
+// for the filesystem that path resides upon.
+func FsInfo(path string) (int64, int64, int64, int64, int64, int64, error) {
 	statfs := &syscall.Statfs_t{}
 	err := syscall.Statfs(path, statfs)
 	if err != nil {
-		return 0, 0, 0, err
+		return 0, 0, 0, 0, 0, 0, err
 	}
-	// TODO(vishh): Include inodes space
+
 	// Available is blocks available * fragment size
 	available := int64(statfs.Bavail) * int64(statfs.Bsize)
 
@@ -45,7 +47,11 @@ func FsInfo(path string) (int64, int64, int64, error) {
 	// Usage is block being used * fragment size (aka block size).
 	usage := (int64(statfs.Blocks) - int64(statfs.Bfree)) * int64(statfs.Bsize)
 
-	return available, capacity, usage, nil
+	inodes := int64(statfs.Files)
+	inodesFree := int64(statfs.Ffree)
+	inodesUsed := inodes - inodesFree
+
+	return available, capacity, usage, inodes, inodesFree, inodesUsed, nil
 }
 
 func Du(path string) (*resource.Quantity, error) {
@@ -62,3 +68,36 @@ func Du(path string) (*resource.Quantity, error) {
 	used.Format = resource.BinarySI
 	return &used, nil
 }
+
+// Find uses the command `find <path> -dev -printf '.' | wc -c` to count files and directories.
+// While this is not an exact measure of inodes used, it is a very good approximation.
+func Find(path string) (int64, error) {
+	var stdout, stdwcerr, stdfinderr bytes.Buffer
+	var err error
+	findCmd := exec.Command("find", path, "-xdev", "-printf", ".")
+	wcCmd := exec.Command("wc", "-c")
+	if wcCmd.Stdin, err = findCmd.StdoutPipe(); err != nil {
+		return 0, fmt.Errorf("failed to setup stdout for cmd %v - %v", findCmd.Args, err)
+	}
+	wcCmd.Stdout, wcCmd.Stderr, findCmd.Stderr = &stdout, &stdwcerr, &stdfinderr
+	if err = findCmd.Start(); err != nil {
+		return 0, fmt.Errorf("failed to exec cmd %v - %v; stderr: %v", findCmd.Args, err, stdfinderr.String())
+	}
+
+	if err = wcCmd.Start(); err != nil {
+		return 0, fmt.Errorf("failed to exec cmd %v - %v; stderr %v", wcCmd.Args, err, stdwcerr.String())
+	}
+	err = findCmd.Wait()
+	if err != nil {
+		return 0, fmt.Errorf("cmd %v failed. stderr: %s; err: %v", findCmd.Args, stdfinderr.String(), err)
+	}
+	err = wcCmd.Wait()
+	if err != nil {
+		return 0, fmt.Errorf("cmd %v failed. stderr: %s; err: %v", wcCmd.Args, stdwcerr.String(), err)
+	}
+	inodeUsage, err := strconv.ParseInt(strings.TrimSpace(stdout.String()), 10, 64)
+	if err != nil {
+		return 0, fmt.Errorf("cannot parse cmds: %v, %v output %s - %s", findCmd.Args, wcCmd.Args, stdout.String(), err)
+	}
+	return inodeUsage, nil
+}

pkg/volume/volume.go

@@ -64,6 +64,20 @@ type Metrics struct {
 	// emptydir, hostpath), this is the available space on the underlying
 	// storage, and is shared with host processes and other Volumes.
 	Available *resource.Quantity
+
+	// InodesUsed represents the total inodes used by the Volume.
+	InodesUsed *resource.Quantity
+
+	// Inodes represents the total number of inodes availible in the volume.
+	// For volumes that share a filesystem with the host (e.g. emptydir, hostpath),
+	// this is the inodes available in the underlying storage,
+	// and will not equal InodesUsed + InodesFree as the fs is shared.
+	Inodes *resource.Quantity
+
+	// InodesFree represent the inodes available for the volume.  For Volues that share
+	// a filesystem with the host (e.g. emptydir, hostpath), this is the free inodes
+	// on the underlying sporage, and is shared with host processes and other volumes
+	InodesFree *resource.Quantity
 }
 
 // Attributes represents the attributes of this mounter.

test/e2e_node/summary_test.go

@@ -138,10 +138,9 @@ var _ = framework.KubeDescribe("Summary API", func() {
 							"AvailableBytes": fsCapacityBounds,
 							"CapacityBytes":  fsCapacityBounds,
 							"UsedBytes":      bounded(kb, 1*mb),
-							// Inodes are not reported for Volumes.
-							"InodesFree": BeNil(),
-							"Inodes":     BeNil(),
-							"InodesUsed": BeNil(),
+							"InodesFree":     bounded(1E4, 1E8),
+							"Inodes":         bounded(1E4, 1E8),
+							"InodesUsed":     bounded(0, 1E8),
 						}),
 					}),
 				}),