force rbd image unlock if the image is not used

pkg/volume/rbd/rbd_util.go

@@ -43,6 +43,7 @@ import (
 
 const (
 	imageWatcherStr = "watcher="
+	kubeLockMagic   = "kubelet_lock_magic_"
 )
 
 // search /sys/bus for rbd device that matches given pool and image
@@ -114,8 +115,15 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error {
 	} else {
 		secret_opt = []string{"-k", b.Keyring}
 	}
+	if len(b.adminId) == 0 {
+		b.adminId = b.Id
+	}
+	if len(b.adminSecret) == 0 {
+		b.adminSecret = b.Secret
+	}
+
 	// construct lock id using host name and a magic prefix
-	lock_id := "kubelet_lock_magic_" + node.GetHostname("")
+	lock_id := kubeLockMagic + node.GetHostname("")
 
 	l := len(b.Mon)
 	// avoid mount storm, pick a host randomly
@@ -130,7 +138,7 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error {
 		cmd, err = b.plugin.execCommand("rbd",
 			append([]string{"lock", "list", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...))
 		output = string(cmd)
-
+		glog.Infof("lock list output %q", output)
 		if err != nil {
 			continue
 		}
@@ -142,6 +150,23 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error {
 				glog.V(1).Infof("rbd: lock already held for %s", lock_id)
 				return nil
 			}
+			// clean up orphaned lock if no watcher on the image
+			used, statusErr := util.rbdStatus(&b)
+			if statusErr == nil && !used {
+				re := regexp.MustCompile("client.* " + kubeLockMagic + ".*")
+				locks := re.FindAllStringSubmatch(output, -1)
+				for _, v := range locks {
+					if len(v) > 0 {
+						lockInfo := strings.Split(v[0], " ")
+						if len(lockInfo) > 2 {
+							cmd, err = b.plugin.execCommand("rbd",
+								append([]string{"lock", "remove", b.Image, lockInfo[1], lockInfo[0], "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...))
+							glog.Infof("remove orphaned locker %s from client %s: err %v, output: %s", lockInfo[1], lockInfo[0], err, string(cmd))
+						}
+					}
+				}
+			}
+
 			// hold a lock: rbd lock add
 			cmd, err = b.plugin.execCommand("rbd",
 				append([]string{"lock", "add", b.Image, lock_id, "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...))
@@ -220,6 +245,20 @@ func (util *RBDUtil) AttachDisk(b rbdMounter) error {
 	var err error
 	var output []byte
 
+	// create mount point
+	globalPDPath := b.manager.MakeGlobalPDName(*b.rbd)
+	notMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath)
+	// in the first time, the path shouldn't exist and IsLikelyNotMountPoint is expected to get NotExist
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("rbd: %s failed to check mountpoint", globalPDPath)
+	}
+	if !notMnt {
+		return nil
+	}
+	if err = os.MkdirAll(globalPDPath, 0750); err != nil {
+		return fmt.Errorf("rbd: failed to mkdir %s, error", globalPDPath)
+	}
+
 	devicePath, found := waitForPath(b.Pool, b.Image, 1)
 	if !found {
 		// modprobe
@@ -227,6 +266,18 @@ func (util *RBDUtil) AttachDisk(b rbdMounter) error {
 		if err != nil {
 			return fmt.Errorf("rbd: failed to modprobe rbd error:%v", err)
 		}
+
+		// fence off other mappers
+		if err = util.fencing(b); err != nil {
+			return fmt.Errorf("rbd: image %s is locked by other nodes", b.Image)
+		}
+		// rbd lock remove needs ceph and image config
+		// but kubelet doesn't get them from apiserver during teardown
+		// so persit rbd config so upon disk detach, rbd lock can be removed
+		// since rbd json is persisted in the same local directory that is used as rbd mountpoint later,
+		// the json file remains invisible during rbd mount and thus won't be removed accidentally.
+		util.persistRBD(b, globalPDPath)
+
 		// rbd map
 		l := len(b.Mon)
 		// avoid mount storm, pick a host randomly
@@ -255,34 +306,8 @@ func (util *RBDUtil) AttachDisk(b rbdMounter) error {
 			return errors.New("Could not map image: Timeout after 10s")
 		}
 	}
-	// mount it
-	globalPDPath := b.manager.MakeGlobalPDName(*b.rbd)
-	notMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath)
-	// in the first time, the path shouldn't exist and IsLikelyNotMountPoint is expected to get NotExist
-	if err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("rbd: %s failed to check mountpoint", globalPDPath)
-	}
-	if !notMnt {
-		return nil
-	}
-
-	if err := os.MkdirAll(globalPDPath, 0750); err != nil {
-		return fmt.Errorf("rbd: failed to mkdir %s, error", globalPDPath)
-	}
-
-	// fence off other mappers
-	if err := util.fencing(b); err != nil {
-		// rbd unmap before exit
-		b.plugin.execCommand("rbd", []string{"unmap", devicePath})
-		return fmt.Errorf("rbd: image %s is locked by other nodes", b.Image)
-	}
-	// rbd lock remove needs ceph and image config
-	// but kubelet doesn't get them from apiserver during teardown
-	// so persit rbd config so upon disk detach, rbd lock can be removed
-	// since rbd json is persisted in the same local directory that is used as rbd mountpoint later,
-	// the json file remains invisible during rbd mount and thus won't be removed accidentally.
-	util.persistRBD(b, globalPDPath)
 
+	// mount it
 	if err = b.mounter.FormatAndMount(devicePath, globalPDPath, b.fsType, nil); err != nil {
 		err = fmt.Errorf("rbd: failed to mount rbd volume %s [%s] to %s, error %v", devicePath, b.fsType, globalPDPath, err)
 	}