Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
K
k3s
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Registry
Registry
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Jacklull
k3s
Commits
7bac6ca7
Commit
7bac6ca7
authored
Nov 01, 2018
by
Jing Xu
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Address comments
This commit addressed the comment and also add a unit test.
parent
562d0fea
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
11 changed files
with
135 additions
and
52 deletions
+135
-52
attach_detach_controller.go
...ontroller/volume/attachdetach/attach_detach_controller.go
+1
-1
attach_detach_controller_test.go
...ller/volume/attachdetach/attach_detach_controller_test.go
+1
-1
actual_state_of_world.go
...roller/volume/attachdetach/cache/actual_state_of_world.go
+29
-25
actual_state_of_world_test.go
...r/volume/attachdetach/cache/actual_state_of_world_test.go
+0
-0
metrics.go
pkg/controller/volume/attachdetach/metrics/metrics.go
+1
-1
reconciler.go
pkg/controller/volume/attachdetach/reconciler/reconciler.go
+3
-3
reconciler_test.go
...troller/volume/attachdetach/reconciler/reconciler_test.go
+77
-10
testvolumespec.go
pkg/controller/volume/attachdetach/testing/testvolumespec.go
+1
-1
testing.go
pkg/volume/testing/testing.go
+12
-4
operation_executor.go
pkg/volume/util/operationexecutor/operation_executor.go
+4
-0
operation_generator.go
pkg/volume/util/operationexecutor/operation_generator.go
+6
-6
No files found.
pkg/controller/volume/attachdetach/attach_detach_controller.go
View file @
7bac6ca7
...
...
@@ -632,7 +632,7 @@ func (adc *attachDetachController) syncPVCByKey(key string) error {
func
(
adc
*
attachDetachController
)
processVolumesInUse
(
nodeName
types
.
NodeName
,
volumesInUse
[]
v1
.
UniqueVolumeName
)
{
klog
.
V
(
4
)
.
Infof
(
"processVolumesInUse for node %q"
,
nodeName
)
for
_
,
attachedVolume
:=
range
adc
.
actualStateOfWorld
.
GetA
ll
VolumesForNode
(
nodeName
)
{
for
_
,
attachedVolume
:=
range
adc
.
actualStateOfWorld
.
GetA
ttached
VolumesForNode
(
nodeName
)
{
mounted
:=
false
for
_
,
volumeInUse
:=
range
volumesInUse
{
if
attachedVolume
.
VolumeName
==
volumeInUse
{
...
...
pkg/controller/volume/attachdetach/attach_detach_controller_test.go
View file @
7bac6ca7
...
...
@@ -278,7 +278,7 @@ func attachDetachRecoveryTestCase(t *testing.T, extraPods1 []*v1.Pod, extraPods2
var
detachedVolumesNum
int
=
0
time
.
Sleep
(
time
.
Second
*
1
)
// Wait for a second
for
_
,
volumeList
:=
range
testPlugin
.
GetA
ll
Volumes
()
{
for
_
,
volumeList
:=
range
testPlugin
.
GetA
ttached
Volumes
()
{
attachedVolumesNum
+=
len
(
volumeList
)
}
for
_
,
volumeList
:=
range
testPlugin
.
GetDetachedVolumes
()
{
...
...
pkg/controller/volume/attachdetach/cache/actual_state_of_world.go
View file @
7bac6ca7
...
...
@@ -96,21 +96,22 @@ type ActualStateOfWorld interface {
// nodes, the volume is also deleted.
DeleteVolumeNode
(
volumeName
v1
.
UniqueVolumeName
,
nodeName
types
.
NodeName
)
//
VolumeNodeExists
returns true if the specified volume/node combo exists
//
IsVolumeAttachedToNode
returns true if the specified volume/node combo exists
// in the underlying store indicating the specified volume is attached to
// the specified node.
IsVolumeAttachedToNode
(
volumeName
v1
.
UniqueVolumeName
,
nodeName
types
.
NodeName
)
bool
// GetAttachedVolumes generates and returns a list of volumes/node pairs
// reflecting which volumes might attached to which nodes based on the
// current actual state of the world.
GetAllVolumes
()
[]
AttachedVolume
// current actual state of the world. This list includes all the volumes which return successful
// attach and also the volumes which return errors during attach.
GetAttachedVolumes
()
[]
AttachedVolume
// GetAttachedVolumes generates and returns a list of volumes that added to
// GetAttachedVolumes
ForNode
generates and returns a list of volumes that added to
// the specified node reflecting which volumes are/or might be attached to that node
// based on the current actual state of the world. This function is currently used by
// attach_detach_controller to process VolumeInUse
GetA
ll
VolumesForNode
(
nodeName
types
.
NodeName
)
[]
AttachedVolume
GetA
ttached
VolumesForNode
(
nodeName
types
.
NodeName
)
[]
AttachedVolume
// GetAttachedVolumesPerNode generates and returns a map of nodes and volumes that added to
// the specified node reflecting which volumes are attached to that node
...
...
@@ -118,9 +119,9 @@ type ActualStateOfWorld interface {
// reconciler to verify whether the volume is still attached to the node.
GetAttachedVolumesPerNode
()
map
[
types
.
NodeName
][]
operationexecutor
.
AttachedVolume
// GetNodesForVolume returns the nodes on which the volume is attached.
// GetNodesFor
Attached
Volume returns the nodes on which the volume is attached.
// This function is used by reconciler for mutli-attach check.
GetNodesForVolume
(
volumeName
v1
.
UniqueVolumeName
)
[]
types
.
NodeName
GetNodesFor
Attached
Volume
(
volumeName
v1
.
UniqueVolumeName
)
[]
types
.
NodeName
// GetVolumesToReportAttached returns a map containing the set of nodes for
// which the VolumesAttached Status field in the Node API object should be
...
...
@@ -193,7 +194,7 @@ type attachedVolume struct {
spec
*
volume
.
Spec
// nodesAttachedTo is a map containing the set of nodes this volume has
// been
trying to be
attached to. The key in this map is the name of the
// been attached to. The key in this map is the name of the
// node and the value is a node object containing more information about
// the node.
nodesAttachedTo
map
[
types
.
NodeName
]
nodeAttachedTo
...
...
@@ -212,8 +213,9 @@ type nodeAttachedTo struct {
// node and is unsafe to detach
mountedByNode
bool
// attached indicates that the volume is confirmed to be attached to this node
attached
bool
// attachConfirmed indicates that the storage system verified the volume has been attached to this node.
// This value is set to false when an attach operation fails and the volume may be attached or not.
attachedConfirmed
bool
// detachRequestedTime used to capture the desire to detach this volume
detachRequestedTime
time
.
Time
...
...
@@ -244,7 +246,7 @@ type nodeToUpdateStatusFor struct {
func
(
asw
*
actualStateOfWorld
)
MarkVolumeAsUncertain
(
uniqueName
v1
.
UniqueVolumeName
,
volumeSpec
*
volume
.
Spec
,
nodeName
types
.
NodeName
)
error
{
_
,
err
:=
asw
.
AddVolumeNode
(
uniqueName
,
volumeSpec
,
nodeName
,
""
,
false
)
_
,
err
:=
asw
.
AddVolumeNode
(
uniqueName
,
volumeSpec
,
nodeName
,
""
,
false
/* isAttached */
)
return
err
}
...
...
@@ -280,6 +282,9 @@ func (asw *actualStateOfWorld) AddVolumeNode(
volumeName
:=
uniqueName
if
volumeName
==
""
{
if
volumeSpec
==
nil
{
return
volumeName
,
fmt
.
Errorf
(
"volumeSpec cannot be nil if volumeName is empty"
)
}
attachableVolumePlugin
,
err
:=
asw
.
volumePluginMgr
.
FindAttachablePluginBySpec
(
volumeSpec
)
if
err
!=
nil
||
attachableVolumePlugin
==
nil
{
return
""
,
fmt
.
Errorf
(
...
...
@@ -316,24 +321,25 @@ func (asw *actualStateOfWorld) AddVolumeNode(
nodeName
,
devicePath
)
}
asw
.
attachedVolumes
[
volumeName
]
=
volumeObj
node
,
nodeExists
:=
volumeObj
.
nodesAttachedTo
[
nodeName
]
if
!
nodeExists
{
// Create object if it doesn't exist.
node
=
nodeAttachedTo
{
nodeName
:
nodeName
,
mountedByNode
:
true
,
// Assume mounted, until proven otherwise
attached
:
isAttached
,
attached
Confirmed
:
isAttached
,
detachRequestedTime
:
time
.
Time
{},
}
}
else
{
node
.
attached
=
isAttached
klog
.
V
(
5
)
.
Infof
(
"Volume %q is already added to attachedVolume list to the node %q"
,
node
.
attached
Confirmed
=
isAttached
klog
.
V
(
5
)
.
Infof
(
"Volume %q is already added to attachedVolume list to the node %q
, the current attach state is %t
"
,
volumeName
,
nodeName
)
nodeName
,
isAttached
)
}
volumeObj
.
nodesAttachedTo
[
nodeName
]
=
node
asw
.
attachedVolumes
[
volumeName
]
=
volumeObj
if
isAttached
{
asw
.
addVolumeToReportAsAttached
(
volumeName
,
nodeName
)
...
...
@@ -529,16 +535,14 @@ func (asw *actualStateOfWorld) IsVolumeAttachedToNode(
volumeObj
,
volumeExists
:=
asw
.
attachedVolumes
[
volumeName
]
if
volumeExists
{
if
node
,
nodeExists
:=
volumeObj
.
nodesAttachedTo
[
nodeName
];
nodeExists
{
if
node
.
attached
==
true
{
return
true
}
return
node
.
attachedConfirmed
}
}
return
false
}
func
(
asw
*
actualStateOfWorld
)
GetA
ll
Volumes
()
[]
AttachedVolume
{
func
(
asw
*
actualStateOfWorld
)
GetA
ttached
Volumes
()
[]
AttachedVolume
{
asw
.
RLock
()
defer
asw
.
RUnlock
()
...
...
@@ -554,7 +558,7 @@ func (asw *actualStateOfWorld) GetAllVolumes() []AttachedVolume {
return
attachedVolumes
}
func
(
asw
*
actualStateOfWorld
)
GetA
ll
VolumesForNode
(
func
(
asw
*
actualStateOfWorld
)
GetA
ttached
VolumesForNode
(
nodeName
types
.
NodeName
)
[]
AttachedVolume
{
asw
.
RLock
()
defer
asw
.
RUnlock
()
...
...
@@ -582,7 +586,7 @@ func (asw *actualStateOfWorld) GetAttachedVolumesPerNode() map[types.NodeName][]
attachedVolumesPerNode
:=
make
(
map
[
types
.
NodeName
][]
operationexecutor
.
AttachedVolume
)
for
_
,
volumeObj
:=
range
asw
.
attachedVolumes
{
for
nodeName
,
nodeObj
:=
range
volumeObj
.
nodesAttachedTo
{
if
nodeObj
.
attached
{
if
nodeObj
.
attached
Confirmed
{
volumes
:=
attachedVolumesPerNode
[
nodeName
]
volumes
=
append
(
volumes
,
getAttachedVolume
(
&
volumeObj
,
&
nodeObj
)
.
AttachedVolume
)
attachedVolumesPerNode
[
nodeName
]
=
volumes
...
...
@@ -593,7 +597,7 @@ func (asw *actualStateOfWorld) GetAttachedVolumesPerNode() map[types.NodeName][]
return
attachedVolumesPerNode
}
func
(
asw
*
actualStateOfWorld
)
GetNodesForVolume
(
volumeName
v1
.
UniqueVolumeName
)
[]
types
.
NodeName
{
func
(
asw
*
actualStateOfWorld
)
GetNodesFor
Attached
Volume
(
volumeName
v1
.
UniqueVolumeName
)
[]
types
.
NodeName
{
asw
.
RLock
()
defer
asw
.
RUnlock
()
...
...
@@ -604,7 +608,7 @@ func (asw *actualStateOfWorld) GetNodesForVolume(volumeName v1.UniqueVolumeName)
nodes
:=
[]
types
.
NodeName
{}
for
k
,
nodesAttached
:=
range
volumeObj
.
nodesAttachedTo
{
if
nodesAttached
.
attached
{
if
nodesAttached
.
attached
Confirmed
{
nodes
=
append
(
nodes
,
k
)
}
}
...
...
pkg/controller/volume/attachdetach/cache/actual_state_of_world_test.go
View file @
7bac6ca7
This diff is collapsed.
Click to expand it.
pkg/controller/volume/attachdetach/metrics/metrics.go
View file @
7bac6ca7
...
...
@@ -185,7 +185,7 @@ func (collector *attachDetachStateCollector) getTotalVolumesCount() volumeCount
stateVolumeMap
.
add
(
"desired_state_of_world"
,
pluginName
)
}
}
for
_
,
v
:=
range
collector
.
asw
.
GetA
ll
Volumes
()
{
for
_
,
v
:=
range
collector
.
asw
.
GetA
ttached
Volumes
()
{
if
plugin
,
err
:=
collector
.
volumePluginMgr
.
FindPluginBySpec
(
v
.
VolumeSpec
);
err
==
nil
{
pluginName
:=
pluginNameNotAvailable
if
plugin
!=
nil
{
...
...
pkg/controller/volume/attachdetach/reconciler/reconciler.go
View file @
7bac6ca7
...
...
@@ -175,7 +175,7 @@ func (rc *reconciler) reconcile() {
// pods that are rescheduled to a different node are detached first.
// Ensure volumes that should be detached are detached.
for
_
,
attachedVolume
:=
range
rc
.
actualStateOfWorld
.
GetA
ll
Volumes
()
{
for
_
,
attachedVolume
:=
range
rc
.
actualStateOfWorld
.
GetA
ttached
Volumes
()
{
if
!
rc
.
desiredStateOfWorld
.
VolumeExists
(
attachedVolume
.
VolumeName
,
attachedVolume
.
NodeName
)
{
// Don't even try to start an operation if there is already one running
...
...
@@ -183,7 +183,7 @@ func (rc *reconciler) reconcile() {
// may pass while at the same time the volume leaves the pending state, resulting in
// double detach attempts
if
rc
.
attacherDetacher
.
IsOperationPending
(
attachedVolume
.
VolumeName
,
""
)
{
klog
.
V
(
5
)
.
Infof
(
"Operation for volume %q is already running. Can't start detach for %q"
,
attachedVolume
.
VolumeName
,
attachedVolume
.
NodeName
)
klog
.
V
(
10
)
.
Infof
(
"Operation for volume %q is already running. Can't start detach for %q"
,
attachedVolume
.
VolumeName
,
attachedVolume
.
NodeName
)
continue
}
...
...
@@ -269,7 +269,7 @@ func (rc *reconciler) attachDesiredVolumes() {
}
if
rc
.
isMultiAttachForbidden
(
volumeToAttach
.
VolumeSpec
)
{
nodes
:=
rc
.
actualStateOfWorld
.
GetNodesForVolume
(
volumeToAttach
.
VolumeName
)
nodes
:=
rc
.
actualStateOfWorld
.
GetNodesFor
Attached
Volume
(
volumeToAttach
.
VolumeName
)
if
len
(
nodes
)
>
0
{
if
!
volumeToAttach
.
MultiAttachErrorReported
{
rc
.
reportMultiAttachError
(
volumeToAttach
,
nodes
)
...
...
pkg/controller/volume/attachdetach/reconciler/reconciler_test.go
View file @
7bac6ca7
...
...
@@ -497,7 +497,7 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteOnce(t *testing.
waitForDetachCallCount
(
t
,
0
/* expectedDetachCallCount */
,
fakePlugin
)
waitForAttachedToNodesCount
(
t
,
1
/* expectedNodeCount */
,
generatedVolumeName
,
asw
)
nodesForVolume
:=
asw
.
GetNodesForVolume
(
generatedVolumeName
)
nodesForVolume
:=
asw
.
GetNodesFor
Attached
Volume
(
generatedVolumeName
)
// check if multiattach is marked
// at least one volume+node should be marked with multiattach error
...
...
@@ -576,10 +576,11 @@ func Test_Run_OneVolumeAttachAndDetachUncertainNodesWithReadWriteOnce(t *testing
t
.
Fatalf
(
"AddPod failed. Expected: <no error> Actual: <%v>"
,
podAddErr
)
}
time
.
Sleep
(
1
*
time
.
Second
)
// Volume is added to asw. Because attach operation fails, volume should not reported as attached to the node.
waitForVolumeAddedToNode
(
t
,
generatedVolumeName
,
nodeName1
,
asw
)
veri
ryVolumeAttachedToNode
(
t
,
generatedVolumeName
,
nodeName1
,
fals
e
,
asw
)
veri
ryVolumeReportedAsAttachedToNode
(
t
,
generatedVolumeName
,
nodeName1
,
fals
e
,
asw
)
veri
fyVolumeAttachedToNode
(
t
,
generatedVolumeName
,
nodeName1
,
tru
e
,
asw
)
veri
fyVolumeReportedAsAttachedToNode
(
t
,
generatedVolumeName
,
nodeName1
,
tru
e
,
asw
)
// When volume is added to the node, it is set to mounted by default. Then the status will be updated by checking node status VolumeInUse.
// Without this, the delete operation will be delayed due to mounted status
...
...
@@ -595,7 +596,73 @@ func Test_Run_OneVolumeAttachAndDetachUncertainNodesWithReadWriteOnce(t *testing
t
.
Fatalf
(
"AddPod failed. Expected: <no error> Actual: <%v>"
,
podAddErr
)
}
waitForVolumeAttachedToNode
(
t
,
generatedVolumeName
,
nodeName2
,
asw
)
veriryVolumeAttachedToNode
(
t
,
generatedVolumeName
,
nodeName2
,
true
,
asw
)
verifyVolumeAttachedToNode
(
t
,
generatedVolumeName
,
nodeName2
,
true
,
asw
)
}
// Creates a volume with accessMode ReadWriteOnce
// First create a pod which will try to attach the volume to the a node named "timeout-node". The attach call for this node will
// fail for timeout, but the volume will be actually attached to the node after the call.
// Secondly, delete the this pod.
// Lastly, create a pod scheduled to a normal node which will trigger attach volume to the node. The attach should return successfully.
func
Test_Run_OneVolumeAttachAndDetachTimeoutNodesWithReadWriteOnce
(
t
*
testing
.
T
)
{
// Arrange
volumePluginMgr
,
_
:=
volumetesting
.
GetTestVolumePluginMgr
(
t
)
dsw
:=
cache
.
NewDesiredStateOfWorld
(
volumePluginMgr
)
asw
:=
cache
.
NewActualStateOfWorld
(
volumePluginMgr
)
fakeKubeClient
:=
controllervolumetesting
.
CreateTestClient
()
fakeRecorder
:=
&
record
.
FakeRecorder
{}
fakeHandler
:=
volumetesting
.
NewBlockVolumePathHandler
()
ad
:=
operationexecutor
.
NewOperationExecutor
(
operationexecutor
.
NewOperationGenerator
(
fakeKubeClient
,
volumePluginMgr
,
fakeRecorder
,
false
,
/* checkNodeCapabilitiesBeforeMount */
fakeHandler
))
nsu
:=
statusupdater
.
NewFakeNodeStatusUpdater
(
false
/* returnError */
)
reconciler
:=
NewReconciler
(
reconcilerLoopPeriod
,
maxWaitForUnmountDuration
,
syncLoopPeriod
,
false
,
dsw
,
asw
,
ad
,
nsu
,
fakeRecorder
)
podName1
:=
"pod-uid1"
podName2
:=
"pod-uid2"
volumeName
:=
v1
.
UniqueVolumeName
(
"volume-name"
)
volumeSpec
:=
controllervolumetesting
.
GetTestVolumeSpec
(
string
(
volumeName
),
volumeName
)
volumeSpec
.
PersistentVolume
.
Spec
.
AccessModes
=
[]
v1
.
PersistentVolumeAccessMode
{
v1
.
ReadWriteOnce
}
nodeName1
:=
k8stypes
.
NodeName
(
volumetesting
.
TimeoutAttachNode
)
nodeName2
:=
k8stypes
.
NodeName
(
"node-name2"
)
dsw
.
AddNode
(
nodeName1
,
false
/*keepTerminatedPodVolumes*/
)
dsw
.
AddNode
(
nodeName2
,
false
/*keepTerminatedPodVolumes*/
)
// Act
ch
:=
make
(
chan
struct
{})
go
reconciler
.
Run
(
ch
)
defer
close
(
ch
)
// Add the pod in which the volume is attached to the timeout node
generatedVolumeName
,
podAddErr
:=
dsw
.
AddPod
(
types
.
UniquePodName
(
podName1
),
controllervolumetesting
.
NewPod
(
podName1
,
podName1
),
volumeSpec
,
nodeName1
)
if
podAddErr
!=
nil
{
t
.
Fatalf
(
"AddPod failed. Expected: <no error> Actual: <%v>"
,
podAddErr
)
}
// Volume is added to asw. Because attach operation fails, volume should not reported as attached to the node.
waitForVolumeAddedToNode
(
t
,
generatedVolumeName
,
nodeName1
,
asw
)
verifyVolumeAttachedToNode
(
t
,
generatedVolumeName
,
nodeName1
,
false
,
asw
)
verifyVolumeReportedAsAttachedToNode
(
t
,
generatedVolumeName
,
nodeName1
,
false
,
asw
)
// When volume is added to the node, it is set to mounted by default. Then the status will be updated by checking node status VolumeInUse.
// Without this, the delete operation will be delayed due to mounted status
asw
.
SetVolumeMountedByNode
(
generatedVolumeName
,
nodeName1
,
false
/* mounted */
)
dsw
.
DeletePod
(
types
.
UniquePodName
(
podName1
),
generatedVolumeName
,
nodeName1
)
waitForVolumeRemovedFromNode
(
t
,
generatedVolumeName
,
nodeName1
,
asw
)
// Add a second pod which tries to attach the volume to a different node.
generatedVolumeName
,
podAddErr
=
dsw
.
AddPod
(
types
.
UniquePodName
(
podName2
),
controllervolumetesting
.
NewPod
(
podName2
,
podName2
),
volumeSpec
,
nodeName2
)
if
podAddErr
!=
nil
{
t
.
Fatalf
(
"AddPod failed. Expected: <no error> Actual: <%v>"
,
podAddErr
)
}
waitForVolumeAttachedToNode
(
t
,
generatedVolumeName
,
nodeName2
,
asw
)
verifyVolumeAttachedToNode
(
t
,
generatedVolumeName
,
nodeName2
,
true
,
asw
)
}
...
...
@@ -935,7 +1002,7 @@ func waitForAttachedToNodesCount(
err
:=
retryWithExponentialBackOff
(
time
.
Duration
(
5
*
time
.
Millisecond
),
func
()
(
bool
,
error
)
{
count
:=
len
(
asw
.
GetNodesForVolume
(
volumeName
))
count
:=
len
(
asw
.
GetNodesFor
Attached
Volume
(
volumeName
))
if
count
==
expectedNodeCount
{
return
true
,
nil
}
...
...
@@ -950,7 +1017,7 @@ func waitForAttachedToNodesCount(
)
if
err
!=
nil
{
count
:=
len
(
asw
.
GetNodesForVolume
(
volumeName
))
count
:=
len
(
asw
.
GetNodesFor
Attached
Volume
(
volumeName
))
t
.
Fatalf
(
"Wrong number of nodes having <%v> attached. Expected: <%v> Actual: <%v>"
,
volumeName
,
...
...
@@ -1010,7 +1077,7 @@ func waitForVolumeAddedToNode(
err
:=
retryWithExponentialBackOff
(
time
.
Duration
(
500
*
time
.
Millisecond
),
func
()
(
bool
,
error
)
{
volumes
:=
asw
.
GetA
ll
Volumes
()
volumes
:=
asw
.
GetA
ttached
Volumes
()
for
_
,
volume
:=
range
volumes
{
if
volume
.
VolumeName
==
volumeName
&&
volume
.
NodeName
==
nodeName
{
return
true
,
nil
...
...
@@ -1042,7 +1109,7 @@ func waitForVolumeRemovedFromNode(
err
:=
retryWithExponentialBackOff
(
time
.
Duration
(
500
*
time
.
Millisecond
),
func
()
(
bool
,
error
)
{
volumes
:=
asw
.
GetA
ll
Volumes
()
volumes
:=
asw
.
GetA
ttached
Volumes
()
exist
:=
false
for
_
,
volume
:=
range
volumes
{
if
volume
.
VolumeName
==
volumeName
&&
volume
.
NodeName
==
nodeName
{
...
...
@@ -1070,7 +1137,7 @@ func waitForVolumeRemovedFromNode(
}
}
func
veri
r
yVolumeAttachedToNode
(
func
veri
f
yVolumeAttachedToNode
(
t
*
testing
.
T
,
volumeName
v1
.
UniqueVolumeName
,
nodeName
k8stypes
.
NodeName
,
...
...
@@ -1089,7 +1156,7 @@ func veriryVolumeAttachedToNode(
}
func
veri
r
yVolumeReportedAsAttachedToNode
(
func
veri
f
yVolumeReportedAsAttachedToNode
(
t
*
testing
.
T
,
volumeName
v1
.
UniqueVolumeName
,
nodeName
k8stypes
.
NodeName
,
...
...
pkg/controller/volume/attachdetach/testing/testvolumespec.go
View file @
7bac6ca7
...
...
@@ -328,7 +328,7 @@ func (plugin *TestPlugin) GetErrorEncountered() bool {
return
plugin
.
ErrorEncountered
}
func
(
plugin
*
TestPlugin
)
GetA
ll
Volumes
()
map
[
string
][]
string
{
func
(
plugin
*
TestPlugin
)
GetA
ttached
Volumes
()
map
[
string
][]
string
{
plugin
.
pluginLock
.
RLock
()
defer
plugin
.
pluginLock
.
RUnlock
()
ret
:=
make
(
map
[
string
][]
string
)
...
...
pkg/volume/testing/testing.go
View file @
7bac6ca7
...
...
@@ -51,8 +51,11 @@ const (
// is expected to fail.
ExpectProvisionFailureKey
=
"expect-provision-failure"
// The node is marked as uncertain. The attach operation will fail and return timeout error
//
but the operation is actually succeeded
.
//
for the first attach call. The following call will return sucesssfully
.
UncertainAttachNode
=
"uncertain-attach-node"
// The node is marked as timeout. The attach operation will always fail and return timeout error
// but the operation is actually succeeded.
TimeoutAttachNode
=
"timeout-attach-node"
// The node is marked as multi-attach which means it is allowed to attach the volume to multiple nodes.
MultiAttachNode
=
"multi-attach-node"
)
...
...
@@ -282,7 +285,12 @@ var _ FSResizableVolumePlugin = &FakeVolumePlugin{}
func
(
plugin
*
FakeVolumePlugin
)
getFakeVolume
(
list
*
[]
*
FakeVolume
)
*
FakeVolume
{
volumeList
:=
*
list
if
list
!=
nil
&&
len
(
volumeList
)
>
0
{
return
volumeList
[
0
]
volume
:=
volumeList
[
0
]
volume
.
Lock
()
defer
volume
.
Unlock
()
volume
.
WaitForAttachHook
=
plugin
.
WaitForAttachHook
volume
.
UnmountDeviceHook
=
plugin
.
UnmountDeviceHook
return
volume
}
volume
:=
&
FakeVolume
{
WaitForAttachHook
:
plugin
.
WaitForAttachHook
,
...
...
@@ -773,7 +781,7 @@ func (fv *FakeVolume) Attach(spec *Spec, nodeName types.NodeName) (string, error
volumeNode
,
exist
:=
fv
.
VolumesAttached
[
volumeName
]
if
exist
{
if
nodeName
==
UncertainAttachNode
{
return
"
"
,
fmt
.
Errorf
(
"Timed out to attach volume %q to node %q"
,
volumeName
,
nodeName
)
return
"
/dev/vdb-test"
,
nil
}
if
volumeNode
==
nodeName
||
volumeNode
==
MultiAttachNode
||
nodeName
==
MultiAttachNode
{
return
"/dev/vdb-test"
,
nil
...
...
@@ -782,7 +790,7 @@ func (fv *FakeVolume) Attach(spec *Spec, nodeName types.NodeName) (string, error
}
fv
.
VolumesAttached
[
volumeName
]
=
nodeName
if
nodeName
==
UncertainAttachNode
{
if
nodeName
==
UncertainAttachNode
||
nodeName
==
TimeoutAttachNode
{
return
""
,
fmt
.
Errorf
(
"Timed out to attach volume %q to node %q"
,
volumeName
,
nodeName
)
}
return
"/dev/vdb-test"
,
nil
...
...
pkg/volume/util/operationexecutor/operation_executor.go
View file @
7bac6ca7
...
...
@@ -192,6 +192,10 @@ type ActualStateOfWorldAttacherUpdater interface {
// volumes. See issue 29695.
MarkVolumeAsAttached
(
volumeName
v1
.
UniqueVolumeName
,
volumeSpec
*
volume
.
Spec
,
nodeName
types
.
NodeName
,
devicePath
string
)
error
// Marks the specified volume as *possibly* attached to the specified node.
// If an attach operation fails, the attach/detach controller does not know for certain if the volume is attached or not.
// If the volume name is supplied, that volume name will be used. If not, the
// volume name is computed using the result from querying the plugin.
MarkVolumeAsUncertain
(
volumeName
v1
.
UniqueVolumeName
,
volumeSpec
*
volume
.
Spec
,
nodeName
types
.
NodeName
)
error
// Marks the specified volume as detached from the specified node
...
...
pkg/volume/util/operationexecutor/operation_generator.go
View file @
7bac6ca7
...
...
@@ -322,12 +322,12 @@ func (og *operationGenerator) GenerateAttachVolumeFunc(
klog
.
Errorf
(
"AttachVolume.MarkVolumeAsAttached failed to fix dangling volume error for volume %q with %s"
,
volumeToAttach
.
VolumeName
,
addErr
)
}
}
addVolumeNode
Err
:=
actualStateOfWorld
.
MarkVolumeAsUncertain
(
v1
.
UniqueVolumeName
(
""
),
volumeToAttach
.
VolumeSpec
,
volumeToAttach
.
NodeName
)
if
addVolumeNode
Err
!=
nil
{
// On failure, return error. Caller will log and retry.
return
volumeToAttach
.
GenerateError
(
"AttachVolume.MarkVolumeAsUncertain failed"
,
addVolumeNodeErr
)
}
else
{
add
Err
:=
actualStateOfWorld
.
MarkVolumeAsUncertain
(
v1
.
UniqueVolumeName
(
""
),
volumeToAttach
.
VolumeSpec
,
volumeToAttach
.
NodeName
)
if
add
Err
!=
nil
{
klog
.
Errorf
(
"AttachVolume.MarkVolumeAsUncertain fail to add the volume %q to actual state with %s"
,
volumeToAttach
.
VolumeName
,
addErr
)
}
}
// On failure, return error. Caller will log and retry.
return
volumeToAttach
.
GenerateError
(
"AttachVolume.Attach failed"
,
attachErr
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment