Adding reaper for deployments

hack/test-cmd.sh

@@ -637,13 +637,13 @@ runTests() {
   # Post-Condition: service "nginx" has configuration annotation
   [[ "$(kubectl get svc nginx -o yaml "${kube_flags[@]}" | grep kubectl.kubernetes.io/last-applied-configuration)" ]]
   # Clean up
-  kubectl delete rc,svc nginx 
+  kubectl delete rc,svc nginx
   ## 6. kubectl autoscale --save-config should generate configuration annotation
   # Pre-Condition: no RC exists, then create the rc "frontend", which shouldn't have configuration annotation
   kube::test::get_object_assert rc "{{range.items}}{{$id_field}}:{{end}}" ''
   kubectl create -f examples/guestbook/frontend-controller.yaml "${kube_flags[@]}"
   ! [[ "$(kubectl get rc frontend -o yaml "${kube_flags[@]}" | grep kubectl.kubernetes.io/last-applied-configuration)" ]]
-  # Command: autoscale rc "frontend" 
+  # Command: autoscale rc "frontend"
   kubectl autoscale -f examples/guestbook/frontend-controller.yaml --save-config "${kube_flags[@]}" --max=2
   # Post-Condition: hpa "frontend" has configuration annotation
   [[ "$(kubectl get hpa frontend -o yaml "${kube_flags[@]}" | grep kubectl.kubernetes.io/last-applied-configuration)" ]]
@@ -653,21 +653,21 @@ runTests() {
   ## kubectl apply should create the resource that doesn't exist yet
   # Pre-Condition: no POD exists
   kube::test::get_object_assert pods "{{range.items}}{{$id_field}}:{{end}}" ''
-  # Command: apply a pod "test-pod" (doesn't exist) should create this pod 
+  # Command: apply a pod "test-pod" (doesn't exist) should create this pod
   kubectl apply -f hack/testdata/pod.yaml "${kube_flags[@]}"
   # Post-Condition: pod "test-pod" is created
   kube::test::get_object_assert 'pods test-pod' "{{${labels_field}.name}}" 'test-pod-label'
   # Post-Condition: pod "test-pod" has configuration annotation
   [[ "$(kubectl get pods test-pod -o yaml "${kube_flags[@]}" | grep kubectl.kubernetes.io/last-applied-configuration)" ]]
-  # Clean up 
+  # Clean up
   kubectl delete pods test-pod "${kube_flags[@]}"
 
-  ## kubectl run should create deployments or jobs 
+  ## kubectl run should create deployments or jobs
   # Pre-Condition: no Job exists
   kube::test::get_object_assert jobs "{{range.items}}{{$id_field}}:{{end}}" ''
   # Command
   kubectl run pi --generator=job/v1beta1 --image=perl --restart=OnFailure -- perl -Mbignum=bpi -wle 'print bpi(20)' "${kube_flags[@]}"
-  # Post-Condition: Job "pi" is created 
+  # Post-Condition: Job "pi" is created
   kube::test::get_object_assert jobs "{{range.items}}{{$id_field}}:{{end}}" 'pi:'
   # Clean up
   kubectl delete jobs pi "${kube_flags[@]}"
@@ -675,11 +675,10 @@ runTests() {
   kube::test::get_object_assert deployment "{{range.items}}{{$id_field}}:{{end}}" ''
   # Command
   kubectl run nginx --image=nginx --generator=deployment/v1beta1 "${kube_flags[@]}"
-  # Post-Condition: Deployment "nginx" is created 
+  # Post-Condition: Deployment "nginx" is created
   kube::test::get_object_assert deployment "{{range.items}}{{$id_field}}:{{end}}" 'nginx:'
-  # Clean up 
+  # Clean up
   kubectl delete deployment nginx "${kube_flags[@]}"
-  kubectl delete rc -l pod-template-hash "${kube_flags[@]}"
 
   ##############
   # Namespaces #
@@ -721,7 +720,7 @@ runTests() {
   # Command
   kubectl delete "${kube_flags[@]}" pod --namespace=other valid-pod --grace-period=0
   # Post-condition: valid-pod POD doesn't exist
-  kube::test::get_object_assert 'pods --namespace=other' "{{range.items}}{{$id_field}}:{{end}}" ''  
+  kube::test::get_object_assert 'pods --namespace=other' "{{range.items}}{{$id_field}}:{{end}}" ''
   # Clean up
   kubectl delete namespace other
 
@@ -966,6 +965,7 @@ __EOF__
   kube::test::get_object_assert 'job pi' "{{$job_parallelism_field}}" '2'
   # Clean-up
   kubectl delete job/pi "${kube_flags[@]}"
+
   # TODO(madhusudancs): Fix this when Scale group issues are resolved (see issue #18528).
   # ### Scale a deployment
   # kubectl create -f examples/extensions/deployment.yaml "${kube_flags[@]}"
@@ -975,8 +975,6 @@ __EOF__
   # kube::test::get_object_assert 'deployment nginx-deployment' "{{$deployment_replicas}}" '1'
   # # Clean-up
   # kubectl delete deployment/nginx-deployment "${kube_flags[@]}"
-  # # TODO: Remove once deployment reaping is implemented
-  # kubectl delete rs --all "${kube_flags[@]}"
 
   ### Expose a deployment as a service
   kubectl create -f docs/user-guide/deployment.yaml "${kube_flags[@]}"
@@ -988,8 +986,6 @@ __EOF__
   kube::test::get_object_assert 'service nginx-deployment' "{{$port_field}}" '80'
   # Clean-up
   kubectl delete deployment/nginx-deployment service/nginx-deployment "${kube_flags[@]}"
-  # TODO: Remove once deployment reaping is implemented
-  kubectl delete rs --all "${kube_flags[@]}"
 
   ### Expose replication controller as service
   kubectl create -f examples/guestbook/frontend-controller.yaml "${kube_flags[@]}"
@@ -1094,7 +1090,7 @@ __EOF__
   # Clean up
   kubectl delete rc frontend "${kube_flags[@]}"
 
-  ### Auto scale deployment 
+  ### Auto scale deployment
   # Pre-condition: no deployment exists
   kube::test::get_object_assert deployment "{{range.items}}{{$id_field}}:{{end}}" ''
   # Command
@@ -1106,9 +1102,8 @@ __EOF__
   # Clean up
   kubectl delete hpa nginx-deployment "${kube_flags[@]}"
   kubectl delete deployment nginx-deployment "${kube_flags[@]}"
-  kubectl delete rs -l pod-template-hash "${kube_flags[@]}"
 
-  ### Rollback a deployment 
+  ### Rollback a deployment
   # Pre-condition: no deployment exists
   kube::test::get_object_assert deployment "{{range.items}}{{$id_field}}:{{end}}" ''
   # Command
@@ -1330,7 +1325,7 @@ __EOF__
     # cleaning
     rm "${temp_editor}"
     # Command
-    # We need to set --overwrite, because otherwise, if the first attempt to run "kubectl label" 
+    # We need to set --overwrite, because otherwise, if the first attempt to run "kubectl label"
     # fails on some, but not all, of the resources, retries will fail because it tries to modify
     # existing labels.
     kubectl-with-retry label -f $file labeled=true --overwrite "${kube_flags[@]}"
@@ -1349,7 +1344,7 @@ __EOF__
     fi
     # Command
     # Command
-    # We need to set --overwrite, because otherwise, if the first attempt to run "kubectl annotate" 
+    # We need to set --overwrite, because otherwise, if the first attempt to run "kubectl annotate"
     # fails on some, but not all, of the resources, retries will fail because it tries to modify
     # existing annotations.
     kubectl-with-retry annotate -f $file annotated=true --overwrite "${kube_flags[@]}"

pkg/kubectl/stop.go

@@ -78,6 +78,9 @@ func ReaperFor(kind unversioned.GroupKind, c client.Interface) (Reaper, error) {
 	case extensions.Kind("Job"):
 		return &JobReaper{c, Interval, Timeout}, nil
 
+	case extensions.Kind("Deployment"):
+		return &DeploymentReaper{c, Interval, Timeout}, nil
+
 	}
 	return nil, &NoSuchReaperError{kind}
 }
@@ -102,6 +105,10 @@ type JobReaper struct {
 	client.Interface
 	pollInterval, timeout time.Duration
 }
+type DeploymentReaper struct {
+	client.Interface
+	pollInterval, timeout time.Duration
+}
 type PodReaper struct {
 	client.Interface
 }
@@ -191,10 +198,7 @@ func (reaper *ReplicationControllerReaper) Stop(namespace, name string, timeout
 			return err
 		}
 	}
-	if err := rc.Delete(name); err != nil {
-		return err
-	}
-	return nil
+	return rc.Delete(name)
 }
 
 // TODO(madhusudancs): Implement it when controllerRef is implemented - https://github.com/kubernetes/kubernetes/issues/2210
@@ -303,10 +307,7 @@ func (reaper *DaemonSetReaper) Stop(namespace, name string, timeout time.Duratio
 		return err
 	}
 
-	if err := reaper.Extensions().DaemonSets(namespace).Delete(name); err != nil {
-		return err
-	}
-	return nil
+	return reaper.Extensions().DaemonSets(namespace).Delete(name)
 }
 
 func (reaper *JobReaper) Stop(namespace, name string, timeout time.Duration, gracePeriod *api.DeleteOptions) error {
@@ -352,10 +353,92 @@ func (reaper *JobReaper) Stop(namespace, name string, timeout time.Duration, gra
 		return utilerrors.NewAggregate(errList)
 	}
 	// once we have all the pods removed we can safely remove the job itself
-	if err := jobs.Delete(name, gracePeriod); err != nil {
+	return jobs.Delete(name, gracePeriod)
+}
+
+func (reaper *DeploymentReaper) Stop(namespace, name string, timeout time.Duration, gracePeriod *api.DeleteOptions) error {
+	deployments := reaper.Extensions().Deployments(namespace)
+	replicaSets := reaper.Extensions().ReplicaSets(namespace)
+	rsReaper, _ := ReaperFor(extensions.Kind("ReplicaSet"), reaper)
+
+	deployment, err := deployments.Get(name)
+	if err != nil {
 		return err
 	}
-	return nil
+
+	// set deployment's history and scale to 0
+	// TODO replace with patch when available: https://github.com/kubernetes/kubernetes/issues/20527
+	zero := 0
+	deployment.Spec.RevisionHistoryLimit = &zero
+	deployment.Spec.Replicas = 0
+	// TODO: un-pausing should not be necessary, remove when this is fixed:
+	// https://github.com/kubernetes/kubernetes/issues/20966
+	// Instead deployment should be Paused at this point and not at next TODO.
+	deployment.Spec.Paused = false
+	deployment, err = deployments.Update(deployment)
+	if err != nil {
+		return err
+	}
+
+	// wait for total no of pods drop to 0
+	if err := wait.Poll(reaper.pollInterval, reaper.timeout, func() (bool, error) {
+		curr, err := deployments.Get(name)
+		// if deployment was not found it must have been deleted, error out
+		if err != nil && errors.IsNotFound(err) {
+			return false, err
+		}
+		// if other errors happen, retry
+		if err != nil {
+			return false, nil
+		}
+		// check if deployment wasn't recreated with the same name
+		// TODO use generations when deployment will have them
+		if curr.UID != deployment.UID {
+			return false, errors.NewNotFound(extensions.Resource("Deployment"), name)
+		}
+		return curr.Status.Replicas == 0, nil
+	}); err != nil {
+		return err
+	}
+
+	// TODO: When deployments will allow running cleanup policy while being
+	// paused, move pausing to above update operation. Without it, we need to
+	// pause deployment before stopping RSs, to prevent creating new RSs.
+	// See https://github.com/kubernetes/kubernetes/issues/20966
+	deployment, err = deployments.Get(name)
+	if err != nil {
+		return err
+	}
+	deployment.Spec.Paused = true
+	deployment, err = deployments.Update(deployment)
+	if err != nil {
+		return err
+	}
+
+	// remove remaining RSs
+	selector, err := unversioned.LabelSelectorAsSelector(deployment.Spec.Selector)
+	if err != nil {
+		return err
+	}
+	options := api.ListOptions{LabelSelector: selector}
+	rsList, err := replicaSets.List(options)
+	if err != nil {
+		return err
+	}
+	errList := []error{}
+	for _, rc := range rsList.Items {
+		if err := rsReaper.Stop(rc.Namespace, rc.Name, timeout, gracePeriod); err != nil {
+			if !errors.IsNotFound(err) {
+				errList = append(errList, err)
+			}
+		}
+	}
+	if len(errList) > 0 {
+		return utilerrors.NewAggregate(errList)
+	}
+
+	// and finally deployment
+	return deployments.Delete(name, gracePeriod)
 }
 
 func (reaper *PodReaper) Stop(namespace, name string, timeout time.Duration, gracePeriod *api.DeleteOptions) error {
@@ -364,11 +447,7 @@ func (reaper *PodReaper) Stop(namespace, name string, timeout time.Duration, gra
 	if err != nil {
 		return err
 	}
-	if err := pods.Delete(name, gracePeriod); err != nil {
-		return err
-	}
-
-	return nil
+	return pods.Delete(name, gracePeriod)
 }
 
 func (reaper *ServiceReaper) Stop(namespace, name string, timeout time.Duration, gracePeriod *api.DeleteOptions) error {
@@ -377,8 +456,5 @@ func (reaper *ServiceReaper) Stop(namespace, name string, timeout time.Duration,
 	if err != nil {
 		return err
 	}
-	if err := services.Delete(name); err != nil {
-		return err
-	}
-	return nil
+	return services.Delete(name)
 }

pkg/kubectl/stop_test.go

@@ -29,6 +29,7 @@ import (
 	client "k8s.io/kubernetes/pkg/client/unversioned"
 	"k8s.io/kubernetes/pkg/client/unversioned/testclient"
 	"k8s.io/kubernetes/pkg/runtime"
+	deploymentutil "k8s.io/kubernetes/pkg/util/deployment"
 )
 
 func TestReplicationControllerStop(t *testing.T) {
@@ -495,6 +496,133 @@ func TestJobStop(t *testing.T) {
 	}
 }
 
+func TestDeploymentStop(t *testing.T) {
+	name := "foo"
+	ns := "default"
+	deployment := extensions.Deployment{
+		ObjectMeta: api.ObjectMeta{
+			Name:      name,
+			Namespace: ns,
+		},
+		Spec: extensions.DeploymentSpec{
+			Replicas: 0,
+			Selector: &unversioned.LabelSelector{MatchLabels: map[string]string{"k1": "v1"}},
+		},
+		Status: extensions.DeploymentStatus{
+			Replicas: 0,
+		},
+	}
+	template := deploymentutil.GetNewReplicaSetTemplate(deployment)
+	tests := []struct {
+		Name            string
+		Objs            []runtime.Object
+		StopError       error
+		ExpectedActions []string
+	}{
+		{
+			Name: "SimpleDeployment",
+			Objs: []runtime.Object{
+				&extensions.Deployment{ // GET
+					ObjectMeta: api.ObjectMeta{
+						Name:      name,
+						Namespace: ns,
+					},
+					Spec: extensions.DeploymentSpec{
+						Replicas: 0,
+						Selector: &unversioned.LabelSelector{MatchLabels: map[string]string{"k1": "v1"}},
+					},
+					Status: extensions.DeploymentStatus{
+						Replicas: 0,
+					},
+				},
+				&extensions.Scale{ // UPDATE
+					ObjectMeta: api.ObjectMeta{
+						Name:      name,
+						Namespace: ns,
+					},
+					Spec: extensions.ScaleSpec{
+						Replicas: 0,
+					},
+					Status: extensions.ScaleStatus{
+						Replicas: 0,
+						Selector: map[string]string{"k1": "v1"},
+					},
+				},
+			},
+			StopError: nil,
+			ExpectedActions: []string{"get:deployments", "update:deployments",
+				"get:deployments", "get:deployments", "update:deployments",
+				"list:replicasets", "delete:deployments"},
+		},
+		{
+			Name: "Deployment with single replicaset",
+			Objs: []runtime.Object{
+				&deployment, // GET
+				&extensions.Scale{ // UPDATE
+					ObjectMeta: api.ObjectMeta{
+						Name:      name,
+						Namespace: ns,
+					},
+					Spec: extensions.ScaleSpec{
+						Replicas: 0,
+					},
+					Status: extensions.ScaleStatus{
+						Replicas: 0,
+						Selector: map[string]string{"k1": "v1"},
+					},
+				},
+				&extensions.ReplicaSetList{ // LIST
+					Items: []extensions.ReplicaSet{
+						{
+							ObjectMeta: api.ObjectMeta{
+								Name:      name,
+								Namespace: ns,
+							},
+							Spec: extensions.ReplicaSetSpec{
+								Template: &template,
+							},
+						},
+					},
+				},
+			},
+			StopError: nil,
+			ExpectedActions: []string{"get:deployments", "update:deployments",
+				"get:deployments", "get:deployments", "update:deployments",
+				"list:replicasets", "get:replicasets", "get:replicasets",
+				"update:replicasets", "get:replicasets", "get:replicasets",
+				"delete:replicasets", "delete:deployments"},
+		},
+	}
+
+	for _, test := range tests {
+		fake := testclient.NewSimpleFake(test.Objs...)
+		reaper := DeploymentReaper{fake, time.Millisecond, time.Millisecond}
+		err := reaper.Stop(ns, name, 0, nil)
+		if !reflect.DeepEqual(err, test.StopError) {
+			t.Errorf("%s unexpected error: %v", test.Name, err)
+			continue
+		}
+
+		actions := fake.Actions()
+		if len(actions) != len(test.ExpectedActions) {
+			t.Errorf("%s unexpected actions: %v, expected %d actions got %d", test.Name, actions, len(test.ExpectedActions), len(actions))
+			continue
+		}
+		for i, expAction := range test.ExpectedActions {
+			action := strings.Split(expAction, ":")
+			if actions[i].GetVerb() != action[0] {
+				t.Errorf("%s unexpected verb: %+v, expected %s", test.Name, actions[i], expAction)
+			}
+			if actions[i].GetResource() != action[1] {
+				t.Errorf("%s unexpected resource: %+v, expected %s", test.Name, actions[i], expAction)
+			}
+			if len(action) == 3 && actions[i].GetSubresource() != action[2] {
+				t.Errorf("%s unexpected subresource: %+v, expected %s", test.Name, actions[i], expAction)
+			}
+		}
+	}
+}
+
 type noSuchPod struct {
 	*testclient.FakePods
 }