Commit 5d1afb49 authored by deads2k's avatar deads2k

add CRD finalizer to remove CRs

parent e9b02c2e
apiVersion: apiregistration.k8s.io/v1alpha1
apiVersion: apiregistration.k8s.io/v1beta1
kind: APIService
metadata:
name: v1alpha1.mygroup.example.com
......
apiVersion: apiregistration.k8s.io/v1alpha1
apiVersion: apiregistration.k8s.io/v1beta1
kind: APIService
metadata:
name: v1alpha1.apiextensions.k8s.io
......
......@@ -5,6 +5,7 @@ licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
......@@ -24,3 +25,11 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
],
)
go_test(
name = "go_default_test",
srcs = ["helpers_test.go"],
library = ":go_default_library",
tags = ["automanaged"],
deps = ["//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library"],
)
......@@ -18,10 +18,10 @@ package apiextensions
// SetCRDCondition sets the status condition. It either overwrites the existing one or
// creates a new one
func SetCRDCondition(customResourceDefinition *CustomResourceDefinition, newCondition CustomResourceDefinitionCondition) {
existingCondition := FindCRDCondition(customResourceDefinition, newCondition.Type)
func SetCRDCondition(crd *CustomResourceDefinition, newCondition CustomResourceDefinitionCondition) {
existingCondition := FindCRDCondition(crd, newCondition.Type)
if existingCondition == nil {
customResourceDefinition.Status.Conditions = append(customResourceDefinition.Status.Conditions, newCondition)
crd.Status.Conditions = append(crd.Status.Conditions, newCondition)
return
}
......@@ -34,11 +34,22 @@ func SetCRDCondition(customResourceDefinition *CustomResourceDefinition, newCond
existingCondition.Message = newCondition.Message
}
// RemoveCRDCondition removes the status condition.
func RemoveCRDCondition(crd *CustomResourceDefinition, conditionType CustomResourceDefinitionConditionType) {
newConditions := []CustomResourceDefinitionCondition{}
for _, condition := range crd.Status.Conditions {
if condition.Type != conditionType {
newConditions = append(newConditions, condition)
}
}
crd.Status.Conditions = newConditions
}
// FindCRDCondition returns the condition you're looking for or nil
func FindCRDCondition(customResourceDefinition *CustomResourceDefinition, conditionType CustomResourceDefinitionConditionType) *CustomResourceDefinitionCondition {
for i := range customResourceDefinition.Status.Conditions {
if customResourceDefinition.Status.Conditions[i].Type == conditionType {
return &customResourceDefinition.Status.Conditions[i]
func FindCRDCondition(crd *CustomResourceDefinition, conditionType CustomResourceDefinitionConditionType) *CustomResourceDefinitionCondition {
for i := range crd.Status.Conditions {
if crd.Status.Conditions[i].Type == conditionType {
return &crd.Status.Conditions[i]
}
}
......@@ -46,18 +57,18 @@ func FindCRDCondition(customResourceDefinition *CustomResourceDefinition, condit
}
// IsCRDConditionTrue indicates if the condition is present and strictly true
func IsCRDConditionTrue(customResourceDefinition *CustomResourceDefinition, conditionType CustomResourceDefinitionConditionType) bool {
return IsCRDConditionPresentAndEqual(customResourceDefinition, conditionType, ConditionTrue)
func IsCRDConditionTrue(crd *CustomResourceDefinition, conditionType CustomResourceDefinitionConditionType) bool {
return IsCRDConditionPresentAndEqual(crd, conditionType, ConditionTrue)
}
// IsCRDConditionFalse indicates if the condition is present and false true
func IsCRDConditionFalse(customResourceDefinition *CustomResourceDefinition, conditionType CustomResourceDefinitionConditionType) bool {
return IsCRDConditionPresentAndEqual(customResourceDefinition, conditionType, ConditionFalse)
func IsCRDConditionFalse(crd *CustomResourceDefinition, conditionType CustomResourceDefinitionConditionType) bool {
return IsCRDConditionPresentAndEqual(crd, conditionType, ConditionFalse)
}
// IsCRDConditionPresentAndEqual indicates if the condition is present and equal to the arg
func IsCRDConditionPresentAndEqual(customResourceDefinition *CustomResourceDefinition, conditionType CustomResourceDefinitionConditionType, status ConditionStatus) bool {
for _, condition := range customResourceDefinition.Status.Conditions {
func IsCRDConditionPresentAndEqual(crd *CustomResourceDefinition, conditionType CustomResourceDefinitionConditionType, status ConditionStatus) bool {
for _, condition := range crd.Status.Conditions {
if condition.Type == conditionType {
return condition.Status == status
}
......@@ -76,3 +87,25 @@ func IsCRDConditionEquivalent(lhs, rhs *CustomResourceDefinitionCondition) bool
return lhs.Message == rhs.Message && lhs.Reason == rhs.Reason && lhs.Status == rhs.Status && lhs.Type == rhs.Type
}
// CRDHasFinalizer returns true if the finalizer is in the list
func CRDHasFinalizer(crd *CustomResourceDefinition, needle string) bool {
for _, finalizer := range crd.Finalizers {
if finalizer == needle {
return true
}
}
return false
}
// CRDRemoveFinalizer removes the finalizer if present
func CRDRemoveFinalizer(crd *CustomResourceDefinition, needle string) {
newFinalizers := []string{}
for _, finalizer := range crd.Finalizers {
if finalizer != needle {
newFinalizers = append(newFinalizers, finalizer)
}
}
crd.Finalizers = newFinalizers
}
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package apiextensions
import (
"reflect"
"testing"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestCRDHasFinalizer(t *testing.T) {
tests := []struct {
name string
crd *CustomResourceDefinition
finalizerToCheck string
expected bool
}{
{
name: "missing",
crd: &CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{Finalizers: []string{"not-it"}},
},
finalizerToCheck: "it",
expected: false,
},
{
name: "present",
crd: &CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{Finalizers: []string{"not-it", "it"}},
},
finalizerToCheck: "it",
expected: true,
},
}
for _, tc := range tests {
actual := CRDHasFinalizer(tc.crd, tc.finalizerToCheck)
if tc.expected != actual {
t.Errorf("%v expected %v, got %v", tc.name, tc.expected, actual)
}
}
}
func TestCRDRemoveFinalizer(t *testing.T) {
tests := []struct {
name string
crd *CustomResourceDefinition
finalizerToCheck string
expected []string
}{
{
name: "missing",
crd: &CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{Finalizers: []string{"not-it"}},
},
finalizerToCheck: "it",
expected: []string{"not-it"},
},
{
name: "present",
crd: &CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{Finalizers: []string{"not-it", "it"}},
},
finalizerToCheck: "it",
expected: []string{"not-it"},
},
}
for _, tc := range tests {
CRDRemoveFinalizer(tc.crd, tc.finalizerToCheck)
if !reflect.DeepEqual(tc.expected, tc.crd.Finalizers) {
t.Errorf("%v expected %v, got %v", tc.name, tc.expected, tc.crd.Finalizers)
}
}
}
......@@ -104,6 +104,10 @@ type CustomResourceDefinitionStatus struct {
AcceptedNames CustomResourceDefinitionNames
}
// CustomResourceCleanupFinalizer is the name of the finalizer which will delete instances of
// a CustomResourceDefinition
const CustomResourceCleanupFinalizer = "customresourcecleanup.apiextensions.k8s.io"
// +genclient=true
// +nonNamespaced=true
......
......@@ -104,6 +104,10 @@ type CustomResourceDefinitionStatus struct {
AcceptedNames CustomResourceDefinitionNames `json:"acceptedNames" protobuf:"bytes,2,opt,name=acceptedNames"`
}
// CustomResourceCleanupFinalizer is the name of the finalizer which will delete instances of
// a CustomResourceDefinition
const CustomResourceCleanupFinalizer = "customresourcecleanup.apiextensions.k8s.io"
// +genclient=true
// +nonNamespaced=true
......
......@@ -43,6 +43,7 @@ go_library(
"//vendor/k8s.io/apiserver/pkg/server:go_default_library",
"//vendor/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
"//vendor/k8s.io/client-go/discovery:go_default_library",
"//vendor/k8s.io/client-go/dynamic:go_default_library",
"//vendor/k8s.io/client-go/tools/cache:go_default_library",
"//vendor/k8s.io/client-go/util/workqueue:go_default_library",
"//vendor/k8s.io/kube-apiextensions-server/pkg/apis/apiextensions:go_default_library",
......@@ -54,6 +55,7 @@ go_library(
"//vendor/k8s.io/kube-apiextensions-server/pkg/client/informers/internalversion:go_default_library",
"//vendor/k8s.io/kube-apiextensions-server/pkg/client/informers/internalversion/apiextensions/internalversion:go_default_library",
"//vendor/k8s.io/kube-apiextensions-server/pkg/client/listers/apiextensions/internalversion:go_default_library",
"//vendor/k8s.io/kube-apiextensions-server/pkg/controller/finalizer:go_default_library",
"//vendor/k8s.io/kube-apiextensions-server/pkg/controller/status:go_default_library",
"//vendor/k8s.io/kube-apiextensions-server/pkg/registry/customresource:go_default_library",
"//vendor/k8s.io/kube-apiextensions-server/pkg/registry/customresourcedefinition:go_default_library",
......
......@@ -31,12 +31,14 @@ import (
genericregistry "k8s.io/apiserver/pkg/registry/generic"
"k8s.io/apiserver/pkg/registry/rest"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/client-go/dynamic"
"k8s.io/kube-apiextensions-server/pkg/apis/apiextensions"
"k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/install"
"k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/v1alpha1"
"k8s.io/kube-apiextensions-server/pkg/client/clientset/internalclientset"
internalinformers "k8s.io/kube-apiextensions-server/pkg/client/informers/internalversion"
"k8s.io/kube-apiextensions-server/pkg/controller/finalizer"
"k8s.io/kube-apiextensions-server/pkg/controller/status"
"k8s.io/kube-apiextensions-server/pkg/registry/customresourcedefinition"
......@@ -157,6 +159,10 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
customResourceDefinitionController := NewDiscoveryController(customResourceDefinitionInformers.Apiextensions().InternalVersion().CustomResourceDefinitions(), versionDiscoveryHandler, groupDiscoveryHandler)
namingController := status.NewNamingConditionController(customResourceDefinitionInformers.Apiextensions().InternalVersion().CustomResourceDefinitions(), customResourceDefinitionClient)
finalizingController := finalizer.NewCRDFinalizer(
customResourceDefinitionInformers.Apiextensions().InternalVersion().CustomResourceDefinitions(),
customResourceDefinitionClient,
dynamic.NewDynamicClientPool(s.GenericAPIServer.LoopbackClientConfig))
s.GenericAPIServer.AddPostStartHook("start-apiextensions-informers", func(context genericapiserver.PostStartHookContext) error {
customResourceDefinitionInformers.Start(context.StopCh)
......@@ -165,6 +171,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
s.GenericAPIServer.AddPostStartHook("start-apiextensions-controllers", func(context genericapiserver.PostStartHookContext) error {
go customResourceDefinitionController.Run(context.StopCh)
go namingController.Run(context.StopCh)
go finalizingController.Run(5, context.StopCh)
return nil
})
......
......@@ -101,12 +101,18 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
}
foundVersion = true
verbs := metav1.Verbs([]string{"delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"})
// if we're terminating we don't allow some verbs
if apiextensions.IsCRDConditionTrue(crd, apiextensions.Terminating) {
verbs = metav1.Verbs([]string{"delete", "deletecollection", "get", "list", "watch"})
}
apiResourcesForDiscovery = append(apiResourcesForDiscovery, metav1.APIResource{
Name: crd.Status.AcceptedNames.Plural,
SingularName: crd.Status.AcceptedNames.Singular,
Namespaced: crd.Spec.Scope == apiextensions.NamespaceScoped,
Kind: crd.Status.AcceptedNames.Kind,
Verbs: metav1.Verbs([]string{"delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"}),
Verbs: verbs,
ShortNames: crd.Status.AcceptedNames.ShortNames,
})
}
......
......@@ -153,6 +153,8 @@ func (r *crdHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
r.delegate.ServeHTTP(w, req)
}
terminating := apiextensions.IsCRDConditionTrue(crd, apiextensions.Terminating)
crdInfo := r.getServingInfoFor(crd)
storage := crdInfo.storage
requestScope := crdInfo.requestScope
......@@ -174,14 +176,26 @@ func (r *crdHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
handler(w, req)
return
case "create":
if terminating {
http.Error(w, fmt.Sprintf("%v not allowed while CustomResourceDefinition is terminating", requestInfo.Verb), http.StatusMethodNotAllowed)
return
}
handler := handlers.CreateResource(storage, requestScope, discovery.NewUnstructuredObjectTyper(nil), r.admission)
handler(w, req)
return
case "update":
if terminating {
http.Error(w, fmt.Sprintf("%v not allowed while CustomResourceDefinition is terminating", requestInfo.Verb), http.StatusMethodNotAllowed)
return
}
handler := handlers.UpdateResource(storage, requestScope, discovery.NewUnstructuredObjectTyper(nil), r.admission)
handler(w, req)
return
case "patch":
if terminating {
http.Error(w, fmt.Sprintf("%v not allowed while CustomResourceDefinition is terminating", requestInfo.Verb), http.StatusMethodNotAllowed)
return
}
handler := handlers.PatchResource(storage, requestScope, r.admission, unstructured.UnstructuredObjectConverter{})
handler(w, req)
return
......@@ -190,6 +204,11 @@ func (r *crdHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
handler := handlers.DeleteResource(storage, allowsOptions, requestScope, r.admission)
handler(w, req)
return
case "deletecollection":
checkBody := true
handler := handlers.DeleteCollection(storage, checkBody, requestScope, r.admission)
handler(w, req)
return
default:
http.Error(w, fmt.Sprintf("unhandled verb %q", requestInfo.Verb), http.StatusMethodNotAllowed)
......
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["crd_finalizer.go"],
tags = ["automanaged"],
deps = [
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/conversion:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/client-go/dynamic:go_default_library",
"//vendor/k8s.io/client-go/tools/cache:go_default_library",
"//vendor/k8s.io/client-go/util/workqueue:go_default_library",
"//vendor/k8s.io/kube-apiextensions-server/pkg/apis/apiextensions:go_default_library",
"//vendor/k8s.io/kube-apiextensions-server/pkg/client/clientset/internalclientset/typed/apiextensions/internalversion:go_default_library",
"//vendor/k8s.io/kube-apiextensions-server/pkg/client/informers/internalversion/apiextensions/internalversion:go_default_library",
"//vendor/k8s.io/kube-apiextensions-server/pkg/client/listers/apiextensions/internalversion:go_default_library",
],
)
......@@ -15,6 +15,8 @@ go_library(
],
tags = ["automanaged"],
deps = [
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
......@@ -24,6 +26,7 @@ go_library(
"//vendor/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library",
"//vendor/k8s.io/apiserver/pkg/registry/rest:go_default_library",
"//vendor/k8s.io/apiserver/pkg/storage:go_default_library",
"//vendor/k8s.io/apiserver/pkg/storage/errors:go_default_library",
"//vendor/k8s.io/apiserver/pkg/storage/names:go_default_library",
"//vendor/k8s.io/kube-apiextensions-server/pkg/apis/apiextensions:go_default_library",
"//vendor/k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/validation:go_default_library",
......
......@@ -17,11 +17,17 @@ limitations under the License.
package customresourcedefinition
import (
"fmt"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/registry/generic"
genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
"k8s.io/apiserver/pkg/registry/rest"
"k8s.io/apiserver/pkg/storage"
storageerr "k8s.io/apiserver/pkg/storage/errors"
"k8s.io/kube-apiextensions-server/pkg/apis/apiextensions"
)
......@@ -55,6 +61,87 @@ func NewREST(scheme *runtime.Scheme, optsGetter generic.RESTOptionsGetter) *REST
return &REST{store}
}
// Delete adds the CRD finalizer to the list
func (r *REST) Delete(ctx genericapirequest.Context, name string, options *metav1.DeleteOptions) (runtime.Object, bool, error) {
obj, err := r.Get(ctx, name, &metav1.GetOptions{})
if err != nil {
return nil, false, err
}
crd := obj.(*apiextensions.CustomResourceDefinition)
// Ensure we have a UID precondition
if options == nil {
options = metav1.NewDeleteOptions(0)
}
if options.Preconditions == nil {
options.Preconditions = &metav1.Preconditions{}
}
if options.Preconditions.UID == nil {
options.Preconditions.UID = &crd.UID
} else if *options.Preconditions.UID != crd.UID {
err = apierrors.NewConflict(
apiextensions.Resource("customresourcedefinitions"),
name,
fmt.Errorf("Precondition failed: UID in precondition: %v, UID in object meta: %v", *options.Preconditions.UID, crd.UID),
)
return nil, false, err
}
// upon first request to delete, add our finalizer and then delegate
if crd.DeletionTimestamp.IsZero() {
key, err := r.Store.KeyFunc(ctx, name)
if err != nil {
return nil, false, err
}
preconditions := storage.Preconditions{UID: options.Preconditions.UID}
out := r.Store.NewFunc()
err = r.Store.Storage.GuaranteedUpdate(
ctx, key, out, false, &preconditions,
storage.SimpleUpdate(func(existing runtime.Object) (runtime.Object, error) {
existingCRD, ok := existing.(*apiextensions.CustomResourceDefinition)
if !ok {
// wrong type
return nil, fmt.Errorf("expected *apiextensions.CustomResourceDefinition, got %v", existing)
}
// Set the deletion timestamp if needed
if existingCRD.DeletionTimestamp.IsZero() {
now := metav1.Now()
existingCRD.DeletionTimestamp = &now
}
if !apiextensions.CRDHasFinalizer(existingCRD, apiextensions.CustomResourceCleanupFinalizer) {
existingCRD.Finalizers = append(existingCRD.Finalizers, apiextensions.CustomResourceCleanupFinalizer)
}
// update the status condition too
apiextensions.SetCRDCondition(existingCRD, apiextensions.CustomResourceDefinitionCondition{
Type: apiextensions.Terminating,
Status: apiextensions.ConditionTrue,
Reason: "InstanceDeletionPending",
Message: "CustomResourceDefinition marked for deletion; CustomResource deletion will begin soon",
})
return existingCRD, nil
}),
)
if err != nil {
err = storageerr.InterpretGetError(err, apiextensions.Resource("customresourcedefinitions"), name)
err = storageerr.InterpretUpdateError(err, apiextensions.Resource("customresourcedefinitions"), name)
if _, ok := err.(*apierrors.StatusError); !ok {
err = apierrors.NewInternalError(err)
}
return nil, false, err
}
return out, false, nil
}
return r.Store.Delete(ctx, name, options)
}
// NewStatusREST makes a RESTStorage for status that has more limited options.
// It is based on the original REST so that we can share the same underlying store
func NewStatusREST(scheme *runtime.Scheme, rest *REST) *StatusREST {
......
......@@ -89,7 +89,6 @@ func (statusStrategy) PrepareForUpdate(ctx genericapirequest.Context, obj, old r
newObj.Spec = oldObj.Spec
newObj.Labels = oldObj.Labels
newObj.Annotations = oldObj.Annotations
newObj.Finalizers = oldObj.Finalizers
newObj.OwnerReferences = oldObj.OwnerReferences
}
......
......@@ -271,11 +271,6 @@ func TestDeRegistrationAndReRegistration(t *testing.T) {
if _, err := instantiateCustomResource(t, testserver.NewNoxuInstance(ns, sameInstanceName), noxuNamespacedResourceClient, noxuDefinition); err != nil {
t.Fatal(err)
}
// Remove sameInstanceName since at the moment there's no finalizers.
// TODO: as soon finalizers will be implemented Delete can be removed.
if err := noxuNamespacedResourceClient.Delete(sameInstanceName, nil); err != nil {
t.Fatal(err)
}
if err := testserver.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil {
t.Fatal(err)
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment