diff --git a/cmd/controller/app/controller.go b/cmd/controller/app/controller.go index 263ca1c13..092378e88 100644 --- a/cmd/controller/app/controller.go +++ b/cmd/controller/app/controller.go @@ -117,6 +117,7 @@ func buildControllerContext(opts *options.ControllerOptions) (*controllers.Conte KubeSharedInformerFactory: kubeSharedInformerFactory, SharedInformerFactory: sharedInformerFactory, Namespace: opts.Namespace, + Features: opts.Features, }, kubeCfg, nil } diff --git a/cmd/controller/app/options/options.go b/cmd/controller/app/options/options.go index c6b474275..f62798863 100644 --- a/cmd/controller/app/options/options.go +++ b/cmd/controller/app/options/options.go @@ -1,9 +1,12 @@ package options import ( + "strings" "time" "github.com/spf13/pflag" + + "github.com/jetstack/navigator/pkg/util/features" ) type ControllerOptions struct { @@ -15,6 +18,9 @@ type ControllerOptions struct { LeaderElectionLeaseDuration time.Duration LeaderElectionRenewDeadline time.Duration LeaderElectionRetryPeriod time.Duration + + FeatureGatesString string + Features map[string]bool } const ( @@ -26,6 +32,8 @@ const ( defaultLeaderElectionLeaseDuration = 15 * time.Second defaultLeaderElectionRenewDeadline = 10 * time.Second defaultLeaderElectionRetryPeriod = 2 * time.Second + + defaultFeatureGatesString = "" ) func NewControllerOptions() *ControllerOptions { @@ -37,6 +45,8 @@ func NewControllerOptions() *ControllerOptions { LeaderElectionLeaseDuration: defaultLeaderElectionLeaseDuration, LeaderElectionRenewDeadline: defaultLeaderElectionRenewDeadline, LeaderElectionRetryPeriod: defaultLeaderElectionRetryPeriod, + FeatureGatesString: defaultFeatureGatesString, + Features: map[string]bool{}, } } @@ -66,6 +76,8 @@ func (s *ControllerOptions) AddFlags(fs *pflag.FlagSet) { fs.DurationVar(&s.LeaderElectionRetryPeriod, "leader-election-retry-period", defaultLeaderElectionRetryPeriod, ""+ "The duration the clients should wait between attempting acquisition and renewal "+ "of a leadership. This is only applicable if leader election is enabled.") + fs.StringVar(&s.FeatureGatesString, "feature-gates", defaultFeatureGatesString, "A set of key=value pairs that describe feature gates for various features. "+ + "Options are:\n"+strings.Join(features.KnownFeatures(&features.InitFeatureGates), "\n")) } func (o *ControllerOptions) Validate() error { diff --git a/cmd/controller/start.go b/cmd/controller/start.go index b21cc94d4..6d23ddd0b 100644 --- a/cmd/controller/start.go +++ b/cmd/controller/start.go @@ -12,6 +12,7 @@ import ( "github.com/jetstack/navigator/cmd/controller/app/options" _ "github.com/jetstack/navigator/pkg/controllers/cassandra" _ "github.com/jetstack/navigator/pkg/controllers/elasticsearch" + "github.com/jetstack/navigator/pkg/util/features" ) type NavigatorControllerOptions struct { @@ -64,6 +65,11 @@ to renew certificates at an appropriate time before expiry.`, func (o NavigatorControllerOptions) Validate(args []string) error { errors := []error{} errors = append(errors, o.ControllerOptions.Validate()) + + f, err := features.NewFeatureGate(&features.InitFeatureGates, o.ControllerOptions.FeatureGatesString) + errors = append(errors, err) + o.ControllerOptions.Features = f + return utilerrors.NewAggregate(errors) } diff --git a/contrib/charts/navigator/templates/controller.yaml b/contrib/charts/navigator/templates/controller.yaml index 341406ccb..8127d0681 100644 --- a/contrib/charts/navigator/templates/controller.yaml +++ b/contrib/charts/navigator/templates/controller.yaml @@ -36,6 +36,9 @@ spec: - navigator-controller {{- if .Values.controller.namespace }} - --namespace={{ .Values.controller.namespace }} +{{- end }} +{{- if .Values.featureGates }} + - --feature-gates={{ .Values.featureGates }} {{- end }} - --leader-election-namespace={{ .Release.Namespace }} - --v={{ .Values.controller.logLevel }} diff --git a/contrib/charts/navigator/values.yaml b/contrib/charts/navigator/values.yaml index 43fb1393b..0e6175bff 100644 --- a/contrib/charts/navigator/values.yaml +++ b/contrib/charts/navigator/values.yaml @@ -6,6 +6,9 @@ createAPIService: true rbac: enabled: true +# A set of key=value pairs that describe feature gates for various features. Use --help to see available flags. +featureGates: "" + apiserver: ## Set to true to skip deploying the apiserver components RBAC policies, ## which require cluster admin access to deploy. diff --git a/pkg/controllers/cassandra/cassandra.go b/pkg/controllers/cassandra/cassandra.go index 43d177644..bf8b4c462 100644 --- a/pkg/controllers/cassandra/cassandra.go +++ b/pkg/controllers/cassandra/cassandra.go @@ -32,6 +32,7 @@ import ( "github.com/jetstack/navigator/pkg/controllers/cassandra/seedlabeller" "github.com/jetstack/navigator/pkg/controllers/cassandra/service" "github.com/jetstack/navigator/pkg/controllers/cassandra/serviceaccount" + "github.com/jetstack/navigator/pkg/util/features" ) // NewCassandra returns a new CassandraController that can be used @@ -55,6 +56,7 @@ type CassandraController struct { roleBindingsListerSynced cache.InformerSynced queue workqueue.RateLimitingInterface recorder record.EventRecorder + features map[string]bool } func NewCassandra( @@ -69,6 +71,7 @@ func NewCassandra( roles rbacinformers.RoleInformer, roleBindings rbacinformers.RoleBindingInformer, recorder record.EventRecorder, + features map[string]bool, ) *CassandraController { queue := workqueue.NewNamedRateLimitingQueue( workqueue.DefaultControllerRateLimiter(), @@ -79,6 +82,7 @@ func NewCassandra( navigatorClient: naviClient, queue: queue, recorder: recorder, + features: features, } cassClusters.Informer().AddEventHandler( &controllers.QueuingEventHandler{Queue: queue}, @@ -166,6 +170,10 @@ func NewCassandra( func (e *CassandraController) Run(workers int, stopCh <-chan struct{}) error { glog.Infof("Starting Cassandra controller") + if features.Enabled(e.features, features.Example) { + glog.Infof("Example feature flag enabled") + } + if !cache.WaitForCacheSync( stopCh, e.cassListerSynced, @@ -333,6 +341,7 @@ func CassandraControllerFromContext(ctx *controllers.Context) *CassandraControll ctx.KubeSharedInformerFactory.Rbac().V1beta1().Roles(), ctx.KubeSharedInformerFactory.Rbac().V1beta1().RoleBindings(), ctx.Recorder, + ctx.Features, ) } diff --git a/pkg/controllers/context.go b/pkg/controllers/context.go index 90b52214c..2a74a31c6 100644 --- a/pkg/controllers/context.go +++ b/pkg/controllers/context.go @@ -18,5 +18,7 @@ type Context struct { KubeSharedInformerFactory kubeinformers.SharedInformerFactory SharedInformerFactory intinformers.SharedInformerFactory + Features map[string]bool + Namespace string } diff --git a/pkg/controllers/elasticsearch/elasticsearch.go b/pkg/controllers/elasticsearch/elasticsearch.go index 5d71bc896..30ee9977f 100644 --- a/pkg/controllers/elasticsearch/elasticsearch.go +++ b/pkg/controllers/elasticsearch/elasticsearch.go @@ -68,6 +68,8 @@ type ElasticsearchController struct { queue workqueue.RateLimitingInterface elasticsearchClusterControl ControlInterface recorder record.EventRecorder + + features map[string]bool } // NewElasticsearch returns a new ElasticsearchController that can be used @@ -89,6 +91,7 @@ func NewElasticsearch( cl kubernetes.Interface, navigatorCl clientset.Interface, recorder record.EventRecorder, + features map[string]bool, ) *ElasticsearchController { queue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "elasticsearchCluster") // create a new ElasticsearchController to manage ElasticsearchCluster resources @@ -97,6 +100,7 @@ func NewElasticsearch( navigatorClient: navigatorCl, queue: queue, recorder: recorder, + features: features, } // add an event handler to the ElasticsearchCluster informer @@ -352,6 +356,7 @@ func init() { ctx.Client, ctx.NavigatorClient, ctx.Recorder, + ctx.Features, ) return e.Run diff --git a/pkg/util/feature/feature_gate.go b/pkg/util/feature/feature_gate.go new file mode 100644 index 000000000..6d878119b --- /dev/null +++ b/pkg/util/feature/feature_gate.go @@ -0,0 +1,307 @@ +/* +Copyright 2016 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// NOTE: this file is based off k8s.io/apiserver/pkg/util/feature + +package feature + +import ( + "fmt" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + + "github.com/golang/glog" + "github.com/spf13/pflag" +) + +type Feature string + +const ( + flagName = "feature-gates" + + // allAlphaGate is a global toggle for alpha features. Per-feature key + // values override the default set by allAlphaGate. Examples: + // AllAlpha=false,NewFeature=true will result in newFeature=true + // AllAlpha=true,NewFeature=false will result in newFeature=false + allAlphaGate Feature = "AllAlpha" +) + +var ( + // The generic features. + defaultFeatures = map[Feature]FeatureSpec{ + allAlphaGate: {Default: false, PreRelease: Alpha}, + } + + // Special handling for a few gates. + specialFeatures = map[Feature]func(known map[Feature]FeatureSpec, enabled map[Feature]bool, val bool){ + allAlphaGate: setUnsetAlphaGates, + } + + // DefaultFeatureGate is a shared global FeatureGate. + DefaultFeatureGate FeatureGate = NewFeatureGate() +) + +type FeatureSpec struct { + Default bool + PreRelease prerelease +} + +type prerelease string + +const ( + // Values for PreRelease. + Alpha = prerelease("ALPHA") + Beta = prerelease("BETA") + GA = prerelease("") + + // Deprecated + Deprecated = prerelease("DEPRECATED") +) + +// FeatureGate parses and stores flag gates for known features from +// a string like feature1=true,feature2=false,... +type FeatureGate interface { + // AddFlag adds a flag for setting global feature gates to the specified FlagSet. + AddFlag(fs *pflag.FlagSet) + // Set parses and stores flag gates for known features + // from a string like feature1=true,feature2=false,... + Set(value string) error + // SetFromMap stores flag gates for known features from a map[string]bool or returns an error + SetFromMap(m map[string]bool) error + // Enabled returns true if the key is enabled. + Enabled(key Feature) bool + // Add adds features to the featureGate. + Add(features map[Feature]FeatureSpec) error + // KnownFeatures returns a slice of strings describing the FeatureGate's known features. + KnownFeatures() []string +} + +// featureGate implements FeatureGate as well as pflag.Value for flag parsing. +type featureGate struct { + special map[Feature]func(map[Feature]FeatureSpec, map[Feature]bool, bool) + + // lock guards writes to known, enabled, and reads/writes of closed + lock sync.Mutex + // known holds a map[Feature]FeatureSpec + known *atomic.Value + // enabled holds a map[Feature]bool + enabled *atomic.Value + // closed is set to true when AddFlag is called, and prevents subsequent calls to Add + closed bool +} + +func setUnsetAlphaGates(known map[Feature]FeatureSpec, enabled map[Feature]bool, val bool) { + for k, v := range known { + if v.PreRelease == Alpha { + if _, found := enabled[k]; !found { + enabled[k] = val + } + } + } +} + +// Set, String, and Type implement pflag.Value +var _ pflag.Value = &featureGate{} + +func NewFeatureGate() *featureGate { + known := map[Feature]FeatureSpec{} + for k, v := range defaultFeatures { + known[k] = v + } + + knownValue := &atomic.Value{} + knownValue.Store(known) + + enabled := map[Feature]bool{} + enabledValue := &atomic.Value{} + enabledValue.Store(enabled) + + f := &featureGate{ + known: knownValue, + special: specialFeatures, + enabled: enabledValue, + } + return f +} + +// Set parses a string of the form "key1=value1,key2=value2,..." into a +// map[string]bool of known keys or returns an error. +func (f *featureGate) Set(value string) error { + f.lock.Lock() + defer f.lock.Unlock() + + // Copy existing state + known := map[Feature]FeatureSpec{} + for k, v := range f.known.Load().(map[Feature]FeatureSpec) { + known[k] = v + } + enabled := map[Feature]bool{} + for k, v := range f.enabled.Load().(map[Feature]bool) { + enabled[k] = v + } + + for _, s := range strings.Split(value, ",") { + if len(s) == 0 { + continue + } + arr := strings.SplitN(s, "=", 2) + k := Feature(strings.TrimSpace(arr[0])) + featureSpec, ok := known[k] + if !ok { + return fmt.Errorf("unrecognized key: %s", k) + } + if len(arr) != 2 { + return fmt.Errorf("missing bool value for %s", k) + } + v := strings.TrimSpace(arr[1]) + boolValue, err := strconv.ParseBool(v) + if err != nil { + return fmt.Errorf("invalid value of %s: %s, err: %v", k, v, err) + } + enabled[k] = boolValue + if boolValue && featureSpec.PreRelease == Deprecated { + glog.Warningf("enabling deprecated feature gate %s", k) + } + + // Handle "special" features like "all alpha gates" + if fn, found := f.special[k]; found { + fn(known, enabled, boolValue) + } + } + + // Persist changes + f.known.Store(known) + f.enabled.Store(enabled) + + glog.Infof("feature gates: %v", enabled) + return nil +} + +// SetFromMap stores flag gates for known features from a map[string]bool or returns an error +func (f *featureGate) SetFromMap(m map[string]bool) error { + f.lock.Lock() + defer f.lock.Unlock() + + // Copy existing state + known := map[Feature]FeatureSpec{} + for k, v := range f.known.Load().(map[Feature]FeatureSpec) { + known[k] = v + } + enabled := map[Feature]bool{} + for k, v := range f.enabled.Load().(map[Feature]bool) { + enabled[k] = v + } + + for k, v := range m { + k := Feature(k) + _, ok := known[k] + if !ok { + return fmt.Errorf("unrecognized key: %s", k) + } + enabled[k] = v + // Handle "special" features like "all alpha gates" + if fn, found := f.special[k]; found { + fn(known, enabled, v) + } + } + + // Persist changes + f.known.Store(known) + f.enabled.Store(enabled) + + glog.Infof("feature gates: %v", f.enabled) + return nil +} + +// String returns a string containing all enabled feature gates, formatted as "key1=value1,key2=value2,...". +func (f *featureGate) String() string { + pairs := []string{} + for k, v := range f.enabled.Load().(map[Feature]bool) { + pairs = append(pairs, fmt.Sprintf("%s=%t", k, v)) + } + sort.Strings(pairs) + return strings.Join(pairs, ",") +} + +func (f *featureGate) Type() string { + return "mapStringBool" +} + +// Add adds features to the featureGate. +func (f *featureGate) Add(features map[Feature]FeatureSpec) error { + f.lock.Lock() + defer f.lock.Unlock() + + if f.closed { + return fmt.Errorf("cannot add a feature gate after adding it to the flag set") + } + + // Copy existing state + known := map[Feature]FeatureSpec{} + for k, v := range f.known.Load().(map[Feature]FeatureSpec) { + known[k] = v + } + + for name, spec := range features { + if existingSpec, found := known[name]; found { + if existingSpec == spec { + continue + } + return fmt.Errorf("feature gate %q with different spec already exists: %v", name, existingSpec) + } + + known[name] = spec + } + + // Persist updated state + f.known.Store(known) + + return nil +} + +// Enabled returns true if the key is enabled. +func (f *featureGate) Enabled(key Feature) bool { + if v, ok := f.enabled.Load().(map[Feature]bool)[key]; ok { + return v + } + return f.known.Load().(map[Feature]FeatureSpec)[key].Default +} + +// AddFlag adds a flag for setting global feature gates to the specified FlagSet. +func (f *featureGate) AddFlag(fs *pflag.FlagSet) { + f.lock.Lock() + f.closed = true + f.lock.Unlock() + + known := f.KnownFeatures() + fs.Var(f, flagName, ""+ + "A set of key=value pairs that describe feature gates for alpha/experimental features. "+ + "Options are:\n"+strings.Join(known, "\n")) +} + +// KnownFeatures returns a slice of strings describing the FeatureGate's known features. +func (f *featureGate) KnownFeatures() []string { + var known []string + for k, v := range f.known.Load().(map[Feature]FeatureSpec) { + pre := "" + if v.PreRelease != GA { + pre = fmt.Sprintf("%s - ", v.PreRelease) + } + known = append(known, fmt.Sprintf("%s=true|false (%sdefault=%t)", k, pre, v.Default)) + } + sort.Strings(known) + return known +} diff --git a/pkg/util/features/features.go b/pkg/util/features/features.go new file mode 100644 index 000000000..3987ccc61 --- /dev/null +++ b/pkg/util/features/features.go @@ -0,0 +1,92 @@ +package features + +import ( + "fmt" + "sort" + "strconv" + "strings" + + utilfeature "github.com/jetstack/navigator/pkg/util/feature" +) + +const ( + Example = "Example" +) + +// InitFeatureGates are the default feature gates for the init command +var InitFeatureGates = FeatureList{ + Example: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, +} + +// FeatureList represents a list of feature gates +type FeatureList map[string]utilfeature.FeatureSpec + +// Enabled indicates whether a feature name has been enabled +func Enabled(featureList map[string]bool, featureName string) bool { + return featureList[string(featureName)] +} + +// Supports indicates whether a feature name is supported on the given +// feature set +func Supports(featureList FeatureList, featureName string) bool { + for k := range featureList { + if featureName == string(k) { + return true + } + } + return false +} + +// Keys returns a slice of feature names for a given feature set +func Keys(featureList FeatureList) []string { + var list []string + for k := range featureList { + list = append(list, string(k)) + } + return list +} + +// KnownFeatures returns a slice of strings describing the FeatureList features. +func KnownFeatures(f *FeatureList) []string { + var known []string + for k, v := range *f { + pre := "" + if v.PreRelease != utilfeature.GA { + pre = fmt.Sprintf("%s - ", v.PreRelease) + } + known = append(known, fmt.Sprintf("%s=true|false (%sdefault=%t)", k, pre, v.Default)) + } + sort.Strings(known) + return known +} + +// NewFeatureGate parses a string of the form "key1=value1,key2=value2,..." into a +// map[string]bool of known keys or returns an error. +func NewFeatureGate(f *FeatureList, value string) (map[string]bool, error) { + featureGate := map[string]bool{} + for _, s := range strings.Split(value, ",") { + if len(s) == 0 { + continue + } + + arr := strings.SplitN(s, "=", 2) + if len(arr) != 2 { + return nil, fmt.Errorf("missing bool value for feature-gate key:%s", s) + } + + k := strings.TrimSpace(arr[0]) + v := strings.TrimSpace(arr[1]) + + if !Supports(*f, k) { + return nil, fmt.Errorf("unrecognized feature-gate key: %s", k) + } + + boolValue, err := strconv.ParseBool(v) + if err != nil { + return nil, fmt.Errorf("invalid value %v for feature-gate key: %s, use true|false instead", v, k) + } + featureGate[k] = boolValue + } + + return featureGate, nil +}