Skip to content

fix: Add client-go QPS and Burst overrides in code and helm chart #997

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: Add client-go QPS and Burst overrides in code and helm chart #997

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions chart/templates/deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,13 @@ spec:
{{- if .Values.controller.configMapOverrides }}
- --config-overrides=chaos-controller-overrides
{{- end }}
- --client-go-qps={{ .Values.controller.clientGoQPS }}
- --client-go-burst={{ .Values.controller.clientGoBurst }}
{{- with .Values.controller.additionalArgs }}
{{- range . }}
- {{ . | quote }}
{{- end }}
{{- end }}
env:
- name: POD_NAMESPACE
valueFrom:
Expand Down
3 changes: 3 additions & 0 deletions chart/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,9 @@ controller:
clusterThreshold: 66
allowNodeFailure: true # if set to false, will prevent the creation of any disruption that applies a nodeFailure, but only if safeMode.enable is true
allowNodeLevel: true # if set to false, will prevent the creation of any disruption at the node level, but only if safeMode.enable is true
additionalArgs: [] # list of additional arguments to pass to the controller
clientGoQPS: 20 # Number of queries per second client-go is allowed to make
clientGoBurst: 30 # Allowed burst queries for client-go
logLevel: DEBUG # sets the log level for the chaos controller
resources: # resources assigned to the controller pod. may need to be increased when deploying to larger scale clusters
cpu: 100m
Expand Down
14 changes: 12 additions & 2 deletions main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ package main

import (
"context"
"flag"
"fmt"
"net/http"
"os"
Expand Down Expand Up @@ -61,7 +62,11 @@ import (
//go:generate mockery --config .local.mockery.yaml
//go:generate mockery --config .vendor.mockery.yaml

var scheme = runtime.NewScheme()
var (
Copy link
Contributor

@aymericDD aymericDD Oct 28, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These client-go rate limiting flags should be moved to the configuration system in config/config.go for consistency. Yhey're defined separately in main.go but
should follow the same pattern as other controller settings - being part of the controllerConfig struct with viper binding support for config files and ConfigMap
overrides.

  1. Add to controllerConfig struct (config/config.go:34-60):
  type controllerConfig struct {
        HealthProbeBindAddr              string                          `json:"healthProbeBindAddr" yaml:"healthProbeBindAddr"`
        MetricsBindAddr                  string                          `json:"metricsBindAddr" yaml:"metricsBindAddr"`
        MetricsSink                      string                          `json:"metricsSink" yaml:"metricsSink"`
        // Add these two fields:
        ClientGoQPS                      float64                         `json:"clientGoQps" yaml:"clientGoQps"`
        ClientGoBurst                    int                             `json:"clientGoBurst" yaml:"clientGoBurst"`
        ExpiredDisruptionGCDelay         time.Duration                   `json:"expiredDisruptionGCDelay" yaml:"expiredDisruptionGCDelay"`
        // ... rest of fields
  }
  1. Add flag binding in New() function (similar to existing patterns around line 139):
  mainFS.Float64Var(&cfg.Controller.ClientGoQPS, "client-go-qps", 20, "Number of queries per second client-go is allowed to make (default 20)")

  if err := viper.BindPFlag("controller.clientGoQps", mainFS.Lookup("client-go-qps")); err != nil {
        return cfg, err
  }

  mainFS.IntVar(&cfg.Controller.ClientGoBurst, "client-go-burst", 30, "Allowed burst queries for client-go (default 30)")

  if err := viper.BindPFlag("controller.clientGoBurst", mainFS.Lookup("client-go-burst")); err != nil {
        return cfg, err
  }
  1. Remove from main.go and use config values:
  // Remove from main.go:
  // clientGoQPS   = flag.Float64("client-go-qps", 20, "...")
  // clientGoBurst = flag.Int("client-go-burst", 30, "...")

  // Use in client initialization:
  // *clientGoQPS -> cfg.Controller.ClientGoQPS
  // *clientGoBurst -> cfg.Controller.ClientGoBurst

This way, these settings can be configured via YAML files, ConfigMaps, and environment variables just like all other controller settings.

clientGoQPS = flag.Float64("client-go-qps", 20, "Number of queries per second client-go is allowed to make (default 20)")
clientGoBurst = flag.Int("client-go-burst", 30, "Allowed burst queries for client-go (default 30)")
scheme = runtime.NewScheme()
)

func init() {
// +kubebuilder:scaffold:scheme
Expand All @@ -70,6 +75,7 @@ func init() {
}

func main() {
flag.Parse()
logger, err := log.NewZapLogger()
if err != nil {
ctrl.Log.WithName("setup").Error(err, "error creating controller logger")
Expand Down Expand Up @@ -105,7 +111,11 @@ func main() {
logger.Fatalw("unable to create a valid configuration", "error", err)
}

mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
ctrlConfig := ctrl.GetConfigOrDie()
ctrlConfig.QPS = float32(*clientGoQPS)
ctrlConfig.Burst = *clientGoBurst

mgr, err := ctrl.NewManager(ctrlConfig, ctrl.Options{
Scheme: scheme,
HealthProbeBindAddress: cfg.Controller.HealthProbeBindAddr,
Metrics: metricsserver.Options{
Expand Down