From c855e8e7c3d43efeb922e064a4f6821fe710ad8c Mon Sep 17 00:00:00 2001 From: Blake Tigges <30013719+blaketigges@users.noreply.github.com> Date: Mon, 12 Jun 2023 11:46:00 -0400 Subject: [PATCH 1/3] Add functions to add webhooks on start/schedule and remove them on shutdown --- docs/configuration_syntax.md | 14 +++++ internal/cmd/run.go | 9 +++ internal/cmd/utils.go | 2 + pkg/config/config.go | 13 +++++ pkg/config/config_test.go | 3 + pkg/controller/controller.go | 3 +- pkg/controller/scheduler.go | 11 +++- pkg/controller/webhooks.go | 108 +++++++++++++++++++++++++++++++++++ pkg/schemas/tasks.go | 3 + 9 files changed, 164 insertions(+), 2 deletions(-) diff --git a/docs/configuration_syntax.md b/docs/configuration_syntax.md index 636d594d..3fe6a9ca 100644 --- a/docs/configuration_syntax.md +++ b/docs/configuration_syntax.md @@ -50,6 +50,20 @@ server: # environment variable) secret_token: 063f51ec-09a4-11eb-adc1-0242ac120002 + add_webhooks: + # Whether to add webhooks to projects from wildcards when + # exporter starts (optional, default: false) + on_init: false + # Whether to add webhooks to projects from wildcards + # on a regular basis (optional, default: false) + scheduled: false + # Interval in seconds to add webhooks to projects + # from wildcards (optional, default: 43200) + interval_seconds: 43200 + + # GCPE Webhook endpoint URL + webhook_url: https://gcpe.example.net/webhook + # Redis configuration, optional and solely useful for an HA setup. # By default the data is held in memory of the exporter redis: diff --git a/internal/cmd/run.go b/internal/cmd/run.go index 8417818f..074e847e 100644 --- a/internal/cmd/run.go +++ b/internal/cmd/run.go @@ -98,6 +98,15 @@ func Run(cliCtx *cli.Context) (int, error) { <-onShutdown log.Info("received signal, attempting to gracefully exit..") + + if c.Config.Server.Webhook.RemoveHooks == true { + if err := c.RemoveWebhooks(ctx); err != nil { + log.WithContext(ctx).WithError(err) + } else { + log.Info("Cleaning up webhooks") + } + } + ctxCancel() httpServerContext, forceHTTPServerShutdown := context.WithTimeout(context.Background(), 5*time.Second) diff --git a/internal/cmd/utils.go b/internal/cmd/utils.go index 427dede5..9668d9d6 100644 --- a/internal/cmd/utils.go +++ b/internal/cmd/utils.go @@ -70,6 +70,8 @@ func configure(ctx *cli.Context) (cfg config.Config, err error) { log.WithFields(config.SchedulerConfig(cfg.Pull.RefsFromProjects).Log()).Info("pull refs from projects") log.WithFields(config.SchedulerConfig(cfg.Pull.Metrics).Log()).Info("pull metrics") + log.WithFields(config.SchedulerConfig(cfg.Server.Webhook.AddWebhooks).Log()).Info("add webhooks") + log.WithFields(config.SchedulerConfig(cfg.GarbageCollect.Projects).Log()).Info("garbage collect projects") log.WithFields(config.SchedulerConfig(cfg.GarbageCollect.Environments).Log()).Info("garbage collect environments") log.WithFields(config.SchedulerConfig(cfg.GarbageCollect.Refs).Log()).Info("garbage collect refs") diff --git a/pkg/config/config.go b/pkg/config/config.go index 4073a8a6..2f0feb2c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -90,6 +90,19 @@ type ServerWebhook struct { // Secret token to authenticate legitimate webhook requests coming from the GitLab server SecretToken string `validate:"required_if=Enabled true" yaml:"secret_token"` + + // Schedule the addition of webhooks to all seleceted projects + AddWebhooks struct { + OnInit bool `default:"false" yaml:"on_init"` + Scheduled bool `default:"false" yaml:"scheduled"` + IntervalSeconds int `default:"43200" validate:"gte=1" yaml:"interval_seconds"` + } `yaml:"add_webhooks"` + + // Webhook URL + URL string `validate:"required_if=AddWebhooks.Scheduled true" yaml:"webhook_url"` + + // Remove webhooks on shutdown + RemoveHooks bool `default:"false" yaml:"remove_webhooks"` } // Gitlab .. diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index cb15964a..dea3231e 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -72,6 +72,9 @@ func TestNew(t *testing.T) { c.ProjectDefaults.Pull.Pipeline.Jobs.RunnerDescription.AggregationRegexp = `shared-runners-manager-(\d*)\.gitlab\.com` c.ProjectDefaults.Pull.Pipeline.Variables.Regexp = `.*` + c.Server.Webhook.AddWebhooks.IntervalSeconds = 43200 + c.Server.Webhook.AddWebhooks.Scheduled = false + assert.Equal(t, c, New()) } diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 8aa123e0..1d58570d 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -62,7 +62,7 @@ func New(ctx context.Context, cfg config.Config, version string) (c Controller, } // Start the scheduler - c.Schedule(ctx, cfg.Pull, cfg.GarbageCollect) + c.Schedule(ctx, cfg.Pull, cfg.GarbageCollect, cfg.Server.Webhook) return } @@ -82,6 +82,7 @@ func (c *Controller) registerTasks() { schemas.TaskTypePullRefMetrics: c.TaskHandlerPullRefMetrics, schemas.TaskTypePullRefsFromProject: c.TaskHandlerPullRefsFromProject, schemas.TaskTypePullRefsFromProjects: c.TaskHandlerPullRefsFromProjects, + schemas.TaskTypeAddWebhooks: c.TaskHandlerAddWebhooks, } { _, _ = c.TaskController.TaskMap.Register(string(n), &taskq.TaskConfig{ Handler: h, diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 7a3ee763..653ea30c 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -302,8 +302,16 @@ func (c *Controller) TaskHandlerGarbageCollectMetrics(ctx context.Context) error return c.GarbageCollectMetrics(ctx) } +// TaskHandlerAddWebhooks .. +func (c *Controller) TaskHandlerAddWebhooks(ctx context.Context) error { + defer c.unqueueTask(ctx, schemas.TaskTypeAddWebhooks, "_") + defer c.TaskController.monitorLastTaskScheduling(schemas.TaskTypeAddWebhooks) + + return c.addWebhooks(ctx) +} + // Schedule .. -func (c *Controller) Schedule(ctx context.Context, pull config.Pull, gc config.GarbageCollect) { +func (c *Controller) Schedule(ctx context.Context, pull config.Pull, gc config.GarbageCollect, wh config.ServerWebhook) { ctx, span := otel.Tracer(tracerName).Start(ctx, "controller:Schedule") defer span.End() @@ -316,6 +324,7 @@ func (c *Controller) Schedule(ctx context.Context, pull config.Pull, gc config.G schemas.TaskTypeGarbageCollectEnvironments: config.SchedulerConfig(gc.Environments), schemas.TaskTypeGarbageCollectRefs: config.SchedulerConfig(gc.Refs), schemas.TaskTypeGarbageCollectMetrics: config.SchedulerConfig(gc.Metrics), + schemas.TaskTypeAddWebhooks: config.SchedulerConfig(wh.AddWebhooks), } { if cfg.OnInit { c.ScheduleTask(ctx, tt, "_") diff --git a/pkg/controller/webhooks.go b/pkg/controller/webhooks.go index f7643e1b..0e00413d 100644 --- a/pkg/controller/webhooks.go +++ b/pkg/controller/webhooks.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" + "github.com/openlyinc/pointy" log "github.com/sirupsen/logrus" goGitlab "github.com/xanzy/go-gitlab" @@ -327,3 +328,110 @@ func isEnvMatchingWilcard(w config.Wildcard, env schemas.Environment) (matches b // Then we check if the ref matches the project pull parameters return isEnvMatchingProjectPullEnvironments(w.Pull.Environments, env) } + +// Add a webhook to every project matching the wildcards. +func (c *Controller) addWebhooks(ctx context.Context) error { + for _, w := range c.Config.Wildcards { + projects, err := c.Gitlab.ListProjects(ctx, w) + if err != nil { + return err + } + + if len(projects) == 0 { // if no wildcards read config.projects + for _, p := range c.Config.Projects { + sp := schemas.Project{Project: p} + projects = append(projects, sp) + } + } + + for _, p := range projects { + hooks, _, err := c.Gitlab.Projects.ListProjectHooks( + p.Name, + &goGitlab.ListProjectHooksOptions{}, + goGitlab.WithContext(ctx), + ) + if err != nil { + return err + } + + WURL := c.Config.Server.Webhook.URL + opts := goGitlab.AddProjectHookOptions{ // options for hook + PushEvents: pointy.Bool(false), + PipelineEvents: pointy.Bool(true), + DeploymentEvents: pointy.Bool(true), + EnableSSLVerification: pointy.Bool(false), + URL: &WURL, + Token: &c.Config.Server.Webhook.SecretToken, + } + + if len(hooks) == 0 { // if no hooks + _, _, err := c.Gitlab.Projects.AddProjectHook( // add hook + p.Name, + &opts, + goGitlab.WithContext(ctx)) + if err != nil { + return err + } + } else { + exists := false + for _, h := range hooks { + if h.URL == WURL { + exists = true + } + } + if exists == false { + _, _, err := c.Gitlab.Projects.AddProjectHook( // else add hook + p.Name, + &opts, + goGitlab.WithContext(ctx)) + if err != nil { + return err + } + } + } + } + } + + return nil +} + +func (c *Controller) RemoveWebhooks(ctx context.Context) error { + for _, w := range c.Config.Wildcards { + projects, err := c.Gitlab.ListProjects(ctx, w) + if err != nil { + return err + } + + if len(projects) == 0 { // if no wildcards read config.projects + for _, p := range c.Config.Projects { + sp := schemas.Project{Project: p} + projects = append(projects, sp) + } + } + + for _, p := range projects { + hooks, _, err := c.Gitlab.Projects.ListProjectHooks( + p.Name, + &goGitlab.ListProjectHooksOptions{}, + goGitlab.WithContext(ctx), + ) + if err != nil { + return err + } + + WURL := c.Config.Server.Webhook.URL + + for _, h := range hooks { + if h.URL == WURL { + c.Gitlab.Projects.DeleteProjectHook( + p.Name, + h.ID, + goGitlab.WithContext(ctx), + ) + } + } + } + } + + return nil +} diff --git a/pkg/schemas/tasks.go b/pkg/schemas/tasks.go index 932b0786..1dd65607 100644 --- a/pkg/schemas/tasks.go +++ b/pkg/schemas/tasks.go @@ -42,6 +42,9 @@ const ( // TaskTypeGarbageCollectMetrics .. TaskTypeGarbageCollectMetrics TaskType = "GarbageCollectMetrics" + + // TaskTypeAddWebhooks .. + TaskTypeAddWebhooks TaskType = "AddWebhooks" ) // Tasks can be used to keep track of tasks. From d242b9d91d4ad8cc34c58cb7f0cf247f97e11649 Mon Sep 17 00:00:00 2001 From: Blake Tigges <30013719+blaketigges@users.noreply.github.com> Date: Tue, 13 Jun 2023 09:28:29 -0400 Subject: [PATCH 2/3] webhook removal config --- docs/configuration_syntax.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/configuration_syntax.md b/docs/configuration_syntax.md index 3fe6a9ca..81b1028f 100644 --- a/docs/configuration_syntax.md +++ b/docs/configuration_syntax.md @@ -64,6 +64,9 @@ server: # GCPE Webhook endpoint URL webhook_url: https://gcpe.example.net/webhook + # Whether to remove webhooks when GCPE shutsdown + remove_webhooks: false + # Redis configuration, optional and solely useful for an HA setup. # By default the data is held in memory of the exporter redis: From 23852b5121a4b8eaa4e30a645a527b926424efdf Mon Sep 17 00:00:00 2001 From: Blake Tigges <30013719+blaketigges@users.noreply.github.com> Date: Wed, 14 Jun 2023 11:41:45 -0400 Subject: [PATCH 3/3] Move webhook api calls to client functions --- pkg/controller/webhooks.go | 31 ++++--------- pkg/gitlab/webhooks.go | 86 +++++++++++++++++++++++++++++++++++++ pkg/gitlab/webhooks_test.go | 64 +++++++++++++++++++++++++++ 3 files changed, 158 insertions(+), 23 deletions(-) create mode 100644 pkg/gitlab/webhooks.go create mode 100644 pkg/gitlab/webhooks_test.go diff --git a/pkg/controller/webhooks.go b/pkg/controller/webhooks.go index 0e00413d..7eb636e9 100644 --- a/pkg/controller/webhooks.go +++ b/pkg/controller/webhooks.go @@ -345,11 +345,7 @@ func (c *Controller) addWebhooks(ctx context.Context) error { } for _, p := range projects { - hooks, _, err := c.Gitlab.Projects.ListProjectHooks( - p.Name, - &goGitlab.ListProjectHooksOptions{}, - goGitlab.WithContext(ctx), - ) + hooks, err := c.Gitlab.GetProjectHooks(ctx, p.Name) if err != nil { return err } @@ -365,10 +361,7 @@ func (c *Controller) addWebhooks(ctx context.Context) error { } if len(hooks) == 0 { // if no hooks - _, _, err := c.Gitlab.Projects.AddProjectHook( // add hook - p.Name, - &opts, - goGitlab.WithContext(ctx)) + _, err := c.Gitlab.AddProjectHook(ctx, p.Name, &opts) if err != nil { return err } @@ -380,10 +373,7 @@ func (c *Controller) addWebhooks(ctx context.Context) error { } } if exists == false { - _, _, err := c.Gitlab.Projects.AddProjectHook( // else add hook - p.Name, - &opts, - goGitlab.WithContext(ctx)) + _, err := c.Gitlab.AddProjectHook(ctx, p.Name, &opts) if err != nil { return err } @@ -410,11 +400,7 @@ func (c *Controller) RemoveWebhooks(ctx context.Context) error { } for _, p := range projects { - hooks, _, err := c.Gitlab.Projects.ListProjectHooks( - p.Name, - &goGitlab.ListProjectHooksOptions{}, - goGitlab.WithContext(ctx), - ) + hooks, err := c.Gitlab.GetProjectHooks(ctx, p.Name) if err != nil { return err } @@ -423,11 +409,10 @@ func (c *Controller) RemoveWebhooks(ctx context.Context) error { for _, h := range hooks { if h.URL == WURL { - c.Gitlab.Projects.DeleteProjectHook( - p.Name, - h.ID, - goGitlab.WithContext(ctx), - ) + err := c.Gitlab.RemoveProjectHook(ctx, p.Name, h.ID) + if err != nil { + return err + } } } } diff --git a/pkg/gitlab/webhooks.go b/pkg/gitlab/webhooks.go new file mode 100644 index 00000000..fc63bfc7 --- /dev/null +++ b/pkg/gitlab/webhooks.go @@ -0,0 +1,86 @@ +package gitlab + +import ( + "context" + + log "github.com/sirupsen/logrus" + goGitlab "github.com/xanzy/go-gitlab" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" +) + +// GetProjectHooks .. +func (c *Client) GetProjectHooks(ctx context.Context, projectName string) (hooks []*goGitlab.ProjectHook, err error) { + ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetProjectHooks") + defer span.End() + span.SetAttributes(attribute.String("project_name", projectName)) + + log.WithField("project_name", projectName).Trace("listing project hooks") + + c.rateLimit(ctx) + + hooks, resp, err := c.Projects.ListProjectHooks( + projectName, + &goGitlab.ListProjectHooksOptions{}, + goGitlab.WithContext(ctx), + ) + if err != nil { + return + } + + c.requestsRemaining(resp) + + return hooks, nil +} + +// AddProjectHook .. +func (c *Client) AddProjectHook(ctx context.Context, projectName string, options *goGitlab.AddProjectHookOptions) (hook *goGitlab.ProjectHook, err error) { + ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:AddProjectHook") + defer span.End() + span.SetAttributes(attribute.String("project_name", projectName)) + + log.WithField("project_name", projectName).Trace("adding project hook") + + c.rateLimit(ctx) + + hook, resp, err := c.Projects.AddProjectHook( + projectName, + options, + goGitlab.WithContext(ctx), + ) + if err != nil { + return + } + + c.requestsRemaining(resp) + + return hook, nil +} + +// RemoveProjectHook .. +func (c *Client) RemoveProjectHook(ctx context.Context, projectName string, hookID int) (err error) { + ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:RemoveProjectHook") + defer span.End() + span.SetAttributes(attribute.String("project_name", projectName)) + span.SetAttributes(attribute.Int("hook_id", hookID)) + + log.WithFields(log.Fields{ + "project_name": projectName, + "hook_id": hookID, + }).Trace("removing project hook") + + c.rateLimit(ctx) + + resp, err := c.Projects.DeleteProjectHook( + projectName, + hookID, + goGitlab.WithContext(ctx), + ) + if err != nil { + return + } + + c.requestsRemaining(resp) + + return nil +} diff --git a/pkg/gitlab/webhooks_test.go b/pkg/gitlab/webhooks_test.go new file mode 100644 index 00000000..765ebfde --- /dev/null +++ b/pkg/gitlab/webhooks_test.go @@ -0,0 +1,64 @@ +package gitlab + +import ( + "fmt" + "net/http" + "net/url" + "testing" + + "github.com/openlyinc/pointy" + "github.com/stretchr/testify/assert" + goGitlab "github.com/xanzy/go-gitlab" +) + +func TestGetProjectHooks(t *testing.T) { + ctx, mux, server, c := getMockedClient() + defer server.Close() + + mux.HandleFunc(fmt.Sprintf("/api/v4/projects/foo/hooks"), + func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "GET", r.Method) + expectedQueryParams := url.Values{} + assert.Equal(t, expectedQueryParams, r.URL.Query()) + fmt.Fprint(w, `[{"id":1}]`) + }) + + hooks, err := c.GetProjectHooks(ctx, "foo") + fmt.Println(hooks) + assert.NoError(t, err) + assert.Len(t, hooks, 1) +} + +func TestAddProjectHook(t *testing.T) { + ctx, mux, server, c := getMockedClient() + defer server.Close() + + mux.HandleFunc(fmt.Sprintf("/api/v4/projects/foo/hooks"), + func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "POST", r.Method) + expectedQueryParams := url.Values{} + assert.Equal(t, expectedQueryParams, r.URL.Query()) + fmt.Fprint(w, `{"id":1, "url":"www.example.com/webhook", "push_events":false, "pipeline_events": true, "deployment_events": true, "enable_ssl_verification": false}`) + }) + + hook, err := c.AddProjectHook(ctx, "foo", &goGitlab.AddProjectHookOptions{ + PushEvents: pointy.Bool(false), + PipelineEvents: pointy.Bool(true), + DeploymentEvents: pointy.Bool(true), + EnableSSLVerification: pointy.Bool(false), // add config for this later + URL: pointy.String("www.example.com/webhook"), + Token: pointy.String("token"), + }) + + h := goGitlab.ProjectHook{ + URL: "www.example.com/webhook", + ID: 1, + PushEvents: false, + PipelineEvents: true, + DeploymentEvents: true, + EnableSSLVerification: false, + } + + assert.NoError(t, err) + assert.Equal(t, &h, hook) +}