diff --git a/docs/metrics.md b/docs/metrics.md index dd6a48f3..7a07bd33 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -23,9 +23,11 @@ | `gitlab_ci_environment_information` | Information about the environment | [project], [environment], [environment_id], [external_url], [kind], [ref], [latest_commit_short_id], [current_commit_short_id], [available], [username] | `project_defaults.pull.environments.enabled` | | `gitlab_ci_pipeline_coverage` | Coverage of the most recent pipeline | [project], [topics], [ref], [kind], [source], [variables] | *available by default* | | `gitlab_ci_pipeline_duration_seconds` | Duration in seconds of the most recent pipeline | [project], [topics], [ref], [kind], [source], [variables] | *available by default* | +| `gitlab_ci_pipeline_duration_total` | Total duration in seconds of all pipelines | [project], [topics], [ref], [kind], [source], [variables] | *available by default* | | `gitlab_ci_pipeline_id` | ID of the most recent pipeline | [project], [topics], [ref], [kind], [source], [variables] | *available by default* | | `gitlab_ci_pipeline_job_artifact_size_bytes` | Artifact size in bytes (sum of all of them) of the most recent job | [project], [topics], [ref], [runner_description], [kind], [source], [variables], [stage], [job_name], [tag_list], [failure_reason] | `project_defaults.pull.pipeline.jobs.enabled` | | `gitlab_ci_pipeline_job_duration_seconds` | Duration in seconds of the most recent job | [project], [topics], [ref], [runner_description], [kind], [source], [variables], [stage], [job_name], [tag_list], [failure_reason] | `project_defaults.pull.pipeline.jobs.enabled` | +| `gitlab_ci_pipeline_job_duration_total` | Duration in seconds of all jobs | [project], [topics], [ref], [runner_description], [kind], [source], [variables], [stage], [job_name], [tag_list], [failure_reason] | `project_defaults.pull.pipeline.jobs.enabled` | | `gitlab_ci_pipeline_job_id` | ID of the most recent job | [project], [topics], [ref], [runner_description], [kind], [source], [variables], [stage], [job_name], [tag_list], [failure_reason] | `project_defaults.pull.pipeline.jobs.enabled` | | `gitlab_ci_pipeline_job_queued_duration_seconds` | Duration in seconds the most recent job has been queued before starting | [project], [topics], [ref], [runner_description], [kind], [source], [variables], [stage], [job_name], [tag_list], [failure_reason] | `project_defaults.pull.pipeline.jobs.enabled` | | `gitlab_ci_pipeline_job_run_count` | Number of executions of a job | [project], [topics], [ref], [runner_description], [kind], [source], [variables], [stage], [job_name], [tag_list], [failure_reason] | `project_defaults.pull.pipeline.jobs.enabled` | diff --git a/pkg/controller/collectors.go b/pkg/controller/collectors.go index a9265cec..6298debb 100644 --- a/pkg/controller/collectors.go +++ b/pkg/controller/collectors.go @@ -134,6 +134,17 @@ func NewCollectorDurationSeconds() prometheus.Collector { ) } +// NewCollectorDurationTotal returns a new collector for the gitlab_ci_pipeline_duration_total metric. +func NewCollectorDurationTotal() prometheus.Collector { + return prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "gitlab_ci_pipeline_duration_total", + Help: "Duration in seconds of all the pipelines", + }, + defaultLabels, + ) +} + // NewCollectorQueuedDurationSeconds returns a new collector for the gitlab_ci_pipeline_queued_duration_seconds metric. func NewCollectorQueuedDurationSeconds() prometheus.Collector { return prometheus.NewGaugeVec( @@ -255,6 +266,17 @@ func NewCollectorJobArtifactSizeBytes() prometheus.Collector { ) } +// NewCollectorJobDurationTotal returns a new collector for the gitlab_ci_pipeline_job_duration_total metric. +func NewCollectorJobDurationTotal() prometheus.Collector { + return prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "gitlab_ci_pipeline_job_duration_total", + Help: "Duration in seconds of all of the jobs", + }, + append(defaultLabels, jobLabels...), + ) +} + // NewCollectorJobDurationSeconds returns a new collector for the gitlab_ci_pipeline_job_duration_seconds metric. func NewCollectorJobDurationSeconds() prometheus.Collector { return prometheus.NewGaugeVec( diff --git a/pkg/controller/collectors_test.go b/pkg/controller/collectors_test.go index f7ff03c0..fb929b33 100644 --- a/pkg/controller/collectors_test.go +++ b/pkg/controller/collectors_test.go @@ -45,6 +45,8 @@ func TestNewCollectorFunctions(t *testing.T) { NewCollectorJobRunCount, NewCollectorRunCount, NewCollectorEnvironmentDeploymentCount, + NewCollectorDurationTotal, + NewCollectorJobDurationTotal, } { c := f() assert.NotNil(t, c) diff --git a/pkg/controller/jobs.go b/pkg/controller/jobs.go index 36cf5ca2..c3669def 100644 --- a/pkg/controller/jobs.go +++ b/pkg/controller/jobs.go @@ -167,6 +167,8 @@ func (c *Controller) ProcessJobMetrics(ctx context.Context, ref schemas.Ref, job storeSetMetric(ctx, c.Store, jobRunCount) + incrementMetric(ctx, c.Store, schemas.MetricKindJobDurationTotal, labels, job.DurationSeconds, projectRefLogFields) + storeSetMetric(ctx, c.Store, schemas.Metric{ Kind: schemas.MetricKindJobArtifactSizeBytes, Labels: labels, diff --git a/pkg/controller/metrics.go b/pkg/controller/metrics.go index a28ae040..c60be9d0 100644 --- a/pkg/controller/metrics.go +++ b/pkg/controller/metrics.go @@ -42,6 +42,7 @@ func NewRegistry(ctx context.Context) *Registry { Collectors: RegistryCollectors{ schemas.MetricKindCoverage: NewCollectorCoverage(), schemas.MetricKindDurationSeconds: NewCollectorDurationSeconds(), + schemas.MetricKindDurationTotal: NewCollectorDurationTotal(), schemas.MetricKindEnvironmentBehindCommitsCount: NewCollectorEnvironmentBehindCommitsCount(), schemas.MetricKindEnvironmentBehindDurationSeconds: NewCollectorEnvironmentBehindDurationSeconds(), schemas.MetricKindEnvironmentDeploymentCount: NewCollectorEnvironmentDeploymentCount(), @@ -53,6 +54,7 @@ func NewRegistry(ctx context.Context) *Registry { schemas.MetricKindID: NewCollectorID(), schemas.MetricKindJobArtifactSizeBytes: NewCollectorJobArtifactSizeBytes(), schemas.MetricKindJobDurationSeconds: NewCollectorJobDurationSeconds(), + schemas.MetricKindJobDurationTotal: NewCollectorJobDurationTotal(), schemas.MetricKindJobID: NewCollectorJobID(), schemas.MetricKindJobQueuedDurationSeconds: NewCollectorJobQueuedDurationSeconds(), schemas.MetricKindJobRunCount: NewCollectorJobRunCount(), @@ -237,3 +239,28 @@ func emitStatusMetric(ctx context.Context, s store.Store, metricKind schemas.Met storeSetMetric(ctx, s, statusMetric) } } + +// incrementMetric increments a counter metric by pulling the current value from the store +func incrementMetric(ctx context.Context, store store.Store, kind schemas.MetricKind, labels prometheus.Labels, inc float64, logFields log.Fields) { + metric := schemas.Metric{ + Kind: kind, + Labels: labels, + } + + // Increase the metric + metricExists, err := store.MetricExists(ctx, metric.Key()) + if err != nil { + log.WithContext(ctx). + WithFields(logFields). + WithError(err). + Error("checking if metric exists in the store") + + return + } + if metricExists { + storeGetMetric(ctx, store, &metric) + } + metric.Value += inc + storeSetMetric(ctx, store, metric) + +} diff --git a/pkg/controller/pipelines.go b/pkg/controller/pipelines.go index 81734343..8396a132 100644 --- a/pkg/controller/pipelines.go +++ b/pkg/controller/pipelines.go @@ -127,6 +127,8 @@ func (c *Controller) PullRefMetrics(ctx context.Context, ref schemas.Ref) error Value: pipeline.DurationSeconds, }) + incrementMetric(ctx, c.Store, schemas.MetricKindDurationTotal, labels, pipeline.DurationSeconds, logFields) + storeSetMetric(ctx, c.Store, schemas.Metric{ Kind: schemas.MetricKindQueuedDurationSeconds, Labels: labels, diff --git a/pkg/schemas/metric.go b/pkg/schemas/metric.go index bbf19752..832df31d 100644 --- a/pkg/schemas/metric.go +++ b/pkg/schemas/metric.go @@ -15,6 +15,9 @@ const ( // MetricKindDurationSeconds .. MetricKindDurationSeconds + // MetricKindDurationTotal .. + MetricKindDurationTotal + // MetricKindEnvironmentBehindCommitsCount .. MetricKindEnvironmentBehindCommitsCount @@ -48,6 +51,9 @@ const ( // MetricKindJobDurationSeconds .. MetricKindJobDurationSeconds + // MetricKindJobDurationTotal .. + MetricKindJobDurationTotal + // MetricKindJobID .. MetricKindJobID @@ -139,7 +145,7 @@ func (m Metric) Key() MetricKey { key := strconv.Itoa(int(m.Kind)) switch m.Kind { - case MetricKindCoverage, MetricKindDurationSeconds, MetricKindID, MetricKindQueuedDurationSeconds, MetricKindRunCount, MetricKindStatus, MetricKindTimestamp, MetricKindTestReportTotalCount, MetricKindTestReportErrorCount, MetricKindTestReportFailedCount, MetricKindTestReportSkippedCount, MetricKindTestReportSuccessCount, MetricKindTestReportTotalTime: + case MetricKindCoverage, MetricKindDurationSeconds, MetricKindJobDurationTotal, MetricKindID, MetricKindQueuedDurationSeconds, MetricKindRunCount, MetricKindStatus, MetricKindTimestamp, MetricKindTestReportTotalCount, MetricKindTestReportErrorCount, MetricKindTestReportFailedCount, MetricKindTestReportSkippedCount, MetricKindTestReportSuccessCount, MetricKindTestReportTotalTime: key += fmt.Sprintf("%v", []string{ m.Labels["project"], m.Labels["kind"],