Skip to content

feat: add duration_total metrics #968

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: add duration_total metrics #968

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/metrics.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,11 @@
| `gitlab_ci_environment_information` | Information about the environment | [project], [environment], [environment_id], [external_url], [kind], [ref], [latest_commit_short_id], [current_commit_short_id], [available], [username] | `project_defaults.pull.environments.enabled` |
| `gitlab_ci_pipeline_coverage` | Coverage of the most recent pipeline | [project], [topics], [ref], [kind], [source], [variables] | *available by default* |
| `gitlab_ci_pipeline_duration_seconds` | Duration in seconds of the most recent pipeline | [project], [topics], [ref], [kind], [source], [variables] | *available by default* |
| `gitlab_ci_pipeline_duration_total` | Total duration in seconds of all pipelines | [project], [topics], [ref], [kind], [source], [variables] | *available by default* |
| `gitlab_ci_pipeline_id` | ID of the most recent pipeline | [project], [topics], [ref], [kind], [source], [variables] | *available by default* |
| `gitlab_ci_pipeline_job_artifact_size_bytes` | Artifact size in bytes (sum of all of them) of the most recent job | [project], [topics], [ref], [runner_description], [kind], [source], [variables], [stage], [job_name], [tag_list], [failure_reason] | `project_defaults.pull.pipeline.jobs.enabled` |
| `gitlab_ci_pipeline_job_duration_seconds` | Duration in seconds of the most recent job | [project], [topics], [ref], [runner_description], [kind], [source], [variables], [stage], [job_name], [tag_list], [failure_reason] | `project_defaults.pull.pipeline.jobs.enabled` |
| `gitlab_ci_pipeline_job_duration_total` | Duration in seconds of all jobs | [project], [topics], [ref], [runner_description], [kind], [source], [variables], [stage], [job_name], [tag_list], [failure_reason] | `project_defaults.pull.pipeline.jobs.enabled` |
| `gitlab_ci_pipeline_job_id` | ID of the most recent job | [project], [topics], [ref], [runner_description], [kind], [source], [variables], [stage], [job_name], [tag_list], [failure_reason] | `project_defaults.pull.pipeline.jobs.enabled` |
| `gitlab_ci_pipeline_job_queued_duration_seconds` | Duration in seconds the most recent job has been queued before starting | [project], [topics], [ref], [runner_description], [kind], [source], [variables], [stage], [job_name], [tag_list], [failure_reason] | `project_defaults.pull.pipeline.jobs.enabled` |
| `gitlab_ci_pipeline_job_run_count` | Number of executions of a job | [project], [topics], [ref], [runner_description], [kind], [source], [variables], [stage], [job_name], [tag_list], [failure_reason] | `project_defaults.pull.pipeline.jobs.enabled` |
Expand Down
22 changes: 22 additions & 0 deletions pkg/controller/collectors.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,17 @@ func NewCollectorDurationSeconds() prometheus.Collector {
)
}

// NewCollectorDurationTotal returns a new collector for the gitlab_ci_pipeline_duration_total metric.
func NewCollectorDurationTotal() prometheus.Collector {
return prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "gitlab_ci_pipeline_duration_total",
Help: "Duration in seconds of all the pipelines",
},
defaultLabels,
)
}

// NewCollectorQueuedDurationSeconds returns a new collector for the gitlab_ci_pipeline_queued_duration_seconds metric.
func NewCollectorQueuedDurationSeconds() prometheus.Collector {
return prometheus.NewGaugeVec(
Expand Down Expand Up @@ -255,6 +266,17 @@ func NewCollectorJobArtifactSizeBytes() prometheus.Collector {
)
}

// NewCollectorJobDurationTotal returns a new collector for the gitlab_ci_pipeline_job_duration_total metric.
func NewCollectorJobDurationTotal() prometheus.Collector {
return prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "gitlab_ci_pipeline_job_duration_total",
Help: "Duration in seconds of all of the jobs",
},
append(defaultLabels, jobLabels...),
)
}

// NewCollectorJobDurationSeconds returns a new collector for the gitlab_ci_pipeline_job_duration_seconds metric.
func NewCollectorJobDurationSeconds() prometheus.Collector {
return prometheus.NewGaugeVec(
Expand Down
2 changes: 2 additions & 0 deletions pkg/controller/collectors_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ func TestNewCollectorFunctions(t *testing.T) {
NewCollectorJobRunCount,
NewCollectorRunCount,
NewCollectorEnvironmentDeploymentCount,
NewCollectorDurationTotal,
NewCollectorJobDurationTotal,
} {
c := f()
assert.NotNil(t, c)
Expand Down
2 changes: 2 additions & 0 deletions pkg/controller/jobs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,8 @@ func (c *Controller) ProcessJobMetrics(ctx context.Context, ref schemas.Ref, job

storeSetMetric(ctx, c.Store, jobRunCount)

incrementMetric(ctx, c.Store, schemas.MetricKindJobDurationTotal, labels, job.DurationSeconds, projectRefLogFields)

storeSetMetric(ctx, c.Store, schemas.Metric{
Kind: schemas.MetricKindJobArtifactSizeBytes,
Labels: labels,
Expand Down
27 changes: 27 additions & 0 deletions pkg/controller/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
Collectors: RegistryCollectors{
schemas.MetricKindCoverage: NewCollectorCoverage(),
schemas.MetricKindDurationSeconds: NewCollectorDurationSeconds(),
schemas.MetricKindDurationTotal: NewCollectorDurationTotal(),
schemas.MetricKindEnvironmentBehindCommitsCount: NewCollectorEnvironmentBehindCommitsCount(),
schemas.MetricKindEnvironmentBehindDurationSeconds: NewCollectorEnvironmentBehindDurationSeconds(),
schemas.MetricKindEnvironmentDeploymentCount: NewCollectorEnvironmentDeploymentCount(),
Expand All @@ -53,6 +54,7 @@
schemas.MetricKindID: NewCollectorID(),
schemas.MetricKindJobArtifactSizeBytes: NewCollectorJobArtifactSizeBytes(),
schemas.MetricKindJobDurationSeconds: NewCollectorJobDurationSeconds(),
schemas.MetricKindJobDurationTotal: NewCollectorJobDurationTotal(),
schemas.MetricKindJobID: NewCollectorJobID(),
schemas.MetricKindJobQueuedDurationSeconds: NewCollectorJobQueuedDurationSeconds(),
schemas.MetricKindJobRunCount: NewCollectorJobRunCount(),
Expand Down Expand Up @@ -237,3 +239,28 @@
storeSetMetric(ctx, s, statusMetric)
}
}

// incrementMetric increments a counter metric by pulling the current value from the store

Check failure on line 243 in pkg/controller/metrics.go

View workflow job for this annotation

GitHub Actions / test (ubuntu-24.04) 

Comment should end in a period (godot)
func incrementMetric(ctx context.Context, store store.Store, kind schemas.MetricKind, labels prometheus.Labels, inc float64, logFields log.Fields) {
metric := schemas.Metric{
Kind: kind,
Labels: labels,
}

// Increase the metric
metricExists, err := store.MetricExists(ctx, metric.Key())
if err != nil {
log.WithContext(ctx).
WithFields(logFields).
WithError(err).
Error("checking if metric exists in the store")

return
}
if metricExists {
storeGetMetric(ctx, store, &metric)
}
metric.Value += inc
storeSetMetric(ctx, store, metric)

}

Check failure on line 266 in pkg/controller/metrics.go

View workflow job for this annotation

GitHub Actions / test (ubuntu-24.04) 

unnecessary trailing newline (whitespace)
2 changes: 2 additions & 0 deletions pkg/controller/pipelines.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,8 @@ func (c *Controller) PullRefMetrics(ctx context.Context, ref schemas.Ref) error
Value: pipeline.DurationSeconds,
})

incrementMetric(ctx, c.Store, schemas.MetricKindDurationTotal, labels, pipeline.DurationSeconds, logFields)

storeSetMetric(ctx, c.Store, schemas.Metric{
Kind: schemas.MetricKindQueuedDurationSeconds,
Labels: labels,
Expand Down
8 changes: 7 additions & 1 deletion pkg/schemas/metric.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ const (
// MetricKindDurationSeconds ..
MetricKindDurationSeconds

// MetricKindDurationTotal ..
MetricKindDurationTotal

// MetricKindEnvironmentBehindCommitsCount ..
MetricKindEnvironmentBehindCommitsCount

Expand Down Expand Up @@ -48,6 +51,9 @@ const (
// MetricKindJobDurationSeconds ..
MetricKindJobDurationSeconds

// MetricKindJobDurationTotal ..
MetricKindJobDurationTotal

// MetricKindJobID ..
MetricKindJobID

Expand Down Expand Up @@ -139,7 +145,7 @@ func (m Metric) Key() MetricKey {
key := strconv.Itoa(int(m.Kind))

switch m.Kind {
case MetricKindCoverage, MetricKindDurationSeconds, MetricKindID, MetricKindQueuedDurationSeconds, MetricKindRunCount, MetricKindStatus, MetricKindTimestamp, MetricKindTestReportTotalCount, MetricKindTestReportErrorCount, MetricKindTestReportFailedCount, MetricKindTestReportSkippedCount, MetricKindTestReportSuccessCount, MetricKindTestReportTotalTime:
case MetricKindCoverage, MetricKindDurationSeconds, MetricKindJobDurationTotal, MetricKindID, MetricKindQueuedDurationSeconds, MetricKindRunCount, MetricKindStatus, MetricKindTimestamp, MetricKindTestReportTotalCount, MetricKindTestReportErrorCount, MetricKindTestReportFailedCount, MetricKindTestReportSkippedCount, MetricKindTestReportSuccessCount, MetricKindTestReportTotalTime:
key += fmt.Sprintf("%v", []string{
m.Labels["project"],
m.Labels["kind"],
Expand Down
Loading