Skip to content

Commit 5c893b0

Browse files
sklochkovsklochkov
committed
feat(collector): add sys user_summary by statement latency/type; clamp negative memory; unit tests
Add two new collectors: - --collect.sys.user_summary_by_statement_latency - --collect.sys.user_summary_by_statement_type Both follow the existing sys collector patterns: - metric names: mysql_sys_user_summary_by_statement_{latency|type}_* - label sets: {user} and {user,statement} - latencies converted from picoseconds to seconds (picoSeconds) - exported as gauges Also harden --collect.sys.user_summary against negative memory values observed on MySQL 8.x by clamping: GREATEST(current_memory, 0) AS current_memory GREATEST(total_memory_allocated, 0) AS total_memory_allocated Tests: - Update sys_user_summary_test.go: * robust SQL regex match * channel-read guard (no nil deref) * column name typo fixed ("statements") * expected values aligned with SQL-side clamping - Add sys_user_summary_by_statement_latency_test.go - Add sys_user_summary_by_statement_type_test.go Notes: - No changes to default enablement; flags must be passed explicitly. - Metric help strings note seconds for latency metrics. Verification: - go test ./collector -run UserSummary - go test ./... (full) Signed-off-by: Sergei Klochkov <[email protected]>
1 parent b56bf64 commit 5c893b0

8 files changed

+481
-5
lines changed

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,8 @@ collect.perf_schema.replication_applier_status_by_worker | 5.7 | C
136136
collect.slave_status | 5.1 | Collect from SHOW SLAVE STATUS (Enabled by default)
137137
collect.slave_hosts | 5.1 | Collect from SHOW SLAVE HOSTS
138138
collect.sys.user_summary | 5.7 | Collect metrics from sys.x$user_summary (disabled by default).
139+
collect.sys.user_summary_by_statement_latency | 5.7 | Collect metrics from sys.x$user_summary_by_statement_latency (disabled by default).
140+
collect.sys.user_summary_by_statement_type | 5.7 | Collects metrics from sys.x$user_summary_by_statement_type (disabled by default).
139141

140142

141143
### General Flags

‎collector/sys_user_summary.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ const sysUserSummaryQuery = `
3131
current_connections,
3232
total_connections,
3333
unique_hosts,
34-
current_memory,
35-
total_memory_allocated
34+
GREATEST(current_memory, 0) AS current_memory,
35+
GREATEST(total_memory_allocated, 0) AS total_memory_allocated
3636
FROM
3737
` + sysSchema + `.x$user_summary
3838
`

‎collector/sys_user_summary_by_statement_latency.go‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
package collector
2+
3+
import (
4+
"context"
5+
"log/slog"
6+
7+
"github.com/prometheus/client_golang/prometheus"
8+
)
9+
10+
type ScrapeSysUserSummaryByStatementLatency struct{}
11+
12+
func (ScrapeSysUserSummaryByStatementLatency) Name() string {
13+
return "sys.user_summary_by_statement_latency"
14+
}
15+
func (ScrapeSysUserSummaryByStatementLatency) Help() string {
16+
return "Collect metrics from sys.x$user_summary_by_statement_latency."
17+
}
18+
func (ScrapeSysUserSummaryByStatementLatency) Version() float64 { return 5.7 }
19+
20+
// Metric name stem to match sys_user_summary.go style.
21+
const userSummaryByStmtLatencyStem = "user_summary_by_statement_latency"
22+
23+
// Descriptors (namespace=sys schema; names include the stem above).
24+
var (
25+
sysUSSBLStatementsTotal = prometheus.NewDesc(
26+
prometheus.BuildFQName(namespace, sysSchema, userSummaryByStmtLatencyStem+"_total"),
27+
"The total number of statements for the user.",
28+
[]string{"user"}, nil,
29+
)
30+
sysUSSBLTotalLatency = prometheus.NewDesc(
31+
prometheus.BuildFQName(namespace, sysSchema, userSummaryByStmtLatencyStem+"_latency"),
32+
"The total wait time of timed statements for the user (seconds).",
33+
[]string{"user"}, nil,
34+
)
35+
sysUSSBLMaxLatency = prometheus.NewDesc(
36+
prometheus.BuildFQName(namespace, sysSchema, userSummaryByStmtLatencyStem+"_max_latency"),
37+
"The maximum single-statement latency for the user (seconds).",
38+
[]string{"user"}, nil,
39+
)
40+
sysUSSBLLockLatency = prometheus.NewDesc(
41+
prometheus.BuildFQName(namespace, sysSchema, userSummaryByStmtLatencyStem+"_lock_latency"),
42+
"The total time spent waiting for locks for the user (seconds).",
43+
[]string{"user"}, nil,
44+
)
45+
sysUSSBLCpuLatency = prometheus.NewDesc(
46+
prometheus.BuildFQName(namespace, sysSchema, userSummaryByStmtLatencyStem+"_cpu_latency"),
47+
"The total CPU time spent by statements for the user (seconds).",
48+
[]string{"user"}, nil,
49+
)
50+
sysUSSBLRowsSent = prometheus.NewDesc(
51+
prometheus.BuildFQName(namespace, sysSchema, userSummaryByStmtLatencyStem+"_rows_sent_total"),
52+
"The total number of rows sent by statements for the user.",
53+
[]string{"user"}, nil,
54+
)
55+
sysUSSBLRowsExamined = prometheus.NewDesc(
56+
prometheus.BuildFQName(namespace, sysSchema, userSummaryByStmtLatencyStem+"_rows_examined_total"),
57+
"The total number of rows examined by statements for the user.",
58+
[]string{"user"}, nil,
59+
)
60+
sysUSSBLRowsAffected = prometheus.NewDesc(
61+
prometheus.BuildFQName(namespace, sysSchema, userSummaryByStmtLatencyStem+"_rows_affected_total"),
62+
"The total number of rows affected by statements for the user.",
63+
[]string{"user"}, nil,
64+
)
65+
sysUSSBLFullScans = prometheus.NewDesc(
66+
prometheus.BuildFQName(namespace, sysSchema, userSummaryByStmtLatencyStem+"_full_scans_total"),
67+
"The total number of full table scans by statements for the user.",
68+
[]string{"user"}, nil,
69+
)
70+
)
71+
72+
func (ScrapeSysUserSummaryByStatementLatency) Scrape(
73+
ctx context.Context,
74+
inst *instance,
75+
ch chan<- prometheus.Metric,
76+
_ *slog.Logger,
77+
) error {
78+
const q = `
79+
SELECT
80+
user,
81+
total,
82+
total_latency,
83+
max_latency,
84+
lock_latency,
85+
cpu_latency,
86+
rows_sent,
87+
rows_examined,
88+
rows_affected,
89+
full_scans
90+
FROM sys.x$user_summary_by_statement_latency`
91+
92+
rows, err := inst.db.QueryContext(ctx, q)
93+
if err != nil {
94+
return err
95+
}
96+
defer rows.Close()
97+
98+
for rows.Next() {
99+
var (
100+
user string
101+
total uint64
102+
totalPs, maxPs, lockPs, cpuPs uint64
103+
rowsSent, rowsExam, rowsAff, fscs uint64
104+
)
105+
if err := rows.Scan(&user, &total, &totalPs, &maxPs, &lockPs, &cpuPs, &rowsSent, &rowsExam, &rowsAff, &fscs); err != nil {
106+
return err
107+
}
108+
109+
ch <- prometheus.MustNewConstMetric(sysUSSBLStatementsTotal, prometheus.GaugeValue, float64(total), user)
110+
ch <- prometheus.MustNewConstMetric(sysUSSBLTotalLatency, prometheus.GaugeValue, float64(totalPs)/picoSeconds, user)
111+
ch <- prometheus.MustNewConstMetric(sysUSSBLMaxLatency, prometheus.GaugeValue, float64(maxPs)/picoSeconds, user)
112+
ch <- prometheus.MustNewConstMetric(sysUSSBLLockLatency, prometheus.GaugeValue, float64(lockPs)/picoSeconds, user)
113+
ch <- prometheus.MustNewConstMetric(sysUSSBLCpuLatency, prometheus.GaugeValue, float64(cpuPs)/picoSeconds, user)
114+
ch <- prometheus.MustNewConstMetric(sysUSSBLRowsSent, prometheus.GaugeValue, float64(rowsSent), user)
115+
ch <- prometheus.MustNewConstMetric(sysUSSBLRowsExamined, prometheus.GaugeValue, float64(rowsExam), user)
116+
ch <- prometheus.MustNewConstMetric(sysUSSBLRowsAffected, prometheus.GaugeValue, float64(rowsAff), user)
117+
ch <- prometheus.MustNewConstMetric(sysUSSBLFullScans, prometheus.GaugeValue, float64(fscs), user)
118+
}
119+
return rows.Err()
120+
}

‎collector/sys_user_summary_by_statement_latency_test.go‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
package collector
2+
3+
import (
4+
"context"
5+
"database/sql/driver"
6+
"strconv"
7+
"testing"
8+
9+
"github.com/DATA-DOG/go-sqlmock"
10+
"github.com/prometheus/client_golang/prometheus"
11+
dto "github.com/prometheus/client_model/go"
12+
"github.com/prometheus/common/promslog"
13+
"github.com/smartystreets/goconvey/convey"
14+
)
15+
16+
func TestScrapeSysUserSummaryByStatementLatency(t *testing.T) {
17+
// Sanity check
18+
if (ScrapeSysUserSummaryByStatementLatency{}).Name() != "sys.user_summary_by_statement_latency" {
19+
t.Fatalf("unexpected Name()")
20+
}
21+
22+
db, mock, err := sqlmock.New()
23+
if err != nil {
24+
t.Fatalf("error opening a stub database connection: %s", err)
25+
}
26+
defer db.Close()
27+
inst := &instance{db: db}
28+
29+
columns := []string{
30+
"user",
31+
"total",
32+
"total_latency",
33+
"max_latency",
34+
"lock_latency",
35+
"cpu_latency",
36+
"rows_sent",
37+
"rows_examined",
38+
"rows_affected",
39+
"full_scans",
40+
}
41+
rows := sqlmock.NewRows(columns)
42+
43+
queryResults := [][]driver.Value{
44+
// user, total, total_latency(ps), max_latency(ps), lock_latency(ps), cpu_latency(ps), rows_sent, rows_examined, rows_affected, full_scans
45+
{"app", "10", "120", "300", "40", "50", "1000", "2000", "300", "7"},
46+
{"background", "2", "0", "0", "0", "0", "0", "0", "0", "0"},
47+
}
48+
for _, r := range queryResults {
49+
rows.AddRow(r...)
50+
}
51+
52+
// Pass regex as STRING (raw literal); sqlmock compiles it internally.
53+
mock.ExpectQuery(`(?s)SELECT\s+.*\s+FROM\s+sys\.x\$user_summary_by_statement_latency\s*`).
54+
WillReturnRows(rows)
55+
56+
// Expected metrics (emission order per row)
57+
expected := []MetricResult{}
58+
for _, r := range queryResults {
59+
u := r[0].(string)
60+
parse := func(s string) float64 {
61+
f, err := strconv.ParseFloat(s, 64)
62+
if err != nil {
63+
t.Fatalf("parse error: %v", err)
64+
}
65+
return f
66+
}
67+
total := parse(r[1].(string))
68+
totalLat := parse(r[2].(string)) / picoSeconds
69+
maxLat := parse(r[3].(string)) / picoSeconds
70+
lockLat := parse(r[4].(string)) / picoSeconds
71+
cpuLat := parse(r[5].(string)) / picoSeconds
72+
rowsSent := parse(r[6].(string))
73+
rowsExam := parse(r[7].(string))
74+
rowsAff := parse(r[8].(string))
75+
fullScans := parse(r[9].(string))
76+
77+
lbl := labelMap{"user": u}
78+
mt := dto.MetricType_GAUGE
79+
80+
expected = append(expected,
81+
MetricResult{labels: lbl, value: total, metricType: mt},
82+
MetricResult{labels: lbl, value: totalLat, metricType: mt},
83+
MetricResult{labels: lbl, value: maxLat, metricType: mt},
84+
MetricResult{labels: lbl, value: lockLat, metricType: mt},
85+
MetricResult{labels: lbl, value: cpuLat, metricType: mt},
86+
MetricResult{labels: lbl, value: rowsSent, metricType: mt},
87+
MetricResult{labels: lbl, value: rowsExam, metricType: mt},
88+
MetricResult{labels: lbl, value: rowsAff, metricType: mt},
89+
MetricResult{labels: lbl, value: fullScans, metricType: mt},
90+
)
91+
}
92+
93+
ch := make(chan prometheus.Metric)
94+
go func() {
95+
if err := (ScrapeSysUserSummaryByStatementLatency{}).Scrape(context.Background(), inst, ch, promslog.NewNopLogger()); err != nil {
96+
t.Errorf("scrape error: %s", err)
97+
}
98+
close(ch)
99+
}()
100+
101+
convey.Convey("Metrics comparison (user_summary_by_statement_latency)", t, func() {
102+
for i, exp := range expected {
103+
m, ok := <-ch
104+
if !ok {
105+
t.Fatalf("metrics channel closed early at index %d", i)
106+
}
107+
got := readMetric(m)
108+
convey.So(exp, convey.ShouldResemble, got)
109+
}
110+
})
111+
112+
if err := mock.ExpectationsWereMet(); err != nil {
113+
t.Errorf("unmet SQL expectations: %s", err)
114+
}
115+
}

‎collector/sys_user_summary_by_statement_type.go‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
package collector
2+
3+
import (
4+
"context"
5+
"log/slog"
6+
7+
"github.com/prometheus/client_golang/prometheus"
8+
)
9+
10+
type ScrapeSysUserSummaryByStatementType struct{}
11+
12+
func (ScrapeSysUserSummaryByStatementType) Name() string { return "sys.user_summary_by_statement_type" }
13+
func (ScrapeSysUserSummaryByStatementType) Help() string {
14+
return "Collect metrics from sys.x$user_summary_by_statement_type."
15+
}
16+
func (ScrapeSysUserSummaryByStatementType) Version() float64 { return 5.7 }
17+
18+
// Metric name stem to match sys_user_summary.go style.
19+
const userSummaryByStmtTypeStem = "user_summary_by_statement_type"
20+
21+
// Descriptors.
22+
var (
23+
sysUSSTStatementsTotal = prometheus.NewDesc(
24+
prometheus.BuildFQName(namespace, sysSchema, userSummaryByStmtTypeStem+"_total"),
25+
"The total number of occurrences of the statement type for the user.",
26+
[]string{"user", "statement"}, nil,
27+
)
28+
sysUSSTTotalLatency = prometheus.NewDesc(
29+
prometheus.BuildFQName(namespace, sysSchema, userSummaryByStmtTypeStem+"_latency"),
30+
"The total wait time of timed occurrences for the user and statement type (seconds).",
31+
[]string{"user", "statement"}, nil,
32+
)
33+
sysUSSTMaxLatency = prometheus.NewDesc(
34+
prometheus.BuildFQName(namespace, sysSchema, userSummaryByStmtTypeStem+"_max_latency"),
35+
"The maximum single-statement latency for the user and statement type (seconds).",
36+
[]string{"user", "statement"}, nil,
37+
)
38+
sysUSSTLockLatency = prometheus.NewDesc(
39+
prometheus.BuildFQName(namespace, sysSchema, userSummaryByStmtTypeStem+"_lock_latency"),
40+
"The total time spent waiting for locks for the user and statement type (seconds).",
41+
[]string{"user", "statement"}, nil,
42+
)
43+
sysUSSTCpuLatency = prometheus.NewDesc(
44+
prometheus.BuildFQName(namespace, sysSchema, userSummaryByStmtTypeStem+"_cpu_latency"),
45+
"The total CPU time for the user and statement type (seconds).",
46+
[]string{"user", "statement"}, nil,
47+
)
48+
sysUSSTRowsSent = prometheus.NewDesc(
49+
prometheus.BuildFQName(namespace, sysSchema, userSummaryByStmtTypeStem+"_rows_sent_total"),
50+
"The total number of rows sent for the user and statement type.",
51+
[]string{"user", "statement"}, nil,
52+
)
53+
sysUSSTRowsExamined = prometheus.NewDesc(
54+
prometheus.BuildFQName(namespace, sysSchema, userSummaryByStmtTypeStem+"_rows_examined_total"),
55+
"The total number of rows examined for the user and statement type.",
56+
[]string{"user", "statement"}, nil,
57+
)
58+
sysUSSTRowsAffected = prometheus.NewDesc(
59+
prometheus.BuildFQName(namespace, sysSchema, userSummaryByStmtTypeStem+"_rows_affected_total"),
60+
"The total number of rows affected for the user and statement type.",
61+
[]string{"user", "statement"}, nil,
62+
)
63+
sysUSSTFullScans = prometheus.NewDesc(
64+
prometheus.BuildFQName(namespace, sysSchema, userSummaryByStmtTypeStem+"_full_scans_total"),
65+
"The total number of full table scans for the user and statement type.",
66+
[]string{"user", "statement"}, nil,
67+
)
68+
)
69+
70+
func (ScrapeSysUserSummaryByStatementType) Scrape(
71+
ctx context.Context,
72+
inst *instance,
73+
ch chan<- prometheus.Metric,
74+
_ *slog.Logger,
75+
) error {
76+
const q = `
77+
SELECT
78+
user,
79+
statement,
80+
total,
81+
total_latency,
82+
max_latency,
83+
lock_latency,
84+
cpu_latency,
85+
rows_sent,
86+
rows_examined,
87+
rows_affected,
88+
full_scans
89+
FROM sys.x$user_summary_by_statement_type`
90+
91+
rows, err := inst.db.QueryContext(ctx, q)
92+
if err != nil {
93+
return err
94+
}
95+
defer rows.Close()
96+
97+
for rows.Next() {
98+
var (
99+
user, stmt string
100+
total uint64
101+
totalPs, maxPs, lockPs, cpuPs uint64
102+
rowsSent, rowsExam, rowsAff, fscs uint64
103+
)
104+
if err := rows.Scan(&user, &stmt, &total, &totalPs, &maxPs, &lockPs, &cpuPs, &rowsSent, &rowsExam, &rowsAff, &fscs); err != nil {
105+
return err
106+
}
107+
108+
ch <- prometheus.MustNewConstMetric(sysUSSTStatementsTotal, prometheus.GaugeValue, float64(total), user, stmt)
109+
ch <- prometheus.MustNewConstMetric(sysUSSTTotalLatency, prometheus.GaugeValue, float64(totalPs)/picoSeconds, user, stmt)
110+
ch <- prometheus.MustNewConstMetric(sysUSSTMaxLatency, prometheus.GaugeValue, float64(maxPs)/picoSeconds, user, stmt)
111+
ch <- prometheus.MustNewConstMetric(sysUSSTLockLatency, prometheus.GaugeValue, float64(lockPs)/picoSeconds, user, stmt)
112+
ch <- prometheus.MustNewConstMetric(sysUSSTCpuLatency, prometheus.GaugeValue, float64(cpuPs)/picoSeconds, user, stmt)
113+
ch <- prometheus.MustNewConstMetric(sysUSSTRowsSent, prometheus.GaugeValue, float64(rowsSent), user, stmt)
114+
ch <- prometheus.MustNewConstMetric(sysUSSTRowsExamined, prometheus.GaugeValue, float64(rowsExam), user, stmt)
115+
ch <- prometheus.MustNewConstMetric(sysUSSTRowsAffected, prometheus.GaugeValue, float64(rowsAff), user, stmt)
116+
ch <- prometheus.MustNewConstMetric(sysUSSTFullScans, prometheus.GaugeValue, float64(fscs), user, stmt)
117+
}
118+
return rows.Err()
119+
}

0 commit comments

Comments
 (0)