Skip to content

Commit 7c56a8b

Browse files
RaduBerindeRaduBerinde
committed
kvserver, storage: add bytes compressed/decompressed metrics
The motivation for these metrics is to enable estimating the CPU usage impact of switching to a stronger compression method (which applies only to L5/L6 data/value blocks). Epic: none Release note: None
1 parent a053744 commit 7c56a8b

File tree

3 files changed

+153
-0
lines changed

3 files changed

+153
-0
lines changed

‎docs/generated/metrics/metrics.yaml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16126,6 +16126,70 @@ layers:
1612616126
unit: COUNT
1612716127
aggregation: AVG
1612816128
derivative: NON_NEGATIVE_DERIVATIVE
16129+
- name: storage.bytes-compressed.l5.data
16130+
exported_name: storage_bytes_compressed_l5_data
16131+
description: Total number of logical bytes compressed for L5 data blocks.
16132+
y_axis_label: Bytes
16133+
type: COUNTER
16134+
unit: BYTES
16135+
aggregation: AVG
16136+
derivative: NON_NEGATIVE_DERIVATIVE
16137+
- name: storage.bytes-compressed.l5.values
16138+
exported_name: storage_bytes_compressed_l5_values
16139+
description: Total number of logical bytes compressed for L5 value blocks.
16140+
y_axis_label: Bytes
16141+
type: COUNTER
16142+
unit: BYTES
16143+
aggregation: AVG
16144+
derivative: NON_NEGATIVE_DERIVATIVE
16145+
- name: storage.bytes-compressed.l6.data
16146+
exported_name: storage_bytes_compressed_l6_data
16147+
description: Total number of logical bytes compressed for L6 data blocks.
16148+
y_axis_label: Bytes
16149+
type: COUNTER
16150+
unit: BYTES
16151+
aggregation: AVG
16152+
derivative: NON_NEGATIVE_DERIVATIVE
16153+
- name: storage.bytes-compressed.l6.values
16154+
exported_name: storage_bytes_compressed_l6_values
16155+
description: Total number of logical bytes compressed for L6 value blocks.
16156+
y_axis_label: Bytes
16157+
type: COUNTER
16158+
unit: BYTES
16159+
aggregation: AVG
16160+
derivative: NON_NEGATIVE_DERIVATIVE
16161+
- name: storage.bytes-decompressed.l5.data
16162+
exported_name: storage_bytes_decompressed_l5_data
16163+
description: Total number of logical bytes decompressed for L5 data blocks.
16164+
y_axis_label: Bytes
16165+
type: COUNTER
16166+
unit: BYTES
16167+
aggregation: AVG
16168+
derivative: NON_NEGATIVE_DERIVATIVE
16169+
- name: storage.bytes-decompressed.l5.values
16170+
exported_name: storage_bytes_decompressed_l5_values
16171+
description: Total number of logical bytes decompressed for L5 value blocks.
16172+
y_axis_label: Bytes
16173+
type: COUNTER
16174+
unit: BYTES
16175+
aggregation: AVG
16176+
derivative: NON_NEGATIVE_DERIVATIVE
16177+
- name: storage.bytes-decompressed.l6.data
16178+
exported_name: storage_bytes_decompressed_l6_data
16179+
description: Total number of logical bytes decompressed for L6 data blocks.
16180+
y_axis_label: Bytes
16181+
type: COUNTER
16182+
unit: BYTES
16183+
aggregation: AVG
16184+
derivative: NON_NEGATIVE_DERIVATIVE
16185+
- name: storage.bytes-decompressed.l6.values
16186+
exported_name: storage_bytes_decompressed_l6_values
16187+
description: Total number of logical bytes decompressed for L6 value blocks.
16188+
y_axis_label: Bytes
16189+
type: COUNTER
16190+
unit: BYTES
16191+
aggregation: AVG
16192+
derivative: NON_NEGATIVE_DERIVATIVE
1612916193
- name: storage.checkpoints
1613016194
exported_name: storage_checkpoints
1613116195
description: |-

‎pkg/kv/kvserver/metrics.go‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1236,6 +1236,54 @@ storage.initial_stats_complete becomes true.
12361236
Measurement: "Ratio",
12371237
Unit: metric.Unit_CONST,
12381238
}
1239+
metaBytesCompressedL5Data = metric.Metadata{
1240+
Name: "storage.bytes-compressed.l5.data",
1241+
Help: "Total number of logical bytes compressed for L5 data blocks.",
1242+
Measurement: "Bytes",
1243+
Unit: metric.Unit_BYTES,
1244+
}
1245+
metaBytesDecompressedL5Data = metric.Metadata{
1246+
Name: "storage.bytes-decompressed.l5.data",
1247+
Help: "Total number of logical bytes decompressed for L5 data blocks.",
1248+
Measurement: "Bytes",
1249+
Unit: metric.Unit_BYTES,
1250+
}
1251+
metaBytesCompressedL5Values = metric.Metadata{
1252+
Name: "storage.bytes-compressed.l5.values",
1253+
Help: "Total number of logical bytes compressed for L5 value blocks.",
1254+
Measurement: "Bytes",
1255+
Unit: metric.Unit_BYTES,
1256+
}
1257+
metaBytesDecompressedL5Values = metric.Metadata{
1258+
Name: "storage.bytes-decompressed.l5.values",
1259+
Help: "Total number of logical bytes decompressed for L5 value blocks.",
1260+
Measurement: "Bytes",
1261+
Unit: metric.Unit_BYTES,
1262+
}
1263+
metaBytesCompressedL6Data = metric.Metadata{
1264+
Name: "storage.bytes-compressed.l6.data",
1265+
Help: "Total number of logical bytes compressed for L6 data blocks.",
1266+
Measurement: "Bytes",
1267+
Unit: metric.Unit_BYTES,
1268+
}
1269+
metaBytesDecompressedL6Data = metric.Metadata{
1270+
Name: "storage.bytes-decompressed.l6.data",
1271+
Help: "Total number of logical bytes decompressed for L6 data blocks.",
1272+
Measurement: "Bytes",
1273+
Unit: metric.Unit_BYTES,
1274+
}
1275+
metaBytesCompressedL6Values = metric.Metadata{
1276+
Name: "storage.bytes-compressed.l6.values",
1277+
Help: "Total number of logical bytes compressed for L6 value blocks.",
1278+
Measurement: "Bytes",
1279+
Unit: metric.Unit_BYTES,
1280+
}
1281+
metaBytesDecompressedL6Values = metric.Metadata{
1282+
Name: "storage.bytes-decompressed.l6.values",
1283+
Help: "Total number of logical bytes decompressed for L6 value blocks.",
1284+
Measurement: "Bytes",
1285+
Unit: metric.Unit_BYTES,
1286+
}
12391287
)
12401288

12411289
var (
@@ -2935,6 +2983,9 @@ Note that the measurement does not include the duration for replicating the eval
29352983
)
29362984

29372985
// StoreMetrics is the set of metrics for a given store.
2986+
//
2987+
// Note: any non-embedded struct fields must implement the metric.Struct
2988+
// interface.
29382989
type StoreMetrics struct {
29392990
registry *metric.Registry
29402991

@@ -3105,6 +3156,7 @@ type StoreMetrics struct {
31053156
SSTableRemoteBytes *metric.Gauge
31063157
SSTableRemoteCount *metric.Gauge
31073158

3159+
// Compression metrics for currently live sstables and blob files.
31083160
CompressionSnappyBytes *metric.Gauge
31093161
CompressionSnappyCR *metric.GaugeFloat64
31103162
CompressionMinLZBytes *metric.Gauge
@@ -3115,6 +3167,16 @@ type StoreMetrics struct {
31153167
CompressionUnknownBytes *metric.Gauge
31163168
CompressionOverallCR *metric.GaugeFloat64
31173169

3170+
// Runtime metrics for compression.
3171+
BytesCompressedL5Values *metric.Counter
3172+
BytesCompressedL5Data *metric.Counter
3173+
BytesCompressedL6Values *metric.Counter
3174+
BytesCompressedL6Data *metric.Counter
3175+
BytesDecompressedL5Values *metric.Counter
3176+
BytesDecompressedL5Data *metric.Counter
3177+
BytesDecompressedL6Values *metric.Counter
3178+
BytesDecompressedL6Data *metric.Counter
3179+
31183180
categoryIterMetrics pebbleCategoryIterMetricsContainer
31193181
categoryDiskWriteMetrics pebbleCategoryDiskWriteMetricsContainer
31203182
ValueSeparationBytesReferenced *metric.Gauge
@@ -3872,6 +3934,15 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics {
38723934
CompressionUnknownBytes: metric.NewGauge(metaCompressionUnknownBytes),
38733935
CompressionOverallCR: metric.NewGaugeFloat64(metaCompressionOverallCR),
38743936

3937+
BytesCompressedL5Data: metric.NewCounter(metaBytesCompressedL5Data),
3938+
BytesCompressedL5Values: metric.NewCounter(metaBytesCompressedL5Values),
3939+
BytesCompressedL6Data: metric.NewCounter(metaBytesCompressedL6Data),
3940+
BytesCompressedL6Values: metric.NewCounter(metaBytesCompressedL6Values),
3941+
BytesDecompressedL5Data: metric.NewCounter(metaBytesDecompressedL5Data),
3942+
BytesDecompressedL5Values: metric.NewCounter(metaBytesDecompressedL5Values),
3943+
BytesDecompressedL6Data: metric.NewCounter(metaBytesDecompressedL6Data),
3944+
BytesDecompressedL6Values: metric.NewCounter(metaBytesDecompressedL6Values),
3945+
38753946
categoryDiskWriteMetrics: pebbleCategoryDiskWriteMetricsContainer{
38763947
registry: storeRegistry,
38773948
},
@@ -4212,6 +4283,7 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics {
42124283
ClosedTimestampPolicyChange: metric.NewCounter(metaClosedTimestampPolicyChange),
42134284
ClosedTimestampLatencyInfoMissing: metric.NewCounter(metaClosedTimestampLatencyInfoMissing),
42144285
}
4286+
42154287
sm.categoryIterMetrics.init(storeRegistry)
42164288

42174289
storeRegistry.AddMetricStruct(sm)
@@ -4378,6 +4450,15 @@ func (sm *StoreMetrics) updateEngineMetrics(m storage.Metrics) {
43784450
// ratio; we estimate it from the data we do have.
43794451
sm.CompressionOverallCR.Update(overall.CompressionRatio())
43804452

4453+
sm.BytesCompressedL5Values.Update(int64(m.CompressionCounters.LogicalBytesCompressed.L5.ValueBlocks))
4454+
sm.BytesCompressedL5Data.Update(int64(m.CompressionCounters.LogicalBytesCompressed.L5.DataBlocks))
4455+
sm.BytesCompressedL6Values.Update(int64(m.CompressionCounters.LogicalBytesCompressed.L6.ValueBlocks))
4456+
sm.BytesCompressedL6Data.Update(int64(m.CompressionCounters.LogicalBytesCompressed.L6.DataBlocks))
4457+
sm.BytesDecompressedL5Values.Update(int64(m.CompressionCounters.LogicalBytesDecompressed.L5.ValueBlocks))
4458+
sm.BytesDecompressedL5Data.Update(int64(m.CompressionCounters.LogicalBytesDecompressed.L5.DataBlocks))
4459+
sm.BytesDecompressedL6Values.Update(int64(m.CompressionCounters.LogicalBytesDecompressed.L6.ValueBlocks))
4460+
sm.BytesDecompressedL6Data.Update(int64(m.CompressionCounters.LogicalBytesDecompressed.L6.DataBlocks))
4461+
43814462
sm.categoryIterMetrics.update(m.CategoryStats)
43824463
sm.categoryDiskWriteMetrics.update(m.DiskWriteStats)
43834464

‎pkg/roachprod/agents/opentelemetry/cockroachdb_metrics.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2283,6 +2283,14 @@ var cockroachdbMetrics = map[string]string{
22832283
"storage_batch_commit_wal_rotation_duration": "storage.batch_commit.wal_rotation.duration",
22842284
"storage_block_load_active": "storage.block_load.active",
22852285
"storage_block_load_queued": "storage.block_load.queued",
2286+
"storage_bytes_compressed_l5_data": "storage.bytes_compressed.l5.data",
2287+
"storage_bytes_compressed_l5_values": "storage.bytes_compressed.l5.values",
2288+
"storage_bytes_compressed_l6_data": "storage.bytes_compressed.l6.data",
2289+
"storage_bytes_compressed_l6_values": "storage.bytes_compressed.l6.values",
2290+
"storage_bytes_decompressed_l5_data": "storage.bytes_decompressed.l5.data",
2291+
"storage_bytes_decompressed_l5_values": "storage.bytes_decompressed.l5.values",
2292+
"storage_bytes_decompressed_l6_data": "storage.bytes_decompressed.l6.data",
2293+
"storage_bytes_decompressed_l6_values": "storage.bytes_decompressed.l6.values",
22862294
"storage_category_pebble_manifest_bytes_written": "storage.category_pebble_manifest.bytes_written",
22872295
"storage_category_pebble_wal_bytes_written": "storage.category_pebble_wal.bytes_written",
22882296
"storage_category_unspecified_bytes_written": "storage.category_unspecified.bytes_written",

0 commit comments

Comments
 (0)