Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

storegateway: Add a metric for chunk pool inuse bytes #6310

Merged
merged 1 commit into from
Nov 4, 2024
Merged

storegateway: Add a metric for chunk pool inuse bytes #6310

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
* [ENHANCEMENT] Ingester/Store Gateway Clients: Introduce an experimental HealthCheck handler to quickly fail requests directed to unhealthy targets. #6225 #6257
* [ENHANCEMENT] Upgrade build image and Go version to 1.23.2. #6261 #6262
* [ENHANCEMENT] Querier/Ruler: Expose `store_gateway_consistency_check_max_attempts` for max retries when querying store gateway in consistency check. #6276
* [ENHANCEMENT] StoreGateway: Add new `cortex_bucket_store_chunk_pool_inuse_bytes` metric to track the usage in chunk pool. #6310
* [BUGFIX] Runtime-config: Handle absolute file paths when working directory is not / #6224

## 1.18.1 2024-10-14
Expand Down
9 changes: 8 additions & 1 deletion pkg/storegateway/chunk_bytes_pool.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ type chunkBytesPool struct {
pool *pool.BucketedPool[byte]

// Metrics.
poolByteStats *prometheus.CounterVec
poolByteStats *prometheus.CounterVec
poolInUseBytes prometheus.GaugeFunc
}

func newChunkBytesPool(minBucketSize, maxBucketSize int, maxChunkPoolBytes uint64, reg prometheus.Registerer) (*chunkBytesPool, error) {
Expand All @@ -25,6 +26,12 @@ func newChunkBytesPool(minBucketSize, maxBucketSize int, maxChunkPoolBytes uint6
Name: "cortex_bucket_store_chunk_pool_operation_bytes_total",
Help: "Total bytes number of bytes pooled by operation.",
}, []string{"operation", "stats"}),
poolInUseBytes: promauto.With(reg).NewGaugeFunc(prometheus.GaugeOpts{
Name: "cortex_bucket_store_chunk_pool_inuse_bytes",
Copy link
Contributor

@yeya24 yeya24 Nov 4, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we just use NewGaugeFunc? This way we don't need the goroutine

Help: "Total bytes in use in the chunk pool.",
}, func() float64 {
return float64(upstream.UsedBytes())
}),
}, nil
}

Expand Down
20 changes: 18 additions & 2 deletions pkg/storegateway/chunk_bytes_pool_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,15 @@ func TestChunkBytesPool_Get(t *testing.T) {
p, err := newChunkBytesPool(cortex_tsdb.ChunkPoolDefaultMinBucketSize, cortex_tsdb.ChunkPoolDefaultMaxBucketSize, 0, reg)
require.NoError(t, err)
testBytes := []byte("test")
_, err = p.Get(store.EstimatedMaxChunkSize - 1)
b0, err := p.Get(store.EstimatedMaxChunkSize - 1)
require.NoError(t, err)

assert.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(fmt.Sprintf(`
# HELP cortex_bucket_store_chunk_pool_inuse_bytes Total bytes in use in the chunk pool.
# TYPE cortex_bucket_store_chunk_pool_inuse_bytes gauge
cortex_bucket_store_chunk_pool_inuse_bytes %d
`, 16000)), "cortex_bucket_store_chunk_pool_inuse_bytes"))

b, err := p.Get(store.EstimatedMaxChunkSize + 1)
require.NoError(t, err)

Expand All @@ -31,11 +37,21 @@ func TestChunkBytesPool_Get(t *testing.T) {
p.Put(b)

assert.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(fmt.Sprintf(`
# HELP cortex_bucket_store_chunk_pool_inuse_bytes Total bytes in use in the chunk pool.
# TYPE cortex_bucket_store_chunk_pool_inuse_bytes gauge
cortex_bucket_store_chunk_pool_inuse_bytes %d
# HELP cortex_bucket_store_chunk_pool_operation_bytes_total Total bytes number of bytes pooled by operation.
# TYPE cortex_bucket_store_chunk_pool_operation_bytes_total counter
cortex_bucket_store_chunk_pool_operation_bytes_total{operation="get",stats="cap"} %d
cortex_bucket_store_chunk_pool_operation_bytes_total{operation="get",stats="requested"} %d
cortex_bucket_store_chunk_pool_operation_bytes_total{operation="put",stats="cap"} %d
cortex_bucket_store_chunk_pool_operation_bytes_total{operation="put",stats="len"} %d
`, store.EstimatedMaxChunkSize*3, store.EstimatedMaxChunkSize*2, store.EstimatedMaxChunkSize*2, len(testBytes)))))
`, 16000, store.EstimatedMaxChunkSize*3, store.EstimatedMaxChunkSize*2, store.EstimatedMaxChunkSize*2, len(testBytes)))))

p.Put(b0)
assert.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(fmt.Sprintf(`
# HELP cortex_bucket_store_chunk_pool_inuse_bytes Total bytes in use in the chunk pool.
# TYPE cortex_bucket_store_chunk_pool_inuse_bytes gauge
cortex_bucket_store_chunk_pool_inuse_bytes %d
`, 0)), "cortex_bucket_store_chunk_pool_inuse_bytes"))
}
Loading