storegateway: Add a metric for inuse bytes

Signed-off-by: 🌲 Harry 🌊 John 🏔 <[email protected]>

CHANGELOG.md

@@ -27,6 +27,7 @@
 * [ENHANCEMENT] Ingester/Store Gateway Clients: Introduce an experimental HealthCheck handler to quickly fail requests directed to unhealthy targets. #6225 #6257
 * [ENHANCEMENT] Upgrade build image and Go version to 1.23.2. #6261 #6262
 * [ENHANCEMENT] Querier/Ruler: Expose `store_gateway_consistency_check_max_attempts` for max retries when querying store gateway in consistency check. #6276
+* [ENHANCEMENT] StoreGateway: Add new `cortex_bucket_store_chunk_pool_inuse_bytes` metric to track the usage in chunk pool. #6310 
 * [BUGFIX] Runtime-config: Handle absolute file paths when working directory is not / #6224
 
 ## 1.18.1 2024-10-14

pkg/storegateway/chunk_bytes_pool.go

@@ -4,13 +4,17 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/thanos-io/thanos/pkg/pool"
+	"time"
 )
 
+const chunkPoolBytesInuseUpdateInterval = 1 * time.Second
+
 type chunkBytesPool struct {
 	pool *pool.BucketedPool[byte]
 
 	// Metrics.
-	poolByteStats *prometheus.CounterVec
+	poolByteStats  *prometheus.CounterVec
+	poolInUseBytes prometheus.GaugeFunc
 }
 
 func newChunkBytesPool(minBucketSize, maxBucketSize int, maxChunkPoolBytes uint64, reg prometheus.Registerer) (*chunkBytesPool, error) {
@@ -25,6 +29,12 @@ func newChunkBytesPool(minBucketSize, maxBucketSize int, maxChunkPoolBytes uint6
 			Name: "cortex_bucket_store_chunk_pool_operation_bytes_total",
 			Help: "Total bytes number of bytes pooled by operation.",
 		}, []string{"operation", "stats"}),
+		poolInUseBytes: promauto.With(reg).NewGaugeFunc(prometheus.GaugeOpts{
+			Name: "cortex_bucket_store_chunk_pool_inuse_bytes",
+			Help: "Total bytes in use in the chunk pool.",
+		}, func() float64 {
+			return float64(upstream.UsedBytes())
+		}),
 	}, nil
 }
 

pkg/storegateway/chunk_bytes_pool_test.go

@@ -3,13 +3,12 @@ package storegateway
 import (
 	"bytes"
 	"fmt"
-	"testing"
-
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/thanos-io/thanos/pkg/store"
+	"testing"
 
 	cortex_tsdb "github.com/cortexproject/cortex/pkg/storage/tsdb"
 )
@@ -20,9 +19,15 @@ func TestChunkBytesPool_Get(t *testing.T) {
 	p, err := newChunkBytesPool(cortex_tsdb.ChunkPoolDefaultMinBucketSize, cortex_tsdb.ChunkPoolDefaultMaxBucketSize, 0, reg)
 	require.NoError(t, err)
 	testBytes := []byte("test")
-	_, err = p.Get(store.EstimatedMaxChunkSize - 1)
+	b0, err := p.Get(store.EstimatedMaxChunkSize - 1)
 	require.NoError(t, err)
 
+	assert.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(fmt.Sprintf(`
+		# HELP cortex_bucket_store_chunk_pool_inuse_bytes Total bytes in use in the chunk pool.
+		# TYPE cortex_bucket_store_chunk_pool_inuse_bytes gauge
+        cortex_bucket_store_chunk_pool_inuse_bytes %d
+	`, 16000)), "cortex_bucket_store_chunk_pool_inuse_bytes"))
+
 	b, err := p.Get(store.EstimatedMaxChunkSize + 1)
 	require.NoError(t, err)
 
@@ -31,11 +36,21 @@ func TestChunkBytesPool_Get(t *testing.T) {
 	p.Put(b)
 
 	assert.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(fmt.Sprintf(`
+		# HELP cortex_bucket_store_chunk_pool_inuse_bytes Total bytes in use in the chunk pool.
+		# TYPE cortex_bucket_store_chunk_pool_inuse_bytes gauge
+        cortex_bucket_store_chunk_pool_inuse_bytes %d
 		# HELP cortex_bucket_store_chunk_pool_operation_bytes_total Total bytes number of bytes pooled by operation.
 		# TYPE cortex_bucket_store_chunk_pool_operation_bytes_total counter
 		cortex_bucket_store_chunk_pool_operation_bytes_total{operation="get",stats="cap"} %d
 		cortex_bucket_store_chunk_pool_operation_bytes_total{operation="get",stats="requested"} %d
 		cortex_bucket_store_chunk_pool_operation_bytes_total{operation="put",stats="cap"} %d
 		cortex_bucket_store_chunk_pool_operation_bytes_total{operation="put",stats="len"} %d
-	`, store.EstimatedMaxChunkSize*3, store.EstimatedMaxChunkSize*2, store.EstimatedMaxChunkSize*2, len(testBytes)))))
+	`, 16000, store.EstimatedMaxChunkSize*3, store.EstimatedMaxChunkSize*2, store.EstimatedMaxChunkSize*2, len(testBytes)))))
+
+	p.Put(b0)
+	assert.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(fmt.Sprintf(`
+		# HELP cortex_bucket_store_chunk_pool_inuse_bytes Total bytes in use in the chunk pool.
+		# TYPE cortex_bucket_store_chunk_pool_inuse_bytes gauge
+        cortex_bucket_store_chunk_pool_inuse_bytes %d
+	`, 0)), "cortex_bucket_store_chunk_pool_inuse_bytes"))
 }