diff --git a/CHANGELOG.md b/CHANGELOG.md index 5432fc045dd..bd8d71ef895 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## master / unreleased +* [CHANGE] Ingester: Remove EnableNativeHistograms config flag and instead gate keep through new per-tenant limit at ingestion. #6718 * [CHANGE] StoreGateway/Alertmanager: Add default 5s connection timeout on client. #6603 * [FEATURE] Query Frontend: Add dynamic interval size for query splitting. This is enabled by configuring experimental flags `querier.max-shards-per-query` and/or `querier.max-fetched-data-duration-per-query`. The split interval size is dynamically increased to maintain a number of shards and total duration fetched below the configured values. #6458 * [FEATURE] Querier/Ruler: Add `query_partial_data` and `rules_partial_data` limits to allow queries/rules to be evaluated with data from a single zone, if other zones are not available. #6526 diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md index 5d09d1169c2..6177adea64f 100644 --- a/docs/blocks-storage/querier.md +++ b/docs/blocks-storage/querier.md @@ -1561,10 +1561,6 @@ blocks_storage: # CLI flag: -blocks-storage.tsdb.out-of-order-cap-max [out_of_order_cap_max: | default = 32] - # [EXPERIMENTAL] True to enable native histogram. - # CLI flag: -blocks-storage.tsdb.enable-native-histograms - [enable_native_histograms: | default = false] - # [EXPERIMENTAL] If enabled, ingesters will cache expanded postings when # querying blocks. Caching can be configured separately for the head and # compacted blocks. diff --git a/docs/blocks-storage/store-gateway.md b/docs/blocks-storage/store-gateway.md index 58b4c08eb3b..5053ea86ee7 100644 --- a/docs/blocks-storage/store-gateway.md +++ b/docs/blocks-storage/store-gateway.md @@ -1679,10 +1679,6 @@ blocks_storage: # CLI flag: -blocks-storage.tsdb.out-of-order-cap-max [out_of_order_cap_max: | default = 32] - # [EXPERIMENTAL] True to enable native histogram. - # CLI flag: -blocks-storage.tsdb.enable-native-histograms - [enable_native_histograms: | default = false] - # [EXPERIMENTAL] If enabled, ingesters will cache expanded postings when # querying blocks. Caching can be configured separately for the head and # compacted blocks. diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 252887a15e5..a64b106ef71 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -2130,10 +2130,6 @@ tsdb: # CLI flag: -blocks-storage.tsdb.out-of-order-cap-max [out_of_order_cap_max: | default = 32] - # [EXPERIMENTAL] True to enable native histogram. - # CLI flag: -blocks-storage.tsdb.enable-native-histograms - [enable_native_histograms: | default = false] - # [EXPERIMENTAL] If enabled, ingesters will cache expanded postings when # querying blocks. Caching can be configured separately for the head and # compacted blocks. @@ -3516,6 +3512,10 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s # [max_series] [limits_per_label_set: | default = []] +# [EXPERIMENTAL] True to enable native histogram. +# CLI flag: -blocks-storage.tsdb.enable-native-histograms +[enable_native_histograms: | default = false] + # The maximum number of active metrics with metadata per user, per ingester. 0 # to disable. # CLI flag: -ingester.max-metadata-per-user diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index e4eebef9911..0dfe0219265 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1347,7 +1347,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte return nil, wrapWithUser(err, userID) } - if i.cfg.BlocksStorageConfig.TSDB.EnableNativeHistograms { + if i.limits.EnableNativeHistograms(userID) { for _, hp := range ts.Histograms { var ( err error @@ -1494,7 +1494,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte i.validateMetrics.DiscardedSamples.WithLabelValues(perLabelsetSeriesLimit, userID).Add(float64(perLabelSetSeriesLimitCount)) } - if !i.cfg.BlocksStorageConfig.TSDB.EnableNativeHistograms && discardedNativeHistogramCount > 0 { + if !i.limits.EnableNativeHistograms(userID) && discardedNativeHistogramCount > 0 { i.validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramSample, userID).Add(float64(discardedNativeHistogramCount)) } @@ -2451,9 +2451,9 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { EnableMemorySnapshotOnShutdown: i.cfg.BlocksStorageConfig.TSDB.MemorySnapshotOnShutdown, OutOfOrderTimeWindow: time.Duration(oooTimeWindow).Milliseconds(), OutOfOrderCapMax: i.cfg.BlocksStorageConfig.TSDB.OutOfOrderCapMax, - EnableOOONativeHistograms: i.cfg.BlocksStorageConfig.TSDB.EnableNativeHistograms, // Automatically enabled when EnableNativeHistograms is true. - EnableOverlappingCompaction: false, // Always let compactors handle overlapped blocks, e.g. OOO blocks. - EnableNativeHistograms: i.cfg.BlocksStorageConfig.TSDB.EnableNativeHistograms, + EnableOOONativeHistograms: true, + EnableOverlappingCompaction: false, // Always let compactors handle overlapped blocks, e.g. OOO blocks. + EnableNativeHistograms: true, // Always enable Native Histograms. Gate keeping is done though a per-tenant limit at ingestion. BlockChunkQuerierFunc: i.blockChunkQuerierFunc(userID), }, nil) if err != nil { diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index dda43c4ba97..0da6a49161c 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -124,6 +124,7 @@ func seriesSetFromResponseStream(s *mockQueryStreamServer) (storage.SeriesSet, e func TestMatcherCache(t *testing.T) { limits := defaultLimitsTestConfig() + limits.EnableNativeHistograms = true userID := "1" tenantLimits := newMockTenantLimits(map[string]*validation.Limits{userID: &limits}) registry := prometheus.NewRegistry() @@ -135,7 +136,7 @@ func TestMatcherCache(t *testing.T) { require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) cfg := defaultIngesterTestConfig(t) cfg.MatchersCacheMaxItems = 50 - ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, tenantLimits, blocksDir, registry, true) + ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, tenantLimits, blocksDir, registry) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) @@ -204,7 +205,7 @@ func TestIngesterDeletionRace(t *testing.T) { require.NoError(t, os.Mkdir(chunksDir, os.ModePerm)) require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) - ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, tenantLimits, blocksDir, registry, false) + ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, tenantLimits, blocksDir, registry) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) defer services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck @@ -254,6 +255,7 @@ func TestIngesterDeletionRace(t *testing.T) { func TestIngesterPerLabelsetLimitExceeded(t *testing.T) { limits := defaultLimitsTestConfig() + limits.EnableNativeHistograms = true userID := "1" registry := prometheus.NewRegistry() @@ -287,7 +289,7 @@ func TestIngesterPerLabelsetLimitExceeded(t *testing.T) { require.NoError(t, os.Mkdir(chunksDir, os.ModePerm)) require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) - ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, tenantLimits, blocksDir, registry, true) + ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, tenantLimits, blocksDir, registry) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) // Wait until it's ACTIVE @@ -630,7 +632,7 @@ func TestIngesterPerLabelsetLimitExceeded(t *testing.T) { // Should persist between restarts services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck registry = prometheus.NewRegistry() - ing, err = prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, tenantLimits, blocksDir, registry, true) + ing, err = prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, tenantLimits, blocksDir, registry) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) ing.updateActiveSeries(ctx) @@ -661,6 +663,7 @@ func TestIngesterPerLabelsetLimitExceeded(t *testing.T) { func TestPushRace(t *testing.T) { cfg := defaultIngesterTestConfig(t) l := defaultLimitsTestConfig() + l.EnableNativeHistograms = true cfg.LabelsStringInterningEnabled = true cfg.LifecyclerConfig.JoinAfter = 0 @@ -686,7 +689,7 @@ func TestPushRace(t *testing.T) { blocksDir := filepath.Join(dir, "blocks") require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) - ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, l, nil, blocksDir, prometheus.NewRegistry(), true) + ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, l, nil, blocksDir, prometheus.NewRegistry()) require.NoError(t, err) defer services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) @@ -747,6 +750,7 @@ func TestPushRace(t *testing.T) { func TestIngesterUserLimitExceeded(t *testing.T) { limits := defaultLimitsTestConfig() + limits.EnableNativeHistograms = true limits.MaxLocalSeriesPerUser = 1 limits.MaxLocalMetricsWithMetadataPerUser = 1 @@ -778,7 +782,7 @@ func TestIngesterUserLimitExceeded(t *testing.T) { require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) blocksIngesterGenerator := func(reg prometheus.Registerer) *Ingester { - ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, nil, blocksDir, reg, true) + ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, nil, blocksDir, reg) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) // Wait until it's ACTIVE @@ -878,6 +882,7 @@ func benchmarkData(nSeries int) (allLabels []labels.Labels, allSamples []cortexp func TestIngesterMetricLimitExceeded(t *testing.T) { limits := defaultLimitsTestConfig() + limits.EnableNativeHistograms = true limits.MaxLocalSeriesPerMetric = 1 limits.MaxLocalMetadataPerMetric = 1 @@ -909,7 +914,7 @@ func TestIngesterMetricLimitExceeded(t *testing.T) { require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) blocksIngesterGenerator := func(reg prometheus.Registerer) *Ingester { - ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, nil, blocksDir, reg, true) + ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, nil, blocksDir, reg) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) // Wait until it's ACTIVE @@ -1933,6 +1938,7 @@ func TestIngester_Push(t *testing.T) { cfg.ActiveSeriesMetricsEnabled = !testData.disableActiveSeries limits := defaultLimitsTestConfig() + limits.EnableNativeHistograms = !testData.disableNativeHistogram limits.MaxExemplars = testData.maxExemplars limits.OutOfOrderTimeWindow = model.Duration(testData.oooTimeWindow) limits.LimitsPerLabelSet = []validation.LimitsPerLabelSet{ @@ -1945,7 +1951,7 @@ func TestIngester_Push(t *testing.T) { Hash: 1, }, } - i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, "", registry, !testData.disableNativeHistogram) + i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, "", registry) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck @@ -2174,7 +2180,8 @@ func TestIngester_PushNativeHistogramErrors(t *testing.T) { cfg.LifecyclerConfig.JoinAfter = 0 limits := defaultLimitsTestConfig() - i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, "", registry, true) + limits.EnableNativeHistograms = true + i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, "", registry) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck @@ -2662,6 +2669,7 @@ func Benchmark_Ingester_PushOnError(b *testing.B) { cfg.LifecyclerConfig.JoinAfter = 0 limits := defaultLimitsTestConfig() + limits.EnableNativeHistograms = true if !testData.prepareConfig(&limits, instanceLimits) { b.SkipNow() } @@ -2670,7 +2678,7 @@ func Benchmark_Ingester_PushOnError(b *testing.B) { return instanceLimits } - ingester, err := prepareIngesterWithBlocksStorageAndLimits(b, cfg, limits, nil, "", registry, true) + ingester, err := prepareIngesterWithBlocksStorageAndLimits(b, cfg, limits, nil, "", registry) require.NoError(b, err) require.NoError(b, services.StartAndAwaitRunning(context.Background(), ingester)) defer services.StopAndAwaitTerminated(context.Background(), ingester) //nolint:errcheck @@ -3947,10 +3955,12 @@ func mockHistogramWriteRequest(t *testing.T, lbls labels.Labels, value int64, ti } func prepareIngesterWithBlocksStorage(t testing.TB, ingesterCfg Config, registerer prometheus.Registerer) (*Ingester, error) { - return prepareIngesterWithBlocksStorageAndLimits(t, ingesterCfg, defaultLimitsTestConfig(), nil, "", registerer, true) + limits := defaultLimitsTestConfig() + limits.EnableNativeHistograms = true + return prepareIngesterWithBlocksStorageAndLimits(t, ingesterCfg, limits, nil, "", registerer) } -func prepareIngesterWithBlocksStorageAndLimits(t testing.TB, ingesterCfg Config, limits validation.Limits, tenantLimits validation.TenantLimits, dataDir string, registerer prometheus.Registerer, nativeHistograms bool) (*Ingester, error) { +func prepareIngesterWithBlocksStorageAndLimits(t testing.TB, ingesterCfg Config, limits validation.Limits, tenantLimits validation.TenantLimits, dataDir string, registerer prometheus.Registerer) (*Ingester, error) { // Create a data dir if none has been provided. if dataDir == "" { dataDir = t.TempDir() @@ -3966,7 +3976,6 @@ func prepareIngesterWithBlocksStorageAndLimits(t testing.TB, ingesterCfg Config, ingesterCfg.BlocksStorageConfig.TSDB.Dir = dataDir ingesterCfg.BlocksStorageConfig.Bucket.Backend = "filesystem" ingesterCfg.BlocksStorageConfig.Bucket.Filesystem.Directory = bucketDir - ingesterCfg.BlocksStorageConfig.TSDB.EnableNativeHistograms = nativeHistograms ingester, err := New(ingesterCfg, overrides, registerer, log.NewNopLogger(), nil) if err != nil { @@ -6432,7 +6441,8 @@ func TestIngester_MaxExemplarsFallBack(t *testing.T) { dir := t.TempDir() blocksDir := filepath.Join(dir, "blocks") limits := defaultLimitsTestConfig() - i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, blocksDir, prometheus.NewRegistry(), true) + limits.EnableNativeHistograms = true + i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, blocksDir, prometheus.NewRegistry()) require.NoError(t, err) maxExemplars := i.getMaxExemplars("someTenant") @@ -6440,7 +6450,7 @@ func TestIngester_MaxExemplarsFallBack(t *testing.T) { // set max exemplars value in limits, and re-initialize the ingester limits.MaxExemplars = 5 - i, err = prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, blocksDir, prometheus.NewRegistry(), true) + i, err = prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, blocksDir, prometheus.NewRegistry()) require.NoError(t, err) // validate this value is picked up now @@ -6815,6 +6825,7 @@ func TestIngester_UpdateLabelSetMetrics(t *testing.T) { cfg.BlocksStorageConfig.TSDB.BlockRanges = []time.Duration{2 * time.Hour} reg := prometheus.NewRegistry() limits := defaultLimitsTestConfig() + limits.EnableNativeHistograms = true userID := "1" ctx := user.InjectOrgID(context.Background(), userID) @@ -6839,7 +6850,7 @@ func TestIngester_UpdateLabelSetMetrics(t *testing.T) { require.NoError(t, os.Mkdir(chunksDir, os.ModePerm)) require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) - i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, tenantLimits, blocksDir, reg, false) + i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, tenantLimits, blocksDir, reg) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index 2a2e16c58d8..13daaff3f4c 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -172,9 +172,6 @@ type TSDBConfig struct { // OutOfOrderCapMax is maximum capacity for OOO chunks (in samples). OutOfOrderCapMax int64 `yaml:"out_of_order_cap_max"` - // Enable native histogram ingestion. - EnableNativeHistograms bool `yaml:"enable_native_histograms"` - // Posting Cache Configuration for TSDB PostingsCache TSDBPostingsCacheConfig `yaml:"expanded_postings_cache" doc:"description=[EXPERIMENTAL] If enabled, ingesters will cache expanded postings when querying blocks. Caching can be configured separately for the head and compacted blocks."` } @@ -204,7 +201,6 @@ func (cfg *TSDBConfig) RegisterFlags(f *flag.FlagSet) { f.IntVar(&cfg.MaxExemplars, "blocks-storage.tsdb.max-exemplars", 0, "Deprecated, use maxExemplars in limits instead. If the MaxExemplars value in limits is set to zero, cortex will fallback on this value. This setting enables support for exemplars in TSDB and sets the maximum number that will be stored. 0 or less means disabled.") f.BoolVar(&cfg.MemorySnapshotOnShutdown, "blocks-storage.tsdb.memory-snapshot-on-shutdown", false, "True to enable snapshotting of in-memory TSDB data on disk when shutting down.") f.Int64Var(&cfg.OutOfOrderCapMax, "blocks-storage.tsdb.out-of-order-cap-max", tsdb.DefaultOutOfOrderCapMax, "[EXPERIMENTAL] Configures the maximum number of samples per chunk that can be out-of-order.") - f.BoolVar(&cfg.EnableNativeHistograms, "blocks-storage.tsdb.enable-native-histograms", false, "[EXPERIMENTAL] True to enable native histogram.") flagext.DeprecatedFlag(f, "blocks-storage.tsdb.wal-compression-enabled", "Deprecated (use blocks-storage.tsdb.wal-compression-type instead): True to enable TSDB WAL compression.", util_log.Logger) diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 84596f17950..03302b4c2b0 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -149,6 +149,7 @@ type Limits struct { MaxGlobalSeriesPerUser int `yaml:"max_global_series_per_user" json:"max_global_series_per_user"` MaxGlobalSeriesPerMetric int `yaml:"max_global_series_per_metric" json:"max_global_series_per_metric"` LimitsPerLabelSet []LimitsPerLabelSet `yaml:"limits_per_label_set" json:"limits_per_label_set" doc:"nocli|description=[Experimental] Enable limits per LabelSet. Supported limits per labelSet: [max_series]"` + EnableNativeHistograms bool `yaml:"enable_native_histograms" json:"enable_native_histograms"` // Metadata MaxLocalMetricsWithMetadataPerUser int `yaml:"max_metadata_per_user" json:"max_metadata_per_user"` @@ -257,6 +258,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.MaxLocalSeriesPerMetric, "ingester.max-series-per-metric", 50000, "The maximum number of active series per metric name, per ingester. 0 to disable.") f.IntVar(&l.MaxGlobalSeriesPerUser, "ingester.max-global-series-per-user", 0, "The maximum number of active series per user, across the cluster before replication. 0 to disable. Supported only if -distributor.shard-by-all-labels is true.") f.IntVar(&l.MaxGlobalSeriesPerMetric, "ingester.max-global-series-per-metric", 0, "The maximum number of active series per metric name, across the cluster before replication. 0 to disable.") + f.BoolVar(&l.EnableNativeHistograms, "blocks-storage.tsdb.enable-native-histograms", false, "[EXPERIMENTAL] True to enable native histogram.") f.IntVar(&l.MaxExemplars, "ingester.max-exemplars", 0, "Enables support for exemplars in TSDB and sets the maximum number that will be stored. less than zero means disabled. If the value is set to zero, cortex will fallback to blocks-storage.tsdb.max-exemplars value.") f.Var(&l.OutOfOrderTimeWindow, "ingester.out-of-order-time-window", "[Experimental] Configures the allowed time window for ingestion of out-of-order samples. Disabled (0s) by default.") @@ -659,6 +661,11 @@ func (o *Overrides) MaxGlobalSeriesPerUser(userID string) int { return o.GetOverridesForUser(userID).MaxGlobalSeriesPerUser } +// EnableNativeHistograms returns whether the Ingester should accept NativeHistograms samples from this user. +func (o *Overrides) EnableNativeHistograms(userID string) bool { + return o.GetOverridesForUser(userID).EnableNativeHistograms +} + // OutOfOrderTimeWindow returns the allowed time window for ingestion of out-of-order samples. func (o *Overrides) OutOfOrderTimeWindow(userID string) model.Duration { return o.GetOverridesForUser(userID).OutOfOrderTimeWindow