Skip to content

Commit 2d3e8d4

Browse files
bwplotkabwplotka
committed
Add WAL segment versioning; add flag (only v1 allowed).
Implementation for prometheus/proposals#40 Signed-off-by: bwplotka <[email protected]>
1 parent 664177b commit 2d3e8d4

File tree

16 files changed

+1517
-1201
lines changed

16 files changed

+1517
-1201
lines changed

‎cmd/prometheus/main.go‎

Lines changed: 38 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -154,21 +154,21 @@ func init() {
154154
// serverOnlyFlag creates server-only kingpin flag.
155155
func serverOnlyFlag(app *kingpin.Application, name, help string) *kingpin.FlagClause {
156156
return app.Flag(name, fmt.Sprintf("%s Use with server mode only.", help)).
157-
PreAction(func(parseContext *kingpin.ParseContext) error {
158-
// This will be invoked only if flag is actually provided by user.
159-
serverOnlyFlags = append(serverOnlyFlags, "--"+name)
160-
return nil
161-
})
157+
PreAction(func(parseContext *kingpin.ParseContext) error {
158+
// This will be invoked only if flag is actually provided by user.
159+
serverOnlyFlags = append(serverOnlyFlags, "--"+name)
160+
return nil
161+
})
162162
}
163163

164164
// agentOnlyFlag creates agent-only kingpin flag.
165165
func agentOnlyFlag(app *kingpin.Application, name, help string) *kingpin.FlagClause {
166166
return app.Flag(name, fmt.Sprintf("%s Use with agent mode only.", help)).
167-
PreAction(func(parseContext *kingpin.ParseContext) error {
168-
// This will be invoked only if flag is actually provided by user.
169-
agentOnlyFlags = append(agentOnlyFlags, "--"+name)
170-
return nil
171-
})
167+
PreAction(func(parseContext *kingpin.ParseContext) error {
168+
// This will be invoked only if flag is actually provided by user.
169+
agentOnlyFlags = append(agentOnlyFlags, "--"+name)
170+
return nil
171+
})
172172
}
173173

174174
type flagConfig struct {
@@ -427,6 +427,9 @@ func main() {
427427
serverOnlyFlag(a, "storage.tsdb.wal-compression-type", "Compression algorithm for the tsdb WAL.").
428428
Hidden().Default(string(wlog.CompressionSnappy)).EnumVar(&cfg.tsdb.WALCompressionType, string(wlog.CompressionSnappy), string(wlog.CompressionZstd))
429429

430+
serverOnlyFlag(a, "storage.tsdb.wal-version", fmt.Sprintf("Version for the new WAL segments. Supported versions: %v", wlog.ReleasedSupportedSegmentVersions())).
431+
Default(fmt.Sprintf("%v", wlog.DefaultSegmentVersion)).Uint32Var(&cfg.tsdb.WALSegmentVersion)
432+
430433
serverOnlyFlag(a, "storage.tsdb.head-chunks-write-queue-size", "Size of the queue through which head chunks are written to the disk to be m-mapped, 0 disables the queue completely. Experimental.").
431434
Default("0").IntVar(&cfg.tsdb.HeadChunksWriteQueueSize)
432435

@@ -443,12 +446,15 @@ func main() {
443446
"Size at which to split WAL segment files. Example: 100MB").
444447
Hidden().PlaceHolder("<bytes>").BytesVar(&cfg.agent.WALSegmentSize)
445448

446-
agentOnlyFlag(a, "storage.agent.wal-compression", "Compress the agent WAL.").
449+
agentOnlyFlag(a, "storage.agent.wal-compression", "Compress the new WAL segments.").
447450
Default("true").BoolVar(&cfg.agent.WALCompression)
448451

449452
agentOnlyFlag(a, "storage.agent.wal-compression-type", "Compression algorithm for the agent WAL.").
450453
Hidden().Default(string(wlog.CompressionSnappy)).EnumVar(&cfg.agent.WALCompressionType, string(wlog.CompressionSnappy), string(wlog.CompressionZstd))
451454

455+
agentOnlyFlag(a, "storage.agent.wal-version", fmt.Sprintf("Version for the new WAL segments. Supported versions: %v", wlog.ReleasedSupportedSegmentVersions())).
456+
Default(fmt.Sprintf("%v", wlog.DefaultSegmentVersion)).Uint32Var(&cfg.agent.WALSegmentVersion)
457+
452458
agentOnlyFlag(a, "storage.agent.wal-truncate-frequency",
453459
"The frequency at which to truncate the WAL and remove old data.").
454460
Hidden().PlaceHolder("<duration>").SetValue(&cfg.agent.TruncateFrequency)
@@ -1229,6 +1235,10 @@ func main() {
12291235
g.Add(
12301236
func() error {
12311237
logger.Info("Starting TSDB ...")
1238+
if !wlog.SegmentVersion(cfg.tsdb.WALSegmentVersion).IsReleased() {
1239+
return fmt.Errorf("flag 'storage.tsdb.wal-version' was set to unsupported WAL segment version %v; supported versions %v", cfg.tsdb.WALSegmentVersion, wlog.ReleasedSupportedSegmentVersions())
1240+
}
1241+
12321242
if cfg.tsdb.WALSegmentSize != 0 {
12331243
if cfg.tsdb.WALSegmentSize < 10*1024*1024 || cfg.tsdb.WALSegmentSize > 256*1024*1024 {
12341244
return errors.New("flag 'storage.tsdb.wal-segment-size' must be set between 10MB and 256MB")
@@ -1285,6 +1295,10 @@ func main() {
12851295
g.Add(
12861296
func() error {
12871297
logger.Info("Starting WAL storage ...")
1298+
if !wlog.SegmentVersion(cfg.agent.WALSegmentVersion).IsReleased() {
1299+
return fmt.Errorf("flag 'storage.agent.wal-version' was set to unsupported WAL segment version %v; supported versions %v", cfg.tsdb.WALSegmentVersion, wlog.ReleasedSupportedSegmentVersions())
1300+
}
1301+
12881302
if cfg.agent.WALSegmentSize != 0 {
12891303
if cfg.agent.WALSegmentSize < 10*1024*1024 || cfg.agent.WALSegmentSize > 256*1024*1024 {
12901304
return errors.New("flag 'storage.agent.wal-segment-size' must be set between 10MB and 256MB")
@@ -1492,7 +1506,7 @@ func reloadConfig(filename string, enableExemplarStorage bool, logger *slog.Logg
14921506

14931507
func startsOrEndsWithQuote(s string) bool {
14941508
return strings.HasPrefix(s, "\"") || strings.HasPrefix(s, "'") ||
1495-
strings.HasSuffix(s, "\"") || strings.HasSuffix(s, "'")
1509+
strings.HasSuffix(s, "\"") || strings.HasSuffix(s, "'")
14961510
}
14971511

14981512
// compileCORSRegexString compiles given string and adds anchors.
@@ -1780,6 +1794,7 @@ func (rm *readyScrapeManager) Get() (*scrape.Manager, error) {
17801794
// This is required as tsdb.Option fields are unit agnostic (time).
17811795
type tsdbOptions struct {
17821796
WALSegmentSize units.Base2Bytes
1797+
WALSegmentVersion uint32
17831798
MaxBlockChunkSegmentSize units.Base2Bytes
17841799
RetentionDuration model.Duration
17851800
MaxBytes units.Base2Bytes
@@ -1804,12 +1819,15 @@ type tsdbOptions struct {
18041819

18051820
func (opts tsdbOptions) ToTSDBOptions() tsdb.Options {
18061821
return tsdb.Options{
1807-
WALSegmentSize: int(opts.WALSegmentSize),
1822+
WALSegment: wlog.SegmentOptions{
1823+
Version: wlog.SegmentVersion(opts.WALSegmentVersion),
1824+
Compression: wlog.ParseCompressionType(opts.WALCompression, opts.WALCompressionType),
1825+
Size: int(opts.WALSegmentSize),
1826+
},
18081827
MaxBlockChunkSegmentSize: int64(opts.MaxBlockChunkSegmentSize),
18091828
RetentionDuration: int64(time.Duration(opts.RetentionDuration) / time.Millisecond),
18101829
MaxBytes: int64(opts.MaxBytes),
18111830
NoLockfile: opts.NoLockfile,
1812-
WALCompression: wlog.ParseCompressionType(opts.WALCompression, opts.WALCompressionType),
18131831
HeadChunksWriteQueueSize: opts.HeadChunksWriteQueueSize,
18141832
SamplesPerChunk: opts.SamplesPerChunk,
18151833
StripeSize: opts.StripeSize,
@@ -1833,6 +1851,7 @@ type agentOptions struct {
18331851
WALSegmentSize units.Base2Bytes
18341852
WALCompression bool
18351853
WALCompressionType string
1854+
WALSegmentVersion uint32
18361855
StripeSize int
18371856
TruncateFrequency model.Duration
18381857
MinWALTime, MaxWALTime model.Duration
@@ -1845,8 +1864,11 @@ func (opts agentOptions) ToAgentOptions(outOfOrderTimeWindow int64) agent.Option
18451864
outOfOrderTimeWindow = 0
18461865
}
18471866
return agent.Options{
1848-
WALSegmentSize: int(opts.WALSegmentSize),
1849-
WALCompression: wlog.ParseCompressionType(opts.WALCompression, opts.WALCompressionType),
1867+
WALSegment: wlog.SegmentOptions{
1868+
Version: wlog.SegmentVersion(opts.WALSegmentVersion),
1869+
Compression: wlog.ParseCompressionType(opts.WALCompression, opts.WALCompressionType),
1870+
Size: int(opts.WALSegmentSize),
1871+
},
18501872
StripeSize: opts.StripeSize,
18511873
TruncateFrequency: time.Duration(opts.TruncateFrequency),
18521874
MinWALTime: durationToInt64Millis(time.Duration(opts.MinWALTime)),

‎tsdb/agent/db.go‎

Lines changed: 33 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -60,13 +60,8 @@ var (
6060

6161
// Options of the WAL storage.
6262
type Options struct {
63-
// Segments (wal files) max size.
64-
// WALSegmentSize <= 0, segment size is default size.
65-
// WALSegmentSize > 0, segment size is WALSegmentSize.
66-
WALSegmentSize int
67-
68-
// WALCompression configures the compression type to use on records in the WAL.
69-
WALCompression wlog.CompressionType
63+
// WALSegment represents WAL segment options.
64+
WALSegment wlog.SegmentOptions
7065

7166
// StripeSize is the size (power of 2) in entries of the series hash map. Reducing the size will save memory but impact performance.
7267
StripeSize int
@@ -89,8 +84,7 @@ type Options struct {
8984
// millisecond-precision timestamps.
9085
func DefaultOptions() *Options {
9186
return &Options{
92-
WALSegmentSize: wlog.DefaultSegmentSize,
93-
WALCompression: wlog.CompressionNone,
87+
WALSegment: wlog.SegmentOptions{}, // wlog.New will set the correct defaults
9488
StripeSize: tsdb.DefaultStripeSize,
9589
TruncateFrequency: DefaultTruncateFrequency,
9690
MinWALTime: DefaultMinWALTime,
@@ -266,7 +260,7 @@ func Open(l *slog.Logger, reg prometheus.Registerer, rs *remote.Storage, dir str
266260
// remote_write expects WAL to be stored in a "wal" subdirectory of the main storage.
267261
dir = filepath.Join(dir, "wal")
268262

269-
w, err := wlog.NewSize(l, reg, dir, opts.WALSegmentSize, opts.WALCompression)
263+
w, err := wlog.New(l, reg, dir, opts.WALSegment)
270264
if err != nil {
271265
return nil, fmt.Errorf("creating WAL: %w", err)
272266
}
@@ -326,14 +320,6 @@ func validateOptions(opts *Options) *Options {
326320
if opts == nil {
327321
opts = DefaultOptions()
328322
}
329-
if opts.WALSegmentSize <= 0 {
330-
opts.WALSegmentSize = wlog.DefaultSegmentSize
331-
}
332-
333-
if opts.WALCompression == "" {
334-
opts.WALCompression = wlog.CompressionNone
335-
}
336-
337323
// Revert StripeSize to DefaultStripeSize if StripeSize is either 0 or not a power of 2.
338324
if opts.StripeSize <= 0 || ((opts.StripeSize & (opts.StripeSize - 1)) != 0) {
339325
opts.StripeSize = tsdb.DefaultStripeSize
@@ -389,14 +375,14 @@ func (db *DB) replayWAL() error {
389375
}
390376

391377
// Find the last segment.
392-
_, last, err := wlog.Segments(db.wal.Dir())
378+
_, last, err := wlog.SegmentsRange(db.wal.Dir())
393379
if err != nil {
394380
return fmt.Errorf("finding WAL segments: %w", err)
395381
}
396382

397383
// Backfill segments from the most recent checkpoint onwards.
398384
for i := startFrom; i <= last; i++ {
399-
seg, err := wlog.OpenReadSegment(wlog.SegmentName(db.wal.Dir(), i))
385+
seg, err := wlog.OpenReadSegmentByIndex(db.wal.Dir(), i)
400386
if err != nil {
401387
return fmt.Errorf("open WAL segment: %d: %w", i, err)
402388
}
@@ -443,10 +429,12 @@ func (db *DB) loadWAL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks.H
443429
series := seriesPool.Get()[:0]
444430
series, err = dec.Series(rec, series)
445431
if err != nil {
432+
i, v := r.Segment()
446433
errCh <- &wlog.CorruptionErr{
447-
Err: fmt.Errorf("decode series: %w", err),
448-
Segment: r.Segment(),
449-
Offset: r.Offset(),
434+
Err: fmt.Errorf("decode series: %w", err),
435+
SegmentIndex: i,
436+
SegmentVersion: v,
437+
Offset: r.Offset(),
450438
}
451439
return
452440
}
@@ -455,10 +443,12 @@ func (db *DB) loadWAL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks.H
455443
samples := samplesPool.Get()[:0]
456444
samples, err = dec.Samples(rec, samples)
457445
if err != nil {
446+
i, v := r.Segment()
458447
errCh <- &wlog.CorruptionErr{
459-
Err: fmt.Errorf("decode samples: %w", err),
460-
Segment: r.Segment(),
461-
Offset: r.Offset(),
448+
Err: fmt.Errorf("decode samples: %w", err),
449+
SegmentIndex: i,
450+
SegmentVersion: v,
451+
Offset: r.Offset(),
462452
}
463453
return
464454
}
@@ -467,10 +457,12 @@ func (db *DB) loadWAL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks.H
467457
histograms := histogramsPool.Get()[:0]
468458
histograms, err = dec.HistogramSamples(rec, histograms)
469459
if err != nil {
460+
i, v := r.Segment()
470461
errCh <- &wlog.CorruptionErr{
471-
Err: fmt.Errorf("decode histogram samples: %w", err),
472-
Segment: r.Segment(),
473-
Offset: r.Offset(),
462+
Err: fmt.Errorf("decode histogram samples: %w", err),
463+
SegmentIndex: i,
464+
SegmentVersion: v,
465+
Offset: r.Offset(),
474466
}
475467
return
476468
}
@@ -479,10 +471,12 @@ func (db *DB) loadWAL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks.H
479471
floatHistograms := floatHistogramsPool.Get()[:0]
480472
floatHistograms, err = dec.FloatHistogramSamples(rec, floatHistograms)
481473
if err != nil {
474+
i, v := r.Segment()
482475
errCh <- &wlog.CorruptionErr{
483-
Err: fmt.Errorf("decode float histogram samples: %w", err),
484-
Segment: r.Segment(),
485-
Offset: r.Offset(),
476+
Err: fmt.Errorf("decode float histogram samples: %w", err),
477+
SegmentIndex: i,
478+
SegmentVersion: v,
479+
Offset: r.Offset(),
486480
}
487481
return
488482
}
@@ -493,10 +487,12 @@ func (db *DB) loadWAL(r *wlog.Reader, multiRef map[chunks.HeadSeriesRef]chunks.H
493487
// stripeSeries.exemplars in the next block by using setLatestExemplar.
494488
continue
495489
default:
490+
i, v := r.Segment()
496491
errCh <- &wlog.CorruptionErr{
497-
Err: fmt.Errorf("invalid record type %v", dec.Type(rec)),
498-
Segment: r.Segment(),
499-
Offset: r.Offset(),
492+
Err: fmt.Errorf("invalid record type %v", dec.Type(rec)),
493+
SegmentIndex: i,
494+
SegmentVersion: v,
495+
Offset: r.Offset(),
500496
}
501497
}
502498
}
@@ -632,7 +628,7 @@ func (db *DB) truncate(mint int64) error {
632628
db.gc(mint)
633629
db.logger.Info("series GC completed", "duration", time.Since(start))
634630

635-
first, last, err := wlog.Segments(db.wal.Dir())
631+
first, last, err := wlog.SegmentsRange(db.wal.Dir())
636632
if err != nil {
637633
return fmt.Errorf("get segment range: %w", err)
638634
}
@@ -711,7 +707,7 @@ func (db *DB) gc(mint int64) {
711707
deleted := db.series.GC(mint)
712708
db.metrics.numActiveSeries.Sub(float64(len(deleted)))
713709

714-
_, last, _ := wlog.Segments(db.wal.Dir())
710+
_, last, _ := wlog.SegmentsRange(db.wal.Dir())
715711

716712
// We want to keep series records for any newly deleted series
717713
// until we've passed the last recorded segment. This prevents

‎tsdb/db.go‎

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ var ErrNotReady = errors.New("TSDB not ready")
7373
// millisecond precision timestamps.
7474
func DefaultOptions() *Options {
7575
return &Options{
76-
WALSegmentSize: wlog.DefaultSegmentSize,
76+
WALSegment: wlog.SegmentOptions{}, // wlog.New sets the correct defaults.
7777
MaxBlockChunkSegmentSize: chunks.DefaultChunkSegmentSize,
7878
RetentionDuration: int64(15 * 24 * time.Hour / time.Millisecond),
7979
MinBlockDuration: DefaultBlockDuration,
@@ -97,11 +97,10 @@ func DefaultOptions() *Options {
9797

9898
// Options of the DB storage.
9999
type Options struct {
100-
// Segments (wal files) max size.
101-
// WALSegmentSize = 0, segment size is default size.
102-
// WALSegmentSize > 0, segment size is WALSegmentSize.
103-
// WALSegmentSize < 0, wal is disabled.
104-
WALSegmentSize int
100+
// WALSegment represents segment options.
101+
WALSegment wlog.SegmentOptions
102+
// DisableWAL disables WAL.
103+
DisableWAL bool
105104

106105
// MaxBlockChunkSegmentSize is the max size of block chunk segment files.
107106
// MaxBlockChunkSegmentSize = 0, chunk segment size is default size.
@@ -929,14 +928,10 @@ func open(dir string, l *slog.Logger, r prometheus.Registerer, opts *Options, rn
929928
}
930929

931930
var wal, wbl *wlog.WL
932-
segmentSize := wlog.DefaultSegmentSize
931+
933932
// Wal is enabled.
934-
if opts.WALSegmentSize >= 0 {
935-
// Wal is set to a custom size.
936-
if opts.WALSegmentSize > 0 {
937-
segmentSize = opts.WALSegmentSize
938-
}
939-
wal, err = wlog.NewSize(l, r, walDir, segmentSize, opts.WALCompression)
933+
if !opts.DisableWAL {
934+
wal, err = wlog.New(l, r, walDir, opts.WALSegment)
940935
if err != nil {
941936
return nil, err
942937
}
@@ -946,7 +941,7 @@ func open(dir string, l *slog.Logger, r prometheus.Registerer, opts *Options, rn
946941
return nil, err
947942
}
948943
if opts.OutOfOrderTimeWindow > 0 || wblSize > 0 {
949-
wbl, err = wlog.NewSize(l, r, wblDir, segmentSize, opts.WALCompression)
944+
wbl, err = wlog.New(l, r, wblDir, opts.WALSegment)
950945
if err != nil {
951946
return nil, err
952947
}
@@ -1159,14 +1154,9 @@ func (db *DB) ApplyConfig(conf *config.Config) error {
11591154
case db.head.wbl != nil:
11601155
// The existing WBL from the disk might have been replayed while OOO was disabled.
11611156
wblog = db.head.wbl
1162-
case !db.oooWasEnabled.Load() && oooTimeWindow > 0 && db.opts.WALSegmentSize >= 0:
1163-
segmentSize := wlog.DefaultSegmentSize
1164-
// Wal is set to a custom size.
1165-
if db.opts.WALSegmentSize > 0 {
1166-
segmentSize = db.opts.WALSegmentSize
1167-
}
1157+
case !db.oooWasEnabled.Load() && oooTimeWindow > 0 && !db.opts.DisableWAL:
11681158
oooWalDir := filepath.Join(db.dir, wlog.WblDirName)
1169-
wblog, err = wlog.NewSize(db.logger, db.registerer, oooWalDir, segmentSize, db.opts.WALCompression)
1159+
wblog, err = wlog.New(db.logger, db.registerer, oooWalDir, db.opts.WALSegment)
11701160
if err != nil {
11711161
return err
11721162
}

0 commit comments

Comments
 (0)