diff --git a/pkg/meta/base.go b/pkg/meta/base.go index 1537e98c1640..0ca841a2b853 100644 --- a/pkg/meta/base.go +++ b/pkg/meta/base.go @@ -174,7 +174,7 @@ type baseMeta struct { usedInodesG prometheus.Gauge txDist prometheus.Histogram txRestart prometheus.Counter - opDist prometheus.Histogram + opDist *prometheus.HistogramVec en engine } @@ -214,11 +214,11 @@ func newBaseMeta(addr string, conf *Config) *baseMeta { Name: "transaction_restart", Help: "The number of times a transaction is restarted.", }), - opDist: prometheus.NewHistogram(prometheus.HistogramOpts{ + opDist: prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: "meta_ops_durations_histogram_seconds", Help: "Operation latency distributions.", Buckets: prometheus.ExponentialBuckets(0.0001, 1.5, 30), - }), + }, []string{"method"}), } } @@ -245,8 +245,8 @@ func (m *baseMeta) InitMetrics(reg prometheus.Registerer) { }() } -func (m *baseMeta) timeit(start time.Time) { - m.opDist.Observe(time.Since(start).Seconds()) +func (m *baseMeta) timeit(method string, start time.Time) { + m.opDist.WithLabelValues(method).Observe(time.Since(start).Seconds()) } func (m *baseMeta) getBase() *baseMeta { @@ -855,7 +855,7 @@ func (m *baseMeta) cleanupSlices() { } func (m *baseMeta) StatFS(ctx Context, ino Ino, totalspace, availspace, iused, iavail *uint64) syscall.Errno { - defer m.timeit(time.Now()) + defer m.timeit("StatFS", time.Now()) if st := m.statRootFs(ctx, totalspace, availspace, iused, iavail); st != 0 { return st } @@ -978,7 +978,7 @@ func (m *baseMeta) Lookup(ctx Context, parent Ino, name string, inode *Ino, attr if inode == nil || attr == nil { return syscall.EINVAL // bad request } - defer m.timeit(time.Now()) + defer m.timeit("Lookup", time.Now()) parent = m.checkRoot(parent) if name == ".." { if parent == m.root { @@ -1144,7 +1144,7 @@ func (m *baseMeta) GetAttr(ctx Context, inode Ino, attr *Attr) syscall.Errno { if m.conf.OpenCache > 0 && m.of.Check(inode, attr) { return 0 } - defer m.timeit(time.Now()) + defer m.timeit("GetAttr", time.Now()) var err syscall.Errno if inode == RootInode { e := utils.WithTimeout(func() error { @@ -1206,7 +1206,7 @@ func (m *baseMeta) Mknod(ctx Context, parent Ino, name string, _type uint8, mode return syscall.ENOENT } - defer m.timeit(time.Now()) + defer m.timeit("Mknod", time.Now()) parent = m.checkRoot(parent) var space, inodes int64 = align4K(0), 1 if err := m.checkQuota(ctx, space, inodes, parent); err != 0 { @@ -1264,7 +1264,7 @@ func (m *baseMeta) Link(ctx Context, inode, parent Ino, name string, attr *Attr) return syscall.ENOENT } - defer m.timeit(time.Now()) + defer m.timeit("Link", time.Now()) if attr == nil { attr = &Attr{} } @@ -1290,7 +1290,7 @@ func (m *baseMeta) ReadLink(ctx Context, inode Ino, path *[]byte) syscall.Errno *path = target.([]byte) return 0 } - defer m.timeit(time.Now()) + defer m.timeit("ReadLink", time.Now()) target, err := m.en.doReadlink(ctx, inode) if err != nil { return errno(err) @@ -1311,7 +1311,7 @@ func (m *baseMeta) Unlink(ctx Context, parent Ino, name string, skipCheckTrash . return syscall.EROFS } - defer m.timeit(time.Now()) + defer m.timeit("Unlink", time.Now()) parent = m.checkRoot(parent) var attr Attr err := m.en.doUnlink(ctx, parent, name, &attr, skipCheckTrash...) @@ -1340,7 +1340,7 @@ func (m *baseMeta) Rmdir(ctx Context, parent Ino, name string, skipCheckTrash .. return syscall.EROFS } - defer m.timeit(time.Now()) + defer m.timeit("Rmdir", time.Now()) parent = m.checkRoot(parent) var inode Ino st := m.en.doRmdir(ctx, parent, name, &inode, skipCheckTrash...) @@ -1377,7 +1377,7 @@ func (m *baseMeta) Rename(ctx Context, parentSrc Ino, nameSrc string, parentDst return syscall.EINVAL } - defer m.timeit(time.Now()) + defer m.timeit("Rename", time.Now()) if inode == nil { inode = new(Ino) } @@ -1534,7 +1534,7 @@ func (m *baseMeta) Readdir(ctx Context, inode Ino, plus uint8, entries *[]*Entry if err := m.GetAttr(ctx, inode, &attr); err != 0 { return err } - defer m.timeit(time.Now()) + defer m.timeit("Readdir", time.Now()) if inode == m.root { attr.Parent = m.root } @@ -1561,7 +1561,7 @@ func (m *baseMeta) SetXattr(ctx Context, inode Ino, name string, value []byte, f return syscall.EINVAL } - defer m.timeit(time.Now()) + defer m.timeit("SetXattr", time.Now()) return m.en.doSetXattr(ctx, m.checkRoot(inode), name, value, flags) } @@ -1573,7 +1573,7 @@ func (m *baseMeta) RemoveXattr(ctx Context, inode Ino, name string) syscall.Errn return syscall.EINVAL } - defer m.timeit(time.Now()) + defer m.timeit("RemoveXattr", time.Now()) return m.en.doRemoveXattr(ctx, m.checkRoot(inode), name) } @@ -2221,7 +2221,7 @@ func (m *baseMeta) Clone(ctx Context, srcIno, parent Ino, name string, cmode uin return syscall.ENOENT } - defer m.timeit(time.Now()) + defer m.timeit("Clone", time.Now()) parent = m.checkRoot(parent) var attr Attr diff --git a/pkg/meta/redis.go b/pkg/meta/redis.go index f669ba30419a..78fbc3d696de 100644 --- a/pkg/meta/redis.go +++ b/pkg/meta/redis.go @@ -755,7 +755,7 @@ func (m *redisMeta) Resolve(ctx Context, parent Ino, path string, inode *Ino, at if len(m.shaResolve) == 0 || m.conf.CaseInsensi || m.prefix != "" { return syscall.ENOTSUP } - defer m.timeit(time.Now()) + defer m.timeit("Resolve", time.Now()) parent = m.checkRoot(parent) args := []string{parent.String(), path, strconv.FormatUint(uint64(ctx.Uid()), 10), @@ -878,7 +878,7 @@ func (m *redisMeta) txn(ctx Context, txf func(tx *redis.Tx) error, keys ...strin } func (m *redisMeta) Truncate(ctx Context, inode Ino, flags uint8, length uint64, attr *Attr) syscall.Errno { - defer m.timeit(time.Now()) + defer m.timeit("Truncate", time.Now()) f := m.of.find(inode) if f != nil { f.Lock() @@ -998,7 +998,7 @@ func (m *redisMeta) Fallocate(ctx Context, inode Ino, mode uint8, off uint64, si if size == 0 { return syscall.EINVAL } - defer m.timeit(time.Now()) + defer m.timeit("Fallocate", time.Now()) f := m.of.find(inode) if f != nil { f.Lock() @@ -1077,7 +1077,7 @@ func (m *redisMeta) Fallocate(ctx Context, inode Ino, mode uint8, off uint64, si } func (m *redisMeta) SetAttr(ctx Context, inode Ino, set uint16, sugidclearmode uint8, attr *Attr) syscall.Errno { - defer m.timeit(time.Now()) + defer m.timeit("SetAttr", time.Now()) inode = m.checkRoot(inode) defer func() { m.of.InvalidateChunk(inode, invalidateAttrOnly) }() return errno(m.txn(ctx, func(tx *redis.Tx) error { @@ -2111,7 +2111,7 @@ func (m *redisMeta) Read(ctx Context, inode Ino, indx uint32, slices *[]Slice) s *slices = ss return 0 } - defer m.timeit(time.Now()) + defer m.timeit("Read", time.Now()) vals, err := m.rdb.LRange(ctx, m.chunkKey(inode, indx), 0, -1).Result() if err != nil { return errno(err) @@ -2129,7 +2129,7 @@ func (m *redisMeta) Read(ctx Context, inode Ino, indx uint32, slices *[]Slice) s } func (m *redisMeta) Write(ctx Context, inode Ino, indx uint32, off uint32, slice Slice) syscall.Errno { - defer m.timeit(time.Now()) + defer m.timeit("Write", time.Now()) f := m.of.find(inode) if f != nil { f.Lock() @@ -2191,7 +2191,7 @@ func (m *redisMeta) Write(ctx Context, inode Ino, indx uint32, off uint32, slice } func (m *redisMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, offOut uint64, size uint64, flags uint32, copied *uint64) syscall.Errno { - defer m.timeit(time.Now()) + defer m.timeit("CopyFileRange", time.Now()) f := m.of.find(fout) if f != nil { f.Lock() @@ -3281,7 +3281,7 @@ func (m *redisMeta) doRepair(ctx Context, inode Ino, attr *Attr) syscall.Errno { } func (m *redisMeta) GetXattr(ctx Context, inode Ino, name string, vbuff *[]byte) syscall.Errno { - defer m.timeit(time.Now()) + defer m.timeit("GetXattr", time.Now()) inode = m.checkRoot(inode) var err error *vbuff, err = m.rdb.HGet(ctx, m.xattrKey(inode), name).Bytes() @@ -3292,7 +3292,7 @@ func (m *redisMeta) GetXattr(ctx Context, inode Ino, name string, vbuff *[]byte) } func (m *redisMeta) ListXattr(ctx Context, inode Ino, names *[]byte) syscall.Errno { - defer m.timeit(time.Now()) + defer m.timeit("ListXattr", time.Now()) inode = m.checkRoot(inode) vals, err := m.rdb.HKeys(ctx, m.xattrKey(inode)).Result() if err != nil { diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go index dfdc6362eacf..b6a9847e6c96 100644 --- a/pkg/meta/sql.go +++ b/pkg/meta/sql.go @@ -867,7 +867,7 @@ func clearSUGIDSQL(ctx Context, cur *node, set *Attr) { } func (m *dbMeta) SetAttr(ctx Context, inode Ino, set uint16, sugidclearmode uint8, attr *Attr) syscall.Errno { - defer m.timeit(time.Now()) + defer m.timeit("SetAttr", time.Now()) inode = m.checkRoot(inode) defer func() { m.of.InvalidateChunk(inode, invalidateAttrOnly) }() return errno(m.txn(func(s *xorm.Session) error { @@ -958,7 +958,7 @@ func (m *dbMeta) appendSlice(s *xorm.Session, inode Ino, indx uint32, buf []byte } func (m *dbMeta) Truncate(ctx Context, inode Ino, flags uint8, length uint64, attr *Attr) syscall.Errno { - defer m.timeit(time.Now()) + defer m.timeit("Truncate", time.Now()) f := m.of.find(inode) if f != nil { f.Lock() @@ -1051,7 +1051,7 @@ func (m *dbMeta) Fallocate(ctx Context, inode Ino, mode uint8, off uint64, size if size == 0 { return syscall.EINVAL } - defer m.timeit(time.Now()) + defer m.timeit("Fallocate", time.Now()) f := m.of.find(inode) if f != nil { f.Lock() @@ -2060,7 +2060,7 @@ func (m *dbMeta) Read(ctx Context, inode Ino, indx uint32, slices *[]Slice) sysc *slices = ss return 0 } - defer m.timeit(time.Now()) + defer m.timeit("Read", time.Now()) var c = chunk{Inode: inode, Indx: indx} err := m.roTxn(func(s *xorm.Session) error { _, err := s.MustCols("indx").Get(&c) @@ -2082,7 +2082,7 @@ func (m *dbMeta) Read(ctx Context, inode Ino, indx uint32, slices *[]Slice) sysc } func (m *dbMeta) Write(ctx Context, inode Ino, indx uint32, off uint32, slice Slice) syscall.Errno { - defer m.timeit(time.Now()) + defer m.timeit("Write", time.Now()) f := m.of.find(inode) if f != nil { f.Lock() @@ -2152,7 +2152,7 @@ func (m *dbMeta) Write(ctx Context, inode Ino, indx uint32, off uint32, slice Sl } func (m *dbMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, offOut uint64, size uint64, flags uint32, copied *uint64) syscall.Errno { - defer m.timeit(time.Now()) + defer m.timeit("CopyFileRange", time.Now()) f := m.of.find(fout) if f != nil { f.Lock() @@ -2962,7 +2962,7 @@ func (m *dbMeta) doRepair(ctx Context, inode Ino, attr *Attr) syscall.Errno { } func (m *dbMeta) GetXattr(ctx Context, inode Ino, name string, vbuff *[]byte) syscall.Errno { - defer m.timeit(time.Now()) + defer m.timeit("GetXattr", time.Now()) inode = m.checkRoot(inode) return errno(m.roTxn(func(s *xorm.Session) error { var x = xattr{Inode: inode, Name: name} @@ -2979,7 +2979,7 @@ func (m *dbMeta) GetXattr(ctx Context, inode Ino, name string, vbuff *[]byte) sy } func (m *dbMeta) ListXattr(ctx Context, inode Ino, names *[]byte) syscall.Errno { - defer m.timeit(time.Now()) + defer m.timeit("ListXattr", time.Now()) inode = m.checkRoot(inode) return errno(m.roTxn(func(s *xorm.Session) error { var xs []xattr diff --git a/pkg/meta/tkv.go b/pkg/meta/tkv.go index b1a218a40a9c..f4e8cde9bef4 100644 --- a/pkg/meta/tkv.go +++ b/pkg/meta/tkv.go @@ -843,7 +843,7 @@ func (m *kvMeta) doGetAttr(ctx Context, inode Ino, attr *Attr) syscall.Errno { } func (m *kvMeta) SetAttr(ctx Context, inode Ino, set uint16, sugidclearmode uint8, attr *Attr) syscall.Errno { - defer m.timeit(time.Now()) + defer m.timeit("SetAttr", time.Now()) inode = m.checkRoot(inode) defer func() { m.of.InvalidateChunk(inode, invalidateAttrOnly) }() return errno(m.txn(func(tx *kvTxn) error { @@ -918,7 +918,7 @@ func (m *kvMeta) SetAttr(ctx Context, inode Ino, set uint16, sugidclearmode uint } func (m *kvMeta) Truncate(ctx Context, inode Ino, flags uint8, length uint64, attr *Attr) syscall.Errno { - defer m.timeit(time.Now()) + defer m.timeit("Truncate", time.Now()) f := m.of.find(inode) if f != nil { f.Lock() @@ -1003,7 +1003,7 @@ func (m *kvMeta) Fallocate(ctx Context, inode Ino, mode uint8, off uint64, size if size == 0 { return syscall.EINVAL } - defer m.timeit(time.Now()) + defer m.timeit("Fallocate", time.Now()) f := m.of.find(inode) if f != nil { f.Lock() @@ -1814,7 +1814,7 @@ func (m *kvMeta) Read(ctx Context, inode Ino, indx uint32, slices *[]Slice) sysc *slices = ss return 0 } - defer m.timeit(time.Now()) + defer m.timeit("Read", time.Now()) val, err := m.get(m.chunkKey(inode, indx)) if err != nil { return errno(err) @@ -1832,7 +1832,7 @@ func (m *kvMeta) Read(ctx Context, inode Ino, indx uint32, slices *[]Slice) sysc } func (m *kvMeta) Write(ctx Context, inode Ino, indx uint32, off uint32, slice Slice) syscall.Errno { - defer m.timeit(time.Now()) + defer m.timeit("Write", time.Now()) f := m.of.find(inode) if f != nil { f.Lock() @@ -1882,7 +1882,7 @@ func (m *kvMeta) Write(ctx Context, inode Ino, indx uint32, off uint32, slice Sl } func (m *kvMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, offOut uint64, size uint64, flags uint32, copied *uint64) syscall.Errno { - defer m.timeit(time.Now()) + defer m.timeit("CopyFileRange", time.Now()) var newLength, newSpace int64 f := m.of.find(fout) if f != nil { @@ -2588,7 +2588,7 @@ func (m *kvMeta) doRepair(ctx Context, inode Ino, attr *Attr) syscall.Errno { } func (m *kvMeta) GetXattr(ctx Context, inode Ino, name string, vbuff *[]byte) syscall.Errno { - defer m.timeit(time.Now()) + defer m.timeit("GetXattr", time.Now()) inode = m.checkRoot(inode) buf, err := m.get(m.xattrKey(inode, name)) if err != nil { @@ -2602,7 +2602,7 @@ func (m *kvMeta) GetXattr(ctx Context, inode Ino, name string, vbuff *[]byte) sy } func (m *kvMeta) ListXattr(ctx Context, inode Ino, names *[]byte) syscall.Errno { - defer m.timeit(time.Now()) + defer m.timeit("ListXattr", time.Now()) inode = m.checkRoot(inode) keys, err := m.scanKeys(m.xattrKey(inode, "")) if err != nil { diff --git a/pkg/vfs/internal.go b/pkg/vfs/internal.go index f21cea98870f..5722e11185a5 100644 --- a/pkg/vfs/internal.go +++ b/pkg/vfs/internal.go @@ -163,8 +163,14 @@ func collectMetrics(registry *prometheus.Registry) []byte { return strconv.FormatFloat(v, 'f', -1, 64) } for _, mf := range mfs { + var name = *mf.Name + if name == "juicefs_meta_ops_durations_histogram_seconds" && *mf.Type == io_prometheus_client.MetricType_HISTOGRAM { + total, sum := mergeHistogramMetrics(mf) + _, _ = fmt.Fprintf(w, "%s_total %d\n", name, total) + _, _ = fmt.Fprintf(w, "%s_sum %s\n", name, format(sum)) + continue + } for _, m := range mf.Metric { - var name string = *mf.Name for _, l := range m.Label { if *l.Name != "mp" && *l.Name != "vol_name" { name += "_" + *l.Value @@ -185,6 +191,16 @@ func collectMetrics(registry *prometheus.Registry) []byte { return w.Bytes() } +func mergeHistogramMetrics(mf *io_prometheus_client.MetricFamily) (uint64, float64) { + var total uint64 + var sum float64 + for _, metric := range mf.Metric { + total += metric.Histogram.GetSampleCount() + sum += metric.Histogram.GetSampleSum() + } + return total, sum +} + func writeProgress(item1, item2 *uint64, out io.Writer, done chan struct{}) { wb := utils.NewBuffer(17) wb.Put8(meta.CPROGRESS)