bug fix and refactoring wrt bytes written stat computation

Author: Thejas-bhat

contentcoder.go

@@ -81,6 +81,7 @@ func newChunkedContentCoder(chunkSize uint64, maxDocNum uint64,
 // and re used. You cannot change the chunk size.
 func (c *chunkedContentCoder) Reset() {
 	c.currChunk = 0
+	c.bytesWritten = 0
 	c.final = c.final[:0]
 	c.chunkBuf.Reset()
 	c.chunkMetaBuf.Reset()

intcoder.go

@@ -58,6 +58,7 @@ func newChunkedIntCoder(chunkSize uint64, maxDocNum uint64) *chunkedIntCoder {
 // from previous use.  you cannot change the chunk size or max doc num.
 func (c *chunkedIntCoder) Reset() {
 	c.final = c.final[:0]
+	c.bytesWritten = 0
 	c.chunkBuf.Reset()
 	c.currChunk = 0
 	for i := range c.chunkLens {
@@ -106,7 +107,6 @@ func (c *chunkedIntCoder) Add(docNum uint64, vals ...uint64) error {
 		if err != nil {
 			return err
 		}
-		c.incrementBytesWritten(uint64(wb))
 	}
 
 	return nil
@@ -129,6 +129,7 @@ func (c *chunkedIntCoder) AddBytes(docNum uint64, buf []byte) error {
 // to be encoded.
 func (c *chunkedIntCoder) Close() {
 	encodingBytes := c.chunkBuf.Bytes()
+	c.incrementBytesWritten(uint64(len(encodingBytes)))
 	c.chunkLens[c.currChunk] = uint64(len(encodingBytes))
 	c.final = append(c.final, encodingBytes...)
 	c.currChunk = uint64(cap(c.chunkLens)) // sentinel to detect double close

new.go

@@ -78,10 +78,13 @@ func (*ZapPlugin) newWithChunkMode(results []index.Document,
 		s.FieldsMap, s.FieldsInv, uint64(len(results)),
 		storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets)
 
+	// get the bytes written before the interim's reset() call
+	// write it to the newly formed segment base.
+	totalBytesWritten := s.getBytesWritten()
 	if err == nil && s.reset() == nil {
 		s.lastNumDocs = len(results)
 		s.lastOutSize = len(br.Bytes())
-		sb.setBytesWritten(s.getBytesWritten())
+		sb.setBytesWritten(totalBytesWritten)
 		interimPool.Put(s)
 	}
 
@@ -192,6 +195,10 @@ func (s *interim) reset() (err error) {
 	s.lastNumDocs = 0
 	s.lastOutSize = 0
 
+	// reset the bytes written stat count
+	// to avoid leaking of bytesWritten across reuse cycles.
+	s.setBytesWritten(0)
+
 	return err
 }
 
@@ -699,7 +706,6 @@ func (s *interim) writeDicts() (fdvIndexOffset uint64, dictOffsets []uint64, err
 					if err != nil {
 						return 0, nil, err
 					}
-					prevBytesWritten := locEncoder.getBytesWritten()
 					for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] {
 						err = locEncoder.Add(docNum,
 							uint64(loc.fieldID), loc.pos, loc.start, loc.end,
@@ -713,9 +719,6 @@ func (s *interim) writeDicts() (fdvIndexOffset uint64, dictOffsets []uint64, err
 							return 0, nil, err
 						}
 					}
-					if locEncoder.getBytesWritten()-prevBytesWritten > 0 {
-						s.incrementBytesWritten(locEncoder.getBytesWritten() - prevBytesWritten)
-					}
 					locOffset += freqNorm.numLocs
 				}
 
@@ -728,6 +731,7 @@ func (s *interim) writeDicts() (fdvIndexOffset uint64, dictOffsets []uint64, err
 
 			tfEncoder.Close()
 			locEncoder.Close()
+			s.incrementBytesWritten(locEncoder.getBytesWritten())
 
 			postingsOffset, err :=
 				writePostings(postingsBS, tfEncoder, locEncoder, nil, s.w, buf)
@@ -801,7 +805,7 @@ func (s *interim) writeDicts() (fdvIndexOffset uint64, dictOffsets []uint64, err
 				return 0, nil, err
 			}
 
-			s.setBytesWritten(s.getBytesWritten())
+			s.incrementBytesWritten(fdvEncoder.getBytesWritten())
 
 			fdvOffsetsStart[fieldID] = uint64(s.w.Count())