diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go
index 3aa821c31..ca3143535 100644
--- a/pkg/sfu/downtrack.go
+++ b/pkg/sfu/downtrack.go
@@ -45,7 +45,6 @@ type DownTrack struct {
 
 	enabled  atomicBool
 	reSync   atomicBool
-	reBaseTs atomicBool
 	snOffset uint16
 	tsOffset uint32
 	lastSSRC uint32
@@ -95,7 +94,6 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters,
 		d.writeStream = t.WriteStream()
 		d.mime = strings.ToLower(codec.MimeType)
 		d.reSync.set(true)
-		d.reBaseTs.set(true)
 		d.enabled.set(true)
 		if rr := d.bufferFactory.GetOrNew(packetio.RTCPBufferPacket, uint32(t.SSRC())).(*buffer.RTCPReader); rr != nil {
 			rr.OnPacket(func(pkt []byte) {
@@ -346,11 +344,10 @@ func (d *DownTrack) writeSimpleRTP(extPkt *buffer.ExtPacket) error {
 				return nil
 			}
 		}
-		if d.reBaseTs.get() {
-			d.snOffset = extPkt.Packet.SequenceNumber - d.lastSN - 1
-			d.tsOffset = extPkt.Packet.Timestamp - d.lastTS - 1
-			d.reBaseTs.set(false)
-		}
+
+		d.snOffset = extPkt.Packet.SequenceNumber - d.lastSN - 1
+		d.tsOffset = extPkt.Packet.Timestamp - d.lastTS - 1
+
 		atomic.StoreUint32(&d.lastSSRC, extPkt.Packet.SSRC)
 		d.reSync.set(false)
 	}