Skip to content

[CORE-12378] fix(QoS): Use QdiscReplace() instead of QdiscAdd() #11899

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
coutinhop merged 1 commit into from
Mar 2, 2026
Merged

[CORE-12378] fix(QoS): Use QdiscReplace() instead of QdiscAdd() #11899

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 6 additions & 30 deletions felix/dataplane/linux/qos/qos.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,20 +242,12 @@ func TeardownIfb(deviceName string) error {
return nil
}

func CreateIngressQdisc(tbs *TokenBucketState, workloadDeviceName string) error {
func CreateOrUpdateIngressQdisc(tbs *TokenBucketState, workloadDeviceName string) error {
workloadDevice, err := netlink.LinkByName(workloadDeviceName)
if err != nil {
return fmt.Errorf("get host device %s: %w", workloadDeviceName, err)
}
return createTBF(tbs, workloadDevice)
}

func UpdateIngressQdisc(tbs *TokenBucketState, workloadDeviceName string) error {
workloadDevice, err := netlink.LinkByName(workloadDeviceName)
if err != nil {
return fmt.Errorf("get host device %s: %w", workloadDeviceName, err)
}
return updateTBF(tbs, workloadDevice)
return createOrUpdateTBF(tbs, workloadDevice)
}

func CreateEgressQdisc(tbs *TokenBucketState, workloadDeviceName string, ifbDeviceName string) error {
Expand Down Expand Up @@ -293,7 +285,7 @@ func CreateEgressQdisc(tbs *TokenBucketState, workloadDeviceName string, ifbDevi
},
}

err = netlink.QdiscAdd(qdisc)
err = netlink.QdiscReplace(qdisc)
if err != nil {
return fmt.Errorf("create ingress qdisc %+v on dev %s: %w", qdisc, workloadDeviceName, err)
}
Expand Down Expand Up @@ -363,7 +355,7 @@ func CreateEgressQdisc(tbs *TokenBucketState, workloadDeviceName string, ifbDevi
}

// throttle traffic on ifb device
err = createTBF(tbs, ifbDevice)
err = createOrUpdateTBF(tbs, ifbDevice)
if err != nil {
return fmt.Errorf("create ifb qdisc on dev %s: %w", ifbDeviceName, err)
}
Expand All @@ -375,7 +367,7 @@ func UpdateEgressQdisc(tbs *TokenBucketState, ifbDeviceName string) error {
if err != nil {
return fmt.Errorf("get ifb device %s: %w", ifbDeviceName, err)
}
return updateTBF(tbs, ifbDevice)
return createOrUpdateTBF(tbs, ifbDevice)
}

func GetTBFValues(rateBitsPerSec, burstBits, peakrateBitsPerSec uint64, minburstBytes uint32) *TokenBucketState {
Expand Down Expand Up @@ -454,23 +446,7 @@ func makeTBF(tbs *TokenBucketState, workloadDevice netlink.Link) (*netlink.Tbf,
return qdisc, nil
}

func createTBF(tbs *TokenBucketState, workloadDevice netlink.Link) error {
// Equivalent to
// tc qdisc add dev link root tbf

qdisc, err := makeTBF(tbs, workloadDevice)
if err != nil {
return fmt.Errorf("make TBF qdisc %+v from tbs %+v: %w", qdisc, tbs, err)
}

err = netlink.QdiscAdd(qdisc)
if err != nil {
return fmt.Errorf("add TBF qdisc %+v, limit: %v, rate %v, buffer %v, peakrate %v, minburst %v: %w", qdisc, qdisc.Limit, qdisc.Rate, qdisc.Buffer, qdisc.Peakrate, qdisc.Minburst, err)
}
return nil
}

func updateTBF(tbs *TokenBucketState, workloadDevice netlink.Link) error {
func createOrUpdateTBF(tbs *TokenBucketState, workloadDevice netlink.Link) error {
// Equivalent to
// tc qdisc replace dev link root tbf

Expand Down
4 changes: 2 additions & 2 deletions felix/dataplane/linux/qos_controls.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ func (m *endpointManager) maybeUpdateQoSBandwidth(old, new *proto.WorkloadEndpoi

if currentIngress == nil && desiredIngress != nil {
// Add.
err := qos.CreateIngressQdisc(desiredIngress, newName)
err := qos.CreateOrUpdateIngressQdisc(desiredIngress, newName)
if err != nil {
errs = append(errs, fmt.Errorf("error adding ingress qdisc to workload %s: %w", newName, err))
}
Expand All @@ -104,7 +104,7 @@ func (m *endpointManager) maybeUpdateQoSBandwidth(old, new *proto.WorkloadEndpoi
}
} else if !currentIngress.Equals(desiredIngress) {
// Update.
err := qos.UpdateIngressQdisc(desiredIngress, newName)
err := qos.CreateOrUpdateIngressQdisc(desiredIngress, newName)
if err != nil {
errs = append(errs, fmt.Errorf("error changing ingress qdisc on workload %s: %w", newName, err))
}
Expand Down
35 changes: 31 additions & 4 deletions felix/fv/qos_controls_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,10 +205,6 @@ var _ = infrastructure.DatastoreDescribe(
infra = getInfra(infrastructure.WithBPFLogByteLimit(16 * 1024 * 1024))
topt = infrastructure.DefaultTopologyOptions()

if bpfLogLevel != "Debug" && !BPFMode() {
Skip("Skipping QoS control tests with non-debug bpfLogLevel on iptables/nftables mode (for deduplication).")
}

switch encap {
case "none":
if !BPFMode() {
Expand All @@ -230,6 +226,7 @@ var _ = infrastructure.DatastoreDescribe(

topt.DelayFelixStart = true
topt.TriggerDelayedFelixStart = true
topt.FelixLogSeverity = "Debug"

@nelljerram nelljerram Feb 26, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you mean to check this change in?

@coutinhop coutinhop Feb 26, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, absolutely! the issue we're fixing only shows up in the debug logs, so I think it makes sense to have them in this test

@nelljerram nelljerram Mar 2, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess it's OK to leave this in. But you only mean for debugging the problem, right? I.e. it is not the case that there was only an issue when log level was debug.

if BPFMode() {
topt.ExtraEnvVars["FELIX_BPFLogLevel"] = bpfLogLevel
}
Expand Down Expand Up @@ -320,7 +317,15 @@ var _ = infrastructure.DatastoreDescribe(
if BPFMode() && BPFAttachType() == "tc" {
Skip("Skipping QoS control bandwidth tests on BPF TC attach mode.")
}

By("Removing all limits from workloads")
for i := range len(w) {
w[i].WorkloadEndpoint.Spec.QoSControls = nil
w[i].UpdateInInfra(infra)
Eventually(tc.Felixes[i].ExecOutputFn("ip", "r", "get", fmt.Sprintf("10.65.%d.2", i)), "10s").Should(ContainSubstring(w[i].InterfaceName))

@nelljerram nelljerram Feb 26, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this check effective? I presume the route was already there even with the QoS in place?

@coutinhop coutinhop Feb 26, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe I arrived at this because the route churns when the workload endpoint is updated (if my memory doesn't fail me), so I added a wait until it shows up again...

@nelljerram nelljerram Mar 2, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, that sounds surprising. I would expect TC updates to be independent of route programming, and that a TC update would not require a route change. Might be a real problem hiding here if you really saw route churn.

@coutinhop coutinhop Mar 2, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think it's related to tc, but the FV test infra, or maybe I was too zealous when writing those into every workload update, I'll merge this PR and investigate separately

}
})

getQdisc := func() string {
out, err := tc.Felixes[1].ExecOutput("tc", "qdisc")
logrus.Infof("tc qdisc output:\n%v", out)
Expand Down Expand Up @@ -407,7 +412,29 @@ var _ = infrastructure.DatastoreDescribe(
Expect(err).NotTo(HaveOccurred())
err = serverCmd.Process.Release()
Expect(err).NotTo(HaveOccurred())
})

It("should correctly apply bandwidth limits when a non-default qdisc exists (handle != 0)", func() {
By("Replacing the default noqueue qdisc with a non-default qdisc (handle 8001)")
out, err := tc.Felixes[1].ExecOutput("tc", "qdisc", "replace", "dev", w[1].InterfaceName, "root", "handle", "8001:", "noqueue")
logrus.Infof("tc qdisc replace output:\n%v", out)
Expect(err).NotTo(HaveOccurred())

By("Waiting for the config to appear in 'tc qdisc'")
Eventually(getQdisc, "10s", "1s").Should(MatchRegexp(`qdisc noqueue 8001: dev ` + regexp.QuoteMeta(w[1].InterfaceName) + ` root refcnt \d+`))

By("Setting 10Mbps limit and 100Mbps peakrate for ingress on workload 1")
w[1].WorkloadEndpoint.Spec.QoSControls = &internalapi.QoSControls{
IngressBandwidth: 10000000,
IngressBurst: 300000000,
IngressPeakrate: 100000000,
}
w[1].UpdateInInfra(infra)
Eventually(tc.Felixes[1].ExecOutputFn("ip", "r", "get", "10.65.1.2"), "10s").Should(ContainSubstring(w[1].InterfaceName))

By("Waiting for the config to appear in 'tc qdisc'")
// ingress config should be present
Eventually(getQdisc, "10s", "1s").Should(MatchRegexp(`qdisc tbf \d+: dev ` + regexp.QuoteMeta(w[1].InterfaceName) + ` root refcnt \d+ rate ` + regexp.QuoteMeta("10Mbit") + `.* peakrate ` + regexp.QuoteMeta("100Mbit")))
})
})

Expand Down