From 844e94d6ca6d78811fbb07cdf5e8fdf66c92fce1 Mon Sep 17 00:00:00 2001 From: Toby Date: Fri, 29 Sep 2023 22:18:24 -0700 Subject: [PATCH] fix: reworked BBR to replace the broken old one --- app/go.mod | 5 +- app/go.sum | 19 +- core/go.mod | 7 +- core/go.sum | 10 +- core/internal/congestion/bbr/bandwidth.go | 6 +- .../congestion/bbr/bandwidth_sampler.go | 814 +++++++++--- core/internal/congestion/bbr/bbr_sender.go | 1152 ++++++++--------- .../bbr/packet_number_indexed_queue.go | 199 +++ core/internal/congestion/bbr/ringbuffer.go | 118 ++ .../congestion/bbr/windowed_filter.go | 172 ++- core/internal/congestion/brutal/brutal.go | 6 +- core/internal/congestion/common/pacer.go | 11 +- core/internal/congestion/utils.go | 5 +- extras/go.mod | 5 +- extras/go.sum | 16 +- go.work | 2 +- 16 files changed, 1668 insertions(+), 879 deletions(-) create mode 100644 core/internal/congestion/bbr/packet_number_indexed_queue.go create mode 100644 core/internal/congestion/bbr/ringbuffer.go diff --git a/app/go.mod b/app/go.mod index 2179881b1d..c7a1043b9b 100644 --- a/app/go.mod +++ b/app/go.mod @@ -1,6 +1,6 @@ module github.com/apernet/hysteria/app -go 1.20 +go 1.21 require ( github.com/apernet/go-tproxy v0.0.0-20230809025308-8f4723fd742f @@ -17,7 +17,7 @@ require ( ) require ( - github.com/apernet/quic-go v0.39.1-0.20230924223134-79ed77c4df4f // indirect + github.com/apernet/quic-go v0.39.1-0.20230930045547-13cecb45baa8 // indirect github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect @@ -46,7 +46,6 @@ require ( github.com/stretchr/objx v0.5.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/txthinking/runnergroup v0.0.0-20210608031112-152c7c4432bf // indirect - github.com/zhangyunhao116/fastrand v0.3.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/mock v0.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect diff --git a/app/go.sum b/app/go.sum index 682ba419b8..d9ec22dd43 100644 --- a/app/go.sum +++ b/app/go.sum @@ -40,8 +40,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/apernet/go-tproxy v0.0.0-20230809025308-8f4723fd742f h1:uVh0qpEslrWjgzx9vOcyCqsOY3c9kofDZ1n+qaw35ZY= github.com/apernet/go-tproxy v0.0.0-20230809025308-8f4723fd742f/go.mod h1:xkkq9D4ygcldQQhKS/w9CadiCKwCngU7K9E3DaKahpM= -github.com/apernet/quic-go v0.39.1-0.20230924223134-79ed77c4df4f h1:NF2a/mJPckBDe0urX7oSwjpcXf3mGQUlH2373zoM8fk= -github.com/apernet/quic-go v0.39.1-0.20230924223134-79ed77c4df4f/go.mod h1:UwsoszQlzTm+dBDuFEwWBYt46K56WqlFEN0RWLvQ0rE= +github.com/apernet/quic-go v0.39.1-0.20230930045547-13cecb45baa8 h1:jZKdY1/SFnt6iRzSBMzaeXmgWKwSsAW2slf/o0+b3bs= +github.com/apernet/quic-go v0.39.1-0.20230930045547-13cecb45baa8/go.mod h1:UwsoszQlzTm+dBDuFEwWBYt46K56WqlFEN0RWLvQ0rE= github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 h1:4NNbNM2Iq/k57qEu7WfL67UrbPq1uFWxW4qODCohi+0= github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6/go.mod h1:J29hk+f9lJrblVIfiJOtTFk+OblBawmib4uz/VdKzlg= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -67,12 +67,14 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -101,6 +103,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -113,6 +116,7 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -151,9 +155,11 @@ github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8t github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/libdns/libdns v0.2.1 h1:Wu59T7wSHRgtA0cfxC+n1c/e+O3upJGWytknkmFEDis= github.com/libdns/libdns v0.2.1/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -168,9 +174,11 @@ github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/oschwald/geoip2-golang v1.9.0 h1:uvD3O6fXAXs+usU+UGExshpdP13GAqp4GBrzN7IgKZc= github.com/oschwald/geoip2-golang v1.9.0/go.mod h1:BHK6TvDyATVQhKNbQBdrj9eAvuwOMi2zSFXizL3K81Y= github.com/oschwald/maxminddb-golang v1.11.0 h1:aSXMqYR/EPNjGE8epgqwDay+P30hCBZIveY0WZbAWh0= @@ -192,6 +200,7 @@ github.com/quic-go/qtls-go1-20 v0.3.4 h1:MfFAPULvst4yoMgY9QmtpYmfij/em7O8UUi+bNV github.com/quic-go/qtls-go1-20 v0.3.4/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= @@ -232,8 +241,6 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zhangyunhao116/fastrand v0.3.0 h1:7bwe124xcckPulX6fxtr2lFdO2KQqaefdtbk+mqO/Ig= -github.com/zhangyunhao116/fastrand v0.3.0/go.mod h1:0v5KgHho0VE6HU192HnY15de/oDS8UrbBChIFjIhBtc= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -245,6 +252,7 @@ go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo= go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= @@ -363,6 +371,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -567,9 +576,11 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/core/go.mod b/core/go.mod index 00f9feffe8..370ef373aa 100644 --- a/core/go.mod +++ b/core/go.mod @@ -1,12 +1,12 @@ module github.com/apernet/hysteria/core -go 1.20 +go 1.21 require ( - github.com/apernet/quic-go v0.39.1-0.20230924223134-79ed77c4df4f + github.com/apernet/quic-go v0.39.1-0.20230930045547-13cecb45baa8 github.com/stretchr/testify v1.8.4 - github.com/zhangyunhao116/fastrand v0.3.0 go.uber.org/goleak v1.2.1 + golang.org/x/exp v0.0.0-20221205204356-47842c84f3db golang.org/x/time v0.3.0 ) @@ -23,7 +23,6 @@ require ( github.com/stretchr/objx v0.5.0 // indirect go.uber.org/mock v0.3.0 // indirect golang.org/x/crypto v0.11.0 // indirect - golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/net v0.12.0 // indirect golang.org/x/sys v0.10.0 // indirect diff --git a/core/go.sum b/core/go.sum index 198dd37212..a8dfbb7ca7 100644 --- a/core/go.sum +++ b/core/go.sum @@ -1,5 +1,5 @@ -github.com/apernet/quic-go v0.39.1-0.20230924223134-79ed77c4df4f h1:NF2a/mJPckBDe0urX7oSwjpcXf3mGQUlH2373zoM8fk= -github.com/apernet/quic-go v0.39.1-0.20230924223134-79ed77c4df4f/go.mod h1:UwsoszQlzTm+dBDuFEwWBYt46K56WqlFEN0RWLvQ0rE= +github.com/apernet/quic-go v0.39.1-0.20230930045547-13cecb45baa8 h1:jZKdY1/SFnt6iRzSBMzaeXmgWKwSsAW2slf/o0+b3bs= +github.com/apernet/quic-go v0.39.1-0.20230930045547-13cecb45baa8/go.mod h1:UwsoszQlzTm+dBDuFEwWBYt46K56WqlFEN0RWLvQ0rE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -8,12 +8,15 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -26,6 +29,7 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= @@ -41,8 +45,6 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/zhangyunhao116/fastrand v0.3.0 h1:7bwe124xcckPulX6fxtr2lFdO2KQqaefdtbk+mqO/Ig= -github.com/zhangyunhao116/fastrand v0.3.0/go.mod h1:0v5KgHho0VE6HU192HnY15de/oDS8UrbBChIFjIhBtc= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo= diff --git a/core/internal/congestion/bbr/bandwidth.go b/core/internal/congestion/bbr/bandwidth.go index 4f0b75ff10..52deb24965 100644 --- a/core/internal/congestion/bbr/bandwidth.go +++ b/core/internal/congestion/bbr/bandwidth.go @@ -7,11 +7,13 @@ import ( "github.com/apernet/quic-go/congestion" ) +const ( + infBandwidth = Bandwidth(math.MaxUint64) +) + // Bandwidth of a connection type Bandwidth uint64 -const infBandwidth Bandwidth = math.MaxUint64 - const ( // BitsPerSecond is 1 bit per second BitsPerSecond Bandwidth = 1 diff --git a/core/internal/congestion/bbr/bandwidth_sampler.go b/core/internal/congestion/bbr/bandwidth_sampler.go index c754f58761..0e770f1e4a 100644 --- a/core/internal/congestion/bbr/bandwidth_sampler.go +++ b/core/internal/congestion/bbr/bandwidth_sampler.go @@ -7,11 +7,17 @@ import ( "github.com/apernet/quic-go/congestion" ) -var InfiniteBandwidth = Bandwidth(math.MaxUint64) +const ( + infRTT = time.Duration(math.MaxInt64) + defaultConnectionStateMapQueueSize = 256 + defaultCandidatesBufferSize = 256 +) + +type roundTripCount uint64 // SendTimeState is a subset of ConnectionStateOnSentPacket which is returned // to the caller when the packet is acked or lost. -type SendTimeState struct { +type sendTimeState struct { // Whether other states in this object is valid. isValid bool // Whether the sender is app limited at the time the packet was sent. @@ -25,16 +31,260 @@ type SendTimeState struct { totalBytesAcked congestion.ByteCount // Total number of lost bytes at the time the packet was sent. totalBytesLost congestion.ByteCount + // Total number of inflight bytes at the time the packet was sent. + // Includes the packet itself. + // It should be equal to |total_bytes_sent| minus the sum of + // |total_bytes_acked|, |total_bytes_lost| and total neutered bytes. + bytesInFlight congestion.ByteCount +} + +func newSendTimeState( + isAppLimited bool, + totalBytesSent congestion.ByteCount, + totalBytesAcked congestion.ByteCount, + totalBytesLost congestion.ByteCount, + bytesInFlight congestion.ByteCount, +) *sendTimeState { + return &sendTimeState{ + isValid: true, + isAppLimited: isAppLimited, + totalBytesSent: totalBytesSent, + totalBytesAcked: totalBytesAcked, + totalBytesLost: totalBytesLost, + bytesInFlight: bytesInFlight, + } +} + +type extraAckedEvent struct { + // The excess bytes acknowlwedged in the time delta for this event. + extraAcked congestion.ByteCount + + // The bytes acknowledged and time delta from the event. + bytesAcked congestion.ByteCount + timeDelta time.Duration + // The round trip of the event. + round roundTripCount +} + +func maxExtraAckedEventFunc(a, b extraAckedEvent) int { + if a.extraAcked > b.extraAcked { + return 1 + } else if a.extraAcked < b.extraAcked { + return -1 + } + return 0 +} + +// BandwidthSample +type bandwidthSample struct { + // The bandwidth at that particular sample. Zero if no valid bandwidth sample + // is available. + bandwidth Bandwidth + // The RTT measurement at this particular sample. Zero if no RTT sample is + // available. Does not correct for delayed ack time. + rtt time.Duration + // |send_rate| is computed from the current packet being acked('P') and an + // earlier packet that is acked before P was sent. + sendRate Bandwidth + // States captured when the packet was sent. + stateAtSend sendTimeState +} + +func newBandwidthSample() *bandwidthSample { + return &bandwidthSample{ + sendRate: infBandwidth, + } +} + +// MaxAckHeightTracker is part of the BandwidthSampler. It is called after every +// ack event to keep track the degree of ack aggregation(a.k.a "ack height"). +type maxAckHeightTracker struct { + // Tracks the maximum number of bytes acked faster than the estimated + // bandwidth. + maxAckHeightFilter *WindowedFilter[extraAckedEvent, roundTripCount] + // The time this aggregation started and the number of bytes acked during it. + aggregationEpochStartTime time.Time + aggregationEpochBytes congestion.ByteCount + // The last sent packet number before the current aggregation epoch started. + lastSentPacketNumberBeforeEpoch congestion.PacketNumber + // The number of ack aggregation epochs ever started, including the ongoing + // one. Stats only. + numAckAggregationEpochs uint64 + ackAggregationBandwidthThreshold float64 + startNewAggregationEpochAfterFullRound bool + reduceExtraAckedOnBandwidthIncrease bool +} + +func newMaxAckHeightTracker(windowLength roundTripCount) *maxAckHeightTracker { + return &maxAckHeightTracker{ + maxAckHeightFilter: NewWindowedFilter(windowLength, maxExtraAckedEventFunc), + lastSentPacketNumberBeforeEpoch: invalidPacketNumber, + ackAggregationBandwidthThreshold: 1.0, + } +} + +func (m *maxAckHeightTracker) Get() congestion.ByteCount { + return m.maxAckHeightFilter.GetBest().extraAcked +} + +func (m *maxAckHeightTracker) Update( + bandwidthEstimate Bandwidth, + isNewMaxBandwidth bool, + roundTripCount roundTripCount, + lastSentPacketNumber congestion.PacketNumber, + lastAckedPacketNumber congestion.PacketNumber, + ackTime time.Time, + bytesAcked congestion.ByteCount, +) congestion.ByteCount { + forceNewEpoch := false + + if m.reduceExtraAckedOnBandwidthIncrease && isNewMaxBandwidth { + // Save and clear existing entries. + best := m.maxAckHeightFilter.GetBest() + secondBest := m.maxAckHeightFilter.GetSecondBest() + thirdBest := m.maxAckHeightFilter.GetThirdBest() + m.maxAckHeightFilter.Clear() + + // Reinsert the heights into the filter after recalculating. + expectedBytesAcked := bytesFromBandwidthAndTimeDelta(bandwidthEstimate, best.timeDelta) + if expectedBytesAcked < best.bytesAcked { + best.extraAcked = best.bytesAcked - expectedBytesAcked + m.maxAckHeightFilter.Update(best, best.round) + } + expectedBytesAcked = bytesFromBandwidthAndTimeDelta(bandwidthEstimate, secondBest.timeDelta) + if expectedBytesAcked < secondBest.bytesAcked { + secondBest.extraAcked = secondBest.bytesAcked - expectedBytesAcked + m.maxAckHeightFilter.Update(secondBest, secondBest.round) + } + expectedBytesAcked = bytesFromBandwidthAndTimeDelta(bandwidthEstimate, thirdBest.timeDelta) + if expectedBytesAcked < thirdBest.bytesAcked { + thirdBest.extraAcked = thirdBest.bytesAcked - expectedBytesAcked + m.maxAckHeightFilter.Update(thirdBest, thirdBest.round) + } + } + + // If any packet sent after the start of the epoch has been acked, start a new + // epoch. + if m.startNewAggregationEpochAfterFullRound && + m.lastSentPacketNumberBeforeEpoch != invalidPacketNumber && + lastAckedPacketNumber != invalidPacketNumber && + lastAckedPacketNumber > m.lastSentPacketNumberBeforeEpoch { + forceNewEpoch = true + } + if m.aggregationEpochStartTime.IsZero() || forceNewEpoch { + m.aggregationEpochBytes = bytesAcked + m.aggregationEpochStartTime = ackTime + m.lastSentPacketNumberBeforeEpoch = lastSentPacketNumber + m.numAckAggregationEpochs++ + return 0 + } + + // Compute how many bytes are expected to be delivered, assuming max bandwidth + // is correct. + aggregationDelta := ackTime.Sub(m.aggregationEpochStartTime) + expectedBytesAcked := bytesFromBandwidthAndTimeDelta(bandwidthEstimate, aggregationDelta) + // Reset the current aggregation epoch as soon as the ack arrival rate is less + // than or equal to the max bandwidth. + if m.aggregationEpochBytes <= congestion.ByteCount(m.ackAggregationBandwidthThreshold*float64(expectedBytesAcked)) { + // Reset to start measuring a new aggregation epoch. + m.aggregationEpochBytes = bytesAcked + m.aggregationEpochStartTime = ackTime + m.lastSentPacketNumberBeforeEpoch = lastSentPacketNumber + m.numAckAggregationEpochs++ + return 0 + } + + m.aggregationEpochBytes += bytesAcked + + // Compute how many extra bytes were delivered vs max bandwidth. + extraBytesAcked := m.aggregationEpochBytes - expectedBytesAcked + newEvent := extraAckedEvent{ + extraAcked: expectedBytesAcked, + bytesAcked: m.aggregationEpochBytes, + timeDelta: aggregationDelta, + } + m.maxAckHeightFilter.Update(newEvent, roundTripCount) + return extraBytesAcked +} + +func (m *maxAckHeightTracker) SetFilterWindowLength(length roundTripCount) { + m.maxAckHeightFilter.SetWindowLength(length) +} + +func (m *maxAckHeightTracker) Reset(newHeight congestion.ByteCount, newTime roundTripCount) { + newEvent := extraAckedEvent{ + extraAcked: newHeight, + round: newTime, + } + m.maxAckHeightFilter.Reset(newEvent, newTime) +} + +func (m *maxAckHeightTracker) SetAckAggregationBandwidthThreshold(threshold float64) { + m.ackAggregationBandwidthThreshold = threshold +} + +func (m *maxAckHeightTracker) SetStartNewAggregationEpochAfterFullRound(value bool) { + m.startNewAggregationEpochAfterFullRound = value +} + +func (m *maxAckHeightTracker) SetReduceExtraAckedOnBandwidthIncrease(value bool) { + m.reduceExtraAckedOnBandwidthIncrease = value +} + +func (m *maxAckHeightTracker) AckAggregationBandwidthThreshold() float64 { + return m.ackAggregationBandwidthThreshold +} + +func (m *maxAckHeightTracker) NumAckAggregationEpochs() uint64 { + return m.numAckAggregationEpochs +} + +// AckPoint represents a point on the ack line. +type ackPoint struct { + ackTime time.Time + totalBytesAcked congestion.ByteCount +} + +// RecentAckPoints maintains the most recent 2 ack points at distinct times. +type recentAckPoints struct { + ackPoints [2]ackPoint +} + +func (r *recentAckPoints) Update(ackTime time.Time, totalBytesAcked congestion.ByteCount) { + if ackTime.Before(r.ackPoints[1].ackTime) { + r.ackPoints[1].ackTime = ackTime + } else if ackTime.After(r.ackPoints[1].ackTime) { + r.ackPoints[0] = r.ackPoints[1] + r.ackPoints[1].ackTime = ackTime + } + + r.ackPoints[1].totalBytesAcked = totalBytesAcked +} + +func (r *recentAckPoints) Clear() { + r.ackPoints[0] = ackPoint{} + r.ackPoints[1] = ackPoint{} +} + +func (r *recentAckPoints) MostRecentPoint() *ackPoint { + return &r.ackPoints[1] +} + +func (r *recentAckPoints) LessRecentPoint() *ackPoint { + if r.ackPoints[0].totalBytesAcked != 0 { + return &r.ackPoints[0] + } + + return &r.ackPoints[1] } // ConnectionStateOnSentPacket represents the information about a sent packet // and the state of the connection at the moment the packet was sent, // specifically the information about the most recently acknowledged packet at // that moment. -type ConnectionStateOnSentPacket struct { - packetNumber congestion.PacketNumber +type connectionStateOnSentPacket struct { // Time at which the packet is sent. - sendTime time.Time + sentTime time.Time // Size of the packet. size congestion.ByteCount // The value of |totalBytesSentAtLastAckedPacket| at the time the @@ -48,25 +298,31 @@ type ConnectionStateOnSentPacket struct { lastAckedPacketAckTime time.Time // Send time states that are returned to the congestion controller when the // packet is acked or lost. - sendTimeState SendTimeState + sendTimeState sendTimeState } -// BandwidthSample -type BandwidthSample struct { - // The bandwidth at that particular sample. Zero if no valid bandwidth sample - // is available. - bandwidth Bandwidth - // The RTT measurement at this particular sample. Zero if no RTT sample is - // available. Does not correct for delayed ack time. - rtt time.Duration - // States captured when the packet was sent. - stateAtSend SendTimeState -} - -func NewBandwidthSample() *BandwidthSample { - return &BandwidthSample{ - // FIXME: the default value of original code is zero. - rtt: InfiniteRTT, +// Snapshot constructor. Records the current state of the bandwidth +// sampler. +// |bytes_in_flight| is the bytes in flight right after the packet is sent. +func newConnectionStateOnSentPacket( + sentTime time.Time, + size congestion.ByteCount, + bytesInFlight congestion.ByteCount, + sampler *bandwidthSampler, +) *connectionStateOnSentPacket { + return &connectionStateOnSentPacket{ + sentTime: sentTime, + size: size, + totalBytesSentAtLastAckedPacket: sampler.totalBytesSentAtLastAckedPacket, + lastAckedPacketSentTime: sampler.lastAckedPacketSentTime, + lastAckedPacketAckTime: sampler.lastAckedPacketAckTime, + sendTimeState: *newSendTimeState( + sampler.isAppLimited, + sampler.totalBytesSent, + sampler.totalBytesAcked, + sampler.totalBytesLost, + bytesInFlight, + ), } } @@ -152,54 +408,162 @@ func NewBandwidthSample() *BandwidthSample { // up until an ack for a packet that was sent after OnAppLimited() was called. // Note that while the scenario above is not the only scenario when the // connection is app-limited, the approach works in other cases too. -type BandwidthSampler struct { + +type congestionEventSample struct { + // The maximum bandwidth sample from all acked packets. + // QuicBandwidth::Zero() if no samples are available. + sampleMaxBandwidth Bandwidth + // Whether |sample_max_bandwidth| is from a app-limited sample. + sampleIsAppLimited bool + // The minimum rtt sample from all acked packets. + // QuicTime::Delta::Infinite() if no samples are available. + sampleRtt time.Duration + // For each packet p in acked packets, this is the max value of INFLIGHT(p), + // where INFLIGHT(p) is the number of bytes acked while p is inflight. + sampleMaxInflight congestion.ByteCount + // The send state of the largest packet in acked_packets, unless it is + // empty. If acked_packets is empty, it's the send state of the largest + // packet in lost_packets. + lastPacketSendState sendTimeState + // The number of extra bytes acked from this ack event, compared to what is + // expected from the flow's bandwidth. Larger value means more ack + // aggregation. + extraAcked congestion.ByteCount +} + +func newCongestionEventSample() *congestionEventSample { + return &congestionEventSample{ + sampleRtt: infRTT, + } +} + +type bandwidthSampler struct { // The total number of congestion controlled bytes sent during the connection. totalBytesSent congestion.ByteCount + // The total number of congestion controlled bytes which were acknowledged. totalBytesAcked congestion.ByteCount + // The total number of congestion controlled bytes which were lost. totalBytesLost congestion.ByteCount - // The value of |totalBytesSent| at the time the last acknowledged packet - // was sent. Valid only when |lastAckedPacketSentTime| is valid. + + // The total number of congestion controlled bytes which have been neutered. + totalBytesNeutered congestion.ByteCount + + // The value of |total_bytes_sent_| at the time the last acknowledged packet + // was sent. Valid only when |last_acked_packet_sent_time_| is valid. totalBytesSentAtLastAckedPacket congestion.ByteCount + // The time at which the last acknowledged packet was sent. Set to // QuicTime::Zero() if no valid timestamp is available. lastAckedPacketSentTime time.Time + // The time at which the most recent packet was acknowledged. lastAckedPacketAckTime time.Time + // The most recently sent packet. - lastSendPacket congestion.PacketNumber + lastSentPacket congestion.PacketNumber + + // The most recently acked packet. + lastAckedPacket congestion.PacketNumber + // Indicates whether the bandwidth sampler is currently in an app-limited // phase. isAppLimited bool + // The packet that will be acknowledged after this one will cause the sampler // to exit the app-limited phase. endOfAppLimitedPhase congestion.PacketNumber + // Record of the connection state at the point where each packet in flight was // sent, indexed by the packet number. - connectionStats *ConnectionStates + connectionStateMap *packetNumberIndexedQueue[connectionStateOnSentPacket] + + recentAckPoints recentAckPoints + a0Candidates RingBuffer[ackPoint] + + // Maximum number of tracked packets. + maxTrackedPackets congestion.ByteCount + + maxAckHeightTracker *maxAckHeightTracker + totalBytesAckedAfterLastAckEvent congestion.ByteCount + + // True if connection option 'BSAO' is set. + overestimateAvoidance bool + + // True if connection option 'BBRB' is set. + limitMaxAckHeightTrackerBySendRate bool +} + +func newBandwidthSampler(maxAckHeightTrackerWindowLength roundTripCount) *bandwidthSampler { + b := &bandwidthSampler{ + maxAckHeightTracker: newMaxAckHeightTracker(maxAckHeightTrackerWindowLength), + connectionStateMap: newPacketNumberIndexedQueue[connectionStateOnSentPacket](defaultConnectionStateMapQueueSize), + lastSentPacket: invalidPacketNumber, + lastAckedPacket: invalidPacketNumber, + endOfAppLimitedPhase: invalidPacketNumber, + } + + b.a0Candidates.Init(defaultCandidatesBufferSize) + + return b +} + +func (b *bandwidthSampler) MaxAckHeight() congestion.ByteCount { + return b.maxAckHeightTracker.Get() +} + +func (b *bandwidthSampler) NumAckAggregationEpochs() uint64 { + return b.maxAckHeightTracker.NumAckAggregationEpochs() +} + +func (b *bandwidthSampler) SetMaxAckHeightTrackerWindowLength(length roundTripCount) { + b.maxAckHeightTracker.SetFilterWindowLength(length) +} + +func (b *bandwidthSampler) ResetMaxAckHeightTracker(newHeight congestion.ByteCount, newTime roundTripCount) { + b.maxAckHeightTracker.Reset(newHeight, newTime) +} + +func (b *bandwidthSampler) SetStartNewAggregationEpochAfterFullRound(value bool) { + b.maxAckHeightTracker.SetStartNewAggregationEpochAfterFullRound(value) } -func NewBandwidthSampler() *BandwidthSampler { - return &BandwidthSampler{ - connectionStats: &ConnectionStates{ - stats: make(map[congestion.PacketNumber]*ConnectionStateOnSentPacket), - }, +func (b *bandwidthSampler) SetLimitMaxAckHeightTrackerBySendRate(value bool) { + b.limitMaxAckHeightTrackerBySendRate = value +} + +func (b *bandwidthSampler) SetReduceExtraAckedOnBandwidthIncrease(value bool) { + b.maxAckHeightTracker.SetReduceExtraAckedOnBandwidthIncrease(value) +} + +func (b *bandwidthSampler) EnableOverestimateAvoidance() { + if b.overestimateAvoidance { + return } + + b.overestimateAvoidance = true + b.maxAckHeightTracker.SetAckAggregationBandwidthThreshold(2.0) } -// OnPacketSent Inputs the sent packet information into the sampler. Assumes that all -// packets are sent in order. The information about the packet will not be -// released from the sampler until it the packet is either acknowledged or -// declared lost. -func (s *BandwidthSampler) OnPacketSent(sentTime time.Time, lastSentPacket congestion.PacketNumber, sentBytes, bytesInFlight congestion.ByteCount, hasRetransmittableData bool) { - s.lastSendPacket = lastSentPacket +func (b *bandwidthSampler) IsOverestimateAvoidanceEnabled() bool { + return b.overestimateAvoidance +} - if !hasRetransmittableData { +func (b *bandwidthSampler) OnPacketSent( + sentTime time.Time, + packetNumber congestion.PacketNumber, + bytes congestion.ByteCount, + bytesInFlight congestion.ByteCount, + isRetransmittable bool, +) { + b.lastSentPacket = packetNumber + + if !isRetransmittable { return } - s.totalBytesSent += sentBytes + b.totalBytesSent += bytes // If there are no packets in flight, the time at which the new transmission // opens can be treated as the A_0 point for the purpose of bandwidth @@ -208,167 +572,303 @@ func (s *BandwidthSampler) OnPacketSent(sentTime time.Time, lastSentPacket conge // samples at important points where we would not have them otherwise, most // importantly at the beginning of the connection. if bytesInFlight == 0 { - s.lastAckedPacketAckTime = sentTime - s.totalBytesSentAtLastAckedPacket = s.totalBytesSent + b.lastAckedPacketAckTime = sentTime + if b.overestimateAvoidance { + b.recentAckPoints.Clear() + b.recentAckPoints.Update(sentTime, b.totalBytesAcked) + b.a0Candidates.Clear() + b.a0Candidates.PushBack(*b.recentAckPoints.MostRecentPoint()) + } + b.totalBytesSentAtLastAckedPacket = b.totalBytesSent // In this situation ack compression is not a concern, set send rate to // effectively infinite. - s.lastAckedPacketSentTime = sentTime + b.lastAckedPacketSentTime = sentTime } - s.connectionStats.Insert(lastSentPacket, sentTime, sentBytes, s) + b.connectionStateMap.Emplace(packetNumber, newConnectionStateOnSentPacket( + sentTime, + bytes, + bytesInFlight+bytes, + b, + )) } -// OnPacketAcked Notifies the sampler that the |lastAckedPacket| is acknowledged. Returns a -// bandwidth sample. If no bandwidth sample is available, -// QuicBandwidth::Zero() is returned. -func (s *BandwidthSampler) OnPacketAcked(ackTime time.Time, lastAckedPacket congestion.PacketNumber) *BandwidthSample { - sentPacketState := s.connectionStats.Get(lastAckedPacket) - if sentPacketState == nil { - return NewBandwidthSample() +func (b *bandwidthSampler) OnCongestionEvent( + ackTime time.Time, + ackedPackets []congestion.AckedPacketInfo, + lostPackets []congestion.LostPacketInfo, + maxBandwidth Bandwidth, + estBandwidthUpperBound Bandwidth, + roundTripCount roundTripCount, +) congestionEventSample { + eventSample := newCongestionEventSample() + + var lastLostPacketSendState sendTimeState + + for _, p := range lostPackets { + sendState := b.OnPacketLost(p.PacketNumber, p.BytesLost) + if sendState.isValid { + lastLostPacketSendState = sendState + } + } + + if len(ackedPackets) == 0 { + // Only populate send state for a loss-only event. + eventSample.lastPacketSendState = lastLostPacketSendState + return *eventSample } - sample := s.onPacketAckedInner(ackTime, lastAckedPacket, sentPacketState) - s.connectionStats.Remove(lastAckedPacket) + var lastAckedPacketSendState sendTimeState + var maxSendRate Bandwidth + + for _, p := range ackedPackets { + sample := b.onPacketAcknowledged(ackTime, p.PacketNumber) + if !sample.stateAtSend.isValid { + continue + } + + lastAckedPacketSendState = sample.stateAtSend + + if sample.rtt != 0 { + eventSample.sampleRtt = min(eventSample.sampleRtt, sample.rtt) + } + if sample.bandwidth > eventSample.sampleMaxBandwidth { + eventSample.sampleMaxBandwidth = sample.bandwidth + eventSample.sampleIsAppLimited = sample.stateAtSend.isAppLimited + } + if sample.sendRate != infBandwidth { + maxSendRate = max(maxSendRate, sample.sendRate) + } + inflightSample := b.totalBytesAcked - lastAckedPacketSendState.totalBytesAcked + if inflightSample > eventSample.sampleMaxInflight { + eventSample.sampleMaxInflight = inflightSample + } + } + + if !lastLostPacketSendState.isValid { + eventSample.lastPacketSendState = lastAckedPacketSendState + } else if !lastAckedPacketSendState.isValid { + eventSample.lastPacketSendState = lastLostPacketSendState + } else { + // If two packets are inflight and an alarm is armed to lose a packet and it + // wakes up late, then the first of two in flight packets could have been + // acknowledged before the wakeup, which re-evaluates loss detection, and + // could declare the later of the two lost. + if lostPackets[len(lostPackets)-1].PacketNumber > ackedPackets[len(ackedPackets)-1].PacketNumber { + eventSample.lastPacketSendState = lastLostPacketSendState + } else { + eventSample.lastPacketSendState = lastAckedPacketSendState + } + } - return sample + isNewMaxBandwidth := eventSample.sampleMaxBandwidth > maxBandwidth + maxBandwidth = max(maxBandwidth, eventSample.sampleMaxBandwidth) + if b.limitMaxAckHeightTrackerBySendRate { + maxBandwidth = max(maxBandwidth, maxSendRate) + } + + eventSample.extraAcked = b.onAckEventEnd(min(estBandwidthUpperBound, maxBandwidth), isNewMaxBandwidth, roundTripCount) + + return *eventSample +} + +func (b *bandwidthSampler) OnPacketLost(packetNumber congestion.PacketNumber, bytesLost congestion.ByteCount) (s sendTimeState) { + b.totalBytesLost += bytesLost + if sentPacketPointer := b.connectionStateMap.GetEntry(packetNumber); sentPacketPointer != nil { + sentPacketToSendTimeState(sentPacketPointer, &s) + } + return s } -// onPacketAckedInner Handles the actual bandwidth calculations, whereas the outer method handles -// retrieving and removing |sentPacket|. -func (s *BandwidthSampler) onPacketAckedInner(ackTime time.Time, lastAckedPacket congestion.PacketNumber, sentPacket *ConnectionStateOnSentPacket) *BandwidthSample { - s.totalBytesAcked += sentPacket.size +func (b *bandwidthSampler) OnPacketNeutered(packetNumber congestion.PacketNumber) { + b.connectionStateMap.Remove(packetNumber, func(sentPacket connectionStateOnSentPacket) { + b.totalBytesNeutered += sentPacket.size + }) +} + +func (b *bandwidthSampler) OnAppLimited() { + b.isAppLimited = true + b.endOfAppLimitedPhase = b.lastSentPacket +} - s.totalBytesSentAtLastAckedPacket = sentPacket.sendTimeState.totalBytesSent - s.lastAckedPacketSentTime = sentPacket.sendTime - s.lastAckedPacketAckTime = ackTime +func (b *bandwidthSampler) RemoveObsoletePackets(leastUnacked congestion.PacketNumber) { + // A packet can become obsolete when it is removed from QuicUnackedPacketMap's + // view of inflight before it is acked or marked as lost. For example, when + // QuicSentPacketManager::RetransmitCryptoPackets retransmits a crypto packet, + // the packet is removed from QuicUnackedPacketMap's inflight, but is not + // marked as acked or lost in the BandwidthSampler. + b.connectionStateMap.RemoveUpTo(leastUnacked) +} + +func (b *bandwidthSampler) TotalBytesSent() congestion.ByteCount { + return b.totalBytesSent +} - // Exit app-limited phase once a packet that was sent while the connection is - // not app-limited is acknowledged. - if s.isAppLimited && lastAckedPacket > s.endOfAppLimitedPhase { - s.isAppLimited = false +func (b *bandwidthSampler) TotalBytesLost() congestion.ByteCount { + return b.totalBytesLost +} + +func (b *bandwidthSampler) TotalBytesAcked() congestion.ByteCount { + return b.totalBytesAcked +} + +func (b *bandwidthSampler) TotalBytesNeutered() congestion.ByteCount { + return b.totalBytesNeutered +} + +func (b *bandwidthSampler) IsAppLimited() bool { + return b.isAppLimited +} + +func (b *bandwidthSampler) EndOfAppLimitedPhase() congestion.PacketNumber { + return b.endOfAppLimitedPhase +} + +func (b *bandwidthSampler) max_ack_height() congestion.ByteCount { + return b.maxAckHeightTracker.Get() +} + +func (b *bandwidthSampler) chooseA0Point(totalBytesAcked congestion.ByteCount, a0 *ackPoint) bool { + if b.a0Candidates.Empty() { + return false + } + + if b.a0Candidates.Len() == 1 { + *a0 = *b.a0Candidates.Front() + return true + } + + for i := 1; i < b.a0Candidates.Len(); i++ { + if b.a0Candidates.Offset(i).totalBytesAcked > totalBytesAcked { + *a0 = *b.a0Candidates.Offset(i - 1) + if i > 1 { + for j := 0; j < i-1; j++ { + b.a0Candidates.PopFront() + } + } + return true + } + } + + *a0 = *b.a0Candidates.Back() + for k := 0; k < b.a0Candidates.Len()-1; k++ { + b.a0Candidates.PopFront() + } + return true +} + +func (b *bandwidthSampler) onPacketAcknowledged(ackTime time.Time, packetNumber congestion.PacketNumber) bandwidthSample { + sample := newBandwidthSample() + b.lastAckedPacket = packetNumber + sentPacketPointer := b.connectionStateMap.GetEntry(packetNumber) + if sentPacketPointer == nil { + return *sample + } + + // OnPacketAcknowledgedInner + b.totalBytesAcked += sentPacketPointer.size + b.totalBytesSentAtLastAckedPacket = sentPacketPointer.sendTimeState.totalBytesSent + b.lastAckedPacketSentTime = sentPacketPointer.sentTime + b.lastAckedPacketAckTime = ackTime + if b.overestimateAvoidance { + b.recentAckPoints.Update(ackTime, b.totalBytesAcked) + } + + if b.isAppLimited { + // Exit app-limited phase in two cases: + // (1) end_of_app_limited_phase_ is not initialized, i.e., so far all + // packets are sent while there are buffered packets or pending data. + // (2) The current acked packet is after the sent packet marked as the end + // of the app limit phase. + if b.endOfAppLimitedPhase == invalidPacketNumber || + packetNumber > b.endOfAppLimitedPhase { + b.isAppLimited = false + } } // There might have been no packets acknowledged at the moment when the // current packet was sent. In that case, there is no bandwidth sample to // make. - if sentPacket.lastAckedPacketSentTime.IsZero() { - return NewBandwidthSample() + if sentPacketPointer.lastAckedPacketSentTime.IsZero() { + return *sample } // Infinite rate indicates that the sampler is supposed to discard the // current send rate sample and use only the ack rate. - sendRate := InfiniteBandwidth - if sentPacket.sendTime.After(sentPacket.lastAckedPacketSentTime) { - sendRate = BandwidthFromDelta(sentPacket.sendTimeState.totalBytesSent-sentPacket.totalBytesSentAtLastAckedPacket, sentPacket.sendTime.Sub(sentPacket.lastAckedPacketSentTime)) + sendRate := infBandwidth + if sentPacketPointer.sentTime.After(sentPacketPointer.lastAckedPacketSentTime) { + sendRate = BandwidthFromDelta( + sentPacketPointer.sendTimeState.totalBytesSent-sentPacketPointer.totalBytesSentAtLastAckedPacket, + sentPacketPointer.sentTime.Sub(sentPacketPointer.lastAckedPacketSentTime)) + } + + var a0 ackPoint + if b.overestimateAvoidance && b.chooseA0Point(sentPacketPointer.sendTimeState.totalBytesAcked, &a0) { + } else { + a0.ackTime = sentPacketPointer.lastAckedPacketAckTime + a0.totalBytesAcked = sentPacketPointer.sendTimeState.totalBytesAcked } // During the slope calculation, ensure that ack time of the current packet is // always larger than the time of the previous packet, otherwise division by // zero or integer underflow can occur. - if !ackTime.After(sentPacket.lastAckedPacketAckTime) { - // TODO(wub): Compare this code count before and after fixing clock jitter - // issue. - // if sentPacket.lastAckedPacketAckTime.Equal(sentPacket.sendTime) { - // This is the 1st packet after quiescense. - // QUIC_CODE_COUNT_N(quic_prev_ack_time_larger_than_current_ack_time, 1, 2); - // } else { - // QUIC_CODE_COUNT_N(quic_prev_ack_time_larger_than_current_ack_time, 2, 2); - // } - - return NewBandwidthSample() + if ackTime.Sub(a0.ackTime) <= 0 { + return *sample } - ackRate := BandwidthFromDelta(s.totalBytesAcked-sentPacket.sendTimeState.totalBytesAcked, - ackTime.Sub(sentPacket.lastAckedPacketAckTime)) + ackRate := BandwidthFromDelta(b.totalBytesAcked-a0.totalBytesAcked, ackTime.Sub(a0.ackTime)) + sample.bandwidth = min(sendRate, ackRate) // Note: this sample does not account for delayed acknowledgement time. This // means that the RTT measurements here can be artificially high, especially // on low bandwidth connections. - sample := &BandwidthSample{ - bandwidth: minBandwidth(sendRate, ackRate), - rtt: ackTime.Sub(sentPacket.sendTime), - } + sample.rtt = ackTime.Sub(sentPacketPointer.sentTime) + sample.sendRate = sendRate + sentPacketToSendTimeState(sentPacketPointer, &sample.stateAtSend) - SentPacketToSendTimeState(sentPacket, &sample.stateAtSend) - return sample + return *sample } -// OnCongestionEvent Informs the sampler that a packet is considered lost and it should no -// longer keep track of it. -func (s *BandwidthSampler) OnCongestionEvent(packetNumber congestion.PacketNumber) SendTimeState { - ok, sentPacket := s.connectionStats.Remove(packetNumber) - sendTimeState := SendTimeState{ - isValid: ok, +func (b *bandwidthSampler) onAckEventEnd( + bandwidthEstimate Bandwidth, + isNewMaxBandwidth bool, + roundTripCount roundTripCount, +) congestion.ByteCount { + newlyAckedBytes := b.totalBytesAcked - b.totalBytesAckedAfterLastAckEvent + if newlyAckedBytes == 0 { + return 0 } - if sentPacket != nil { - s.totalBytesLost += sentPacket.size - SentPacketToSendTimeState(sentPacket, &sendTimeState) + b.totalBytesAckedAfterLastAckEvent = b.totalBytesAcked + extraAcked := b.maxAckHeightTracker.Update( + bandwidthEstimate, + isNewMaxBandwidth, + roundTripCount, + b.lastSentPacket, + b.lastAckedPacket, + b.lastAckedPacketAckTime, + newlyAckedBytes) + // If |extra_acked| is zero, i.e. this ack event marks the start of a new ack + // aggregation epoch, save LessRecentPoint, which is the last ack point of the + // previous epoch, as a A0 candidate. + if b.overestimateAvoidance && extraAcked == 0 { + b.a0Candidates.PushBack(*b.recentAckPoints.LessRecentPoint()) } - - return sendTimeState -} - -// OnAppLimited Informs the sampler that the connection is currently app-limited, causing -// the sampler to enter the app-limited phase. The phase will expire by -// itself. -func (s *BandwidthSampler) OnAppLimited() { - s.isAppLimited = true - s.endOfAppLimitedPhase = s.lastSendPacket + return extraAcked } -// SentPacketToSendTimeState Copy a subset of the (private) ConnectionStateOnSentPacket to the (public) -// SendTimeState. Always set send_time_state->is_valid to true. -func SentPacketToSendTimeState(sentPacket *ConnectionStateOnSentPacket, sendTimeState *SendTimeState) { - sendTimeState.isAppLimited = sentPacket.sendTimeState.isAppLimited - sendTimeState.totalBytesSent = sentPacket.sendTimeState.totalBytesSent - sendTimeState.totalBytesAcked = sentPacket.sendTimeState.totalBytesAcked - sendTimeState.totalBytesLost = sentPacket.sendTimeState.totalBytesLost +func sentPacketToSendTimeState(sentPacket *connectionStateOnSentPacket, sendTimeState *sendTimeState) { + *sendTimeState = sentPacket.sendTimeState sendTimeState.isValid = true } -// ConnectionStates Record of the connection state at the point where each packet in flight was -// sent, indexed by the packet number. -// FIXME: using LinkedList replace map to fast remove all the packets lower than the specified packet number. -type ConnectionStates struct { - stats map[congestion.PacketNumber]*ConnectionStateOnSentPacket -} - -func (s *ConnectionStates) Insert(packetNumber congestion.PacketNumber, sentTime time.Time, bytes congestion.ByteCount, sampler *BandwidthSampler) bool { - if _, ok := s.stats[packetNumber]; ok { - return false - } - - s.stats[packetNumber] = NewConnectionStateOnSentPacket(packetNumber, sentTime, bytes, sampler) - return true -} - -func (s *ConnectionStates) Get(packetNumber congestion.PacketNumber) *ConnectionStateOnSentPacket { - return s.stats[packetNumber] -} - -func (s *ConnectionStates) Remove(packetNumber congestion.PacketNumber) (bool, *ConnectionStateOnSentPacket) { - state, ok := s.stats[packetNumber] - if ok { - delete(s.stats, packetNumber) - } - return ok, state +// BytesFromBandwidthAndTimeDelta calculates the bytes +// from a bandwidth(bits per second) and a time delta +func bytesFromBandwidthAndTimeDelta(bandwidth Bandwidth, delta time.Duration) congestion.ByteCount { + return (congestion.ByteCount(bandwidth) * congestion.ByteCount(delta)) / + (congestion.ByteCount(time.Second) * 8) } -func NewConnectionStateOnSentPacket(packetNumber congestion.PacketNumber, sentTime time.Time, bytes congestion.ByteCount, sampler *BandwidthSampler) *ConnectionStateOnSentPacket { - return &ConnectionStateOnSentPacket{ - packetNumber: packetNumber, - sendTime: sentTime, - size: bytes, - lastAckedPacketSentTime: sampler.lastAckedPacketSentTime, - lastAckedPacketAckTime: sampler.lastAckedPacketAckTime, - totalBytesSentAtLastAckedPacket: sampler.totalBytesSentAtLastAckedPacket, - sendTimeState: SendTimeState{ - isValid: true, - isAppLimited: sampler.isAppLimited, - totalBytesSent: sampler.totalBytesSent, - totalBytesAcked: sampler.totalBytesAcked, - totalBytesLost: sampler.totalBytesLost, - }, - } +func timeDeltaFromBytesAndBandwidth(bytes congestion.ByteCount, bandwidth Bandwidth) time.Duration { + return time.Duration(bytes*8) * time.Second / time.Duration(bandwidth) } diff --git a/core/internal/congestion/bbr/bbr_sender.go b/core/internal/congestion/bbr/bbr_sender.go index 783b7846a1..48e6e4b3a6 100644 --- a/core/internal/congestion/bbr/bbr_sender.go +++ b/core/internal/congestion/bbr/bbr_sender.go @@ -1,198 +1,195 @@ package bbr -// src from https://quiche.googlesource.com/quiche.git/+/66dea072431f94095dfc3dd2743cb94ef365f7ef/quic/core/congestion_control/bbr_sender.cc - import ( "fmt" - "math" + "math/rand" "net" "time" - "github.com/apernet/hysteria/core/internal/congestion/common" "github.com/apernet/quic-go/congestion" - "github.com/zhangyunhao116/fastrand" -) -const ( - InitialPacketSizeIPv4 = 1252 - InitialPacketSizeIPv6 = 1232 - InitialCongestionWindow = 32 - DefaultBBRMaxCongestionWindow = 10000 + "github.com/apernet/hysteria/core/internal/congestion/common" ) -func GetInitialPacketSize(addr net.Addr) congestion.ByteCount { - maxSize := congestion.ByteCount(1200) - // If this is not a UDP address, we don't know anything about the MTU. - // Use the minimum size of an Initial packet as the max packet size. - if udpAddr, ok := addr.(*net.UDPAddr); ok { - if udpAddr.IP.To4() != nil { - maxSize = InitialPacketSizeIPv4 - } else { - maxSize = InitialPacketSizeIPv6 - } - } - return congestion.ByteCount(maxSize) -} - -var ( - - // Default initial rtt used before any samples are received. - InitialRtt = 100 * time.Millisecond - - // The gain used for the STARTUP, equal to 4*ln(2). - DefaultHighGain = 2.77 +// BbrSender implements BBR congestion control algorithm. BBR aims to estimate +// the current available Bottleneck Bandwidth and RTT (hence the name), and +// regulates the pacing rate and the size of the congestion window based on +// those signals. +// +// BBR relies on pacing in order to function properly. Do not use BBR when +// pacing is disabled. +// - // The gain used in STARTUP after loss has been detected. - // 1.5 is enough to allow for 25% exogenous loss and still observe a 25% growth - // in measured bandwidth. - StartupAfterLossGain = 1.5 +const ( + invalidPacketNumber = -1 + initialCongestionWindowPackets = 32 + + // Constants based on TCP defaults. + // The minimum CWND to ensure delayed acks don't reduce bandwidth measurements. + // Does not inflate the pacing rate. + defaultMinimumCongestionWindow = 4 * congestion.ByteCount(congestion.InitialPacketSizeIPv4) + + // The gain used for the STARTUP, equal to 2/ln(2). + defaultHighGain = 2.885 + // The newly derived gain for STARTUP, equal to 4 * ln(2) + derivedHighGain = 2.773 + // The newly derived CWND gain for STARTUP, 2. + derivedHighCWNDGain = 2.0 +) - // The cycle of gains used during the PROBE_BW stage. - PacingGain = []float64{1.25, 0.75, 1, 1, 1, 1, 1, 1} +// The cycle of gains used during the PROBE_BW stage. +var pacingGain = [...]float64{1.25, 0.75, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0} +const ( // The length of the gain cycle. - GainCycleLength = len(PacingGain) - + gainCycleLength = len(pacingGain) // The size of the bandwidth filter window, in round-trips. - BandwidthWindowSize = GainCycleLength + 2 + bandwidthWindowSize = gainCycleLength + 2 // The time after which the current min_rtt value expires. - MinRttExpiry = 10 * time.Second - + minRttExpiry = 10 * time.Second // The minimum time the connection can spend in PROBE_RTT mode. - ProbeRttTime = time.Millisecond * 200 - + probeRttTime = 200 * time.Millisecond // If the bandwidth does not increase by the factor of |kStartupGrowthTarget| // within |kRoundTripsWithoutGrowthBeforeExitingStartup| rounds, the connection // will exit the STARTUP mode. - StartupGrowthTarget = 1.25 - RoundTripsWithoutGrowthBeforeExitingStartup = int64(3) + startupGrowthTarget = 1.25 + roundTripsWithoutGrowthBeforeExitingStartup = int64(3) - // Coefficient of target congestion window to use when basing PROBE_RTT on BDP. - ModerateProbeRttMultiplier = 0.75 - - // Coefficient to determine if a new RTT is sufficiently similar to min_rtt that - // we don't need to enter PROBE_RTT. - SimilarMinRttThreshold = 1.125 - - // Congestion window gain for QUIC BBR during PROBE_BW phase. - DefaultCongestionWindowGainConst = 2.0 + // Flag. + defaultStartupFullLossCount = 8 + quicBbr2DefaultLossThreshold = 0.02 + maxBbrBurstPackets = 3 ) type bbrMode int const ( // Startup phase of the connection. - STARTUP = iota + bbrModeStartup = iota // After achieving the highest possible bandwidth during the startup, lower // the pacing rate in order to drain the queue. - DRAIN + bbrModeDrain // Cruising mode. - PROBE_BW + bbrModeProbeBw // Temporarily slow down sending in order to empty the buffer and measure // the real minimum RTT. - PROBE_RTT + bbrModeProbeRtt ) +// Indicates how the congestion control limits the amount of bytes in flight. type bbrRecoveryState int const ( // Do not limit. - NOT_IN_RECOVERY = iota - + bbrRecoveryStateNotInRecovery = iota // Allow an extra outstanding byte for each byte acknowledged. - CONSERVATION - + bbrRecoveryStateConservation // Allow two extra outstanding bytes for each byte acknowledged (slow // start). - GROWTH + bbrRecoveryStateGrowth ) type bbrSender struct { - mode bbrMode - clock Clock - rttStats congestion.RTTStatsProvider - bytesInFlight congestion.ByteCount - // return total bytes of unacked packets. - // GetBytesInFlight func() congestion.ByteCount + rttStats congestion.RTTStatsProvider + clock Clock + pacer *common.Pacer + + mode bbrMode + // Bandwidth sampler provides BBR with the bandwidth measurements at // individual points. - sampler *BandwidthSampler + sampler *bandwidthSampler + // The number of the round trips that have occurred during the connection. - roundTripCount int64 + roundTripCount roundTripCount + // The packet number of the most recently sent packet. - lastSendPacket congestion.PacketNumber + lastSentPacket congestion.PacketNumber // Acknowledgement of any packet after |current_round_trip_end_| will cause // the round trip counter to advance. currentRoundTripEnd congestion.PacketNumber + + // Number of congestion events with some losses, in the current round. + numLossEventsInRound uint64 + + // Number of total bytes lost in the current round. + bytesLostInRound congestion.ByteCount + // The filter that tracks the maximum bandwidth over the multiple recent // round-trips. - maxBandwidth *WindowedFilter - // Tracks the maximum number of bytes acked faster than the sending rate. - maxAckHeight *WindowedFilter - // The time this aggregation started and the number of bytes acked during it. - aggregationEpochStartTime time.Time - aggregationEpochBytes congestion.ByteCount + maxBandwidth *WindowedFilter[Bandwidth, roundTripCount] + // Minimum RTT estimate. Automatically expires within 10 seconds (and // triggers PROBE_RTT mode) if no new value is sampled during that period. minRtt time.Duration // The time at which the current value of |min_rtt_| was assigned. minRttTimestamp time.Time + // The maximum allowed number of bytes in flight. congestionWindow congestion.ByteCount + // The initial value of the |congestion_window_|. initialCongestionWindow congestion.ByteCount + // The largest value the |congestion_window_| can achieve. - initialMaxCongestionWindow congestion.ByteCount + maxCongestionWindow congestion.ByteCount + // The smallest value the |congestion_window_| can achieve. - // minCongestionWindow congestion.ByteCount + minCongestionWindow congestion.ByteCount + // The pacing gain applied during the STARTUP phase. highGain float64 + // The CWND gain applied during the STARTUP phase. highCwndGain float64 + // The pacing gain applied during the DRAIN phase. drainGain float64 + // The current pacing rate of the connection. pacingRate Bandwidth + // The gain currently applied to the pacing rate. pacingGain float64 // The gain currently applied to the congestion window. congestionWindowGain float64 + // The gain used for the congestion window during PROBE_BW. Latched from // quic_bbr_cwnd_gain flag. - congestionWindowGainConst float64 + congestionWindowGainConstant float64 // The number of RTTs to stay in STARTUP mode. Defaults to 3. numStartupRtts int64 - // If true, exit startup if 1RTT has passed with no bandwidth increase and - // the connection is in recovery. - exitStartupOnLoss bool + // Number of round-trips in PROBE_BW mode, used for determining the current // pacing gain cycle. cycleCurrentOffset int // The time at which the last pacing gain cycle was started. lastCycleStart time.Time + // Indicates whether the connection has reached the full bandwidth mode. isAtFullBandwidth bool // Number of rounds during which there was no significant bandwidth increase. roundsWithoutBandwidthGain int64 // The bandwidth compared to which the increase is measured. bandwidthAtLastRound Bandwidth + // Set to true upon exiting quiescence. exitingQuiescence bool + // Time at which PROBE_RTT has to be exited. Setting it to zero indicates // that the time is yet unknown as the number of packets in flight has not // reached the required value. exitProbeRttAt time.Time // Indicates whether a round-trip has passed since PROBE_RTT became active. probeRttRoundPassed bool + // Indicates whether the most recent bandwidth sample was marked as // app-limited. lastSampleIsAppLimited bool // Indicates whether any non app-limited samples have been recorded. hasNoAppLimitedSample bool - // Indicates app-limited calls should be ignored as long as there's - // enough data inflight to see more bandwidth when necessary. - flexibleAppLimited bool + // Current state of recovery. recoveryState bbrRecoveryState // Receiving acknowledgement of a packet after |end_recovery_at_| will cause @@ -202,656 +199,598 @@ type bbrSender struct { // A window used to limit the number of bytes in flight during loss recovery. recoveryWindow congestion.ByteCount // If true, consider all samples in recovery app-limited. - isAppLimitedRecovery bool + isAppLimitedRecovery bool // not used + // When true, pace at 1.5x and disable packet conservation in STARTUP. - slowerStartup bool + slowerStartup bool // not used // When true, disables packet conservation in STARTUP. - rateBasedStartup bool - // When non-zero, decreases the rate in STARTUP by the total number of bytes - // lost in STARTUP divided by CWND. - startupRateReductionMultiplier int64 - // Sum of bytes lost in STARTUP. - startupBytesLost congestion.ByteCount + rateBasedStartup bool // not used + // When true, add the most recent ack aggregation measurement during STARTUP. enableAckAggregationDuringStartup bool // When true, expire the windowed ack aggregation values in STARTUP when // bandwidth increases more than 25%. expireAckAggregationInStartup bool + // If true, will not exit low gain mode until bytes_in_flight drops below BDP // or it's time for high gain mode. drainToTarget bool - // If true, use a CWND of 0.75*BDP during probe_rtt instead of 4 packets. - probeRttBasedOnBdp bool - // If true, skip probe_rtt and update the timestamp of the existing min_rtt to - // now if min_rtt over the last cycle is within 12.5% of the current min_rtt. - // Even if the min_rtt is 12.5% too low, the 25% gain cycling and 2x CWND gain - // should overcome an overly small min_rtt. - probeRttSkippedIfSimilarRtt bool - // If true, disable PROBE_RTT entirely as long as the connection was recently - // app limited. - probeRttDisabledIfAppLimited bool - appLimitedSinceLastProbeRtt bool - minRttSinceLastProbeRtt time.Duration - // Latched value of --quic_always_get_bw_sample_when_acked. - alwaysGetBwSampleWhenAcked bool - - pacer *common.Pacer + // If true, slow down pacing rate in STARTUP when overshooting is detected. + detectOvershooting bool + // Bytes lost while detect_overshooting_ is true. + bytesLostWhileDetectingOvershooting congestion.ByteCount + // Slow down pacing rate if + // bytes_lost_while_detecting_overshooting_ * + // bytes_lost_multiplier_while_detecting_overshooting_ > IW. + bytesLostMultiplierWhileDetectingOvershooting uint8 + // When overshooting is detected, do not drop pacing_rate_ below this value / + // min_rtt. + cwndToCalculateMinPacingRate congestion.ByteCount + + // Max congestion window when adjusting network parameters. + maxCongestionWindowWithNetworkParametersAdjusted congestion.ByteCount // not used + + // Params. maxDatagramSize congestion.ByteCount + // Recorded on packet sent. equivalent |unacked_packets_->bytes_in_flight()| + bytesInFlight congestion.ByteCount } -func NewBBRSender( +var _ congestion.CongestionControl = &bbrSender{} + +func NewBbrSender( + clock Clock, + initialMaxDatagramSize congestion.ByteCount, +) *bbrSender { + return newBbrSender( + clock, + initialMaxDatagramSize, + initialCongestionWindowPackets*initialMaxDatagramSize, + congestion.MaxCongestionWindowPackets*initialMaxDatagramSize, + ) +} + +func newBbrSender( clock Clock, initialMaxDatagramSize, initialCongestionWindow, initialMaxCongestionWindow congestion.ByteCount, ) *bbrSender { b := &bbrSender{ - mode: STARTUP, - clock: clock, - sampler: NewBandwidthSampler(), - maxBandwidth: NewWindowedFilter(int64(BandwidthWindowSize), MaxFilter), - maxAckHeight: NewWindowedFilter(int64(BandwidthWindowSize), MaxFilter), - congestionWindow: initialCongestionWindow, - initialCongestionWindow: initialCongestionWindow, - highGain: DefaultHighGain, - highCwndGain: DefaultHighGain, - drainGain: 1.0 / DefaultHighGain, - pacingGain: 1.0, - congestionWindowGain: 1.0, - congestionWindowGainConst: DefaultCongestionWindowGainConst, - numStartupRtts: RoundTripsWithoutGrowthBeforeExitingStartup, - recoveryState: NOT_IN_RECOVERY, - recoveryWindow: initialMaxCongestionWindow, - minRttSinceLastProbeRtt: InfiniteRTT, - maxDatagramSize: initialMaxDatagramSize, + clock: clock, + mode: bbrModeStartup, + sampler: newBandwidthSampler(roundTripCount(bandwidthWindowSize)), + lastSentPacket: invalidPacketNumber, + currentRoundTripEnd: invalidPacketNumber, + maxBandwidth: NewWindowedFilter(roundTripCount(bandwidthWindowSize), MaxFilter[Bandwidth]), + congestionWindow: initialCongestionWindow, + initialCongestionWindow: initialCongestionWindow, + maxCongestionWindow: initialMaxCongestionWindow, + minCongestionWindow: defaultMinimumCongestionWindow, + highGain: defaultHighGain, + highCwndGain: defaultHighGain, + drainGain: 1.0 / defaultHighGain, + pacingGain: 1.0, + congestionWindowGain: 1.0, + congestionWindowGainConstant: 2.0, + numStartupRtts: roundTripsWithoutGrowthBeforeExitingStartup, + recoveryState: bbrRecoveryStateNotInRecovery, + endRecoveryAt: invalidPacketNumber, + recoveryWindow: initialMaxCongestionWindow, + bytesLostMultiplierWhileDetectingOvershooting: 2, + cwndToCalculateMinPacingRate: initialCongestionWindow, + maxCongestionWindowWithNetworkParametersAdjusted: initialMaxCongestionWindow, + maxDatagramSize: initialMaxDatagramSize, } b.pacer = common.NewPacer(func() congestion.ByteCount { - return congestion.ByteCount(b.BandwidthEstimate() / BytesPerSecond) + // Pacer wants bytes per second, but Bandwidth is in bits per second. + return congestion.ByteCount(float64(b.bandwidthEstimate()) * b.congestionWindowGain / float64(BytesPerSecond)) }) - return b -} -func (b *bbrSender) maxCongestionWindow() congestion.ByteCount { - return b.maxDatagramSize * DefaultBBRMaxCongestionWindow -} + /* + if b.tracer != nil { + b.lastState = logging.CongestionStateStartup + b.tracer.UpdatedCongestionState(logging.CongestionStateStartup) + } + */ + + b.enterStartupMode(b.clock.Now()) + b.setHighCwndGain(derivedHighCWNDGain) -func (b *bbrSender) minCongestionWindow() congestion.ByteCount { - return b.maxDatagramSize * b.initialCongestionWindow + return b } func (b *bbrSender) SetRTTStatsProvider(provider congestion.RTTStatsProvider) { b.rttStats = provider } -func (b *bbrSender) GetBytesInFlight() congestion.ByteCount { - return b.bytesInFlight -} - -// TimeUntilSend returns when the next packet should be sent. +// TimeUntilSend implements the SendAlgorithm interface. func (b *bbrSender) TimeUntilSend(bytesInFlight congestion.ByteCount) time.Time { - b.bytesInFlight = bytesInFlight return b.pacer.TimeUntilSend() } +// HasPacingBudget implements the SendAlgorithm interface. func (b *bbrSender) HasPacingBudget(now time.Time) bool { return b.pacer.Budget(now) >= b.maxDatagramSize } +// OnPacketSent implements the SendAlgorithm interface. +func (b *bbrSender) OnPacketSent( + sentTime time.Time, + bytesInFlight congestion.ByteCount, + packetNumber congestion.PacketNumber, + bytes congestion.ByteCount, + isRetransmittable bool, +) { + b.pacer.SentPacket(sentTime, bytes) + + b.lastSentPacket = packetNumber + b.bytesInFlight = bytesInFlight + + if bytesInFlight == 0 { + b.exitingQuiescence = true + } + + b.sampler.OnPacketSent(sentTime, packetNumber, bytes, bytesInFlight, isRetransmittable) +} + +// CanSend implements the SendAlgorithm interface. +func (b *bbrSender) CanSend(bytesInFlight congestion.ByteCount) bool { + return bytesInFlight < b.GetCongestionWindow() +} + +// MaybeExitSlowStart implements the SendAlgorithm interface. +func (b *bbrSender) MaybeExitSlowStart() { + // Do nothing +} + +// OnPacketAcked implements the SendAlgorithm interface. +func (b *bbrSender) OnPacketAcked(number congestion.PacketNumber, ackedBytes, priorInFlight congestion.ByteCount, eventTime time.Time) { + // Do nothing. +} + +// OnPacketLost implements the SendAlgorithm interface. +func (b *bbrSender) OnPacketLost(number congestion.PacketNumber, lostBytes, priorInFlight congestion.ByteCount) { + // Do nothing. +} + +// OnRetransmissionTimeout implements the SendAlgorithm interface. +func (b *bbrSender) OnRetransmissionTimeout(packetsRetransmitted bool) { + // Do nothing. +} + +// SetMaxDatagramSize implements the SendAlgorithm interface. func (b *bbrSender) SetMaxDatagramSize(s congestion.ByteCount) { if s < b.maxDatagramSize { panic(fmt.Sprintf("congestion BUG: decreased max datagram size from %d to %d", b.maxDatagramSize, s)) } - cwndIsMinCwnd := b.congestionWindow == b.minCongestionWindow() + cwndIsMinCwnd := b.congestionWindow == b.minCongestionWindow b.maxDatagramSize = s if cwndIsMinCwnd { - b.congestionWindow = b.minCongestionWindow() + b.congestionWindow = b.minCongestionWindow } b.pacer.SetMaxDatagramSize(s) } -func (b *bbrSender) OnPacketSent(sentTime time.Time, bytesInFlight congestion.ByteCount, packetNumber congestion.PacketNumber, bytes congestion.ByteCount, isRetransmittable bool) { - b.pacer.SentPacket(sentTime, bytes) - b.lastSendPacket = packetNumber - - b.bytesInFlight = bytesInFlight - if bytesInFlight == 0 && b.sampler.isAppLimited { - b.exitingQuiescence = true - } - - if b.aggregationEpochStartTime.IsZero() { - b.aggregationEpochStartTime = sentTime - } - - b.sampler.OnPacketSent(sentTime, packetNumber, bytes, bytesInFlight, isRetransmittable) +// InSlowStart implements the SendAlgorithmWithDebugInfos interface. +func (b *bbrSender) InSlowStart() bool { + return b.mode == bbrModeStartup } -func (b *bbrSender) CanSend(bytesInFlight congestion.ByteCount) bool { - b.bytesInFlight = bytesInFlight - return bytesInFlight < b.GetCongestionWindow() +// InRecovery implements the SendAlgorithmWithDebugInfos interface. +func (b *bbrSender) InRecovery() bool { + return b.recoveryState != bbrRecoveryStateNotInRecovery } +// GetCongestionWindow implements the SendAlgorithmWithDebugInfos interface. func (b *bbrSender) GetCongestionWindow() congestion.ByteCount { - if b.mode == PROBE_RTT { - return b.ProbeRttCongestionWindow() + if b.mode == bbrModeProbeRtt { + return b.probeRttCongestionWindow() } - if b.InRecovery() && !(b.rateBasedStartup && b.mode == STARTUP) { - return minByteCount(b.congestionWindow, b.recoveryWindow) + if b.InRecovery() { + return min(b.congestionWindow, b.recoveryWindow) } return b.congestionWindow } -func (b *bbrSender) MaybeExitSlowStart() { +func (b *bbrSender) OnCongestionEvent(number congestion.PacketNumber, lostBytes, priorInFlight congestion.ByteCount) { + // Do nothing. } -func (b *bbrSender) OnPacketAcked(number congestion.PacketNumber, ackedBytes, priorInFlight congestion.ByteCount, eventTime time.Time) { - totalBytesAckedBefore := b.sampler.totalBytesAcked - isRoundStart, minRttExpired := false, false - lastAckedPacket := number +func (b *bbrSender) OnCongestionEventEx(priorInFlight congestion.ByteCount, eventTime time.Time, ackedPackets []congestion.AckedPacketInfo, lostPackets []congestion.LostPacketInfo) { + totalBytesAckedBefore := b.sampler.TotalBytesAcked() + totalBytesLostBefore := b.sampler.TotalBytesLost() - isRoundStart = b.UpdateRoundTripCounter(lastAckedPacket) - minRttExpired = b.UpdateBandwidthAndMinRtt(eventTime, number, ackedBytes) - b.UpdateRecoveryState(false, isRoundStart) - bytesAcked := b.sampler.totalBytesAcked - totalBytesAckedBefore - excessAcked := b.UpdateAckAggregationBytes(eventTime, bytesAcked) + var isRoundStart, minRttExpired bool + var excessAcked, bytesLost congestion.ByteCount - // Handle logic specific to STARTUP and DRAIN modes. - if isRoundStart && !b.isAtFullBandwidth { - b.CheckIfFullBandwidthReached() + // The send state of the largest packet in acked_packets, unless it is + // empty. If acked_packets is empty, it's the send state of the largest + // packet in lost_packets. + var lastPacketSendState sendTimeState + + b.maybeApplimited(priorInFlight) + + // Update bytesInFlight + b.bytesInFlight = priorInFlight + for _, p := range ackedPackets { + b.bytesInFlight -= p.BytesAcked + } + for _, p := range lostPackets { + b.bytesInFlight -= p.BytesLost } - b.MaybeExitStartupOrDrain(eventTime) - // Handle logic specific to PROBE_RTT. - b.MaybeEnterOrExitProbeRtt(eventTime, isRoundStart, minRttExpired) + if len(ackedPackets) != 0 { + lastAckedPacket := ackedPackets[len(ackedPackets)-1].PacketNumber + isRoundStart = b.updateRoundTripCounter(lastAckedPacket) + b.updateRecoveryState(lastAckedPacket, len(lostPackets) != 0, isRoundStart) + } - // After the model is updated, recalculate the pacing rate and congestion - // window. - b.CalculatePacingRate() - b.CalculateCongestionWindow(bytesAcked, excessAcked) - b.CalculateRecoveryWindow(bytesAcked, congestion.ByteCount(0)) -} + sample := b.sampler.OnCongestionEvent(eventTime, + ackedPackets, lostPackets, b.maxBandwidth.GetBest(), infBandwidth, b.roundTripCount) + if sample.lastPacketSendState.isValid { + b.lastSampleIsAppLimited = sample.lastPacketSendState.isAppLimited + b.hasNoAppLimitedSample = b.hasNoAppLimitedSample || !b.lastSampleIsAppLimited + } + // Avoid updating |max_bandwidth_| if a) this is a loss-only event, or b) all + // packets in |acked_packets| did not generate valid samples. (e.g. ack of + // ack-only packets). In both cases, sampler_.total_bytes_acked() will not + // change. + if totalBytesAckedBefore != b.sampler.TotalBytesAcked() { + if !sample.sampleIsAppLimited || sample.sampleMaxBandwidth > b.maxBandwidth.GetBest() { + b.maxBandwidth.Update(sample.sampleMaxBandwidth, b.roundTripCount) + } + } -func (b *bbrSender) OnCongestionEvent(number congestion.PacketNumber, lostBytes, priorInFlight congestion.ByteCount) { - eventTime := time.Now() - totalBytesAckedBefore := b.sampler.totalBytesAcked - isRoundStart, minRttExpired := false, false + if sample.sampleRtt != infRTT { + minRttExpired = b.maybeUpdateMinRtt(eventTime, sample.sampleRtt) + } + bytesLost = b.sampler.TotalBytesLost() - totalBytesLostBefore - b.DiscardLostPackets(number, lostBytes) + excessAcked = sample.extraAcked + lastPacketSendState = sample.lastPacketSendState - // Input the new data into the BBR model of the connection. - var excessAcked congestion.ByteCount + if len(lostPackets) != 0 { + b.numLossEventsInRound++ + b.bytesLostInRound += bytesLost + } // Handle logic specific to PROBE_BW mode. - if b.mode == PROBE_BW { - b.UpdateGainCyclePhase(time.Now(), priorInFlight, true) + if b.mode == bbrModeProbeBw { + b.updateGainCyclePhase(eventTime, priorInFlight, len(lostPackets) != 0) } // Handle logic specific to STARTUP and DRAIN modes. - b.MaybeExitStartupOrDrain(eventTime) + if isRoundStart && !b.isAtFullBandwidth { + b.checkIfFullBandwidthReached(&lastPacketSendState) + } + + b.maybeExitStartupOrDrain(eventTime) // Handle logic specific to PROBE_RTT. - b.MaybeEnterOrExitProbeRtt(eventTime, isRoundStart, minRttExpired) + b.maybeEnterOrExitProbeRtt(eventTime, isRoundStart, minRttExpired) // Calculate number of packets acked and lost. - bytesAcked := b.sampler.totalBytesAcked - totalBytesAckedBefore - bytesLost := lostBytes + bytesAcked := b.sampler.TotalBytesAcked() - totalBytesAckedBefore // After the model is updated, recalculate the pacing rate and congestion // window. - b.CalculatePacingRate() - b.CalculateCongestionWindow(bytesAcked, excessAcked) - b.CalculateRecoveryWindow(bytesAcked, bytesLost) + b.calculatePacingRate(bytesLost) + b.calculateCongestionWindow(bytesAcked, excessAcked) + b.calculateRecoveryWindow(bytesAcked, bytesLost) + + // Cleanup internal state. + if len(lostPackets) != 0 { + lastLostPacket := lostPackets[len(lostPackets)-1].PacketNumber + b.sampler.RemoveObsoletePackets(lastLostPacket) + } + if isRoundStart { + b.numLossEventsInRound = 0 + b.bytesLostInRound = 0 + } } -//func (b *bbrSender) OnCongestionEvent(priorInFlight congestion.ByteCount, eventTime time.Time, ackedPackets, lostPackets []*congestion.Packet) { -// totalBytesAckedBefore := b.sampler.totalBytesAcked -// isRoundStart, minRttExpired := false, false -// -// if lostPackets != nil { -// b.DiscardLostPackets(lostPackets) -// } -// -// // Input the new data into the BBR model of the connection. -// var excessAcked congestion.ByteCount -// if len(ackedPackets) > 0 { -// lastAckedPacket := ackedPackets[len(ackedPackets)-1].PacketNumber -// isRoundStart = b.UpdateRoundTripCounter(lastAckedPacket) -// minRttExpired = b.UpdateBandwidthAndMinRtt(eventTime, ackedPackets) -// b.UpdateRecoveryState(lastAckedPacket, len(lostPackets) > 0, isRoundStart) -// bytesAcked := b.sampler.totalBytesAcked - totalBytesAckedBefore -// excessAcked = b.UpdateAckAggregationBytes(eventTime, bytesAcked) -// } -// -// // Handle logic specific to PROBE_BW mode. -// if b.mode == PROBE_BW { -// b.UpdateGainCyclePhase(eventTime, priorInFlight, len(lostPackets) > 0) -// } -// -// // Handle logic specific to STARTUP and DRAIN modes. -// if isRoundStart && !b.isAtFullBandwidth { -// b.CheckIfFullBandwidthReached() -// } -// b.MaybeExitStartupOrDrain(eventTime) -// -// // Handle logic specific to PROBE_RTT. -// b.MaybeEnterOrExitProbeRtt(eventTime, isRoundStart, minRttExpired) -// -// // Calculate number of packets acked and lost. -// bytesAcked := b.sampler.totalBytesAcked - totalBytesAckedBefore -// bytesLost := congestion.ByteCount(0) -// for _, packet := range lostPackets { -// bytesLost += packet.Length -// } -// -// // After the model is updated, recalculate the pacing rate and congestion -// // window. -// b.CalculatePacingRate() -// b.CalculateCongestionWindow(bytesAcked, excessAcked) -// b.CalculateRecoveryWindow(bytesAcked, bytesLost) -//} - -//func (b *bbrSender) SetNumEmulatedConnections(n int) { -// -//} +func (b *bbrSender) PacingRate() Bandwidth { + if b.pacingRate == 0 { + return Bandwidth(b.highGain * float64( + BandwidthFromDelta(b.initialCongestionWindow, b.getMinRtt()))) + } -func (b *bbrSender) OnRetransmissionTimeout(packetsRetransmitted bool) { + return b.pacingRate } -//func (b *bbrSender) OnConnectionMigration() { -// -//} - -//// Experiments -//func (b *bbrSender) SetSlowStartLargeReduction(enabled bool) { -// -//} +func (b *bbrSender) hasGoodBandwidthEstimateForResumption() bool { + return b.hasNonAppLimitedSample() +} -//func (b *bbrSender) BandwidthEstimate() Bandwidth { -// return Bandwidth(b.maxBandwidth.GetBest()) -//} +func (b *bbrSender) hasNonAppLimitedSample() bool { + return b.hasNoAppLimitedSample +} -// BandwidthEstimate returns the current bandwidth estimate -func (b *bbrSender) BandwidthEstimate() Bandwidth { - if b.rttStats == nil { - return infBandwidth +// Sets the pacing gain used in STARTUP. Must be greater than 1. +func (b *bbrSender) setHighGain(highGain float64) { + b.highGain = highGain + if b.mode == bbrModeStartup { + b.pacingGain = highGain } - srtt := b.rttStats.SmoothedRTT() - if srtt == 0 { - // If we haven't measured an rtt, the bandwidth estimate is unknown. - return infBandwidth - } - return BandwidthFromDelta(b.GetCongestionWindow(), srtt) } -//func (b *bbrSender) HybridSlowStart() *HybridSlowStart { -// return nil -//} - -//func (b *bbrSender) SlowstartThreshold() congestion.ByteCount { -// return 0 -//} +// Sets the CWND gain used in STARTUP. Must be greater than 1. +func (b *bbrSender) setHighCwndGain(highCwndGain float64) { + b.highCwndGain = highCwndGain + if b.mode == bbrModeStartup { + b.congestionWindowGain = highCwndGain + } +} -//func (b *bbrSender) RenoBeta() float32 { -// return 0.0 -//} +// Sets the gain used in DRAIN. Must be less than 1. +func (b *bbrSender) setDrainGain(drainGain float64) { + b.drainGain = drainGain +} -func (b *bbrSender) InRecovery() bool { - return b.recoveryState != NOT_IN_RECOVERY +// What's the current estimated bandwidth in bytes per second. +func (b *bbrSender) bandwidthEstimate() Bandwidth { + return b.maxBandwidth.GetBest() } -func (b *bbrSender) InSlowStart() bool { - return b.mode == STARTUP -} - -//func (b *bbrSender) ShouldSendProbingPacket() bool { -// if b.pacingGain <= 1 { -// return false -// } -// // TODO(b/77975811): If the pipe is highly under-utilized, consider not -// // sending a probing transmission, because the extra bandwidth is not needed. -// // If flexible_app_limited is enabled, check if the pipe is sufficiently full. -// if b.flexibleAppLimited { -// return !b.IsPipeSufficientlyFull() -// } else { -// return true -// } -//} - -//func (b *bbrSender) IsPipeSufficientlyFull() bool { -// // See if we need more bytes in flight to see more bandwidth. -// if b.mode == STARTUP { -// // STARTUP exits if it doesn't observe a 25% bandwidth increase, so the CWND -// // must be more than 25% above the target. -// return b.GetBytesInFlight() >= b.GetTargetCongestionWindow(1.5) -// } -// if b.pacingGain > 1 { -// // Super-unity PROBE_BW doesn't exit until 1.25 * BDP is achieved. -// return b.GetBytesInFlight() >= b.GetTargetCongestionWindow(b.pacingGain) -// } -// // If bytes_in_flight are above the target congestion window, it should be -// // possible to observe the same or more bandwidth if it's available. -// return b.GetBytesInFlight() >= b.GetTargetCongestionWindow(1.1) -//} - -//func (b *bbrSender) SetFromConfig() { -// // TODO: not impl. -//} - -func (b *bbrSender) UpdateRoundTripCounter(lastAckedPacket congestion.PacketNumber) bool { - if b.currentRoundTripEnd == 0 || lastAckedPacket > b.currentRoundTripEnd { - b.currentRoundTripEnd = lastAckedPacket - b.roundTripCount++ - // if b.rttStats != nil && b.InSlowStart() { - // TODO: ++stats_->slowstart_num_rtts; - // } - return true +// Returns the current estimate of the RTT of the connection. Outside of the +// edge cases, this is minimum RTT. +func (b *bbrSender) getMinRtt() time.Duration { + if b.minRtt != 0 { + return b.minRtt + } + // min_rtt could be available if the handshake packet gets neutered then + // gets acknowledged. This could only happen for QUIC crypto where we do not + // drop keys. + minRtt := b.rttStats.MinRTT() + if minRtt == 0 { + return 100 * time.Millisecond + } else { + return minRtt } - return false } -func (b *bbrSender) UpdateBandwidthAndMinRtt(now time.Time, number congestion.PacketNumber, ackedBytes congestion.ByteCount) bool { - sampleMinRtt := InfiniteRTT +// Computes the target congestion window using the specified gain. +func (b *bbrSender) getTargetCongestionWindow(gain float64) congestion.ByteCount { + bdp := bdpFromRttAndBandwidth(b.getMinRtt(), b.bandwidthEstimate()) + congestionWindow := congestion.ByteCount(gain * float64(bdp)) - if !b.alwaysGetBwSampleWhenAcked && ackedBytes == 0 { - // Skip acked packets with 0 in flight bytes when updating bandwidth. - return false - } - bandwidthSample := b.sampler.OnPacketAcked(now, number) - if b.alwaysGetBwSampleWhenAcked && !bandwidthSample.stateAtSend.isValid { - // From the sampler's perspective, the packet has never been sent, or the - // packet has been acked or marked as lost previously. - return false - } - b.lastSampleIsAppLimited = bandwidthSample.stateAtSend.isAppLimited - // has_non_app_limited_sample_ |= - // !bandwidth_sample.state_at_send.is_app_limited; - if !bandwidthSample.stateAtSend.isAppLimited { - b.hasNoAppLimitedSample = true - } - if bandwidthSample.rtt > 0 { - sampleMinRtt = minRtt(sampleMinRtt, bandwidthSample.rtt) - } - if !bandwidthSample.stateAtSend.isAppLimited || bandwidthSample.bandwidth > b.BandwidthEstimate() { - b.maxBandwidth.Update(int64(bandwidthSample.bandwidth), b.roundTripCount) + // BDP estimate will be zero if no bandwidth samples are available yet. + if congestionWindow == 0 { + congestionWindow = congestion.ByteCount(gain * float64(b.initialCongestionWindow)) } - // If none of the RTT samples are valid, return immediately. - if sampleMinRtt == InfiniteRTT { - return false - } + return max(congestionWindow, b.minCongestionWindow) +} + +// The target congestion window during PROBE_RTT. +func (b *bbrSender) probeRttCongestionWindow() congestion.ByteCount { + return b.minCongestionWindow +} - b.minRttSinceLastProbeRtt = minRtt(b.minRttSinceLastProbeRtt, sampleMinRtt) +func (b *bbrSender) maybeUpdateMinRtt(now time.Time, sampleMinRtt time.Duration) bool { // Do not expire min_rtt if none was ever available. - minRttExpired := b.minRtt > 0 && (now.After(b.minRttTimestamp.Add(MinRttExpiry))) + minRttExpired := b.minRtt != 0 && now.After(b.minRttTimestamp.Add(minRttExpiry)) if minRttExpired || sampleMinRtt < b.minRtt || b.minRtt == 0 { - if minRttExpired && b.ShouldExtendMinRttExpiry() { - minRttExpired = false - } else { - b.minRtt = sampleMinRtt - } + b.minRtt = sampleMinRtt b.minRttTimestamp = now - // Reset since_last_probe_rtt fields. - b.minRttSinceLastProbeRtt = InfiniteRTT - b.appLimitedSinceLastProbeRtt = false } return minRttExpired } -func (b *bbrSender) ShouldExtendMinRttExpiry() bool { - if b.probeRttDisabledIfAppLimited && b.appLimitedSinceLastProbeRtt { - // Extend the current min_rtt if we've been app limited recently. - return true - } - - minRttIncreasedSinceLastProbe := b.minRttSinceLastProbeRtt > time.Duration(float64(b.minRtt)*SimilarMinRttThreshold) - if b.probeRttSkippedIfSimilarRtt && b.appLimitedSinceLastProbeRtt && !minRttIncreasedSinceLastProbe { - // Extend the current min_rtt if we've been app limited recently and an rtt - // has been measured in that time that's less than 12.5% more than the - // current min_rtt. - return true - } - - return false +// Enters the STARTUP mode. +func (b *bbrSender) enterStartupMode(now time.Time) { + b.mode = bbrModeStartup + // b.maybeTraceStateChange(logging.CongestionStateStartup) + b.pacingGain = b.highGain + b.congestionWindowGain = b.highCwndGain } -func (b *bbrSender) DiscardLostPackets(number congestion.PacketNumber, lostBytes congestion.ByteCount) { - b.sampler.OnCongestionEvent(number) - if b.mode == STARTUP { - // if b.rttStats != nil { - // TODO: slow start. - // } - if b.startupRateReductionMultiplier != 0 { - b.startupBytesLost += lostBytes - } - } -} +// Enters the PROBE_BW mode. +func (b *bbrSender) enterProbeBandwidthMode(now time.Time) { + b.mode = bbrModeProbeBw + // b.maybeTraceStateChange(logging.CongestionStateProbeBw) + b.congestionWindowGain = b.congestionWindowGainConstant -func (b *bbrSender) UpdateRecoveryState(hasLosses, isRoundStart bool) { - // Exit recovery when there are no losses for a round. - if !hasLosses { - b.endRecoveryAt = b.lastSendPacket - } - switch b.recoveryState { - case NOT_IN_RECOVERY: - // Enter conservation on the first loss. - if hasLosses { - b.recoveryState = CONSERVATION - // This will cause the |recovery_window_| to be set to the correct - // value in CalculateRecoveryWindow(). - b.recoveryWindow = 0 - // Since the conservation phase is meant to be lasting for a whole - // round, extend the current round as if it were started right now. - b.currentRoundTripEnd = b.lastSendPacket - if false && b.lastSampleIsAppLimited { - b.isAppLimitedRecovery = true - } - } - case CONSERVATION: - if isRoundStart { - b.recoveryState = GROWTH - } - fallthrough - case GROWTH: - // Exit recovery if appropriate. - if !hasLosses && b.lastSendPacket > b.endRecoveryAt { - b.recoveryState = NOT_IN_RECOVERY - b.isAppLimitedRecovery = false - } + // Pick a random offset for the gain cycle out of {0, 2..7} range. 1 is + // excluded because in that case increased gain and decreased gain would not + // follow each other. + b.cycleCurrentOffset = int(rand.Int31n(congestion.PacketsPerConnectionID)) % (gainCycleLength - 1) + if b.cycleCurrentOffset >= 1 { + b.cycleCurrentOffset += 1 } - if b.recoveryState != NOT_IN_RECOVERY && b.isAppLimitedRecovery { - b.sampler.OnAppLimited() - } + b.lastCycleStart = now + b.pacingGain = pacingGain[b.cycleCurrentOffset] } -func (b *bbrSender) UpdateAckAggregationBytes(ackTime time.Time, ackedBytes congestion.ByteCount) congestion.ByteCount { - // Compute how many bytes are expected to be delivered, assuming max bandwidth - // is correct. - expectedAckedBytes := congestion.ByteCount(b.maxBandwidth.GetBest()) * - congestion.ByteCount((ackTime.Sub(b.aggregationEpochStartTime))) - // Reset the current aggregation epoch as soon as the ack arrival rate is less - // than or equal to the max bandwidth. - if b.aggregationEpochBytes <= expectedAckedBytes { - // Reset to start measuring a new aggregation epoch. - b.aggregationEpochBytes = ackedBytes - b.aggregationEpochStartTime = ackTime - return 0 +// Updates the round-trip counter if a round-trip has passed. Returns true if +// the counter has been advanced. +func (b *bbrSender) updateRoundTripCounter(lastAckedPacket congestion.PacketNumber) bool { + if b.currentRoundTripEnd == invalidPacketNumber || lastAckedPacket > b.currentRoundTripEnd { + b.roundTripCount++ + b.currentRoundTripEnd = b.lastSentPacket + return true } - // Compute how many extra bytes were delivered vs max bandwidth. - // Include the bytes most recently acknowledged to account for stretch acks. - b.aggregationEpochBytes += ackedBytes - b.maxAckHeight.Update(int64(b.aggregationEpochBytes-expectedAckedBytes), b.roundTripCount) - return b.aggregationEpochBytes - expectedAckedBytes + return false } -func (b *bbrSender) UpdateGainCyclePhase(now time.Time, priorInFlight congestion.ByteCount, hasLosses bool) { - bytesInFlight := b.GetBytesInFlight() +// Updates the current gain used in PROBE_BW mode. +func (b *bbrSender) updateGainCyclePhase(now time.Time, priorInFlight congestion.ByteCount, hasLosses bool) { // In most cases, the cycle is advanced after an RTT passes. - shouldAdvanceGainCycling := now.Sub(b.lastCycleStart) > b.GetMinRtt() - + shouldAdvanceGainCycling := now.After(b.lastCycleStart.Add(b.getMinRtt())) // If the pacing gain is above 1.0, the connection is trying to probe the // bandwidth by increasing the number of bytes in flight to at least // pacing_gain * BDP. Make sure that it actually reaches the target, as long // as there are no losses suggesting that the buffers are not able to hold // that much. - if b.pacingGain > 1.0 && !hasLosses && priorInFlight < b.GetTargetCongestionWindow(b.pacingGain) { + if b.pacingGain > 1.0 && !hasLosses && priorInFlight < b.getTargetCongestionWindow(b.pacingGain) { shouldAdvanceGainCycling = false } + // If pacing gain is below 1.0, the connection is trying to drain the extra // queue which could have been incurred by probing prior to it. If the number // of bytes in flight falls down to the estimated BDP value earlier, conclude // that the queue has been successfully drained and exit this cycle early. - if b.pacingGain < 1.0 && bytesInFlight <= b.GetTargetCongestionWindow(1.0) { + if b.pacingGain < 1.0 && b.bytesInFlight <= b.getTargetCongestionWindow(1) { shouldAdvanceGainCycling = true } if shouldAdvanceGainCycling { - b.cycleCurrentOffset = (b.cycleCurrentOffset + 1) % GainCycleLength + b.cycleCurrentOffset = (b.cycleCurrentOffset + 1) % gainCycleLength b.lastCycleStart = now // Stay in low gain mode until the target BDP is hit. // Low gain mode will be exited immediately when the target BDP is achieved. - if b.drainToTarget && b.pacingGain < 1.0 && PacingGain[b.cycleCurrentOffset] == 1.0 && - bytesInFlight > b.GetTargetCongestionWindow(1.0) { + if b.drainToTarget && b.pacingGain < 1 && + pacingGain[b.cycleCurrentOffset] == 1 && + b.bytesInFlight > b.getTargetCongestionWindow(1) { return } - b.pacingGain = PacingGain[b.cycleCurrentOffset] + b.pacingGain = pacingGain[b.cycleCurrentOffset] } } -func (b *bbrSender) GetTargetCongestionWindow(gain float64) congestion.ByteCount { - bdp := congestion.ByteCount(b.GetMinRtt()) * congestion.ByteCount(b.BandwidthEstimate()) - congestionWindow := congestion.ByteCount(gain * float64(bdp)) - - // BDP estimate will be zero if no bandwidth samples are available yet. - if congestionWindow == 0 { - congestionWindow = congestion.ByteCount(gain * float64(b.initialCongestionWindow)) - } - - return maxByteCount(congestionWindow, b.minCongestionWindow()) -} - -func (b *bbrSender) CheckIfFullBandwidthReached() { +// Tracks for how many round-trips the bandwidth has not increased +// significantly. +func (b *bbrSender) checkIfFullBandwidthReached(lastPacketSendState *sendTimeState) { if b.lastSampleIsAppLimited { return } - target := Bandwidth(float64(b.bandwidthAtLastRound) * StartupGrowthTarget) - if b.BandwidthEstimate() >= target { - b.bandwidthAtLastRound = b.BandwidthEstimate() + target := Bandwidth(float64(b.bandwidthAtLastRound) * startupGrowthTarget) + if b.bandwidthEstimate() >= target { + b.bandwidthAtLastRound = b.bandwidthEstimate() b.roundsWithoutBandwidthGain = 0 if b.expireAckAggregationInStartup { // Expire old excess delivery measurements now that bandwidth increased. - b.maxAckHeight.Reset(0, b.roundTripCount) + b.sampler.ResetMaxAckHeightTracker(0, b.roundTripCount) } return } + b.roundsWithoutBandwidthGain++ - if b.roundsWithoutBandwidthGain >= b.numStartupRtts || (b.exitStartupOnLoss && b.InRecovery()) { + if b.roundsWithoutBandwidthGain >= b.numStartupRtts || + b.shouldExitStartupDueToLoss(lastPacketSendState) { b.isAtFullBandwidth = true } } -func (b *bbrSender) MaybeExitStartupOrDrain(now time.Time) { - if b.mode == STARTUP && b.isAtFullBandwidth { - b.OnExitStartup(now) - b.mode = DRAIN - b.pacingGain = b.drainGain - b.congestionWindowGain = b.highCwndGain +func (b *bbrSender) maybeApplimited(bytesInFlight congestion.ByteCount) { + congestionWindow := b.GetCongestionWindow() + if bytesInFlight >= congestionWindow { + return } - if b.mode == DRAIN && b.GetBytesInFlight() <= b.GetTargetCongestionWindow(1) { - b.EnterProbeBandwidthMode(now) + availableBytes := congestionWindow - bytesInFlight + drainLimited := b.mode == bbrModeDrain && bytesInFlight > congestionWindow/2 + if !drainLimited || availableBytes > maxBbrBurstPackets*b.maxDatagramSize { + b.sampler.OnAppLimited() } } -func (b *bbrSender) EnterProbeBandwidthMode(now time.Time) { - b.mode = PROBE_BW - b.congestionWindowGain = b.congestionWindowGainConst - - // Pick a random offset for the gain cycle out of {0, 2..7} range. 1 is - // excluded because in that case increased gain and decreased gain would not - // follow each other. - b.cycleCurrentOffset = fastrand.Int() % (GainCycleLength - 1) - if b.cycleCurrentOffset >= 1 { - b.cycleCurrentOffset += 1 +// Transitions from STARTUP to DRAIN and from DRAIN to PROBE_BW if +// appropriate. +func (b *bbrSender) maybeExitStartupOrDrain(now time.Time) { + if b.mode == bbrModeStartup && b.isAtFullBandwidth { + b.mode = bbrModeDrain + // b.maybeTraceStateChange(logging.CongestionStateDrain) + b.pacingGain = b.drainGain + b.congestionWindowGain = b.highCwndGain + } + if b.mode == bbrModeDrain && b.bytesInFlight <= b.getTargetCongestionWindow(1) { + b.enterProbeBandwidthMode(now) } - - b.lastCycleStart = now - b.pacingGain = PacingGain[b.cycleCurrentOffset] } -func (b *bbrSender) MaybeEnterOrExitProbeRtt(now time.Time, isRoundStart, minRttExpired bool) { - if minRttExpired && !b.exitingQuiescence && b.mode != PROBE_RTT { - if b.InSlowStart() { - b.OnExitStartup(now) - } - b.mode = PROBE_RTT +// Decides whether to enter or exit PROBE_RTT. +func (b *bbrSender) maybeEnterOrExitProbeRtt(now time.Time, isRoundStart, minRttExpired bool) { + if minRttExpired && !b.exitingQuiescence && b.mode != bbrModeProbeRtt { + b.mode = bbrModeProbeRtt + // b.maybeTraceStateChange(logging.CongestionStateProbRtt) b.pacingGain = 1.0 // Do not decide on the time to exit PROBE_RTT until the |bytes_in_flight| // is at the target small value. b.exitProbeRttAt = time.Time{} } - if b.mode == PROBE_RTT { + if b.mode == bbrModeProbeRtt { b.sampler.OnAppLimited() + // b.maybeTraceStateChange(logging.CongestionStateApplicationLimited) + if b.exitProbeRttAt.IsZero() { // If the window has reached the appropriate size, schedule exiting // PROBE_RTT. The CWND during PROBE_RTT is kMinimumCongestionWindow, but // we allow an extra packet since QUIC checks CWND before sending a // packet. - if b.GetBytesInFlight() < b.ProbeRttCongestionWindow()+b.maxDatagramSize { - b.exitProbeRttAt = now.Add(ProbeRttTime) + if b.bytesInFlight < b.probeRttCongestionWindow()+congestion.MaxPacketBufferSize { + b.exitProbeRttAt = now.Add(probeRttTime) b.probeRttRoundPassed = false } } else { if isRoundStart { b.probeRttRoundPassed = true } - if !now.Before(b.exitProbeRttAt) && b.probeRttRoundPassed { + if now.Sub(b.exitProbeRttAt) >= 0 && b.probeRttRoundPassed { b.minRttTimestamp = now if !b.isAtFullBandwidth { - b.EnterStartupMode(now) + b.enterStartupMode(now) } else { - b.EnterProbeBandwidthMode(now) + b.enterProbeBandwidthMode(now) } } } } + b.exitingQuiescence = false } -func (b *bbrSender) ProbeRttCongestionWindow() congestion.ByteCount { - if b.probeRttBasedOnBdp { - return b.GetTargetCongestionWindow(ModerateProbeRttMultiplier) - } else { - return b.minCongestionWindow() +// Determines whether BBR needs to enter, exit or advance state of the +// recovery. +func (b *bbrSender) updateRecoveryState(lastAckedPacket congestion.PacketNumber, hasLosses, isRoundStart bool) { + // Disable recovery in startup, if loss-based exit is enabled. + if !b.isAtFullBandwidth { + return } -} -func (b *bbrSender) EnterStartupMode(now time.Time) { - // if b.rttStats != nil { - // TODO: slow start. - // } - b.mode = STARTUP - b.pacingGain = b.highGain - b.congestionWindowGain = b.highCwndGain -} + // Exit recovery when there are no losses for a round. + if hasLosses { + b.endRecoveryAt = b.lastSentPacket + } -func (b *bbrSender) OnExitStartup(now time.Time) { - if b.rttStats == nil { - return + switch b.recoveryState { + case bbrRecoveryStateNotInRecovery: + if hasLosses { + b.recoveryState = bbrRecoveryStateConservation + // This will cause the |recovery_window_| to be set to the correct + // value in CalculateRecoveryWindow(). + b.recoveryWindow = 0 + // Since the conservation phase is meant to be lasting for a whole + // round, extend the current round as if it were started right now. + b.currentRoundTripEnd = b.lastSentPacket + } + case bbrRecoveryStateConservation: + if isRoundStart { + b.recoveryState = bbrRecoveryStateGrowth + } + fallthrough + case bbrRecoveryStateGrowth: + // Exit recovery if appropriate. + if !hasLosses && lastAckedPacket > b.endRecoveryAt { + b.recoveryState = bbrRecoveryStateNotInRecovery + } } - // TODO: slow start. } -func (b *bbrSender) CalculatePacingRate() { - if b.BandwidthEstimate() == 0 { +// Determines the appropriate pacing rate for the connection. +func (b *bbrSender) calculatePacingRate(bytesLost congestion.ByteCount) { + if b.bandwidthEstimate() == 0 { return } - targetRate := Bandwidth(b.pacingGain * float64(b.BandwidthEstimate())) + targetRate := Bandwidth(b.pacingGain * float64(b.bandwidthEstimate())) if b.isAtFullBandwidth { b.pacingRate = targetRate return @@ -859,39 +798,42 @@ func (b *bbrSender) CalculatePacingRate() { // Pace at the rate of initial_window / RTT as soon as RTT measurements are // available. - if b.pacingRate == 0 && b.rttStats.MinRTT() > 0 { + if b.pacingRate == 0 && b.rttStats.MinRTT() != 0 { b.pacingRate = BandwidthFromDelta(b.initialCongestionWindow, b.rttStats.MinRTT()) return } - // Slow the pacing rate in STARTUP once loss has ever been detected. - hasEverDetectedLoss := b.endRecoveryAt > 0 - if b.slowerStartup && hasEverDetectedLoss && b.hasNoAppLimitedSample { - b.pacingRate = Bandwidth(StartupAfterLossGain * float64(b.BandwidthEstimate())) - return - } - // Slow the pacing rate in STARTUP by the bytes_lost / CWND. - if b.startupRateReductionMultiplier != 0 && hasEverDetectedLoss && b.hasNoAppLimitedSample { - b.pacingRate = Bandwidth((1.0 - (float64(b.startupBytesLost) * float64(b.startupRateReductionMultiplier) / float64(b.congestionWindow))) * float64(targetRate)) - // Ensure the pacing rate doesn't drop below the startup growth target times - // the bandwidth estimate. - b.pacingRate = maxBandwidth(b.pacingRate, Bandwidth(StartupGrowthTarget*float64(b.BandwidthEstimate()))) - return + if b.detectOvershooting { + b.bytesLostWhileDetectingOvershooting += bytesLost + // Check for overshooting with network parameters adjusted when pacing rate + // > target_rate and loss has been detected. + if b.pacingRate > targetRate && b.bytesLostWhileDetectingOvershooting > 0 { + if b.hasNoAppLimitedSample || + b.bytesLostWhileDetectingOvershooting*congestion.ByteCount(b.bytesLostMultiplierWhileDetectingOvershooting) > b.initialCongestionWindow { + // We are fairly sure overshoot happens if 1) there is at least one + // non app-limited bw sample or 2) half of IW gets lost. Slow pacing + // rate. + b.pacingRate = max(targetRate, BandwidthFromDelta(b.cwndToCalculateMinPacingRate, b.rttStats.MinRTT())) + b.bytesLostWhileDetectingOvershooting = 0 + b.detectOvershooting = false + } + } } // Do not decrease the pacing rate during startup. - b.pacingRate = maxBandwidth(b.pacingRate, targetRate) + b.pacingRate = max(b.pacingRate, targetRate) } -func (b *bbrSender) CalculateCongestionWindow(ackedBytes, excessAcked congestion.ByteCount) { - if b.mode == PROBE_RTT { +// Determines the appropriate congestion window for the connection. +func (b *bbrSender) calculateCongestionWindow(bytesAcked, excessAcked congestion.ByteCount) { + if b.mode == bbrModeProbeRtt { return } - targetWindow := b.GetTargetCongestionWindow(b.congestionWindowGain) + targetWindow := b.getTargetCongestionWindow(b.congestionWindowGain) if b.isAtFullBandwidth { // Add the max recently measured ack aggregation to CWND. - targetWindow += congestion.ByteCount(b.maxAckHeight.GetBest()) + targetWindow += b.sampler.MaxAckHeight() } else if b.enableAckAggregationDuringStartup { // Add the most recent excess acked. Because CWND never decreases in // STARTUP, this will automatically create a very localized max filter. @@ -901,101 +843,83 @@ func (b *bbrSender) CalculateCongestionWindow(ackedBytes, excessAcked congestion // Instead of immediately setting the target CWND as the new one, BBR grows // the CWND towards |target_window| by only increasing it |bytes_acked| at a // time. - addBytesAcked := true || !b.InRecovery() if b.isAtFullBandwidth { - b.congestionWindow = minByteCount(targetWindow, b.congestionWindow+ackedBytes) - } else if addBytesAcked && (b.congestionWindow < targetWindow || b.sampler.totalBytesAcked < b.initialCongestionWindow) { + b.congestionWindow = min(targetWindow, b.congestionWindow+bytesAcked) + } else if b.congestionWindow < targetWindow || + b.sampler.TotalBytesAcked() < b.initialCongestionWindow { // If the connection is not yet out of startup phase, do not decrease the // window. - b.congestionWindow += ackedBytes + b.congestionWindow += bytesAcked } // Enforce the limits on the congestion window. - b.congestionWindow = maxByteCount(b.congestionWindow, b.minCongestionWindow()) - b.congestionWindow = minByteCount(b.congestionWindow, b.maxCongestionWindow()) + b.congestionWindow = max(b.congestionWindow, b.minCongestionWindow) + b.congestionWindow = min(b.congestionWindow, b.maxCongestionWindow) } -func (b *bbrSender) CalculateRecoveryWindow(ackedBytes, lostBytes congestion.ByteCount) { - if b.rateBasedStartup && b.mode == STARTUP { - return - } - - if b.recoveryState == NOT_IN_RECOVERY { +// Determines the appropriate window that constrains the in-flight during recovery. +func (b *bbrSender) calculateRecoveryWindow(bytesAcked, bytesLost congestion.ByteCount) { + if b.recoveryState == bbrRecoveryStateNotInRecovery { return } // Set up the initial recovery window. if b.recoveryWindow == 0 { - b.recoveryWindow = maxByteCount(b.GetBytesInFlight()+ackedBytes, b.minCongestionWindow()) + b.recoveryWindow = b.bytesInFlight + bytesAcked + b.recoveryWindow = max(b.minCongestionWindow, b.recoveryWindow) return } // Remove losses from the recovery window, while accounting for a potential // integer underflow. - if b.recoveryWindow >= lostBytes { - b.recoveryWindow -= lostBytes + if b.recoveryWindow >= bytesLost { + b.recoveryWindow = b.recoveryWindow - bytesLost } else { - b.recoveryWindow = congestion.ByteCount(b.maxDatagramSize) + b.recoveryWindow = b.maxDatagramSize } + // In CONSERVATION mode, just subtracting losses is sufficient. In GROWTH, // release additional |bytes_acked| to achieve a slow-start-like behavior. - if b.recoveryState == GROWTH { - b.recoveryWindow += ackedBytes + if b.recoveryState == bbrRecoveryStateGrowth { + b.recoveryWindow += bytesAcked } - // Sanity checks. Ensure that we always allow to send at least an MSS or - // |bytes_acked| in response, whichever is larger. - b.recoveryWindow = maxByteCount(b.recoveryWindow, b.GetBytesInFlight()+ackedBytes) - b.recoveryWindow = maxByteCount(b.recoveryWindow, b.minCongestionWindow()) -} - -var _ congestion.CongestionControl = (*bbrSender)(nil) -func (b *bbrSender) GetMinRtt() time.Duration { - if b.minRtt > 0 { - return b.minRtt - } else { - return InitialRtt - } + // Always allow sending at least |bytes_acked| in response. + b.recoveryWindow = max(b.recoveryWindow, b.bytesInFlight+bytesAcked) + b.recoveryWindow = max(b.minCongestionWindow, b.recoveryWindow) } -func minRtt(a, b time.Duration) time.Duration { - if a < b { - return a - } else { - return b +// Return whether we should exit STARTUP due to excessive loss. +func (b *bbrSender) shouldExitStartupDueToLoss(lastPacketSendState *sendTimeState) bool { + if b.numLossEventsInRound < defaultStartupFullLossCount || !lastPacketSendState.isValid { + return false } -} -func minBandwidth(a, b Bandwidth) Bandwidth { - if a < b { - return a - } else { - return b - } -} + inflightAtSend := lastPacketSendState.bytesInFlight -func maxBandwidth(a, b Bandwidth) Bandwidth { - if a > b { - return a - } else { - return b + if inflightAtSend > 0 && b.bytesLostInRound > 0 { + if b.bytesLostInRound > congestion.ByteCount(float64(inflightAtSend)*quicBbr2DefaultLossThreshold) { + return true + } + return false } + return false } -func maxByteCount(a, b congestion.ByteCount) congestion.ByteCount { - if a > b { - return a - } else { - return b - } +func bdpFromRttAndBandwidth(rtt time.Duration, bandwidth Bandwidth) congestion.ByteCount { + return congestion.ByteCount(rtt) * congestion.ByteCount(bandwidth) / congestion.ByteCount(BytesPerSecond) / congestion.ByteCount(time.Second) } -func minByteCount(a, b congestion.ByteCount) congestion.ByteCount { - if a < b { - return a +func GetInitialPacketSize(addr net.Addr) congestion.ByteCount { + // If this is not a UDP address, we don't know anything about the MTU. + // Use the minimum size of an Initial packet as the max packet size. + if udpAddr, ok := addr.(*net.UDPAddr); ok { + if udpAddr.IP.To4() != nil { + return congestion.InitialPacketSizeIPv4 + } else { + return congestion.InitialPacketSizeIPv6 + } } else { - return b + return congestion.MinInitialPacketSize } } - -var InfiniteRTT = time.Duration(math.MaxInt64) diff --git a/core/internal/congestion/bbr/packet_number_indexed_queue.go b/core/internal/congestion/bbr/packet_number_indexed_queue.go new file mode 100644 index 0000000000..86fe52d2be --- /dev/null +++ b/core/internal/congestion/bbr/packet_number_indexed_queue.go @@ -0,0 +1,199 @@ +package bbr + +import ( + "github.com/apernet/quic-go/congestion" +) + +// packetNumberIndexedQueue is a queue of mostly continuous numbered entries +// which supports the following operations: +// - adding elements to the end of the queue, or at some point past the end +// - removing elements in any order +// - retrieving elements +// If all elements are inserted in order, all of the operations above are +// amortized O(1) time. +// +// Internally, the data structure is a deque where each element is marked as +// present or not. The deque starts at the lowest present index. Whenever an +// element is removed, it's marked as not present, and the front of the deque is +// cleared of elements that are not present. +// +// The tail of the queue is not cleared due to the assumption of entries being +// inserted in order, though removing all elements of the queue will return it +// to its initial state. +// +// Note that this data structure is inherently hazardous, since an addition of +// just two entries will cause it to consume all of the memory available. +// Because of that, it is not a general-purpose container and should not be used +// as one. + +type entryWrapper[T any] struct { + present bool + entry T +} + +type packetNumberIndexedQueue[T any] struct { + entries RingBuffer[entryWrapper[T]] + numberOfPresentEntries int + firstPacket congestion.PacketNumber +} + +func newPacketNumberIndexedQueue[T any](size int) *packetNumberIndexedQueue[T] { + q := &packetNumberIndexedQueue[T]{ + firstPacket: invalidPacketNumber, + } + + q.entries.Init(size) + + return q +} + +// Emplace inserts data associated |packet_number| into (or past) the end of the +// queue, filling up the missing intermediate entries as necessary. Returns +// true if the element has been inserted successfully, false if it was already +// in the queue or inserted out of order. +func (p *packetNumberIndexedQueue[T]) Emplace(packetNumber congestion.PacketNumber, entry *T) bool { + if packetNumber == invalidPacketNumber || entry == nil { + return false + } + + if p.IsEmpty() { + p.entries.PushBack(entryWrapper[T]{ + present: true, + entry: *entry, + }) + p.numberOfPresentEntries = 1 + p.firstPacket = packetNumber + return true + } + + // Do not allow insertion out-of-order. + if packetNumber <= p.LastPacket() { + return false + } + + // Handle potentially missing elements. + offset := int(packetNumber - p.FirstPacket()) + if gap := offset - p.entries.Len(); gap > 0 { + for i := 0; i < gap; i++ { + p.entries.PushBack(entryWrapper[T]{}) + } + } + + p.entries.PushBack(entryWrapper[T]{ + present: true, + entry: *entry, + }) + p.numberOfPresentEntries++ + return true +} + +// GetEntry Retrieve the entry associated with the packet number. Returns the pointer +// to the entry in case of success, or nullptr if the entry does not exist. +func (p *packetNumberIndexedQueue[T]) GetEntry(packetNumber congestion.PacketNumber) *T { + ew := p.getEntryWraper(packetNumber) + if ew == nil { + return nil + } + + return &ew.entry +} + +// Remove, Same as above, but if an entry is present in the queue, also call f(entry) +// before removing it. +func (p *packetNumberIndexedQueue[T]) Remove(packetNumber congestion.PacketNumber, f func(T)) bool { + ew := p.getEntryWraper(packetNumber) + if ew == nil { + return false + } + if f != nil { + f(ew.entry) + } + ew.present = false + p.numberOfPresentEntries-- + + if packetNumber == p.FirstPacket() { + p.clearup() + } + + return true +} + +// RemoveUpTo, but not including |packet_number|. +// Unused slots in the front are also removed, which means when the function +// returns, |first_packet()| can be larger than |packet_number|. +func (p *packetNumberIndexedQueue[T]) RemoveUpTo(packetNumber congestion.PacketNumber) { + for !p.entries.Empty() && + p.firstPacket != invalidPacketNumber && + p.firstPacket < packetNumber { + if p.entries.Front().present { + p.numberOfPresentEntries-- + } + p.entries.PopFront() + p.firstPacket++ + } + p.clearup() + + return +} + +// IsEmpty return if queue is empty. +func (p *packetNumberIndexedQueue[T]) IsEmpty() bool { + return p.numberOfPresentEntries == 0 +} + +// NumberOfPresentEntries returns the number of entries in the queue. +func (p *packetNumberIndexedQueue[T]) NumberOfPresentEntries() int { + return p.numberOfPresentEntries +} + +// EntrySlotsUsed returns the number of entries allocated in the underlying deque. This is +// proportional to the memory usage of the queue. +func (p *packetNumberIndexedQueue[T]) EntrySlotsUsed() int { + return p.entries.Len() +} + +// LastPacket returns packet number of the first entry in the queue. +func (p *packetNumberIndexedQueue[T]) FirstPacket() (packetNumber congestion.PacketNumber) { + return p.firstPacket +} + +// LastPacket returns packet number of the last entry ever inserted in the queue. Note that the +// entry in question may have already been removed. Zero if the queue is +// empty. +func (p *packetNumberIndexedQueue[T]) LastPacket() (packetNumber congestion.PacketNumber) { + if p.IsEmpty() { + return invalidPacketNumber + } + + return p.firstPacket + congestion.PacketNumber(p.entries.Len()-1) +} + +func (p *packetNumberIndexedQueue[T]) clearup() { + for !p.entries.Empty() && !p.entries.Front().present { + p.entries.PopFront() + p.firstPacket++ + } + if p.entries.Empty() { + p.firstPacket = invalidPacketNumber + } +} + +func (p *packetNumberIndexedQueue[T]) getEntryWraper(packetNumber congestion.PacketNumber) *entryWrapper[T] { + if packetNumber == invalidPacketNumber || + p.IsEmpty() || + packetNumber < p.firstPacket { + return nil + } + + offset := int(packetNumber - p.firstPacket) + if offset >= p.entries.Len() { + return nil + } + + ew := p.entries.Offset(offset) + if ew == nil || !ew.present { + return nil + } + + return ew +} diff --git a/core/internal/congestion/bbr/ringbuffer.go b/core/internal/congestion/bbr/ringbuffer.go new file mode 100644 index 0000000000..ed92d4ce01 --- /dev/null +++ b/core/internal/congestion/bbr/ringbuffer.go @@ -0,0 +1,118 @@ +package bbr + +// A RingBuffer is a ring buffer. +// It acts as a heap that doesn't cause any allocations. +type RingBuffer[T any] struct { + ring []T + headPos, tailPos int + full bool +} + +// Init preallocs a buffer with a certain size. +func (r *RingBuffer[T]) Init(size int) { + r.ring = make([]T, size) +} + +// Len returns the number of elements in the ring buffer. +func (r *RingBuffer[T]) Len() int { + if r.full { + return len(r.ring) + } + if r.tailPos >= r.headPos { + return r.tailPos - r.headPos + } + return r.tailPos - r.headPos + len(r.ring) +} + +// Empty says if the ring buffer is empty. +func (r *RingBuffer[T]) Empty() bool { + return !r.full && r.headPos == r.tailPos +} + +// PushBack adds a new element. +// If the ring buffer is full, its capacity is increased first. +func (r *RingBuffer[T]) PushBack(t T) { + if r.full || len(r.ring) == 0 { + r.grow() + } + r.ring[r.tailPos] = t + r.tailPos++ + if r.tailPos == len(r.ring) { + r.tailPos = 0 + } + if r.tailPos == r.headPos { + r.full = true + } +} + +// PopFront returns the next element. +// It must not be called when the buffer is empty, that means that +// callers might need to check if there are elements in the buffer first. +func (r *RingBuffer[T]) PopFront() T { + if r.Empty() { + panic("github.com/quic-go/quic-go/internal/utils/ringbuffer: pop from an empty queue") + } + r.full = false + t := r.ring[r.headPos] + r.ring[r.headPos] = *new(T) + r.headPos++ + if r.headPos == len(r.ring) { + r.headPos = 0 + } + return t +} + +// Offset returns the offset element. +// It must not be called when the buffer is empty, that means that +// callers might need to check if there are elements in the buffer first +// and check if the index larger than buffer length. +func (r *RingBuffer[T]) Offset(index int) *T { + if r.Empty() || index >= r.Len() { + panic("github.com/quic-go/quic-go/internal/utils/ringbuffer: offset from invalid index") + } + offset := (r.headPos + index) % len(r.ring) + return &r.ring[offset] +} + +// Front returns the front element. +// It must not be called when the buffer is empty, that means that +// callers might need to check if there are elements in the buffer first. +func (r *RingBuffer[T]) Front() *T { + if r.Empty() { + panic("github.com/quic-go/quic-go/internal/utils/ringbuffer: front from an empty queue") + } + return &r.ring[r.headPos] +} + +// Back returns the back element. +// It must not be called when the buffer is empty, that means that +// callers might need to check if there are elements in the buffer first. +func (r *RingBuffer[T]) Back() *T { + if r.Empty() { + panic("github.com/quic-go/quic-go/internal/utils/ringbuffer: back from an empty queue") + } + return r.Offset(r.Len() - 1) +} + +// Grow the maximum size of the queue. +// This method assume the queue is full. +func (r *RingBuffer[T]) grow() { + oldRing := r.ring + newSize := len(oldRing) * 2 + if newSize == 0 { + newSize = 1 + } + r.ring = make([]T, newSize) + headLen := copy(r.ring, oldRing[r.headPos:]) + copy(r.ring[headLen:], oldRing[:r.headPos]) + r.headPos, r.tailPos, r.full = 0, len(oldRing), false +} + +// Clear removes all elements. +func (r *RingBuffer[T]) Clear() { + var zeroValue T + for i := range r.ring { + r.ring[i] = zeroValue + } + r.headPos, r.tailPos, r.full = 0, 0, false +} diff --git a/core/internal/congestion/bbr/windowed_filter.go b/core/internal/congestion/bbr/windowed_filter.go index 92a41aa3f0..4773bce597 100644 --- a/core/internal/congestion/bbr/windowed_filter.go +++ b/core/internal/congestion/bbr/windowed_filter.go @@ -1,132 +1,162 @@ package bbr -// WindowedFilter Use the following to construct a windowed filter object of type T. -// For example, a min filter using QuicTime as the time type: -// -// WindowedFilter, QuicTime, QuicTime::Delta> ObjectName; -// -// A max filter using 64-bit integers as the time type: +import ( + "golang.org/x/exp/constraints" +) + +// Implements Kathleen Nichols' algorithm for tracking the minimum (or maximum) +// estimate of a stream of samples over some fixed time interval. (E.g., +// the minimum RTT over the past five minutes.) The algorithm keeps track of +// the best, second best, and third best min (or max) estimates, maintaining an +// invariant that the measurement time of the n'th best >= n-1'th best. + +// The algorithm works as follows. On a reset, all three estimates are set to +// the same sample. The second best estimate is then recorded in the second +// quarter of the window, and a third best estimate is recorded in the second +// half of the window, bounding the worst case error when the true min is +// monotonically increasing (or true max is monotonically decreasing) over the +// window. // -// WindowedFilter, uint64_t, int64_t> ObjectName; +// A new best sample replaces all three estimates, since the new best is lower +// (or higher) than everything else in the window and it is the most recent. +// The window thus effectively gets reset on every new min. The same property +// holds true for second best and third best estimates. Specifically, when a +// sample arrives that is better than the second best but not better than the +// best, it replaces the second and third best estimates but not the best +// estimate. Similarly, a sample that is better than the third best estimate +// but not the other estimates replaces only the third best estimate. // -// Specifically, this template takes four arguments: -// 1. T -- type of the measurement that is being filtered. -// 2. Compare -- MinFilter or MaxFilter, depending on the type of filter -// desired. -// 3. TimeT -- the type used to represent timestamps. -// 4. TimeDeltaT -- the type used to represent continuous time intervals between -// two timestamps. Has to be the type of (a - b) if both |a| and |b| are -// of type TimeT. -type WindowedFilter struct { +// Finally, when the best expires, it is replaced by the second best, which in +// turn is replaced by the third best. The newest sample replaces the third +// best. + +type WindowedFilterValue interface { + any +} + +type WindowedFilterTime interface { + constraints.Integer | constraints.Float +} + +type WindowedFilter[V WindowedFilterValue, T WindowedFilterTime] struct { // Time length of window. - windowLength int64 - estimates []Sample - comparator func(int64, int64) bool + windowLength T + estimates []entry[V, T] + comparator func(V, V) int } -type Sample struct { - sample int64 - time int64 +type entry[V WindowedFilterValue, T WindowedFilterTime] struct { + sample V + time T } // Compares two values and returns true if the first is greater than or equal // to the second. -func MaxFilter(a, b int64) bool { - return a >= b +func MaxFilter[O constraints.Ordered](a, b O) int { + if a > b { + return 1 + } else if a < b { + return -1 + } + return 0 } // Compares two values and returns true if the first is less than or equal // to the second. -func MinFilter(a, b int64) bool { - return a <= b +func MinFilter[O constraints.Ordered](a, b O) int { + if a < b { + return 1 + } else if a > b { + return -1 + } + return 0 } -func NewWindowedFilter(windowLength int64, comparator func(int64, int64) bool) *WindowedFilter { - return &WindowedFilter{ +func NewWindowedFilter[V WindowedFilterValue, T WindowedFilterTime](windowLength T, comparator func(V, V) int) *WindowedFilter[V, T] { + return &WindowedFilter[V, T]{ windowLength: windowLength, - estimates: make([]Sample, 3), + estimates: make([]entry[V, T], 3, 3), comparator: comparator, } } // Changes the window length. Does not update any current samples. -func (f *WindowedFilter) SetWindowLength(windowLength int64) { +func (f *WindowedFilter[V, T]) SetWindowLength(windowLength T) { f.windowLength = windowLength } -func (f *WindowedFilter) GetBest() int64 { +func (f *WindowedFilter[V, T]) GetBest() V { return f.estimates[0].sample } -func (f *WindowedFilter) GetSecondBest() int64 { +func (f *WindowedFilter[V, T]) GetSecondBest() V { return f.estimates[1].sample } -func (f *WindowedFilter) GetThirdBest() int64 { +func (f *WindowedFilter[V, T]) GetThirdBest() V { return f.estimates[2].sample } -func (f *WindowedFilter) Update(sample, time int64) { - if f.estimates[0].time == 0 || f.comparator(sample, f.estimates[0].sample) || (time-f.estimates[2].time) > f.windowLength { - f.Reset(sample, time) +// Updates best estimates with |sample|, and expires and updates best +// estimates as necessary. +func (f *WindowedFilter[V, T]) Update(newSample V, newTime T) { + // Reset all estimates if they have not yet been initialized, if new sample + // is a new best, or if the newest recorded estimate is too old. + if f.comparator(f.estimates[0].sample, *new(V)) == 0 || + f.comparator(newSample, f.estimates[0].sample) >= 0 || + newTime-f.estimates[2].time > f.windowLength { + f.Reset(newSample, newTime) return } - if f.comparator(sample, f.estimates[1].sample) { - f.estimates[1].sample = sample - f.estimates[1].time = time - f.estimates[2].sample = sample - f.estimates[2].time = time - } else if f.comparator(sample, f.estimates[2].sample) { - f.estimates[2].sample = sample - f.estimates[2].time = time + if f.comparator(newSample, f.estimates[1].sample) >= 0 { + f.estimates[1] = entry[V, T]{newSample, newTime} + f.estimates[2] = f.estimates[1] + } else if f.comparator(newSample, f.estimates[2].sample) >= 0 { + f.estimates[2] = entry[V, T]{newSample, newTime} } // Expire and update estimates as necessary. - if time-f.estimates[0].time > f.windowLength { + if newTime-f.estimates[0].time > f.windowLength { // The best estimate hasn't been updated for an entire window, so promote // second and third best estimates. - f.estimates[0].sample = f.estimates[1].sample - f.estimates[0].time = f.estimates[1].time - f.estimates[1].sample = f.estimates[2].sample - f.estimates[1].time = f.estimates[2].time - f.estimates[2].sample = sample - f.estimates[2].time = time + f.estimates[0] = f.estimates[1] + f.estimates[1] = f.estimates[2] + f.estimates[2] = entry[V, T]{newSample, newTime} // Need to iterate one more time. Check if the new best estimate is // outside the window as well, since it may also have been recorded a // long time ago. Don't need to iterate once more since we cover that // case at the beginning of the method. - if time-f.estimates[0].time > f.windowLength { - f.estimates[0].sample = f.estimates[1].sample - f.estimates[0].time = f.estimates[1].time - f.estimates[1].sample = f.estimates[2].sample - f.estimates[1].time = f.estimates[2].time + if newTime-f.estimates[0].time > f.windowLength { + f.estimates[0] = f.estimates[1] + f.estimates[1] = f.estimates[2] } return } - if f.estimates[1].sample == f.estimates[0].sample && time-f.estimates[1].time > f.windowLength>>2 { + if f.comparator(f.estimates[1].sample, f.estimates[0].sample) == 0 && + newTime-f.estimates[1].time > f.windowLength/4 { // A quarter of the window has passed without a better sample, so the // second-best estimate is taken from the second quarter of the window. - f.estimates[1].sample = sample - f.estimates[1].time = time - f.estimates[2].sample = sample - f.estimates[2].time = time + f.estimates[1] = entry[V, T]{newSample, newTime} + f.estimates[2] = f.estimates[1] return } - if f.estimates[2].sample == f.estimates[1].sample && time-f.estimates[2].time > f.windowLength>>1 { + if f.comparator(f.estimates[2].sample, f.estimates[1].sample) == 0 && + newTime-f.estimates[2].time > f.windowLength/2 { // We've passed a half of the window without a better estimate, so take // a third-best estimate from the second half of the window. - f.estimates[2].sample = sample - f.estimates[2].time = time + f.estimates[2] = entry[V, T]{newSample, newTime} } } -func (f *WindowedFilter) Reset(newSample, newTime int64) { - f.estimates[0].sample = newSample - f.estimates[0].time = newTime - f.estimates[1].sample = newSample - f.estimates[1].time = newTime - f.estimates[2].sample = newSample - f.estimates[2].time = newTime +// Resets all estimates to new sample. +func (f *WindowedFilter[V, T]) Reset(newSample V, newTime T) { + f.estimates[2] = entry[V, T]{newSample, newTime} + f.estimates[1] = f.estimates[2] + f.estimates[0] = f.estimates[1] +} + +func (f *WindowedFilter[V, T]) Clear() { + f.estimates = make([]entry[V, T], 3, 3) } diff --git a/core/internal/congestion/brutal/brutal.go b/core/internal/congestion/brutal/brutal.go index 5539f7deb4..2e93dfebbd 100644 --- a/core/internal/congestion/brutal/brutal.go +++ b/core/internal/congestion/brutal/brutal.go @@ -35,7 +35,7 @@ type pktInfo struct { func NewBrutalSender(bps uint64) *BrutalSender { bs := &BrutalSender{ bps: congestion.ByteCount(bps), - maxDatagramSize: common.InitMaxDatagramSize, + maxDatagramSize: congestion.InitialPacketSizeIPv4, ackRate: 1, } bs.pacer = common.NewPacer(func() congestion.ByteCount { @@ -106,6 +106,10 @@ func (b *BrutalSender) OnCongestionEvent(number congestion.PacketNumber, lostByt b.updateAckRate(currentTimestamp) } +func (b *BrutalSender) OnCongestionEventEx(priorInFlight congestion.ByteCount, eventTime time.Time, ackedPackets []congestion.AckedPacketInfo, lostPackets []congestion.LostPacketInfo) { + // Stub +} + func (b *BrutalSender) SetMaxDatagramSize(size congestion.ByteCount) { b.maxDatagramSize = size b.pacer.SetMaxDatagramSize(size) diff --git a/core/internal/congestion/common/pacer.go b/core/internal/congestion/common/pacer.go index 5b5079c125..b2b301b495 100644 --- a/core/internal/congestion/common/pacer.go +++ b/core/internal/congestion/common/pacer.go @@ -8,10 +8,7 @@ import ( ) const ( - InitMaxDatagramSize = 1252 - maxBurstPackets = 10 - minPacingDelay = time.Millisecond ) // Pacer implements a token bucket pacing algorithm. @@ -24,8 +21,8 @@ type Pacer struct { func NewPacer(getBandwidth func() congestion.ByteCount) *Pacer { p := &Pacer{ - budgetAtLastSent: maxBurstPackets * InitMaxDatagramSize, - maxDatagramSize: InitMaxDatagramSize, + budgetAtLastSent: maxBurstPackets * congestion.InitialPacketSizeIPv4, + maxDatagramSize: congestion.InitialPacketSizeIPv4, getBandwidth: getBandwidth, } return p @@ -51,7 +48,7 @@ func (p *Pacer) Budget(now time.Time) congestion.ByteCount { func (p *Pacer) maxBurstSize() congestion.ByteCount { return maxByteCount( - congestion.ByteCount((minPacingDelay+time.Millisecond).Nanoseconds())*p.getBandwidth()/1e9, + congestion.ByteCount((congestion.MinPacingDelay+time.Millisecond).Nanoseconds())*p.getBandwidth()/1e9, maxBurstPackets*p.maxDatagramSize, ) } @@ -63,7 +60,7 @@ func (p *Pacer) TimeUntilSend() time.Time { return time.Time{} } return p.lastSentTime.Add(maxDuration( - minPacingDelay, + congestion.MinPacingDelay, time.Duration(math.Ceil(float64(p.maxDatagramSize-p.budgetAtLastSent)*1e9/ float64(p.getBandwidth())))*time.Nanosecond, )) diff --git a/core/internal/congestion/utils.go b/core/internal/congestion/utils.go index 6a71da1590..48ddaa2ca4 100644 --- a/core/internal/congestion/utils.go +++ b/core/internal/congestion/utils.go @@ -3,16 +3,13 @@ package congestion import ( "github.com/apernet/hysteria/core/internal/congestion/bbr" "github.com/apernet/hysteria/core/internal/congestion/brutal" - "github.com/apernet/hysteria/core/internal/congestion/common" "github.com/apernet/quic-go" ) func UseBBR(conn quic.Connection) { - conn.SetCongestionControl(bbr.NewBBRSender( + conn.SetCongestionControl(bbr.NewBbrSender( bbr.DefaultClock{}, bbr.GetInitialPacketSize(conn.RemoteAddr()), - bbr.InitialCongestionWindow*common.InitMaxDatagramSize, - bbr.DefaultBBRMaxCongestionWindow*common.InitMaxDatagramSize, )) } diff --git a/extras/go.mod b/extras/go.mod index c82e888d68..993306fa11 100644 --- a/extras/go.mod +++ b/extras/go.mod @@ -1,6 +1,6 @@ module github.com/apernet/hysteria/extras -go 1.20 +go 1.21 require ( github.com/apernet/hysteria/core v0.0.0-00010101000000-000000000000 @@ -14,7 +14,7 @@ require ( ) require ( - github.com/apernet/quic-go v0.39.1-0.20230924223134-79ed77c4df4f // indirect + github.com/apernet/quic-go v0.39.1-0.20230930045547-13cecb45baa8 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect @@ -26,7 +26,6 @@ require ( github.com/quic-go/qtls-go1-20 v0.3.4 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/txthinking/runnergroup v0.0.0-20210608031112-152c7c4432bf // indirect - github.com/zhangyunhao116/fastrand v0.3.0 // indirect go.uber.org/mock v0.3.0 // indirect golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect golang.org/x/mod v0.12.0 // indirect diff --git a/extras/go.sum b/extras/go.sum index d369dc5b66..ffccd93883 100644 --- a/extras/go.sum +++ b/extras/go.sum @@ -1,5 +1,5 @@ -github.com/apernet/quic-go v0.39.1-0.20230924223134-79ed77c4df4f h1:NF2a/mJPckBDe0urX7oSwjpcXf3mGQUlH2373zoM8fk= -github.com/apernet/quic-go v0.39.1-0.20230924223134-79ed77c4df4f/go.mod h1:UwsoszQlzTm+dBDuFEwWBYt46K56WqlFEN0RWLvQ0rE= +github.com/apernet/quic-go v0.39.1-0.20230930045547-13cecb45baa8 h1:jZKdY1/SFnt6iRzSBMzaeXmgWKwSsAW2slf/o0+b3bs= +github.com/apernet/quic-go v0.39.1-0.20230930045547-13cecb45baa8/go.mod h1:UwsoszQlzTm+dBDuFEwWBYt46K56WqlFEN0RWLvQ0rE= github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 h1:4NNbNM2Iq/k57qEu7WfL67UrbPq1uFWxW4qODCohi+0= github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6/go.mod h1:J29hk+f9lJrblVIfiJOtTFk+OblBawmib4uz/VdKzlg= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -9,23 +9,29 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4= github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/miekg/dns v1.1.51/go.mod h1:2Z9d3CP1LQWihRZUf29mQ19yDThaI4DAYzte2CaQW5c= github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/oschwald/geoip2-golang v1.9.0 h1:uvD3O6fXAXs+usU+UGExshpdP13GAqp4GBrzN7IgKZc= github.com/oschwald/geoip2-golang v1.9.0/go.mod h1:BHK6TvDyATVQhKNbQBdrj9eAvuwOMi2zSFXizL3K81Y= github.com/oschwald/maxminddb-golang v1.11.0 h1:aSXMqYR/EPNjGE8epgqwDay+P30hCBZIveY0WZbAWh0= @@ -52,9 +58,8 @@ github.com/txthinking/runnergroup v0.0.0-20210608031112-152c7c4432bf/go.mod h1:C github.com/txthinking/socks5 v0.0.0-20230325130024-4230056ae301 h1:d/Wr/Vl/wiJHc3AHYbYs5I3PucJvRuw3SvbmlIRf+oM= github.com/txthinking/socks5 v0.0.0-20230325130024-4230056ae301/go.mod h1:ntmMHL/xPq1WLeKiw8p/eRATaae6PiVRNipHFJxI8PM= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zhangyunhao116/fastrand v0.3.0 h1:7bwe124xcckPulX6fxtr2lFdO2KQqaefdtbk+mqO/Ig= -github.com/zhangyunhao116/fastrand v0.3.0/go.mod h1:0v5KgHho0VE6HU192HnY15de/oDS8UrbBChIFjIhBtc= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo= go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -77,6 +82,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -103,8 +109,10 @@ golang.org/x/tools v0.11.1 h1:ojD5zOW8+7dOGzdnNgersm8aPfcDjhMp12UfG93NIMc= golang.org/x/tools v0.11.1/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go.work b/go.work index 4dd2b2f641..bde9a9c52b 100644 --- a/go.work +++ b/go.work @@ -1,4 +1,4 @@ -go 1.20 +go 1.21 use ( ./app