Increasingly slow ethernet after boot on Orange Pi R1 Plus LTS

[DaveWK]

Hi,
I have been checking out the images for the Orange Pi R1 Plus LTS, and after it finishes booting, when I attempt to do anything that requires network connectivity such as running apt update, within a few seconds the connection gets increasingly slow to the point it no longer is functional. The YT8531 is detected by all of the images:

[   14.527865] usbcore: registered new interface driver cdc_ether
[   14.708545] rk_gmac-dwmac ff540000.ethernet: IRQ eth_wake_irq not found
[   14.709215] rk_gmac-dwmac ff540000.ethernet: IRQ eth_lpi not found
[   14.743766] rk_gmac-dwmac ff540000.ethernet: PTP uses main clock
[   14.744949] rk_gmac-dwmac ff540000.ethernet: clock input or output? (input).
[   14.745824] rk_gmac-dwmac ff540000.ethernet: TX delay(0x19).
[   14.746376] rk_gmac-dwmac ff540000.ethernet: RX delay(0x5).
[   14.746917] rk_gmac-dwmac ff540000.ethernet: integrated PHY? (no).
[   14.752152] rk_gmac-dwmac ff540000.ethernet: clock input from PHY
[   14.757829] rk_gmac-dwmac ff540000.ethernet: init for RGMII
[   14.768972] rk_gmac-dwmac ff540000.ethernet: User ID: 0x10, Synopsys ID: 0x35
[   14.769745] rk_gmac-dwmac ff540000.ethernet: 	DWMAC1000
[   14.770250] rk_gmac-dwmac ff540000.ethernet: DMA HW capability register supported
[   14.770941] rk_gmac-dwmac ff540000.ethernet: RX Checksum Offload Engine supported
[   14.771703] rk_gmac-dwmac ff540000.ethernet: COE Type 2
[   14.772201] rk_gmac-dwmac ff540000.ethernet: TX Checksum insertion supported
[   14.773027] rk_gmac-dwmac ff540000.ethernet: Wake-Up On Lan supported
[   14.774117] rk_gmac-dwmac ff540000.ethernet: Normal descriptors
[   14.774714] rk_gmac-dwmac ff540000.ethernet: Ring mode enabled
[   14.775262] rk_gmac-dwmac ff540000.ethernet: Enable RX Mitigation via HW Watchdog Timer
[   15.035812] rk_gmac-dwmac ff540000.ethernet end0: renamed from eth0
[   15.179382] rk_gmac-dwmac ff540000.ethernet end0: Register MEM_TYPE_PAGE_POOL RxQ-0
[   15.243032] rk_gmac-dwmac ff540000.ethernet end0: PHY [stmmac-0:00] driver [YT8531 Gigabit Ethernet] (irq=POLL)
[   15.253610] rk_gmac-dwmac ff540000.ethernet end0: No Safety Features support found
[   15.254333] rk_gmac-dwmac ff540000.ethernet end0: PTP not supported by HW
[   15.255482] rk_gmac-dwmac ff540000.ethernet end0: configuring for phy/rgmii link mode
[   18.334667] rk_gmac-dwmac ff540000.ethernet end0: Link is Up - 1Gbps/Full - flow control off

and gets an IPv4 address from DHCP. Within a few seconds the connection just gets slower and slower, down to a few bits a second. I have tried different cables, as well as multiple routers, as well as crossover to rule out any external hardware issue. I did notice when running wireshard that it appears to be sending out a lot of TCP retransmissions when this is occurring.

I have tried all 4 of the images, and they all have the same issue. I have also noticed I can't seem to get the second NIC to be detected by the OS or powered on at all.
I also was able to validate with a USB Wireless Cellular connection that it is able to connect to the internet and download things fine (using a serial connection since eventually SSH drops off)

I suspect it may be a u-boot problem since it happens in all of the images, but am not sure how to debug any potential power-up/regulator issues that might be occurring.

Here's my output from lsmod on trixie:

Module                  Size  Used by
sd_mod                 65536  0
t10_pi                 16384  1 sd_mod
crc64_rocksoft_generic    12288  1
crc64_rocksoft         16384  1 t10_pi
crc_t10dif             16384  1 t10_pi
crct10dif_generic      12288  0
crc64                  16384  2 crc64_rocksoft,crc64_rocksoft_generic
sg                     32768  0
ofpart                 16384  0
spi_nor               102400  0
evdev                  20480  1
mtd                    69632  3 spi_nor,ofpart
motorcomm              24576  1
polyval_ce             12288  0
polyval_generic        12288  1 polyval_ce
dwmac_rk               28672  0
stmmac_platform        20480  1 dwmac_rk
ghash_ce               16384  0
gf128mul               12288  2 polyval_generic,ghash_ce
sha2_ce                12288  0
stmmac                241664  3 stmmac_platform,dwmac_rk
uas                    24576  0
hantro_vpu            262144  0
usb_storage            73728  1 uas
sha256_arm64           24576  1 sha2_ce
pcs_xpcs               16384  1 stmmac
rockchip_vdec          40960  0
rndis_host             20480  0
des_generic            12288  0
phylink                49152  2 stmmac,pcs_xpcs
sha1_ce                12288  0
v4l2_vp9               20480  2 rockchip_vdec,hantro_vpu
ecb                    12288  0
cdc_ether              16384  1 rndis_host
scsi_mod              233472  4 sd_mod,usb_storage,uas,sg
v4l2_h264              16384  2 rockchip_vdec,hantro_vpu
of_mdio                16384  2 stmmac_platform,stmmac
usbnet                 40960  2 rndis_host,cdc_ether
fixed_phy              12288  1 of_mdio
videobuf2_dma_contig    20480  2 rockchip_vdec,hantro_vpu
mii                    16384  1 usbnet
scsi_common            12288  5 scsi_mod,sd_mod,usb_storage,uas,sg
fwnode_mdio            16384  1 of_mdio
aes_neon_bs            24576  0
v4l2_mem2mem           32768  2 rockchip_vdec,hantro_vpu
libphy                114688  7 fwnode_mdio,of_mdio,stmmac,fixed_phy,motorcomm,phylink,pcs_xpcs
leds_gpio              12288  0
videobuf2_memops       16384  1 videobuf2_dma_contig
videobuf2_v4l2         20480  3 rockchip_vdec,hantro_vpu,v4l2_mem2mem
aes_neon_blk           24576  1 aes_neon_bs
lima                   65536  0
videodev              245760  4 rockchip_vdec,videobuf2_v4l2,hantro_vpu,v4l2_mem2mem
aes_ce_blk             28672  0
aes_ce_cipher          12288  1 aes_ce_blk
videobuf2_common       53248  6 rockchip_vdec,videobuf2_dma_contig,videobuf2_v4l2,hantro_vpu,v4l2_mem2mem,videobuf2_memops
gpu_sched              45056  1 lima
drm_shmem_helper       16384  1 lima
mc                     57344  6 rockchip_vdec,videodev,videobuf2_v4l2,hantro_vpu,videobuf2_common,v4l2_mem2mem
rk805_pwrkey           12288  0
syscon_reboot_mode     12288  0
nvmem_rockchip_efuse    12288  0
reboot_mode            12288  1 syscon_reboot_mode
dw_wdt                 12288  0
rockchip_thermal       24576  0
rk_crypto              28672  0
pl330                  28672  2
crypto_engine          16384  1 rk_crypto
libdes                 20480  2 des_generic,rk_crypto
cfg80211              831488  0
cpufreq_dt             16384  0
rfkill                 32768  2 cfg80211
fuse                  135168  1
loop                   32768  0
drm                   466944  4 gpu_sched,lima,drm_shmem_helper
efi_pstore             12288  0
dm_mod                143360  0
dax                    32768  1 dm_mod
configfs               49152  1
nfnetlink              16384  2
ip_tables              24576  0
x_tables               32768  1 ip_tables
autofs4                45056  2
ext4                  782336  1
crc16                  12288  1 ext4
mbcache                16384  1 ext4
jbd2                  135168  1 ext4
crc32c_generic         12288  2
xhci_plat_hcd          16384  0
xhci_hcd              249856  1 xhci_plat_hcd
rk808_regulator        49152  12
gpio_rockchip          20480  7
ohci_platform          12288  0
dwc2                  241664  0
ohci_hcd               53248  1 ohci_platform
ehci_platform          16384  0
ehci_hcd               86016  1 ehci_platform
dwc3                  163840  0
crct10dif_ce           12288  1
crct10dif_common       12288  3 crct10dif_ce,crct10dif_generic,crc_t10dif
usbcore               270336  12 ohci_platform,xhci_hcd,ohci_hcd,ehci_platform,usbnet,dwc2,usb_storage,rndis_host,ehci_hcd,cdc_ether,uas,xhci_plat_hcd
udc_core               53248  2 dwc2,dwc3
dw_mmc_rockchip        16384  0
fixed                  16384  7
pinctrl_rockchip      114688  52
dw_mmc_pltfm           12288  1 dw_mmc_rockchip
roles                  12288  2 dwc2,dwc3
phy_rockchip_inno_usb2    40960  6
ulpi                   16384  1 dwc3
usb_common             12288  11 ohci_platform,xhci_hcd,ohci_hcd,ehci_platform,phy_rockchip_inno_usb2,dwc2,usbcore,ehci_hcd,xhci_plat_hcd,dwc3,udc_core
dw_mmc                 45056  2 dw_mmc_pltfm,dw_mmc_rockchip
gpio_syscon            16384  0
pwm_rockchip           12288  0
io_domain              20480  0
spi_rockchip           16384  0
i2c_rk3x               20480  0

Here's also my output from ifconfig:

        inet 192.168.88.242  netmask 255.255.255.0  broadcast 192.168.88.255
        inet6 fe80::84e0:c0ff:feea:faa9  prefixlen 64  scopeid 0x20<link>
        ether 86:e0:c0:ea:fa:a9  txqueuelen 1000  (Ethernet)
        RX packets 1744  bytes 109074 (106.5 KiB)
        RX errors 0  dropped 1703  overruns 0  frame 0
        TX packets 66  bytes 8163 (7.9 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
        device interrupt 56  

which has that entry for device interrupt at the end which looks suspicious, but it may be a symptom rather than the cause.

[johang]

Do you have packet loss? You can confirm this with ping. Try forcing 100 Mbps and 10 Mbps and see if that improves it. Sounds like a HW issue to be honest.

[DaveWK]

Hi,
I confirmed with ping and wireshark that I am getting packet loss when running these sd-images. I have tried a few different cords, and switches to confirm they were in working order. Each time when I get into a root shell and run apt update it will have stalled out in a few seconds due to retransmission and packet loss.

But in counter-point to it being a hardware issue,
when I test using the latest Armbian Trixie rolling release (6.6.10). I am able to finish an initial apt update successfully before the network connection slows down substantially.

I noticed this merge in arbian commit that seems to give the nics some extra time to power up: armbian/build@ac9373a

I suspect that the issue may be the issue?

[johang]

https://lore.kernel.org/linux-rockchip/[email protected]/T/#u

Possibly related.

[DaveWK]

Interesting -- I will give that a shot to see if that improves anything, but my current armbian dts isn't using that.. here's my live one:

 ethernet@ff540000 {
                compatible = "rockchip,rk3328-gmac";
                reg = <0x00 0xff540000 0x00 0x10000>;
                interrupts = <0x00 0x18 0x04>;
                interrupt-names = "macirq";
                clocks = <0x02 0x64 0x02 0x57 0x02 0x58 0x02 0x5a 0x02 0x59 0x02 0x96 0x02 0xdf>;
                clock-names = "stmmaceth\0mac_clk_rx\0mac_clk_tx\0clk_mac_ref\0clk_mac_refout\0aclk_mac\0pclk_mac";
                resets = <0x02 0x63>;
                reset-names = "stmmaceth";
                rockchip,grf = <0x3d>;
                snps,txpbl = <0x04>;
                status = "okay";
                assigned-clocks = <0x02 0x64 0x02 0x66>;
                assigned-clock-parents = <0x54 0x54>;
                clock_in_out = "input";
                phy-mode = "rgmii";
                phy-supply = <0x1e>;
                pinctrl-0 = <0x55>;
                pinctrl-names = "default";
                snps,aal;
                phy-handle = <0x56>;
                tx_delay = <0x19>;
                rx_delay = <0x05>;
                phandle = <0xa1>;

                mdio {
                        compatible = "snps,dwmac-mdio";
                        #address-cells = <0x01>;
                        #size-cells = <0x00>;
ethernet-phy@0 {
                                compatible = "ethernet-phy-id4f51.e91b\0ethernet-phy-ieee802.3-c22";
                                reg = <0x00>;
                                motorcomm,clk-out-frequency-hz = <0x7735940>;
                                motorcomm,keep-pll-enabled;
                                motorcomm,auto-sleep-disabled;
                                pinctrl-0 = <0x57>;
                                pinctrl-names = "default";
                                reset-assert-us = <0x3a98>;
                                reset-deassert-us = <0xc350>;
                                reset-gpios = <0x2b 0x12 0x01>;
                                phandle = <0x56>;
                        };
               };
        };