diff --git a/lib/net/nvnetdrv/nvnetdrv.c b/lib/net/nvnetdrv/nvnetdrv.c index e29e7e178..7f2ea1fa7 100644 --- a/lib/net/nvnetdrv/nvnetdrv.c +++ b/lib/net/nvnetdrv/nvnetdrv.c @@ -7,13 +7,13 @@ #include "nvnetdrv_regs.h" #include #include -#include #include #include +#include #include #define NVNET_MIN(a, b) (((a) < (b)) ? (a) : (b)) -#define NVNET_RX_EMPTY (NULL) +#define NVNET_RX_EMPTY (NULL) struct __attribute__((packed)) descriptor_t { @@ -56,13 +56,16 @@ struct nvnetdrv_stats_t uint32_t rx_missedFrameError; }; static struct nvnetdrv_stats_t nvnetdrv_stats; -#define INC_STAT(statname, val) do {nvnetdrv_stats.statname += (val);} while(0); +#define INC_STAT(statname, val) \ + do { \ + nvnetdrv_stats.statname += (val); \ + } while (0); #else #define INC_STAT(statname, val) #endif // FIXME -#define BASE ((void *)0xFEF00000) +#define BASE ((void *)0xFEF00000) #define reg32(offset) (*((volatile uint32_t *)((uintptr_t)BASE + (offset)))) // Manage NIC @@ -76,41 +79,50 @@ static KINTERRUPT g_interrupt; static HANDLE g_irqThread; static KEVENT g_irqEvent; -// Manage RX ring -static volatile struct descriptor_t *g_rxRingDescriptors; -static KSEMAPHORE g_rxRingFreeDescriptors; -static HANDLE g_rxRingRequeueThread; -static size_t g_rxRingBeginIndex; -static size_t g_rxRingEndIndex; -static uint8_t *g_rxRingBuffers; +// Manage RX and TX rings +static volatile struct descriptor_t *g_rxRing; +static volatile struct descriptor_t *g_txRing; +static size_t g_rxRingSize; +static size_t g_txRingSize; +static size_t g_rxRingHead; +static size_t g_txRingHead; +static atomic_size_t g_txPendingCount; +static KSEMAPHORE g_rxPendingCount; +static atomic_size_t g_rxRingTail; +static atomic_size_t g_txRingTail; +static KSEMAPHORE g_txRingFreeCount; +struct tx_misc_t g_txData[TX_RING_SIZE]; +static uint8_t *g_rxRingUserBuffers; static uint32_t g_rxRingBufferVtoP; -// Manage RX buffer callbacks to user network stack +// Manage RX buffers +static KSEMAPHORE g_rxRingFreeDescriptors; +static HANDLE g_rxRingRequeueThread; static nvnetdrv_rx_callback_t g_rxCallback; -static KSEMAPHORE g_rxCallbackQueued; static HANDLE g_rxCallbackThread; struct rx_misc_t *g_rxCallbackQueue; -static size_t g_rxCallbackEndIndex; - -// Manage RX buffer pool to supply RX ring +static size_t g_rxCallbackTail; static void **g_rxBuffPool; static size_t g_rxBuffPoolHead; -static size_t g_rxBuffCnt; static RTL_CRITICAL_SECTION g_rxBuffPoolLock; static KSEMAPHORE g_rxFreeBuffers; -// Manage TX ring -static volatile struct descriptor_t *g_txRingDescriptors; -static size_t g_txRingBeginIndex; -static atomic_size_t g_txRingEndIndex; -static atomic_size_t g_txRingDescriptorsInUseCount; -static KSEMAPHORE g_txRingFreeDescriptors; -struct tx_misc_t tx_misc[TX_RING_SIZE]; - // Time constants used in nvnetdrv -static LARGE_INTEGER no_sleep = {.QuadPart = 0}; -static LARGE_INTEGER tenmicros = {.QuadPart = -100}; -static LARGE_INTEGER fiftymicros = {.QuadPart = -500}; +#define NO_SLEEP \ + &(LARGE_INTEGER) \ + { \ + .QuadPart = 0 \ + } +#define TEN_MICRO \ + &(LARGE_INTEGER) \ + { \ + .QuadPart = -100 \ + } +#define FIFTY_MICRO \ + &(LARGE_INTEGER) \ + { \ + .QuadPart = -500 \ + } static void nvnetdrv_rx_push (void *buffer_virt) { @@ -166,19 +178,18 @@ static void nvnetdrv_rx_requeue (size_t buffer_index) { assert(buffer_index < RX_RING_SIZE); void *rx_buff = nvnetdrv_rx_pop(); - assert (rx_buff != NULL); - g_rxRingDescriptors[buffer_index].paddr = nvnetdrv_rx_vtop((uint32_t)rx_buff); - g_rxRingDescriptors[buffer_index].length = NVNET_RX_BUFF_LEN; - g_rxRingDescriptors[buffer_index].flags = NV_RX_AVAIL; + assert(rx_buff != NULL); + g_rxRing[buffer_index].paddr = nvnetdrv_rx_vtop((uint32_t)rx_buff); + g_rxRing[buffer_index].length = NVNET_RX_BUFF_LEN; + g_rxRing[buffer_index].flags = NV_RX_AVAIL; } static void nvnetdrv_handle_rx_irq (void) { LONG freed_descriptors = 0; - while (g_rxRingDescriptors[g_rxRingBeginIndex].paddr != NVNET_RX_EMPTY) { - - volatile struct descriptor_t *rx_packet = &g_rxRingDescriptors[g_rxRingBeginIndex]; + while (g_rxRing[g_rxRingHead].paddr != NVNET_RX_EMPTY) { + volatile struct descriptor_t *rx_packet = &g_rxRing[g_rxRingHead]; uint16_t flags = rx_packet->flags; if (flags & NV_RX_AVAIL) { @@ -190,13 +201,14 @@ static void nvnetdrv_handle_rx_irq (void) goto release_packet; } - if (!(flags & NV_RX_ERROR) || (flags & NV_RX_FRAMINGERR)) { + if ((flags & NV_RX_ERROR_MASK) == 0) { uint16_t packet_length = rx_packet->length; if (flags & NV_RX_SUBTRACT1) { INC_STAT(rx_extraByteErrors, 1); - if (packet_length > 0) + if (packet_length > 0) { packet_length--; + } } if (flags & NV_RX_FRAMINGERR) { @@ -207,34 +219,46 @@ static void nvnetdrv_handle_rx_irq (void) // Queue a pending RX callback. We dont want to call it directly from the IRQ thread // incase the users network stack blocks in the callback. - g_rxCallbackQueue[g_rxCallbackEndIndex].length = packet_length; - g_rxCallbackQueue[g_rxCallbackEndIndex].bufAddr = (void *)nvnetdrv_rx_ptov(rx_packet->paddr); - g_rxCallbackEndIndex = (g_rxCallbackEndIndex + 1) % g_rxBuffCnt; - KeReleaseSemaphore(&g_rxCallbackQueued, IO_NETWORK_INCREMENT, 1, FALSE); + g_rxCallbackQueue[g_rxCallbackTail].length = packet_length; + g_rxCallbackQueue[g_rxCallbackTail].bufAddr = (void *)nvnetdrv_rx_ptov(rx_packet->paddr); + g_rxCallbackTail = (g_rxCallbackTail + 1) % g_rxRingSize; + KeReleaseSemaphore(&g_rxPendingCount, IO_NETWORK_INCREMENT, 1, FALSE); goto next_packet; - } - else - { - if (flags & NV_RX_MISSEDFRAME) INC_STAT(rx_missedFrameError, 1); - if (flags & NV_RX_OVERFLOW) INC_STAT(rx_overflowError, 1); - if (flags & NV_RX_CRCERR) INC_STAT(rx_crcError, 1); - if (flags & NV_RX_ERROR4) INC_STAT(rx_error4, 1); - if (flags & NV_RX_ERROR3) INC_STAT(rx_error3, 1); - if (flags & NV_RX_ERROR2) INC_STAT(rx_error2, 1); - if (flags & NV_RX_ERROR1) INC_STAT(rx_error1, 1); + } else { + if (flags & NV_RX_MISSEDFRAME) { + INC_STAT(rx_missedFrameError, 1); + } + if (flags & NV_RX_OVERFLOW) { + INC_STAT(rx_overflowError, 1); + } + if (flags & NV_RX_CRCERR) { + INC_STAT(rx_crcError, 1); + } + if (flags & NV_RX_ERROR4) { + INC_STAT(rx_error4, 1); + } + if (flags & NV_RX_ERROR3) { + INC_STAT(rx_error3, 1); + } + if (flags & NV_RX_ERROR2) { + INC_STAT(rx_error2, 1); + } + if (flags & NV_RX_ERROR1) { + INC_STAT(rx_error1, 1); + } goto release_packet; } - //On error drop packet and release buffer + // On error drop packet and release buffer release_packet: nvnetdrv_rx_release((void *)nvnetdrv_rx_ptov(rx_packet->paddr)); // Fallthrough - //A successful RX the packet is passed to user but we clear it from the RX ring which will be repopulated with - //a spare RX buffer if available to keep data flowing while the user holds their buffer. + // A successful RX the packet is passed to user but we clear it from the RX ring which will be repopulated with + // a spare RX buffer if available to keep data flowing while the user holds their buffer. next_packet: rx_packet->paddr = NVNET_RX_EMPTY; freed_descriptors++; - g_rxRingBeginIndex = (g_rxRingBeginIndex + 1) % RX_RING_SIZE; + g_rxRingHead = (g_rxRingHead + 1) % RX_RING_SIZE; } KeReleaseSemaphore(&g_rxRingFreeDescriptors, IO_NETWORK_INCREMENT, freed_descriptors, FALSE); INC_STAT(rx_interrupts, 1); @@ -244,8 +268,8 @@ static void nvnetdrv_handle_tx_irq (void) { LONG freed_descriptors = 0; - while (g_txRingDescriptorsInUseCount > 0) { - uint16_t flags = g_txRingDescriptors[g_txRingBeginIndex].flags; + while (g_txPendingCount > 0) { + uint16_t flags = g_txRing[g_txRingHead].flags; if (flags & NV_TX_VALID) { // We reached a descriptor that wasn't processed by hw yet @@ -255,19 +279,19 @@ static void nvnetdrv_handle_tx_irq (void) } // Buffers get locked before sending and unlocked after sending - MmLockUnlockBufferPages(tx_misc[g_txRingBeginIndex].bufAddr, tx_misc[g_txRingBeginIndex].length, TRUE); + MmLockUnlockBufferPages(g_txData[g_txRingHead].bufAddr, g_txData[g_txRingHead].length, TRUE); // If registered, call the users tx complete callback funciton. - if (tx_misc[g_txRingBeginIndex].callback) { - tx_misc[g_txRingBeginIndex].callback(tx_misc[g_txRingBeginIndex].userdata); + if (g_txData[g_txRingHead].callback) { + g_txData[g_txRingHead].callback(g_txData[g_txRingHead].userdata); } freed_descriptors++; - g_txRingBeginIndex = (g_txRingBeginIndex + 1) % TX_RING_SIZE; - g_txRingDescriptorsInUseCount--; + g_txRingHead = (g_txRingHead + 1) % TX_RING_SIZE; + g_txPendingCount--; } - KeReleaseSemaphore(&g_txRingFreeDescriptors, IO_NETWORK_INCREMENT, freed_descriptors, FALSE); + KeReleaseSemaphore(&g_txRingFreeCount, IO_NETWORK_INCREMENT, freed_descriptors, FALSE); INC_STAT(tx_interrupts, 1); } @@ -301,14 +325,14 @@ static void nvnetdrv_handle_mii_irq (uint32_t miiStatus, bool init) static void nvnetdrv_handle_irq (void) { - while (true) - { + while (true) { uint32_t irq = reg32(NvRegIrqStatus); uint32_t mii = reg32(NvRegMIIStatus); - //No interrupts left to handle. Leave - if (!irq && !mii) + // No interrupts left to handle. Leave + if (!irq && !mii) { break; + } // Acknowledge interrupts reg32(NvRegMIIStatus) = mii; @@ -327,14 +351,16 @@ static void nvnetdrv_handle_irq (void) } } -static void NTAPI irq_thread(PKSTART_ROUTINE StartRoutine, PVOID StartContext) +static void NTAPI irq_thread (PKSTART_ROUTINE StartRoutine, PVOID StartContext) { (void)StartRoutine; (void)StartContext; while (true) { KeWaitForSingleObject(&g_irqEvent, Executive, KernelMode, FALSE, NULL); - if (!g_running) break; + if (!g_running) { + break; + } nvnetdrv_handle_irq(); nvnetdrv_irq_enable(); @@ -343,28 +369,31 @@ static void NTAPI irq_thread(PKSTART_ROUTINE StartRoutine, PVOID StartContext) PsTerminateSystemThread(0); } -static void NTAPI rxrequeue_thread(PKSTART_ROUTINE StartRoutine, PVOID StartContext) +static void NTAPI rxrequeue_thread (PKSTART_ROUTINE StartRoutine, PVOID StartContext) { (void)StartRoutine; (void)StartContext; while (true) { - //Sleep until there is an empty descriptor in the RX ring - if (!g_running) break; + // Sleep until there is an empty descriptor in the RX ring + if (!g_running) { + break; + } KeWaitForSingleObject(&g_rxRingFreeDescriptors, Executive, KernelMode, FALSE, NULL); - if (!g_running) break; + if (!g_running) { + break; + } do { - nvnetdrv_rx_requeue(g_rxRingEndIndex); - g_rxRingEndIndex = (g_rxRingEndIndex + 1) % RX_RING_SIZE; - } while (g_running && - KeWaitForSingleObject(&g_rxRingFreeDescriptors, Executive, - KernelMode, FALSE, &no_sleep) == STATUS_SUCCESS); + nvnetdrv_rx_requeue(g_rxRingTail); + g_rxRingTail = (g_rxRingTail + 1) % RX_RING_SIZE; + } while (g_running && KeWaitForSingleObject(&g_rxRingFreeDescriptors, Executive, KernelMode, FALSE, NO_SLEEP) == + STATUS_SUCCESS); } PsTerminateSystemThread(0); } -static void NTAPI rxcallback_thread(PKSTART_ROUTINE StartRoutine, PVOID StartContext) +static void NTAPI rxcallback_thread (PKSTART_ROUTINE StartRoutine, PVOID StartContext) { (void)StartRoutine; (void)StartContext; @@ -372,17 +401,20 @@ static void NTAPI rxcallback_thread(PKSTART_ROUTINE StartRoutine, PVOID StartCon size_t idx = 0; while (true) { - //Sleep until there is an RX callback that needs processing - if (!g_running) break; - KeWaitForSingleObject(&g_rxCallbackQueued, Executive, KernelMode, FALSE, NULL); - if (!g_running) break; + // Sleep until there is an RX callback that needs processing + if (!g_running) { + break; + } + KeWaitForSingleObject(&g_rxPendingCount, Executive, KernelMode, FALSE, NULL); + if (!g_running) { + break; + } do { g_rxCallback(g_rxCallbackQueue[idx].bufAddr, g_rxCallbackQueue[idx].length); - idx = (idx + 1) % g_rxBuffCnt; + idx = (idx + 1) % g_rxRingSize; } while (g_running && - KeWaitForSingleObject(&g_rxCallbackQueued, Executive, - KernelMode, FALSE, &no_sleep) == STATUS_SUCCESS); + KeWaitForSingleObject(&g_rxPendingCount, Executive, KernelMode, FALSE, NO_SLEEP) == STATUS_SUCCESS); } PsTerminateSystemThread(0); } @@ -392,14 +424,14 @@ const uint8_t *nvnetdrv_get_ethernet_addr () return g_ethAddr; } -int nvnetdrv_init(size_t rx_buffer_count, nvnetdrv_rx_callback_t rx_callback) +int nvnetdrv_init (size_t rx_buffer_count, nvnetdrv_rx_callback_t rx_callback) { assert(!g_running); assert(rx_callback); assert(rx_buffer_count > 1); g_rxCallback = rx_callback; - g_rxBuffCnt = rx_buffer_count; + g_rxRingSize = rx_buffer_count; // Get Mac Address from EEPROM ULONG type; @@ -409,47 +441,47 @@ int nvnetdrv_init(size_t rx_buffer_count, nvnetdrv_rx_callback_t rx_callback) } // Allocate memory for TX and RX ring descriptors. - void *descriptors = MmAllocateContiguousMemoryEx((RX_RING_SIZE + TX_RING_SIZE) * sizeof(struct descriptor_t), - 0, 0xFFFFFFFF, 0, PAGE_READWRITE); + void *descriptors = MmAllocateContiguousMemoryEx((RX_RING_SIZE + TX_RING_SIZE) * sizeof(struct descriptor_t), 0, + 0xFFFFFFFF, 0, PAGE_READWRITE); if (!descriptors) { return NVNET_NO_MEM; } // Allocate memory for RX buffers. TX buffers are supplied by the user. - g_rxRingBuffers = MmAllocateContiguousMemoryEx(g_rxBuffCnt * NVNET_RX_BUFF_LEN, - 0, 0xFFFFFFFF, 0, PAGE_READWRITE); - if (!g_rxRingBuffers) { + g_rxRingUserBuffers = + MmAllocateContiguousMemoryEx(g_rxRingSize * NVNET_RX_BUFF_LEN, 0, 0xFFFFFFFF, 0, PAGE_READWRITE); + if (!g_rxRingUserBuffers) { MmFreeContiguousMemory(descriptors); return NVNET_NO_MEM; } // Allocate memory for storing our push/pop stack for spare RX buffer - g_rxBuffPool = (void **)malloc(g_rxBuffCnt * sizeof(void *)); + g_rxBuffPool = (void **)malloc(g_rxRingSize * sizeof(void *)); if (!g_rxBuffPool) { MmFreeContiguousMemory(descriptors); - MmFreeContiguousMemory(g_rxRingBuffers); + MmFreeContiguousMemory(g_rxRingUserBuffers); return NVNET_NO_MEM; } // Allocate memory for storing complete RX transfers, ready for callbacks. - g_rxCallbackQueue = malloc(g_rxBuffCnt * sizeof(struct rx_misc_t)); + g_rxCallbackQueue = malloc(g_rxRingSize * sizeof(struct rx_misc_t)); if (!g_rxCallbackQueue) { MmFreeContiguousMemory(descriptors); - MmFreeContiguousMemory(g_rxRingBuffers); + MmFreeContiguousMemory(g_rxRingUserBuffers); free(g_rxBuffPool); return NVNET_NO_MEM; } RtlZeroMemory(descriptors, (RX_RING_SIZE + TX_RING_SIZE) * sizeof(struct descriptor_t)); - RtlZeroMemory(g_rxCallbackQueue, g_rxBuffCnt * sizeof(struct rx_misc_t)); - RtlZeroMemory(g_rxBuffPool, g_rxBuffCnt * sizeof(void *)); + RtlZeroMemory(g_rxCallbackQueue, g_rxRingSize * sizeof(struct rx_misc_t)); + RtlZeroMemory(g_rxBuffPool, g_rxRingSize * sizeof(void *)); // Reset NIC. MSDash delays 10us here nvnetdrv_stop_txrx(); reg32(NvRegTxRxControl) = NVREG_TXRXCTL_RESET; - KeDelayExecutionThread(KernelMode, FALSE, &tenmicros); + KeDelayExecutionThread(KernelMode, FALSE, TEN_MICRO); reg32(NvRegTxRxControl) = 0; - KeDelayExecutionThread(KernelMode, FALSE, &tenmicros); + KeDelayExecutionThread(KernelMode, FALSE, TEN_MICRO); // Disable interrupts while we initialise the NIC reg32(NvRegIrqMask) = 0; @@ -468,19 +500,19 @@ int nvnetdrv_init(size_t rx_buffer_count, nvnetdrv_rx_callback_t rx_callback) reg32(NvRegLinkSpeed) = 0; // Reset local ring tracking variables - g_rxRingBeginIndex = 0; - g_txRingBeginIndex = 0; - g_txRingEndIndex = 0; - g_txRingDescriptorsInUseCount = 0; - g_rxCallbackEndIndex = 0; - RtlZeroMemory(tx_misc, sizeof(tx_misc)); + g_rxRingHead = 0; + g_txRingHead = 0; + g_txRingTail = 0; + g_txPendingCount = 0; + g_rxCallbackTail = 0; + RtlZeroMemory(g_txData, sizeof(g_txData)); // Setup the TX and RX ring descriptor pointers - g_rxRingDescriptors = (struct descriptor_t *)descriptors; - g_txRingDescriptors = (struct descriptor_t *)descriptors + RX_RING_SIZE; + g_rxRing = (volatile struct descriptor_t *)descriptors; + g_txRing = (volatile struct descriptor_t *)descriptors + RX_RING_SIZE; // Remember the offset between virtual and physical address - g_rxRingBufferVtoP = ((uint32_t)g_rxRingBuffers) - (uint32_t)MmGetPhysicalAddress(g_rxRingBuffers); + g_rxRingBufferVtoP = ((uint32_t)g_rxRingUserBuffers) - (uint32_t)MmGetPhysicalAddress(g_rxRingUserBuffers); // Setup some fixed registers for the NIC reg32(NvRegMacAddrA) = (g_ethAddr[0] << 0) | (g_ethAddr[1] << 8) | (g_ethAddr[2] << 16) | (g_ethAddr[3] << 24); @@ -493,38 +525,37 @@ int nvnetdrv_init(size_t rx_buffer_count, nvnetdrv_rx_callback_t rx_callback) reg32(NvRegPacketFilterFlags) = NVREG_PFF_ALWAYS_MYADDR; reg32(NvRegDuplexMode) = NVREG_DUPLEX_MODE_FORCEH; - //Pseudo random slot time to minimise collisions + // Pseudo random slot time to minimise collisions reg32(NvRegSlotTime) = ((rand() % 0xFF) & NVREG_SLOTTIME_MASK) | NVREG_SLOTTIME_10_100_FULL; reg32(NvRegTxDeferral) = NVREG_TX_DEFERRAL_RGMII_10_100; reg32(NvRegRxDeferral) = NVREG_RX_DEFERRAL_DEFAULT; // MS Dash does this and sets up both these registers with 0x300010) - reg32(NvRegUnknownSetupReg7) = NVREG_UNKSETUP7_VAL1; //RxWatermark? + reg32(NvRegUnknownSetupReg7) = NVREG_UNKSETUP7_VAL1; // RxWatermark? reg32(NvRegTxWatermark) = NVREG_UNKSETUP7_VAL1; // Point the NIC to our TX and RX ring buffers. NIC expects Ring size as size-1. - reg32(NvRegTxRingPhysAddr) = MmGetPhysicalAddress((void *)g_txRingDescriptors); - reg32(NvRegRxRingPhysAddr) = MmGetPhysicalAddress((void *)g_rxRingDescriptors); - reg32(NvRegRingSizes) = ((RX_RING_SIZE - 1) << NVREG_RINGSZ_RXSHIFT) | - ((TX_RING_SIZE - 1) << NVREG_RINGSZ_TXSHIFT); + reg32(NvRegTxRingPhysAddr) = MmGetPhysicalAddress((void *)g_txRing); + reg32(NvRegRxRingPhysAddr) = MmGetPhysicalAddress((void *)g_rxRing); + reg32(NvRegRingSizes) = ((RX_RING_SIZE - 1) << NVREG_RINGSZ_RXSHIFT) | ((TX_RING_SIZE - 1) << NVREG_RINGSZ_TXSHIFT); // Prepare for Phy Init reg32(NvRegAdapterControl) = (1 << NVREG_ADAPTCTL_PHYSHIFT) | NVREG_ADAPTCTL_PHYVALID; reg32(NvRegMIISpeed) = NVREG_MIISPEED_BIT8 | NVREG_MIIDELAY; reg32(NvRegMIIMask) = NVREG_MII_LINKCHANGE; - KeDelayExecutionThread(KernelMode, FALSE, &fiftymicros); + KeDelayExecutionThread(KernelMode, FALSE, FIFTY_MICRO); // Initialise the transceiver if (PhyInitialize(FALSE, NULL) != STATUS_SUCCESS) { MmFreeContiguousMemory(descriptors); - MmFreeContiguousMemory(g_rxRingBuffers); + MmFreeContiguousMemory(g_rxRingUserBuffers); free(g_rxCallbackQueue); return NVNET_PHY_ERR; } // Short delay to allow the phy to startup. MSDash delays 50us reg32(NvRegAdapterControl) |= NVREG_ADAPTCTL_RUNNING; - KeDelayExecutionThread(KernelMode, FALSE, &fiftymicros); + KeDelayExecutionThread(KernelMode, FALSE, FIFTY_MICRO); // The NIC hardware IRQ queues a DPC. The DPC then sets g_irqEvent. // g_irqEvent is monitored by irqthread to handle the IRQ @@ -535,28 +566,28 @@ int nvnetdrv_init(size_t rx_buffer_count, nvnetdrv_rx_callback_t rx_callback) // We use semaphores to track the number of free TX and RX ring descriptors, the number of free RX buffers // available to be queued in the RX ring and the number of pending rx callbacks. - KeInitializeSemaphore(&g_txRingFreeDescriptors, TX_RING_SIZE, TX_RING_SIZE); + KeInitializeSemaphore(&g_txRingFreeCount, TX_RING_SIZE, TX_RING_SIZE); KeInitializeSemaphore(&g_rxRingFreeDescriptors, RX_RING_SIZE, RX_RING_SIZE); - KeInitializeSemaphore(&g_rxFreeBuffers, 0, g_rxBuffCnt); - KeInitializeSemaphore(&g_rxCallbackQueued, 0, g_rxBuffCnt); + KeInitializeSemaphore(&g_rxFreeBuffers, 0, g_rxRingSize); + KeInitializeSemaphore(&g_rxPendingCount, 0, g_rxRingSize); // Setup the push/pop stack for all our RX buffers. RX buffers are stored here until they can // be pushed into the RX ring. We need to handle the case where the user supplies less buffers // than can fit in the RX ring, also if excess (spare) buffers are supplied. g_rxBuffPoolHead = -1; RtlInitializeCriticalSection(&g_rxBuffPoolLock); - for (uint32_t i = 0; i < g_rxBuffCnt; i++) { - nvnetdrv_rx_push(g_rxRingBuffers + (i * NVNET_RX_BUFF_LEN)); + for (uint32_t i = 0; i < g_rxRingSize; i++) { + nvnetdrv_rx_push(g_rxRingUserBuffers + (i * NVNET_RX_BUFF_LEN)); } - // Fill our rx ring descriptor. g_rxBuffCnt may be less than the ring size, so only fill what we can. - for (uint32_t i = 0; i < NVNET_MIN(g_rxBuffCnt, RX_RING_SIZE); i++) { + // Fill our rx ring descriptor. g_rxRingSize may be less than the ring size, so only fill what we can. + for (uint32_t i = 0; i < NVNET_MIN(g_rxRingSize, RX_RING_SIZE); i++) { KeWaitForSingleObject(&g_rxRingFreeDescriptors, Executive, KernelMode, FALSE, NULL); nvnetdrv_rx_requeue(i); } - // g_rxRingEndIndex stores the position in the RX ring that is empty or about to be empty on the next packet. - g_rxRingEndIndex = NVNET_MIN(g_rxBuffCnt, RX_RING_SIZE) % RX_RING_SIZE; + // g_rxRingTail stores the position in the RX ring that is empty or about to be empty on the next packet. + g_rxRingTail = NVNET_MIN(g_rxRingSize, RX_RING_SIZE) % RX_RING_SIZE; // Get link speed settings from Phy nvnetdrv_handle_mii_irq(0, true); @@ -588,7 +619,7 @@ int nvnetdrv_init(size_t rx_buffer_count, nvnetdrv_rx_callback_t rx_callback) return NVNET_OK; } -void nvnetdrv_stop(void) +void nvnetdrv_stop (void) { assert(g_running); @@ -600,13 +631,13 @@ void nvnetdrv_stop(void) if (reg32(NvRegTxRxControl) & NVREG_TXRXCTL_IDLE) { break; } - KeDelayExecutionThread(KernelMode, FALSE, &fiftymicros); + KeDelayExecutionThread(KernelMode, FALSE, FIFTY_MICRO); } - //Stop NIC processing rings + // Stop NIC processing rings nvnetdrv_stop_txrx(); - //Clear the nvnet running flag so threads know to end + // Clear the nvnet running flag so threads know to end bool prev_value = atomic_exchange(&g_running, false); assert(prev_value); @@ -617,34 +648,34 @@ void nvnetdrv_stop(void) // Pass back all TX buffers to user. for (int i = 0; i < TX_RING_SIZE; i++) { - if (tx_misc[i].callback) { - tx_misc[i].callback(tx_misc[i].userdata); + if (g_txData[i].callback) { + g_txData[i].callback(g_txData[i].userdata); } } - // Free all TX descriptors g_txRingFreeDescriptors so nvnetdrv_acquire_tx_descriptors will return. - KeReleaseSemaphore(&g_txRingFreeDescriptors, IO_NETWORK_INCREMENT, g_txRingDescriptorsInUseCount, NULL); + // Free all TX descriptors g_txRingFreeCount so nvnetdrv_acquire_tx_descriptors will return. + KeReleaseSemaphore(&g_txRingFreeCount, IO_NETWORK_INCREMENT, g_txPendingCount, NULL); // End rxrequeue_thread - nvnetdrv_rx_push(g_rxRingBuffers); //Just push a buffer into stack so we dont get stuck waiting for one + nvnetdrv_rx_push(g_rxRingUserBuffers); // Just push a buffer into stack so we dont get stuck waiting for one KeReleaseSemaphore(&g_rxRingFreeDescriptors, IO_NETWORK_INCREMENT, 1, NULL); NtWaitForSingleObject(g_rxRingRequeueThread, FALSE, NULL); NtClose(g_rxRingRequeueThread); // End rxcallback_thread - KeReleaseSemaphore(&g_rxCallbackQueued, IO_NETWORK_INCREMENT, 1, NULL); + KeReleaseSemaphore(&g_rxPendingCount, IO_NETWORK_INCREMENT, 1, NULL); NtWaitForSingleObject(g_rxCallbackThread, FALSE, NULL); NtClose(g_rxCallbackThread); // Reset TX & RX control reg32(NvRegTxRxControl) = NVREG_TXRXCTL_DISABLE | NVREG_TXRXCTL_RESET; - KeDelayExecutionThread(KernelMode, FALSE, &tenmicros); + KeDelayExecutionThread(KernelMode, FALSE, TEN_MICRO); reg32(NvRegTxRxControl) = NVREG_TXRXCTL_DISABLE; // Free all memory allocated by nvnetdrv RtlDeleteCriticalSection(&g_rxBuffPoolLock); - MmFreeContiguousMemory((void *)g_rxRingDescriptors); - MmFreeContiguousMemory((void *)g_rxRingBuffers); + MmFreeContiguousMemory((void *)g_rxRing); + MmFreeContiguousMemory((void *)g_rxRingUserBuffers); free(g_rxCallbackQueue); free(g_rxBuffPool); } @@ -662,13 +693,13 @@ void nvnetdrv_stop_txrx (void) reg32(NvRegReceiverControl) &= ~NVREG_RCVCTL_START; reg32(NvRegTransmitterControl) &= ~NVREG_XMITCTL_START; - //Wait for active TX and RX descriptors to finish + // Wait for active TX and RX descriptors to finish for (int i = 0; i < 50000; i++) { if (!((reg32(NvRegReceiverStatus) & NVREG_RCVSTAT_BUSY) || (reg32(NvRegTransmitterStatus) & NVREG_XMITSTAT_BUSY))) { break; } - KeDelayExecutionThread(KernelMode, FALSE, &tenmicros); + KeDelayExecutionThread(KernelMode, FALSE, TEN_MICRO); } reg32(NvRegLinkSpeed) = 0; @@ -684,26 +715,29 @@ int nvnetdrv_acquire_tx_descriptors (size_t count) // Avoid excessive requests assert(count <= 4); - if (!g_running) + if (!g_running) { return false; + } while (true) { // Wait for TX descriptors to become available - KeWaitForSingleObject(&g_txRingFreeDescriptors, Executive, KernelMode, FALSE, NULL); + KeWaitForSingleObject(&g_txRingFreeCount, Executive, KernelMode, FALSE, NULL); - if (!g_running) return false; + if (!g_running) { + return false; + } // We want to try claim all tx descriptors at once without sleeping. size_t i = 0; for (i = 0; i < count - 1; i++) { // Try to acquire remaining descriptors without sleeping - status = KeWaitForSingleObject(&g_txRingFreeDescriptors, Executive, KernelMode, FALSE, &no_sleep); + status = KeWaitForSingleObject(&g_txRingFreeCount, Executive, KernelMode, FALSE, NO_SLEEP); if (!NT_SUCCESS(status) || status == STATUS_TIMEOUT) { // Couldn't acquire all at once, back off - KeReleaseSemaphore(&g_txRingFreeDescriptors, IO_NETWORK_INCREMENT, i + 1, NULL); + KeReleaseSemaphore(&g_txRingFreeCount, IO_NETWORK_INCREMENT, i + 1, NULL); if (status == STATUS_TIMEOUT) { // Sleep for 10 microseconds to avoid immediate re-locking - KeDelayExecutionThread(UserMode, FALSE, &tenmicros); + KeDelayExecutionThread(UserMode, FALSE, TEN_MICRO); // Retry break; } else { @@ -712,11 +746,14 @@ int nvnetdrv_acquire_tx_descriptors (size_t count) } } - if (!g_running) return false; + if (!g_running) { + return false; + } - //If we have claimed all the tx descriptors. We are done. - if (i == (count - 1)) + // If we have claimed all the tx descriptors. We are done. + if (i == (count - 1)) { break; + } } return true; } @@ -728,8 +765,9 @@ void nvnetdrv_submit_tx_descriptors (nvnetdrv_descriptor_t *buffers, size_t coun // Avoid excessive requests assert(count <= 4); - if (!g_running) + if (!g_running) { return; + } // Check that no buffer crosses a page boundary for (size_t i = 0; i < count; i++) { @@ -738,33 +776,34 @@ void nvnetdrv_submit_tx_descriptors (nvnetdrv_descriptor_t *buffers, size_t coun } // We don't check for buffer overrun here, because the Semaphore already protects us - size_t descriptors_index = g_txRingEndIndex; - while (!atomic_compare_exchange_weak(&g_txRingEndIndex, &descriptors_index, - (descriptors_index + count) % TX_RING_SIZE)); + size_t descriptors_index = g_txRingTail; + while ( + !atomic_compare_exchange_weak(&g_txRingTail, &descriptors_index, (descriptors_index + count) % TX_RING_SIZE)) + ; for (size_t i = 0; i < count; i++) { size_t current_descriptor_index = (descriptors_index + i) % TX_RING_SIZE; - tx_misc[current_descriptor_index].bufAddr = buffers[i].addr; - tx_misc[current_descriptor_index].length = buffers[i].length; - tx_misc[current_descriptor_index].userdata = buffers[i].userdata; - tx_misc[current_descriptor_index].callback = buffers[i].callback; + g_txData[current_descriptor_index].bufAddr = buffers[i].addr; + g_txData[current_descriptor_index].length = buffers[i].length; + g_txData[current_descriptor_index].userdata = buffers[i].userdata; + g_txData[current_descriptor_index].callback = buffers[i].callback; // Buffers get locked before sending and unlocked after sending MmLockUnlockBufferPages(buffers[i].addr, buffers[i].length, FALSE); - g_txRingDescriptors[current_descriptor_index].paddr = MmGetPhysicalAddress(buffers[i].addr); - g_txRingDescriptors[current_descriptor_index].length = buffers[i].length - 1; - g_txRingDescriptors[current_descriptor_index].flags = (i != 0 ? NV_TX_VALID : 0); + g_txRing[current_descriptor_index].paddr = MmGetPhysicalAddress(buffers[i].addr); + g_txRing[current_descriptor_index].length = buffers[i].length - 1; + g_txRing[current_descriptor_index].flags = (i != 0 ? NV_TX_VALID : 0); } // Terminate descriptor chain - g_txRingDescriptors[(descriptors_index + count - 1) % TX_RING_SIZE].flags |= NV_TX_LASTPACKET; + g_txRing[(descriptors_index + count - 1) % TX_RING_SIZE].flags |= NV_TX_LASTPACKET; // Enable first descriptor last to keep the NIC from sending incomplete packets - g_txRingDescriptors[descriptors_index].flags |= NV_TX_VALID; + g_txRing[descriptors_index].flags |= NV_TX_VALID; // Keep track of how many descriptors are in use - g_txRingDescriptorsInUseCount += count; + g_txPendingCount += count; // Inform that NIC that we have TX packet waiting reg32(NvRegTxRxControl) = NVREG_TXRXCTL_KICK; @@ -774,8 +813,9 @@ void nvnetdrv_rx_release (void *buffer_virt) { assert(buffer_virt != NULL); - if (!g_running) + if (!g_running) { return; + } nvnetdrv_rx_push(buffer_virt); } diff --git a/lib/net/nvnetdrv/nvnetdrv.h b/lib/net/nvnetdrv/nvnetdrv.h index 2af915e57..1435f1398 100644 --- a/lib/net/nvnetdrv/nvnetdrv.h +++ b/lib/net/nvnetdrv/nvnetdrv.h @@ -17,18 +17,18 @@ #define TX_RING_SIZE 64 #endif -// Must be greated than max thernet frame size. A multiple of page size prevents page boundary crossing +// Must be greater than max ethernet frame size. A multiple of page size prevents page boundary crossing #define NVNET_RX_BUFF_LEN (PAGE_SIZE / 2) // NVNET error codes -#define NVNET_OK 0 -#define NVNET_NO_MEM -1 -#define NVNET_NO_MAC -2 +#define NVNET_OK 0 +#define NVNET_NO_MEM -1 +#define NVNET_NO_MAC -2 #define NVNET_PHY_ERR -3 #define NVNET_SYS_ERR -4 -typedef void (*nvnetdrv_rx_callback_t) (void *buffer, uint16_t length); -typedef void (*nvnetdrv_tx_callback_t) (void *userdata); +typedef void (*nvnetdrv_rx_callback_t)(void *buffer, uint16_t length); +typedef void (*nvnetdrv_tx_callback_t)(void *userdata); typedef struct _nvnetdrv_descriptor_t { @@ -66,7 +66,7 @@ void nvnetdrv_stop (void); * Returns the ethernet MAC Address. * @return A pointer to an array containing the 6 byte ethernet MAC address. */ -const uint8_t *nvnetdrv_get_ethernet_addr (); +const uint8_t *nvnetdrv_get_ethernet_addr (void); /** * Reserves 1-4 descriptors. If the requested number is not immediately available, @@ -92,6 +92,6 @@ void nvnetdrv_submit_tx_descriptors (nvnetdrv_descriptor_t *buffers, size_t coun * This function is thread-safe. * @param buffer_virt Pointer to the buffer given out by nvnetdrv. */ -void nvnetdrv_rx_release(void *buffer_virt); +void nvnetdrv_rx_release (void *buffer_virt); -#endif \ No newline at end of file +#endif diff --git a/lib/net/nvnetdrv/nvnetdrv_lwip.c b/lib/net/nvnetdrv/nvnetdrv_lwip.c index 3c01d5887..d755879a4 100644 --- a/lib/net/nvnetdrv/nvnetdrv_lwip.c +++ b/lib/net/nvnetdrv/nvnetdrv_lwip.c @@ -5,27 +5,26 @@ // SPDX-FileCopyrightText: 2022 Stefan Schmidt // SPDX-FileCopyrightText: 2022 Ryan Wendland -#include "lwip/opt.h" #include "lwip/def.h" +#include "lwip/ethip6.h" #include "lwip/mem.h" +#include "lwip/mld6.h" +#include "lwip/opt.h" #include "lwip/pbuf.h" +#include "lwip/snmp.h" #include "lwip/stats.h" #include "lwip/sys.h" -#include "lwip/snmp.h" -#include "lwip/ethip6.h" -#include "lwip/mld6.h" #include "netif/etharp.h" #include "netif/ppp/pppoe.h" #include "nvnetdrv.h" -#include #include +#include /* Define those to better describe your network interface. */ -#define IFNAME0 'x' -#define IFNAME1 'b' +#define IFNAME0 'x' +#define IFNAME1 'b' #define RX_BUFF_CNT (RX_RING_SIZE) - #define LINK_SPEED_OF_YOUR_NETIF_IN_BPS 100 * 1000 * 1000 /* 100 Mbps */ static struct netif *g_pnetif; @@ -54,25 +53,25 @@ typedef struct } rx_pbuf_t; LWIP_MEMPOOL_DECLARE(RX_POOL, RX_BUFF_CNT, sizeof(rx_pbuf_t), "Zero-copy RX PBUF pool"); -void rx_pbuf_free_callback(struct pbuf *p) +void rx_pbuf_free_callback (struct pbuf *p) { rx_pbuf_t *rx_pbuf = (rx_pbuf_t *)p; nvnetdrv_rx_release(rx_pbuf->buff); LWIP_MEMPOOL_FREE(RX_POOL, rx_pbuf); } -void rx_callback(void *buffer, uint16_t length) +void rx_callback (void *buffer, uint16_t length) { rx_pbuf_t *rx_pbuf = (rx_pbuf_t *)LWIP_MEMPOOL_ALLOC(RX_POOL); LWIP_ASSERT("RX_POOL full\n", rx_pbuf != NULL); rx_pbuf->p.custom_free_function = rx_pbuf_free_callback; rx_pbuf->buff = buffer; struct pbuf *p = pbuf_alloced_custom(PBUF_RAW, - length + ETH_PAD_SIZE, - PBUF_REF, - &rx_pbuf->p, - buffer - ETH_PAD_SIZE, - NVNET_RX_BUFF_LEN - ETH_PAD_SIZE); + length + ETH_PAD_SIZE, + PBUF_REF, + &rx_pbuf->p, + buffer - ETH_PAD_SIZE, + NVNET_RX_BUFF_LEN - ETH_PAD_SIZE); if (g_pnetif->input(p, g_pnetif) != ERR_OK) { pbuf_free(p); @@ -89,7 +88,7 @@ void rx_callback(void *buffer, uint16_t length) * @return ERR_OK if low level initiliazation succeeds * ERR_IF if any failure */ -static err_t low_level_init(struct netif *netif) +static err_t low_level_init (struct netif *netif) { if (nvnetdrv_init(RX_BUFF_CNT, rx_callback) < 0) { return ERR_IF; @@ -114,8 +113,7 @@ static err_t low_level_init(struct netif *netif) * All-nodes link-local is handled by default, so we must let the hardware know * to allow multicast packets in. * Should set mld_mac_filter previously. */ - if (netif->mld_mac_filter != NULL) - { + if (netif->mld_mac_filter != NULL) { ip6_addr_t ip6_allnodes_ll; ip6_addr_set_allnodes_linklocal(&ip6_allnodes_ll); netif->mld_mac_filter(netif, &ip6_allnodes_ll, NETIF_ADD_MAC_FILTER); @@ -132,7 +130,7 @@ static err_t low_level_init(struct netif *netif) * * @param userdata the pbuf address, supplied by low_level_output */ -void tx_pbuf_free_callback(void *userdata) +void tx_pbuf_free_callback (void *userdata) { struct pbuf *p = (struct pbuf *)userdata; pbuf_free(p); @@ -154,7 +152,7 @@ void tx_pbuf_free_callback(void *userdata) * dropped because of memory failure (except for the TCP timers). */ -static err_t low_level_output(struct netif *netif, struct pbuf *p) +static err_t low_level_output (struct netif *netif, struct pbuf *p) { #if ETH_PAD_SIZE pbuf_header(p, -ETH_PAD_SIZE); /* drop the padding word */ @@ -162,16 +160,16 @@ static err_t low_level_output(struct netif *netif, struct pbuf *p) nvnetdrv_descriptor_t descriptors[4]; size_t pbufCount = 0; - for (struct pbuf *q = p; q != NULL; q = q->next) - { + for (struct pbuf *q = p; q != NULL; q = q->next) { assert(p->len < 4096); descriptors[pbufCount].addr = q->payload; descriptors[pbufCount].length = q->len; descriptors[pbufCount].callback = NULL; pbufCount++; - if (pbufCount > 4) + if (pbufCount > 4) { return ERR_MEM; + } const uint32_t addr_start = (uint32_t)q->payload; const uint32_t addr_end = ((uint32_t)q->payload + q->len); @@ -182,8 +180,9 @@ static err_t low_level_output(struct netif *netif, struct pbuf *p) const uint32_t length_b = addr_end - addr_boundary; // Buffer ends right at page boundary, so no problem - if (length_b == 0) + if (length_b == 0) { continue; + } // Fixup the descriptor descriptors[pbufCount - 1].length = length_a; @@ -194,8 +193,9 @@ static err_t low_level_output(struct netif *netif, struct pbuf *p) descriptors[pbufCount].callback = NULL; pbufCount++; - if (pbufCount > 4) + if (pbufCount > 4) { return ERR_MEM; + } } } @@ -234,7 +234,7 @@ static err_t low_level_output(struct netif *netif, struct pbuf *p) * ERR_MEM if private data couldn't be allocated * any other err_t on error */ -err_t nvnetif_init(struct netif *netif) +err_t nvnetif_init (struct netif *netif) { struct nforceif *nforceif; diff --git a/lib/net/nvnetdrv/nvnetdrv_regs.h b/lib/net/nvnetdrv/nvnetdrv_regs.h index a6caeb45c..9371e5f7f 100644 --- a/lib/net/nvnetdrv/nvnetdrv_regs.h +++ b/lib/net/nvnetdrv/nvnetdrv_regs.h @@ -202,4 +202,5 @@ enum { #define NV_RX_OVERFLOW (1 << 12) #define NV_RX_FRAMINGERR (1 << 13) #define NV_RX_ERROR (1 << 14) -#define NV_RX_AVAIL (1 << 15) \ No newline at end of file +#define NV_RX_AVAIL (1 << 15) +#define NV_RX_ERROR_MASK (NV_RX_ERROR1|NV_RX_ERROR2|NV_RX_ERROR3|NV_RX_ERROR4|NV_RX_CRCERR|NV_RX_OVERFLOW)