Skip to content

Commit

Permalink
dmaengine: idxd: add work queue drain support
Browse files Browse the repository at this point in the history 
Add wq drain support. When a wq is being released, it needs to wait for
all in-flight operation to complete.  A device control function
idxd_wq_drain() has been added to facilitate this. A wq drain call
is added to the char dev on release to make sure all user operations are
complete. A wq drain is also added before the wq is being disabled.

A drain command can take an unpredictable period of time. Interrupt support
for device commands is added to allow waiting on the command to
finish. If a previous command is in progress, the new submitter can block
until the current command is finished before proceeding. The interrupt
based submission will submit the command and then wait until a command
completion interrupt happens to complete. All commands are moved to the
interrupt based command submission except for the device reset during
probe, which will be polled.

Fixes: 42d279f ("dmaengine: idxd: add char driver to expose submission portal to userland")
Signed-off-by: Dave Jiang <[email protected]>
Reviewed-by: Tony Luck <[email protected]>
Reviewed-by: Dan Williams <[email protected]>
Link: https://lore.kernel.org/r/159319502515.69593.13451647706946040301.stgit@djiang5-desk3.ch.intel.com
Signed-off-by: Vinod Koul <[email protected]>
  • Loading branch information
@davejiang @vinodkoul
davejiang authored and vinodkoul committed Jul 13, 2020
1 parent d12ea55 commit 0d5c10b
Show file tree
Hide file tree
Showing 6 changed files with 112 additions and 132 deletions.
3 changes: 3 additions & 0 deletions drivers/dma/idxd/cdev.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,9 @@ static int idxd_cdev_release(struct inode *node, struct file *filep)
dev_dbg(dev, "%s called\n", __func__);
filep->private_data = NULL;

/* Wait for in-flight operations to complete. */
idxd_wq_drain(wq);

kfree(ctx);
idxd_wq_put(wq);
return 0;
Expand Down
155 changes: 70 additions & 85 deletions drivers/dma/idxd/device.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@
#include "idxd.h"
#include "registers.h"

static int idxd_cmd_wait(struct idxd_device *idxd, u32 *status, int timeout);
static int idxd_cmd_send(struct idxd_device *idxd, int cmd_code, u32 operand);
static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand,
u32 *status);

/* Interrupt control bits */
int idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id)
Expand Down Expand Up @@ -233,21 +233,13 @@ int idxd_wq_enable(struct idxd_wq *wq)
struct idxd_device *idxd = wq->idxd;
struct device *dev = &idxd->pdev->dev;
u32 status;
int rc;

lockdep_assert_held(&idxd->dev_lock);

if (wq->state == IDXD_WQ_ENABLED) {
dev_dbg(dev, "WQ %d already enabled\n", wq->id);
return -ENXIO;
}

rc = idxd_cmd_send(idxd, IDXD_CMD_ENABLE_WQ, wq->id);
if (rc < 0)
return rc;
rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
if (rc < 0)
return rc;
idxd_cmd_exec(idxd, IDXD_CMD_ENABLE_WQ, wq->id, &status);

if (status != IDXD_CMDSTS_SUCCESS &&
status != IDXD_CMDSTS_ERR_WQ_ENABLED) {
Expand All @@ -265,9 +257,7 @@ int idxd_wq_disable(struct idxd_wq *wq)
struct idxd_device *idxd = wq->idxd;
struct device *dev = &idxd->pdev->dev;
u32 status, operand;
int rc;

lockdep_assert_held(&idxd->dev_lock);
dev_dbg(dev, "Disabling WQ %d\n", wq->id);

if (wq->state != IDXD_WQ_ENABLED) {
Expand All @@ -276,12 +266,7 @@ int idxd_wq_disable(struct idxd_wq *wq)
}

operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
rc = idxd_cmd_send(idxd, IDXD_CMD_DISABLE_WQ, operand);
if (rc < 0)
return rc;
rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
if (rc < 0)
return rc;
idxd_cmd_exec(idxd, IDXD_CMD_DISABLE_WQ, operand, &status);

if (status != IDXD_CMDSTS_SUCCESS) {
dev_dbg(dev, "WQ disable failed: %#x\n", status);
Expand All @@ -293,6 +278,22 @@ int idxd_wq_disable(struct idxd_wq *wq)
return 0;
}

void idxd_wq_drain(struct idxd_wq *wq)
{
struct idxd_device *idxd = wq->idxd;
struct device *dev = &idxd->pdev->dev;
u32 operand;

if (wq->state != IDXD_WQ_ENABLED) {
dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);
return;
}

dev_dbg(dev, "Draining WQ %d\n", wq->id);
operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
idxd_cmd_exec(idxd, IDXD_CMD_DRAIN_WQ, operand, NULL);
}

int idxd_wq_map_portal(struct idxd_wq *wq)
{
struct idxd_device *idxd = wq->idxd;
Expand Down Expand Up @@ -330,66 +331,79 @@ static inline bool idxd_is_enabled(struct idxd_device *idxd)
return false;
}

static int idxd_cmd_wait(struct idxd_device *idxd, u32 *status, int timeout)
/*
* This is function is only used for reset during probe and will
* poll for completion. Once the device is setup with interrupts,
* all commands will be done via interrupt completion.
*/
void idxd_device_init_reset(struct idxd_device *idxd)
{
u32 sts, to = timeout;

lockdep_assert_held(&idxd->dev_lock);
sts = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET);
while (sts & IDXD_CMDSTS_ACTIVE && --to) {
cpu_relax();
sts = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET);
}
struct device *dev = &idxd->pdev->dev;
union idxd_command_reg cmd;
unsigned long flags;

if (to == 0 && sts & IDXD_CMDSTS_ACTIVE) {
dev_warn(&idxd->pdev->dev, "%s timed out!\n", __func__);
*status = 0;
return -EBUSY;
}
memset(&cmd, 0, sizeof(cmd));
cmd.cmd = IDXD_CMD_RESET_DEVICE;
dev_dbg(dev, "%s: sending reset for init.\n", __func__);
spin_lock_irqsave(&idxd->dev_lock, flags);
iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET);

*status = sts;
return 0;
while (ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET) &
IDXD_CMDSTS_ACTIVE)
cpu_relax();
spin_unlock_irqrestore(&idxd->dev_lock, flags);
}

static int idxd_cmd_send(struct idxd_device *idxd, int cmd_code, u32 operand)
static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand,
u32 *status)
{
union idxd_command_reg cmd;
int rc;
u32 status;

lockdep_assert_held(&idxd->dev_lock);
rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
if (rc < 0)
return rc;
DECLARE_COMPLETION_ONSTACK(done);
unsigned long flags;

memset(&cmd, 0, sizeof(cmd));
cmd.cmd = cmd_code;
cmd.operand = operand;
cmd.int_req = 1;

spin_lock_irqsave(&idxd->dev_lock, flags);
wait_event_lock_irq(idxd->cmd_waitq,
!test_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags),
idxd->dev_lock);

dev_dbg(&idxd->pdev->dev, "%s: sending cmd: %#x op: %#x\n",
__func__, cmd_code, operand);

__set_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags);
idxd->cmd_done = &done;
iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET);

return 0;
/*
* After command submitted, release lock and go to sleep until
* the command completes via interrupt.
*/
spin_unlock_irqrestore(&idxd->dev_lock, flags);
wait_for_completion(&done);
spin_lock_irqsave(&idxd->dev_lock, flags);
if (status)
*status = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET);
__clear_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags);
/* Wake up other pending commands */
wake_up(&idxd->cmd_waitq);
spin_unlock_irqrestore(&idxd->dev_lock, flags);
}

int idxd_device_enable(struct idxd_device *idxd)
{
struct device *dev = &idxd->pdev->dev;
int rc;
u32 status;

lockdep_assert_held(&idxd->dev_lock);
if (idxd_is_enabled(idxd)) {
dev_dbg(dev, "Device already enabled\n");
return -ENXIO;
}

rc = idxd_cmd_send(idxd, IDXD_CMD_ENABLE_DEVICE, 0);
if (rc < 0)
return rc;
rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
if (rc < 0)
return rc;
idxd_cmd_exec(idxd, IDXD_CMD_ENABLE_DEVICE, 0, &status);

/* If the command is successful or if the device was enabled */
if (status != IDXD_CMDSTS_SUCCESS &&
Expand All @@ -405,58 +419,29 @@ int idxd_device_enable(struct idxd_device *idxd)
int idxd_device_disable(struct idxd_device *idxd)
{
struct device *dev = &idxd->pdev->dev;
int rc;
u32 status;

lockdep_assert_held(&idxd->dev_lock);
if (!idxd_is_enabled(idxd)) {
dev_dbg(dev, "Device is not enabled\n");
return 0;
}

rc = idxd_cmd_send(idxd, IDXD_CMD_DISABLE_DEVICE, 0);
if (rc < 0)
return rc;
rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
if (rc < 0)
return rc;
idxd_cmd_exec(idxd, IDXD_CMD_DISABLE_DEVICE, 0, &status);

/* If the command is successful or if the device was disabled */
if (status != IDXD_CMDSTS_SUCCESS &&
!(status & IDXD_CMDSTS_ERR_DIS_DEV_EN)) {
dev_dbg(dev, "%s: err_code: %#x\n", __func__, status);
rc = -ENXIO;
return rc;
return -ENXIO;
}

idxd->state = IDXD_DEV_CONF_READY;
return 0;
}

int __idxd_device_reset(struct idxd_device *idxd)
{
u32 status;
int rc;

rc = idxd_cmd_send(idxd, IDXD_CMD_RESET_DEVICE, 0);
if (rc < 0)
return rc;
rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
if (rc < 0)
return rc;

return 0;
}

int idxd_device_reset(struct idxd_device *idxd)
void idxd_device_reset(struct idxd_device *idxd)
{
unsigned long flags;
int rc;

spin_lock_irqsave(&idxd->dev_lock, flags);
rc = __idxd_device_reset(idxd);
spin_unlock_irqrestore(&idxd->dev_lock, flags);
return rc;
idxd_cmd_exec(idxd, IDXD_CMD_RESET_DEVICE, 0, NULL);
}

/* Device configuration bits */
Expand Down
11 changes: 8 additions & 3 deletions drivers/dma/idxd/idxd.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@ enum idxd_device_state {

enum idxd_device_flag {
IDXD_FLAG_CONFIGURABLE = 0,
IDXD_FLAG_CMD_RUNNING,
};

struct idxd_device {
Expand All @@ -158,6 +159,7 @@ struct idxd_device {
void __iomem *reg_base;

spinlock_t dev_lock; /* spinlock for device */
struct completion *cmd_done;
struct idxd_group *groups;
struct idxd_wq *wqs;
struct idxd_engine *engines;
Expand All @@ -180,12 +182,14 @@ struct idxd_device {
int nr_tokens; /* non-reserved tokens */

union sw_err_reg sw_err;

wait_queue_head_t cmd_waitq;
struct msix_entry *msix_entries;
int num_wq_irqs;
struct idxd_irq_entry *irq_entries;

struct dma_device dma_dev;
struct workqueue_struct *wq;
struct work_struct work;
};

/* IDXD software descriptor */
Expand Down Expand Up @@ -273,10 +277,10 @@ int idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id);
int idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id);

/* device control */
void idxd_device_init_reset(struct idxd_device *idxd);
int idxd_device_enable(struct idxd_device *idxd);
int idxd_device_disable(struct idxd_device *idxd);
int idxd_device_reset(struct idxd_device *idxd);
int __idxd_device_reset(struct idxd_device *idxd);
void idxd_device_reset(struct idxd_device *idxd);
void idxd_device_cleanup(struct idxd_device *idxd);
int idxd_device_config(struct idxd_device *idxd);
void idxd_device_wqs_clear_state(struct idxd_device *idxd);
Expand All @@ -286,6 +290,7 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq);
void idxd_wq_free_resources(struct idxd_wq *wq);
int idxd_wq_enable(struct idxd_wq *wq);
int idxd_wq_disable(struct idxd_wq *wq);
void idxd_wq_drain(struct idxd_wq *wq);
int idxd_wq_map_portal(struct idxd_wq *wq);
void idxd_wq_unmap_portal(struct idxd_wq *wq);

Expand Down
14 changes: 8 additions & 6 deletions drivers/dma/idxd/init.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ static int idxd_setup_internals(struct idxd_device *idxd)
struct device *dev = &idxd->pdev->dev;
int i;

init_waitqueue_head(&idxd->cmd_waitq);
idxd->groups = devm_kcalloc(dev, idxd->max_groups,
sizeof(struct idxd_group), GFP_KERNEL);
if (!idxd->groups)
Expand Down Expand Up @@ -182,6 +183,10 @@ static int idxd_setup_internals(struct idxd_device *idxd)
idxd->engines[i].id = i;
}

idxd->wq = create_workqueue(dev_name(dev));
if (!idxd->wq)
return -ENOMEM;

return 0;
}

Expand Down Expand Up @@ -277,9 +282,7 @@ static int idxd_probe(struct idxd_device *idxd)
int rc;

dev_dbg(dev, "%s entered and resetting device\n", __func__);
rc = idxd_device_reset(idxd);
if (rc < 0)
return rc;
idxd_device_init_reset(idxd);
dev_dbg(dev, "IDXD reset complete\n");

idxd_read_caps(idxd);
Expand Down Expand Up @@ -414,11 +417,8 @@ static void idxd_shutdown(struct pci_dev *pdev)
int rc, i;
struct idxd_irq_entry *irq_entry;
int msixcnt = pci_msix_vec_count(pdev);
unsigned long flags;

spin_lock_irqsave(&idxd->dev_lock, flags);
rc = idxd_device_disable(idxd);
spin_unlock_irqrestore(&idxd->dev_lock, flags);
if (rc)
dev_err(&pdev->dev, "Disabling device failed\n");

Expand All @@ -434,6 +434,8 @@ static void idxd_shutdown(struct pci_dev *pdev)
idxd_flush_pending_llist(irq_entry);
idxd_flush_work_list(irq_entry);
}

destroy_workqueue(idxd->wq);
}

static void idxd_remove(struct pci_dev *pdev)
Expand Down
Loading

0 comments on commit 0d5c10b

Please sign in to comment.