How to handle interrupts from a PCI device that already have a non shareable handler in a Linux kernel module?

Question

I'm learning Linux kernel PCI development with QEMU using the educational edu PCI device.

If I use the device with:

-device edu

it is inserted since boot, and my interrupts get number 11 and work well.

However, I started playing with inserting the device after boot with the monitor command:

device_add edu

followed by:

echo 1 > /sys/bus/pci/rescan

If I do that, the IRQ is getting assigned to an interrupt 0, which already has a non-shareable interrupt on it (timer), and my:

 request_irq(pci_irq, irq_handler, IRQF_SHARED, "pci_irq_handler0", &major)

fails with message:

 genirq: Flags mismatch irq 0. 00000080 (pci_irq_handler0) vs. 00015a00 (timer)

From the kernel source, we see that 0x80 is the shareable flag, which is not present in the timer.

Is this a bug in the edu device, or is there something I can do about it in my kernel module?

The state with -device on boot can :

device      BDF      IRQ
==========  =======  ===
edu         00:04.0  10
virtio-pci  00:05.0  11

while device_add gives:

device      BDF      IRQ
==========  =======  ===
virtio-pci  00:04.0  10
edu         00:05.0   0

so we see that edu and virtio-pci swapped places on the probe, but not IRQs unfortunately.

The full device code with all boilerplate is on GitHub, and here is a minimized reproduction version:

#include <asm/uaccess.h>
#include <linux/cdev.h>
#include <linux/fs.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>

#define BAR 0
#define CDEV_NAME "lkmc_pci"
#define EDU_DEVICE_ID 0x11e8
#define IO_IRQ_ACK 0x64
#define IO_IRQ_STATUS 0x24
#define QEMU_VENDOR_ID 0x1234

MODULE_LICENSE("GPL");

static struct pci_device_id pci_ids[] = {
    { PCI_DEVICE(QEMU_VENDOR_ID, EDU_DEVICE_ID), },
    { 0, }
};
MODULE_DEVICE_TABLE(pci, pci_ids);

static int major;
static int pci_irq;
static struct pci_dev *pdev;
static void __iomem *mmio;

static struct file_operations fops = {
    .owner   = THIS_MODULE,
};

static irqreturn_t irq_handler(int irq, void *dev)
{
    int devi;
    irqreturn_t ret;
    u32 irq_status;

    devi = *(int *)dev;
    irq_status = ioread32(mmio + IO_IRQ_STATUS);
    pr_info("irq_handler irq = %d dev = %d irq_status = %llx\n",
            irq, devi, (unsigned long long)irq_status);
    iowrite32(irq_status, mmio + IO_IRQ_ACK);
    ret = IRQ_HANDLED;
    return ret;
}

static int pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
{
    u8 val;

    pr_info("pci_probe\n");
    major = register_chrdev(0, CDEV_NAME, &fops);
    pdev = dev;
    if (pci_enable_device(dev) < 0) {
        dev_err(&(pdev->dev), "pci_enable_device\n");
        goto error;
    }
    if (pci_request_region(dev, BAR, "myregion0")) {
        dev_err(&(pdev->dev), "pci_request_region\n");
        goto error;
    }
    mmio = pci_iomap(pdev, BAR, pci_resource_len(pdev, BAR));
    pci_read_config_byte(dev, PCI_INTERRUPT_LINE, &val);
    pci_irq = val;
    if (request_irq(pci_irq, irq_handler, IRQF_SHARED, "pci_irq_handler0", &major) < 0) {
        dev_err(&(dev->dev), "request_irq\n");
        goto error;
    }
    /* This makes the device generate an interrupt. */
    iowrite32(0x12345678, mmio + 0x60);
    return 0;
error:
    return 1;
}

static void pci_remove(struct pci_dev *dev)
{
    pr_info("pci_remove\n");
    free_irq(pci_irq, &major);
    pci_release_region(dev, BAR);
    unregister_chrdev(major, CDEV_NAME);
}

static struct pci_driver pci_driver = {
    .name     = "lkmc_pci",
    .id_table = pci_ids,
    .probe    = pci_probe,
    .remove   = pci_remove,
};

static int myinit(void)
{
    if (pci_register_driver(&pci_driver) < 0) {
        return 1;
    }
    return 0;
}

static void myexit(void)
{
    pci_unregister_driver(&pci_driver);
}

module_init(myinit);
module_exit(myexit);

Possibly related: https://serverfault.com/questions/70585/manually-assign-a-pci-card-to-an-interrupt

Answer 1

As mentioned by 0andriy, we need to use dev->pci instead of reading the config with pci_read_config_byte(dev, PCI_INTERRUPT_LINE, &val); . TODO follow kernel code and understand exactly why. This works:

#include <asm/uaccess.h>
#include <linux/cdev.h>
#include <linux/fs.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>

#define BAR 0
#define CDEV_NAME "lkmc_hw_pci_min"
#define EDU_DEVICE_ID 0x11e9
#define QEMU_VENDOR_ID 0x1234

MODULE_LICENSE("GPL");

static struct pci_device_id id_table[] = {
    { PCI_DEVICE(QEMU_VENDOR_ID, EDU_DEVICE_ID), },
    { 0, }
};
MODULE_DEVICE_TABLE(pci, id_table);
static int major;
static struct pci_dev *pdev;
static void __iomem *mmio;
static struct file_operations fops = {
    .owner   = THIS_MODULE,
};

static irqreturn_t irq_handler(int irq, void *dev)
{
    pr_info("irq_handler irq = %d dev = %d\n", irq, *(int *)dev);
    iowrite32(0, mmio + 4);
    return IRQ_HANDLED;
}

static int probe(struct pci_dev *dev, const struct pci_device_id *id)
{
    pr_info("probe\n");
    major = register_chrdev(0, CDEV_NAME, &fops);
    pdev = dev;
    if (pci_enable_device(dev) < 0) {
        dev_err(&(pdev->dev), "pci_enable_device\n");
        goto error;
    }
    if (pci_request_region(dev, BAR, "myregion0")) {
        dev_err(&(pdev->dev), "pci_request_region\n");
        goto error;
    }
    mmio = pci_iomap(pdev, BAR, pci_resource_len(pdev, BAR));
    pr_info("dev->irq = %u\n", dev->irq);
    if (request_irq(dev->irq, irq_handler, IRQF_SHARED, "pci_irq_handler0", &major) < 0) {
        dev_err(&(dev->dev), "request_irq\n");
        goto error;
    }
    iowrite32(0x12345678, mmio);
    return 0;
error:
    return 1;
}

static void remove(struct pci_dev *dev)
{
    pr_info("remove\n");
    free_irq(dev->irq, &major);
    pci_release_region(dev, BAR);
    unregister_chrdev(major, CDEV_NAME);
}

static struct pci_driver pci_driver = {
    .name     = CDEV_NAME,
    .id_table = id_table,
    .probe    = probe,
    .remove   = remove,
};

static int myinit(void)
{
    if (pci_register_driver(&pci_driver) < 0) {
        return 1;
    }
    return 0;
}

static void myexit(void)
{
    pci_unregister_driver(&pci_driver);
}

module_init(myinit);
module_exit(myexit);