linux-imx

/*

 * Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra

 *

 * Provides a framework for enqueueing and running callbacks from hardirq

 * context. The enqueueing is NMI-safe.

 */

​

#include <linux/bug.h>

#include <linux/kernel.h>

#include <linux/export.h>

#include <linux/irq_work.h>

#include <linux/percpu.h>

#include <linux/hardirq.h>

#include <linux/irqflags.h>

#include <linux/sched.h>

#include <linux/tick.h>

#include <linux/cpu.h>

#include <linux/notifier.h>

#include <linux/smp.h>

#include <asm/processor.h>

​

​

static DEFINE_PER_CPU(struct llist_head, raised_list);

static DEFINE_PER_CPU(struct llist_head, lazy_list);

​

/*

 * Claim the entry so that no one else will poke at it.

 */

static bool irq_work_claim(struct irq_work *work)

{

    unsigned long flags, oflags, nflags;

​

    /*

     * Start with our best wish as a premise but only trust any

     * flag value after cmpxchg() result.

     */

    flags = work->flags & ~IRQ_WORK_PENDING;

    for (;;) {

        nflags = flags | IRQ_WORK_CLAIMED;

        oflags = cmpxchg(&work->flags, flags, nflags);

        if (oflags == flags)

            break;

        if (oflags & IRQ_WORK_PENDING)

            return false;

        flags = oflags;

        cpu_relax();

    }

​

    return true;

}

​

void __weak arch_irq_work_raise(void)

{

    /*

     * Lame architectures will get the timer tick callback

     */

}

​

/*

 * Enqueue the irq_work @work on @cpu unless it's already pending

 * somewhere.

 *

 * Can be re-enqueued while the callback is still in progress.

 */

bool irq_work_queue_on(struct irq_work *work, int cpu)

{

    /* All work should have been flushed before going offline */

    WARN_ON_ONCE(cpu_is_offline(cpu));

​

#ifdef CONFIG_SMP

​

    /* Arch remote IPI send/receive backend aren't NMI safe */

    WARN_ON_ONCE(in_nmi());

​

    /* Only queue if not already pending */

    if (!irq_work_claim(work))

        return false;

​

    if (llist_add(&work->llnode, &per_cpu(raised_list, cpu)))

        arch_send_call_function_single_ipi(cpu);

​

#else /* #ifdef CONFIG_SMP */

    irq_work_queue(work);

#endif /* #else #ifdef CONFIG_SMP */

​

    return true;

}

​

/* Enqueue the irq work @work on the current CPU */

bool irq_work_queue(struct irq_work *work)

{

    /* Only queue if not already pending */