linux-imx

/*

 * Copyright (C) 2010-2017 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>

 *

 * membarrier system call

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 */

#include "sched.h"

​

/*

 * Bitmask made from a "or" of all commands within enum membarrier_cmd,

 * except MEMBARRIER_CMD_QUERY.

 */

#ifdef CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE

#define MEMBARRIER_PRIVATE_EXPEDITED_SYNC_CORE_BITMASK          \

    (MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE         \

    | MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE)

#else

#define MEMBARRIER_PRIVATE_EXPEDITED_SYNC_CORE_BITMASK  0

#endif

​

#define MEMBARRIER_CMD_BITMASK                      \

    (MEMBARRIER_CMD_GLOBAL | MEMBARRIER_CMD_GLOBAL_EXPEDITED    \

    | MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED          \

    | MEMBARRIER_CMD_PRIVATE_EXPEDITED              \

    | MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED         \

    | MEMBARRIER_PRIVATE_EXPEDITED_SYNC_CORE_BITMASK)

​

static void ipi_mb(void *info)

{

    smp_mb();   /* IPIs should be serializing but paranoid. */

}

​

static int membarrier_global_expedited(void)

{

    int cpu;

    bool fallback = false;

    cpumask_var_t tmpmask;

​

    if (num_online_cpus() == 1)

        return 0;

​

    /*

     * Matches memory barriers around rq->curr modification in

     * scheduler.

     */

    smp_mb();   /* system call entry is not a mb. */

​

    /*

     * Expedited membarrier commands guarantee that they won't

     * block, hence the GFP_NOWAIT allocation flag and fallback

     * implementation.

     */

    if (!zalloc_cpumask_var(&tmpmask, GFP_NOWAIT)) {

        /* Fallback for OOM. */

        fallback = true;

    }

​

    cpus_read_lock();

    for_each_online_cpu(cpu) {

        struct task_struct *p;

​

        /*

         * Skipping the current CPU is OK even through we can be

         * migrated at any point. The current CPU, at the point

         * where we read raw_smp_processor_id(), is ensured to

         * be in program order with respect to the caller

         * thread. Therefore, we can skip this CPU from the

         * iteration.

         */

        if (cpu == raw_smp_processor_id())

            continue;

​

        rcu_read_lock();

        p = task_rcu_dereference(&cpu_rq(cpu)->curr);

        if (p && p->mm && (atomic_read(&p->mm->membarrier_state) &

                   MEMBARRIER_STATE_GLOBAL_EXPEDITED)) {

            if (!fallback)

                __cpumask_set_cpu(cpu, tmpmask);

            else

                smp_call_function_single(cpu, ipi_mb, NULL, 1);

        }

        rcu_read_unlock();

    }