linux-imx

// SPDX-License-Identifier: GPL-2.0-only

/*

 * Copyright (C) 2016 Facebook

 * Copyright (C) 2013-2014 Jens Axboe

 */

​

#include <linux/sched.h>

#include <linux/random.h>

#include <linux/sbitmap.h>

#include <linux/seq_file.h>

​

/*

 * See if we have deferred clears that we can batch move

 */

static inline bool sbitmap_deferred_clear(struct sbitmap *sb, int index)

{

    unsigned long mask, val;

    bool ret = false;

    unsigned long flags;

​

    spin_lock_irqsave(&sb->map[index].swap_lock, flags);

​

    if (!sb->map[index].cleared)

        goto out_unlock;

​

    /*

     * First get a stable cleared mask, setting the old mask to 0.

     */

    do {

        mask = sb->map[index].cleared;

    } while (cmpxchg(&sb->map[index].cleared, mask, 0) != mask);

​

    /*

     * Now clear the masked bits in our free word

     */

    do {

        val = sb->map[index].word;

    } while (cmpxchg(&sb->map[index].word, val, val & ~mask) != val);

​

    ret = true;

out_unlock:

    spin_unlock_irqrestore(&sb->map[index].swap_lock, flags);

    return ret;

}

​

int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift,

              gfp_t flags, int node)

{

    unsigned int bits_per_word;

    unsigned int i;

​

    if (shift < 0) {

        shift = ilog2(BITS_PER_LONG);

        /*

         * If the bitmap is small, shrink the number of bits per word so

         * we spread over a few cachelines, at least. If less than 4

         * bits, just forget about it, it's not going to work optimally

         * anyway.

         */

        if (depth >= 4) {

            while ((4U << shift) > depth)

                shift--;

        }

    }

    bits_per_word = 1U << shift;

    if (bits_per_word > BITS_PER_LONG)

        return -EINVAL;

​

    sb->shift = shift;

    sb->depth = depth;

    sb->map_nr = DIV_ROUND_UP(sb->depth, bits_per_word);

​

    if (depth == 0) {

        sb->map = NULL;

        return 0;

    }

​

    sb->map = kcalloc_node(sb->map_nr, sizeof(*sb->map), flags, node);

    if (!sb->map)

        return -ENOMEM;

​

    for (i = 0; i < sb->map_nr; i++) {

        sb->map[i].depth = min(depth, bits_per_word);

        depth -= sb->map[i].depth;

        spin_lock_init(&sb->map[i].swap_lock);

    }

    return 0;

}

EXPORT_SYMBOL_GPL(sbitmap_init_node);

​

void sbitmap_resize(struct sbitmap *sb, unsigned int depth)

{