linux-imx

// SPDX-License-Identifier: GPL-2.0

/*

 * SLUB: A slab allocator that limits cache line use instead of queuing

 * objects in per cpu and per node lists.

 *

 * The allocator synchronizes using per slab locks or atomic operatios

 * and only uses a centralized lock to manage a pool of partial slabs.

 *

 * (C) 2007 SGI, Christoph Lameter

 * (C) 2011 Linux Foundation, Christoph Lameter

 */

​

#include <linux/mm.h>

#include <linux/swap.h> /* struct reclaim_state */

#include <linux/module.h>

#include <linux/bit_spinlock.h>

#include <linux/interrupt.h>

#include <linux/bitops.h>

#include <linux/slab.h>

#include "slab.h"

#include <linux/proc_fs.h>

#include <linux/seq_file.h>

#include <linux/kasan.h>

#include <linux/cpu.h>

#include <linux/cpuset.h>

#include <linux/mempolicy.h>

#include <linux/ctype.h>

#include <linux/debugobjects.h>

#include <linux/kallsyms.h>

#include <linux/memory.h>

#include <linux/math64.h>

#include <linux/fault-inject.h>

#include <linux/stacktrace.h>

#include <linux/prefetch.h>

#include <linux/memcontrol.h>

#include <linux/random.h>

​

#include <trace/events/kmem.h>

​

#include "internal.h"

​

/*

 * Lock order:

 *   1. slab_mutex (Global Mutex)

 *   2. node->list_lock

 *   3. slab_lock(page) (Only on some arches and for debugging)

 *

 *   slab_mutex

 *

 *   The role of the slab_mutex is to protect the list of all the slabs

 *   and to synchronize major metadata changes to slab cache structures.

 *

 *   The slab_lock is only used for debugging and on arches that do not

 *   have the ability to do a cmpxchg_double. It only protects:

 *  A. page->freelist   -> List of object free in a page

 *  B. page->inuse      -> Number of objects in use

 *  C. page->objects    -> Number of objects in page

 *  D. page->frozen     -> frozen state

 *

 *   If a slab is frozen then it is exempt from list management. It is not

 *   on any list. The processor that froze the slab is the one who can

 *   perform list operations on the page. Other processors may put objects

 *   onto the freelist but the processor that froze the slab is the only

 *   one that can retrieve the objects from the page's freelist.

 *

 *   The list_lock protects the partial and full list on each node and

 *   the partial slab counter. If taken then no new slabs may be added or

 *   removed from the lists nor make the number of partial slabs be modified.

 *   (Note that the total number of slabs is an atomic value that may be

 *   modified without taking the list lock).

 *

 *   The list_lock is a centralized lock and thus we avoid taking it as

 *   much as possible. As long as SLUB does not have to handle partial

 *   slabs, operations can continue without any centralized lock. F.e.

 *   allocating a long series of objects that fill up slabs does not require

 *   the list lock.

 *   Interrupts are disabled during allocation and deallocation in order to

 *   make the slab allocator safe to use in the context of an irq. In addition

 *   interrupts are disabled to ensure that the processor does not change

 *   while handling per_cpu slabs, due to kernel preemption.

 *

 * SLUB assigns one slab for allocation to each processor.

 * Allocations only occur from these slabs called cpu slabs.

 *

 * Slabs with free elements are kept on a partial list and during regular

 * operations no list for full slabs is used. If an object in a full slab is

 * freed then the slab will show up again on the partial lists.

 * We track full slabs for debugging purposes though because otherwise we

 * cannot scan all objects.

 *

 * Slabs are freed when they become empty. Teardown and setup is

 * minimal so we rely on the page allocators per cpu caches for