linux-imx

/*

 *  linux/mm/vmalloc.c

 *

 *  Copyright (C) 1993  Linus Torvalds

 *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999

 *  SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000

 *  Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002

 *  Numa awareness, Christoph Lameter, SGI, June 2005

 */

​

#include <linux/vmalloc.h>

#include <linux/mm.h>

#include <linux/module.h>

#include <linux/highmem.h>

#include <linux/sched/signal.h>

#include <linux/slab.h>

#include <linux/spinlock.h>

#include <linux/interrupt.h>

#include <linux/proc_fs.h>

#include <linux/seq_file.h>

#include <linux/debugobjects.h>

#include <linux/kallsyms.h>

#include <linux/list.h>

#include <linux/notifier.h>

#include <linux/rbtree.h>

#include <linux/radix-tree.h>

#include <linux/rcupdate.h>

#include <linux/pfn.h>

#include <linux/kmemleak.h>

#include <linux/atomic.h>

#include <linux/compiler.h>

#include <linux/llist.h>

#include <linux/bitops.h>

​

#include <linux/uaccess.h>

#include <asm/tlbflush.h>

#include <asm/shmparam.h>

​

#include "internal.h"

​

struct vfree_deferred {

    struct llist_head list;

    struct work_struct wq;

};

static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred);

​

static void __vunmap(const void *, int);

​

static void free_work(struct work_struct *w)

{

    struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq);

    struct llist_node *t, *llnode;

​

    llist_for_each_safe(llnode, t, llist_del_all(&p->list))

        __vunmap((void *)llnode, 1);

}

​

/*** Page table manipulation functions ***/

​

static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)

{

    pte_t *pte;

​

    pte = pte_offset_kernel(pmd, addr);

    do {

        pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);

        WARN_ON(!pte_none(ptent) && !pte_present(ptent));

    } while (pte++, addr += PAGE_SIZE, addr != end);

}

​

static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end)

{

    pmd_t *pmd;

    unsigned long next;

​

    pmd = pmd_offset(pud, addr);

    do {

        next = pmd_addr_end(addr, end);

        if (pmd_clear_huge(pmd))

            continue;

        if (pmd_none_or_clear_bad(pmd))

            continue;

        vunmap_pte_range(pmd, addr, next);

    } while (pmd++, addr = next, addr != end);

}

​

static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end)

{

    pud_t *pud;

    unsigned long next;

​

    pud = pud_offset(p4d, addr);