linux-imx

/*

 * mm/percpu-vm.c - vmalloc area based chunk allocation

 *

 * Copyright (C) 2010       SUSE Linux Products GmbH

 * Copyright (C) 2010       Tejun Heo <tj@kernel.org>

 *

 * This file is released under the GPLv2.

 *

 * Chunks are mapped into vmalloc areas and populated page by page.

 * This is the default chunk allocator.

 */

​

static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk,

                    unsigned int cpu, int page_idx)

{

    /* must not be used on pre-mapped chunk */

    WARN_ON(chunk->immutable);

​

    return vmalloc_to_page((void *)pcpu_chunk_addr(chunk, cpu, page_idx));

}

​

/**

 * pcpu_get_pages - get temp pages array

 *

 * Returns pointer to array of pointers to struct page which can be indexed

 * with pcpu_page_idx().  Note that there is only one array and accesses

 * should be serialized by pcpu_alloc_mutex.

 *

 * RETURNS:

 * Pointer to temp pages array on success.

 */

static struct page **pcpu_get_pages(void)

{

    static struct page **pages;

    size_t pages_size = pcpu_nr_units * pcpu_unit_pages * sizeof(pages[0]);

​

    lockdep_assert_held(&pcpu_alloc_mutex);

​

    if (!pages)

        pages = pcpu_mem_zalloc(pages_size, GFP_KERNEL);

    return pages;

}

​

/**

 * pcpu_free_pages - free pages which were allocated for @chunk

 * @chunk: chunk pages were allocated for

 * @pages: array of pages to be freed, indexed by pcpu_page_idx()

 * @page_start: page index of the first page to be freed

 * @page_end: page index of the last page to be freed + 1

 *

 * Free pages [@page_start and @page_end) in @pages for all units.

 * The pages were allocated for @chunk.

 */

static void pcpu_free_pages(struct pcpu_chunk *chunk,

                struct page **pages, int page_start, int page_end)

{

    unsigned int cpu;

    int i;

​

    for_each_possible_cpu(cpu) {

        for (i = page_start; i < page_end; i++) {

            struct page *page = pages[pcpu_page_idx(cpu, i)];

​

            if (page)

                __free_page(page);

        }

    }

}

​

/**

 * pcpu_alloc_pages - allocates pages for @chunk

 * @chunk: target chunk

 * @pages: array to put the allocated pages into, indexed by pcpu_page_idx()

 * @page_start: page index of the first page to be allocated

 * @page_end: page index of the last page to be allocated + 1

 * @gfp: allocation flags passed to the underlying allocator

 *

 * Allocate pages [@page_start,@page_end) into @pages for all units.

 * The allocation is for @chunk.  Percpu core doesn't care about the

 * content of @pages and will pass it verbatim to pcpu_map_pages().

 */

static int pcpu_alloc_pages(struct pcpu_chunk *chunk,

                struct page **pages, int page_start, int page_end,

                gfp_t gfp)

{

    unsigned int cpu, tcpu;

    int i;

​

    gfp |= __GFP_HIGHMEM;

​

    for_each_possible_cpu(cpu) {

        for (i = page_start; i < page_end; i++) {