linux-imx

// SPDX-License-Identifier: GPL-2.0

#include <linux/mm.h>

#include <linux/highmem.h>

#include <linux/sched.h>

#include <linux/hugetlb.h>

​

static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,

              struct mm_walk *walk)

{

    pte_t *pte;

    int err = 0;

​

    pte = pte_offset_map(pmd, addr);

    for (;;) {

        err = walk->pte_entry(pte, addr, addr + PAGE_SIZE, walk);

        if (err)

               break;

        addr += PAGE_SIZE;

        if (addr == end)

            break;

        pte++;

    }

​

    pte_unmap(pte);

    return err;

}

​

static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,

              struct mm_walk *walk)

{

    pmd_t *pmd;

    unsigned long next;

    int err = 0;

​

    pmd = pmd_offset(pud, addr);

    do {

again:

        next = pmd_addr_end(addr, end);

        if (pmd_none(*pmd) || !walk->vma) {

            if (walk->pte_hole)

                err = walk->pte_hole(addr, next, walk);

            if (err)

                break;

            continue;

        }

        /*

         * This implies that each ->pmd_entry() handler

         * needs to know about pmd_trans_huge() pmds

         */

        if (walk->pmd_entry)

            err = walk->pmd_entry(pmd, addr, next, walk);

        if (err)

            break;

​

        /*

         * Check this here so we only break down trans_huge

         * pages when we _need_ to

         */

        if (!walk->pte_entry)

            continue;

​

        split_huge_pmd(walk->vma, pmd, addr);

        if (pmd_trans_unstable(pmd))

            goto again;

        err = walk_pte_range(pmd, addr, next, walk);

        if (err)

            break;

    } while (pmd++, addr = next, addr != end);

​

    return err;

}

​

static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,

              struct mm_walk *walk)

{

    pud_t *pud;

    unsigned long next;

    int err = 0;

​

    pud = pud_offset(p4d, addr);

    do {

 again:

        next = pud_addr_end(addr, end);

        if (pud_none(*pud) || !walk->vma) {

            if (walk->pte_hole)

                err = walk->pte_hole(addr, next, walk);

            if (err)

                break;

            continue;

        }

​

        if (walk->pud_entry) {