linux-imx

/*

 * Copyright 2013 Red Hat Inc.

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * Authors: Jérôme Glisse <jglisse@redhat.com>

 */

/*

 * Refer to include/linux/hmm.h for information about heterogeneous memory

 * management or HMM for short.

 */

#include <linux/mm.h>

#include <linux/hmm.h>

#include <linux/init.h>

#include <linux/rmap.h>

#include <linux/swap.h>

#include <linux/slab.h>

#include <linux/sched.h>

#include <linux/mmzone.h>

#include <linux/pagemap.h>

#include <linux/swapops.h>

#include <linux/hugetlb.h>

#include <linux/memremap.h>

#include <linux/jump_label.h>

#include <linux/mmu_notifier.h>

#include <linux/memory_hotplug.h>

​

#define PA_SECTION_SIZE (1UL << PA_SECTION_SHIFT)

​

#if IS_ENABLED(CONFIG_HMM_MIRROR)

static const struct mmu_notifier_ops hmm_mmu_notifier_ops;

​

/*

 * struct hmm - HMM per mm struct

 *

 * @mm: mm struct this HMM struct is bound to

 * @lock: lock protecting ranges list

 * @ranges: list of range being snapshotted

 * @mirrors: list of mirrors for this mm

 * @mmu_notifier: mmu notifier to track updates to CPU page table

 * @mirrors_sem: read/write semaphore protecting the mirrors list

 */

struct hmm {

    struct mm_struct    *mm;

    spinlock_t      lock;

    struct list_head    ranges;

    struct list_head    mirrors;

    struct mmu_notifier mmu_notifier;

    struct rw_semaphore mirrors_sem;

};

​

/*

 * hmm_register - register HMM against an mm (HMM internal)

 *

 * @mm: mm struct to attach to

 *

 * This is not intended to be used directly by device drivers. It allocates an

 * HMM struct if mm does not have one, and initializes it.

 */

static struct hmm *hmm_register(struct mm_struct *mm)

{

    struct hmm *hmm = READ_ONCE(mm->hmm);

    bool cleanup = false;

​

    /*

     * The hmm struct can only be freed once the mm_struct goes away,

     * hence we should always have pre-allocated an new hmm struct

     * above.

     */

    if (hmm)

        return hmm;

​

    hmm = kmalloc(sizeof(*hmm), GFP_KERNEL);

    if (!hmm)

        return NULL;

    INIT_LIST_HEAD(&hmm->mirrors);

    init_rwsem(&hmm->mirrors_sem);

    hmm->mmu_notifier.ops = NULL;

    INIT_LIST_HEAD(&hmm->ranges);

    spin_lock_init(&hmm->lock);

    hmm->mm = mm;

​

    spin_lock(&mm->page_table_lock);

    if (!mm->hmm)