linux-imx

// SPDX-License-Identifier: GPL-2.0

/*

 * Block multiqueue core code

 *

 * Copyright (C) 2013-2014 Jens Axboe

 * Copyright (C) 2013-2014 Christoph Hellwig

 */

#include <linux/kernel.h>

#include <linux/module.h>

#include <linux/backing-dev.h>

#include <linux/bio.h>

#include <linux/blkdev.h>

#include <linux/kmemleak.h>

#include <linux/mm.h>

#include <linux/init.h>

#include <linux/slab.h>

#include <linux/workqueue.h>

#include <linux/smp.h>

#include <linux/llist.h>

#include <linux/list_sort.h>

#include <linux/cpu.h>

#include <linux/cache.h>

#include <linux/sched/sysctl.h>

#include <linux/sched/topology.h>

#include <linux/sched/signal.h>

#include <linux/delay.h>

#include <linux/crash_dump.h>

#include <linux/prefetch.h>

​

#include <trace/events/block.h>

​

#include <linux/blk-mq.h>

#include "blk.h"

#include "blk-mq.h"

#include "blk-mq-debugfs.h"

#include "blk-mq-tag.h"

#include "blk-pm.h"

#include "blk-stat.h"

#include "blk-mq-sched.h"

#include "blk-rq-qos.h"

​

static void blk_mq_poll_stats_start(struct request_queue *q);

static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb);

​

static int blk_mq_poll_stats_bkt(const struct request *rq)

{

    int ddir, bytes, bucket;

​

    ddir = rq_data_dir(rq);

    bytes = blk_rq_bytes(rq);

​

    bucket = ddir + 2*(ilog2(bytes) - 9);

​

    if (bucket < 0)

        return -1;

    else if (bucket >= BLK_MQ_POLL_STATS_BKTS)

        return ddir + BLK_MQ_POLL_STATS_BKTS - 2;

​

    return bucket;

}

​

/*

 * Check if any of the ctx, dispatch list or elevator

 * have pending work in this hardware queue.

 */

static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)

{

    return !list_empty_careful(&hctx->dispatch) ||

        sbitmap_any_bit_set(&hctx->ctx_map) ||

            blk_mq_sched_has_work(hctx);

}

​

/*

 * Mark this ctx as having pending work in this hardware queue

 */

static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,

                     struct blk_mq_ctx *ctx)

{

    const int bit = ctx->index_hw[hctx->type];

​

    if (!sbitmap_test_bit(&hctx->ctx_map, bit))

        sbitmap_set_bit(&hctx->ctx_map, bit);

}

​

static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,

                      struct blk_mq_ctx *ctx)

{

    const int bit = ctx->index_hw[hctx->type];

​

    sbitmap_clear_bit(&hctx->ctx_map, bit);

}

​