linux-imx

    u64 latency_targets[KYBER_OTHER];

};

​

struct kyber_hctx_data {

    spinlock_t lock;

    struct list_head rqs[KYBER_NUM_DOMAINS];

    unsigned int cur_domain;

    unsigned int batching;

    struct kyber_ctx_queue *kcqs;

    struct sbitmap kcq_map[KYBER_NUM_DOMAINS];

    struct sbq_wait_state *domain_ws[KYBER_NUM_DOMAINS];

    atomic_t wait_index[KYBER_NUM_DOMAINS];

};

​

static int kyber_domain_wake(wait_queue_entry_t *wait, unsigned mode, int flags,

                 void *key);

​

static unsigned int kyber_sched_domain(unsigned int op)

{

    switch (op & REQ_OP_MASK) {

            while (--i >= 0)

                sbitmap_free(&khd->kcq_map[i]);

            goto err_kcqs;

        }

    }

​

    spin_lock_init(&khd->lock);

​

    for (i = 0; i < KYBER_NUM_DOMAINS; i++) {

        INIT_LIST_HEAD(&khd->rqs[i]);

                      kyber_domain_wake);

        atomic_set(&khd->wait_index[i], 0);

    }

​

    khd->cur_domain = 0;

    khd->batching = 0;

​

    hctx->sched_data = khd;

    sbitmap_queue_min_shallow_depth(&hctx->sched_tags->bitmap_tags,

                    kqd->async_depth);

​

    struct flush_kcq_data data = {

        .khd = khd,

        .sched_domain = sched_domain,

        .list = list,

    };

​

    sbitmap_for_each_set(&khd->kcq_map[sched_domain],

                 flush_busy_kcq, &data);

}

​

                 void *key)

{

​

    blk_mq_run_hw_queue(hctx, true);

    return 1;

}

​

static int kyber_get_domain_token(struct kyber_queue_data *kqd,

                  struct kyber_hctx_data *khd,

                  struct blk_mq_hw_ctx *hctx)

{

    unsigned int sched_domain = khd->cur_domain;

    struct sbitmap_queue *domain_tokens = &kqd->domain_tokens[sched_domain];

    struct sbq_wait_state *ws;

    int nr;

​

    nr = __sbitmap_queue_get(domain_tokens);

​

    /*

     * If we failed to get a domain token, make sure the hardware queue is

     * run when one becomes available. Note that this is serialized on

     * khd->lock, but we still need to be careful about the waker.

     */

        ws = sbq_wait_ptr(domain_tokens,

                  &khd->wait_index[sched_domain]);

        khd->domain_ws[sched_domain] = ws;

​

        /*

         * Try again in case a token was freed before we got on the wait

         * queue.

         */

        nr = __sbitmap_queue_get(domain_tokens);

    }

​

    /*

     * If we got a token while we were on the wait queue, remove ourselves

     * from the wait queue to ensure that all wake ups make forward

     * progress. It's possible that the waker already deleted the entry

     * between the !list_empty_careful() check and us grabbing the lock, but

     * list_del_init() is okay with that.

     */

        ws = khd->domain_ws[sched_domain];

        spin_lock_irq(&ws->wait.lock);

        spin_unlock_irq(&ws->wait.lock);

    }

​

    return nr;

}

​

static struct request *

kyber_dispatch_cur_domain(struct kyber_queue_data *kqd,

              struct kyber_hctx_data *khd,

              struct blk_mq_hw_ctx *hctx)

    .start  = kyber_##name##_rqs_start,             \

    .next   = kyber_##name##_rqs_next,              \

    .stop   = kyber_##name##_rqs_stop,              \

    .show   = blk_mq_debugfs_rq_show,               \

};                                  \

                                    \

static int kyber_##name##_waiting_show(void *data, struct seq_file *m)  \

{                                   \

    struct blk_mq_hw_ctx *hctx = data;              \

    struct kyber_hctx_data *khd = hctx->sched_data;         \

                                    \

    seq_printf(m, "%d\n", !list_empty_careful(&wait->entry));   \

    return 0;                           \

}

KYBER_DEBUGFS_DOMAIN_ATTRS(KYBER_READ, read)

KYBER_DEBUGFS_DOMAIN_ATTRS(KYBER_WRITE, write)

KYBER_DEBUGFS_DOMAIN_ATTRS(KYBER_DISCARD, discard)

KYBER_DEBUGFS_DOMAIN_ATTRS(KYBER_OTHER, other)

#undef KYBER_DEBUGFS_DOMAIN_ATTRS

​