linux-imx

// SPDX-License-Identifier: GPL-2.0

/*

 * Copyright (C) 2016 CNEX Labs

 * Initial release: Javier Gonzalez <javier@cnexlabs.com>

 *                  Matias Bjorling <matias@cnexlabs.com>

 *

 * This program is free software; you can redistribute it and/or

 * modify it under the terms of the GNU General Public License version

 * 2 as published by the Free Software Foundation.

 *

 * This program is distributed in the hope that it will be useful, but

 * WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

 * General Public License for more details.

 *

 * pblk-write.c - pblk's write path from write buffer to media

 */

​

#include "pblk.h"

#include "pblk-trace.h"

​

static unsigned long pblk_end_w_bio(struct pblk *pblk, struct nvm_rq *rqd,

                    struct pblk_c_ctx *c_ctx)

{

    struct bio *original_bio;

    struct pblk_rb *rwb = &pblk->rwb;

    unsigned long ret;

    int i;

​

    for (i = 0; i < c_ctx->nr_valid; i++) {

        struct pblk_w_ctx *w_ctx;

        int pos = c_ctx->sentry + i;

        int flags;

​

        w_ctx = pblk_rb_w_ctx(rwb, pos);

        flags = READ_ONCE(w_ctx->flags);

​

        if (flags & PBLK_FLUSH_ENTRY) {

            flags &= ~PBLK_FLUSH_ENTRY;

            /* Release flags on context. Protect from writes */

            smp_store_release(&w_ctx->flags, flags);

​

#ifdef CONFIG_NVM_PBLK_DEBUG

            atomic_dec(&rwb->inflight_flush_point);

#endif

        }

​

        while ((original_bio = bio_list_pop(&w_ctx->bios)))

            bio_endio(original_bio);

    }

​

    if (c_ctx->nr_padded)

        pblk_bio_free_pages(pblk, rqd->bio, c_ctx->nr_valid,

                            c_ctx->nr_padded);

​

#ifdef CONFIG_NVM_PBLK_DEBUG

    atomic_long_add(rqd->nr_ppas, &pblk->sync_writes);

#endif

​

    ret = pblk_rb_sync_advance(&pblk->rwb, c_ctx->nr_valid);

​

    bio_put(rqd->bio);

    pblk_free_rqd(pblk, rqd, PBLK_WRITE);

​

    return ret;

}

​

static unsigned long pblk_end_queued_w_bio(struct pblk *pblk,

                       struct nvm_rq *rqd,

                       struct pblk_c_ctx *c_ctx)

{

    list_del(&c_ctx->list);

    return pblk_end_w_bio(pblk, rqd, c_ctx);

}

​

static void pblk_complete_write(struct pblk *pblk, struct nvm_rq *rqd,

                struct pblk_c_ctx *c_ctx)

{

    struct pblk_c_ctx *c, *r;

    unsigned long flags;

    unsigned long pos;

​

#ifdef CONFIG_NVM_PBLK_DEBUG

    atomic_long_sub(c_ctx->nr_valid, &pblk->inflight_writes);

#endif

    pblk_up_rq(pblk, c_ctx->lun_bitmap);

​

    pos = pblk_rb_sync_init(&pblk->rwb, &flags);

    if (pos == c_ctx->sentry) {

        pos = pblk_end_w_bio(pblk, rqd, c_ctx);

​

retry: