linux-imx

/*

 * Copyright (C) 2012 Intel Corporation

 * Author: Jim Kukunas <james.t.kukunas@linux.intel.com>

 *

 * This program is free software; you can redistribute it and/or

 * modify it under the terms of the GNU General Public License

 * as published by the Free Software Foundation; version 2

 * of the License.

 */

​

#ifdef CONFIG_AS_AVX2

​

#include <linux/raid/pq.h>

#include "x86.h"

​

static int raid6_has_avx2(void)

{

    return boot_cpu_has(X86_FEATURE_AVX2) &&

        boot_cpu_has(X86_FEATURE_AVX);

}

​

static void raid6_2data_recov_avx2(int disks, size_t bytes, int faila,

        int failb, void **ptrs)

{

    u8 *p, *q, *dp, *dq;

    const u8 *pbmul;    /* P multiplier table for B data */

    const u8 *qmul;     /* Q multiplier table (for both) */

    const u8 x0f = 0x0f;

​

    p = (u8 *)ptrs[disks-2];

    q = (u8 *)ptrs[disks-1];

​

    /* Compute syndrome with zero for the missing data pages

       Use the dead data pages as temporary storage for

       delta p and delta q */

    dp = (u8 *)ptrs[faila];

    ptrs[faila] = (void *)raid6_empty_zero_page;

    ptrs[disks-2] = dp;

    dq = (u8 *)ptrs[failb];

    ptrs[failb] = (void *)raid6_empty_zero_page;

    ptrs[disks-1] = dq;

​

    raid6_call.gen_syndrome(disks, bytes, ptrs);

​

    /* Restore pointer table */

    ptrs[faila]   = dp;

    ptrs[failb]   = dq;

    ptrs[disks-2] = p;

    ptrs[disks-1] = q;

​

    /* Now, pick the proper data tables */

    pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];

    qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^

        raid6_gfexp[failb]]];

​

    kernel_fpu_begin();

​

    /* ymm0 = x0f[16] */

    asm volatile("vpbroadcastb %0, %%ymm7" : : "m" (x0f));

​

    while (bytes) {

#ifdef CONFIG_X86_64

        asm volatile("vmovdqa %0, %%ymm1" : : "m" (q[0]));

        asm volatile("vmovdqa %0, %%ymm9" : : "m" (q[32]));

        asm volatile("vmovdqa %0, %%ymm0" : : "m" (p[0]));

        asm volatile("vmovdqa %0, %%ymm8" : : "m" (p[32]));

        asm volatile("vpxor %0, %%ymm1, %%ymm1" : : "m" (dq[0]));

        asm volatile("vpxor %0, %%ymm9, %%ymm9" : : "m" (dq[32]));

        asm volatile("vpxor %0, %%ymm0, %%ymm0" : : "m" (dp[0]));

        asm volatile("vpxor %0, %%ymm8, %%ymm8" : : "m" (dp[32]));

​

        /*

         * 1 = dq[0]  ^ q[0]

         * 9 = dq[32] ^ q[32]

         * 0 = dp[0]  ^ p[0]

         * 8 = dp[32] ^ p[32]

         */

​

        asm volatile("vbroadcasti128 %0, %%ymm4" : : "m" (qmul[0]));

        asm volatile("vbroadcasti128 %0, %%ymm5" : : "m" (qmul[16]));

​

        asm volatile("vpsraw $4, %ymm1, %ymm3");

        asm volatile("vpsraw $4, %ymm9, %ymm12");

        asm volatile("vpand %ymm7, %ymm1, %ymm1");

        asm volatile("vpand %ymm7, %ymm9, %ymm9");

        asm volatile("vpand %ymm7, %ymm3, %ymm3");

        asm volatile("vpand %ymm7, %ymm12, %ymm12");

        asm volatile("vpshufb %ymm9, %ymm4, %ymm14");

        asm volatile("vpshufb %ymm1, %ymm4, %ymm4");

        asm volatile("vpshufb %ymm12, %ymm5, %ymm15");

        asm volatile("vpshufb %ymm3, %ymm5, %ymm5");

        asm volatile("vpxor %ymm14, %ymm15, %ymm15");