/*************************************************************************
* Copyright (C) 2020 Intel Corporation
*
* Licensed under the Apache License,  Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* 	http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law  or agreed  to  in  writing,  software
* distributed under  the License  is  distributed  on  an  "AS IS"  BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the  specific  language  governing  permissions  and
* limitations under the License.
*************************************************************************/

/*
//
//  Purpose:
//     Cryptography Primitive.
//     SMS4 CFB decryption
//
//  Contents:
//     cpSMS4_CFB_gfni512()
//     cpSMS4_CFB_gfni512x48()
//     cpSMS4_CFB_gfni512x32()
//     cpSMS4_CFB_gfni512x16()
//     cpSMS4_CFB_gfni128x12()
//     cpSMS4_CFB_gfni128x8()
//     cpSMS4_CFB_gfni128x4()
//
*/

#include "owndefs.h"
#include "owncp.h"
#include "sms4/pcpsms4.h"
#include "pcptool.h"

#if (_IPP32E >= _IPP32E_K1)
#if defined(__INTEL_COMPILER) || defined(__INTEL_LLVM_COMPILER) || !defined(_MSC_VER) || \
    (_MSC_VER >= 1920)

#include "sms4/pcpsms4_gfni.h"

static void cpSMS4_CFB_dec_gfni512x48(Ipp8u* pDst,
                                      const Ipp8u* pSrc,
                                      int len,
                                      int cfbBlkSize,
                                      const Ipp32u* pRKey,
                                      Ipp8u* pIV);
static void cpSMS4_CFB_dec_gfni512x32(Ipp8u* pDst,
                                      const Ipp8u* pSrc,
                                      int len,
                                      int cfbBlkSize,
                                      const Ipp32u* pRKey,
                                      Ipp8u* pIV);
static void cpSMS4_CFB_dec_gfni512x16(Ipp8u* pDst,
                                      const Ipp8u* pSrc,
                                      int len,
                                      int cfbBlkSize,
                                      const Ipp32u* pRKey,
                                      Ipp8u* pIV);
static void cpSMS4_CFB_dec_gfni128x12(Ipp8u* pDst,
                                      const Ipp8u* pSrc,
                                      int len,
                                      int cfbBlkSize,
                                      const Ipp32u* pRKey,
                                      Ipp8u* pIV);
static void cpSMS4_CFB_dec_gfni128x8(Ipp8u* pDst,
                                     const Ipp8u* pSrc,
                                     int len,
                                     int cfbBlkSize,
                                     const Ipp32u* pRKey,
                                     Ipp8u* pIV);
static void cpSMS4_CFB_dec_gfni128x4(Ipp8u* pDst,
                                     const Ipp8u* pSrc,
                                     int len,
                                     int cfbBlkSize,
                                     const Ipp32u* pRKey,
                                     Ipp8u* pIV);

IPPCP_FORCEINLINE Ipp64u broadcast_16to64(Ipp16u mask16)
{
    Ipp64u mask64 = (Ipp64u)mask16;
    mask64        = (mask64 << 48) | (mask64 << 32) | (mask64 << 16) | mask64;
    return mask64;
}

IPPCP_FORCEINLINE __m512i getInputBlocks(__m128i* const currentState,
                                         const __m512i* const pCipherBlocks,
                                         __mmask16 blocksCompressMask)
{
    /* extract 128-bit cipher blocks */
    __m128i c0 = _mm512_extracti64x2_epi64(*pCipherBlocks, 0);
    __m128i c1 = _mm512_extracti64x2_epi64(*pCipherBlocks, 1);
    __m128i c2 = _mm512_extracti64x2_epi64(*pCipherBlocks, 2);
    __m128i c3 = _mm512_extracti64x2_epi64(*pCipherBlocks, 3);

    /*  InpBlk_j = LSB_(b-s)(InpBlk_(j-1)) | C_(j-1); b = 128 bit, s = 8*cfbBlkSize bit */
    __m128i inpBlk0 = *currentState;
    /* drop MSB bits (size = cfbBlkSize) from the inpBlk_i (by mask) and append c_(i-1) to LSB bits */
    __m128i inpBlk1 = _mm_mask_compress_epi8(c0, blocksCompressMask, inpBlk0);
    __m128i inpBlk2 = _mm_mask_compress_epi8(c1, blocksCompressMask, inpBlk1);
    __m128i inpBlk3 = _mm_mask_compress_epi8(c2, blocksCompressMask, inpBlk2);

    /* next InputBlock ready */
    *currentState = _mm_mask_compress_epi8(c3, blocksCompressMask, inpBlk3);

    /* inserts */
    __m512i inpBlk512 = _mm512_setzero_si512();
    inpBlk512         = _mm512_mask_broadcast_i64x2(inpBlk512, 0x03, inpBlk0); /* 0000 0011 */
    inpBlk512         = _mm512_mask_broadcast_i64x2(inpBlk512, 0x0C, inpBlk1); /* 0000 1100 */
    inpBlk512         = _mm512_mask_broadcast_i64x2(inpBlk512, 0x30, inpBlk2); /* 0011 0000 */
    inpBlk512         = _mm512_mask_broadcast_i64x2(inpBlk512, 0xC0, inpBlk3); /* 1100 0000 */

    return inpBlk512;
}

/*
// 64*cfbBlkSize bytes processing
*/
/* clang-format off */
IPP_OWN_DEFN(void, cpSMS4_CFB_dec_gfni512, (Ipp8u* pDst,
                                            const Ipp8u* pSrc,
                                            int len,
                                            int cfbBlkSize,
                                            const Ipp32u* pRKey,
                                            Ipp8u* pIV))
/* clang-format on */
{
    __ALIGN16 __m512i TMP[24];

    /*
   // TMP[0..15]  - decrypted text blocks
   // TMP[16..19] - key
   // TMP[20..23] - temp out
   */

    const int bytesPerLoad512       = 4 * cfbBlkSize;
    const Ipp16u blocksCompressMask = (Ipp16u)(0xFFFF << cfbBlkSize);

    int processedLen = len - (len % (64 * cfbBlkSize));
    int blocks;

    /* load masks; allows to load 4 source blocks of cfbBlkSize each (in bytes) to LSB parts of 128-bit lanes in 512-bit register */
    __mmask64 kLsbMask64 = broadcast_16to64((Ipp16u)(0xFFFF << (16 - cfbBlkSize)));
    /* same mask to load in MSB parts */
    __mmask64 kMsbMask64 = broadcast_16to64((Ipp16u)~blocksCompressMask);

    /* load IV */
    __m128i currentState = _mm_maskz_loadu_epi64(0x03 /* load 128-bit */, pIV);

    for (blocks = len / cfbBlkSize; blocks >= (64); blocks -= (64)) {
        /* load cipher blocks to LSB parts of registers */
        __m512i ciphBytes0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 0 * bytesPerLoad512);
        __m512i ciphBytes1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 1 * bytesPerLoad512);
        __m512i ciphBytes2 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 2 * bytesPerLoad512);
        __m512i ciphBytes3 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 3 * bytesPerLoad512);

        /* prepare InputBlocks for decryption */
        ciphBytes0 = getInputBlocks(&currentState, &ciphBytes0, blocksCompressMask);
        ciphBytes1 = getInputBlocks(&currentState, &ciphBytes1, blocksCompressMask);
        ciphBytes2 = getInputBlocks(&currentState, &ciphBytes2, blocksCompressMask);
        ciphBytes3 = getInputBlocks(&currentState, &ciphBytes3, blocksCompressMask);

        ciphBytes0 = _mm512_shuffle_epi8(ciphBytes0, M512(swapBytes));
        ciphBytes1 = _mm512_shuffle_epi8(ciphBytes1, M512(swapBytes));
        ciphBytes2 = _mm512_shuffle_epi8(ciphBytes2, M512(swapBytes));
        ciphBytes3 = _mm512_shuffle_epi8(ciphBytes3, M512(swapBytes));
        /* clang-format off */
        TRANSPOSE_INP_512(TMP[0], TMP[1], TMP[2], TMP[3], ciphBytes0, ciphBytes1, ciphBytes2, ciphBytes3);
        /* clang-format on */

        /* load cipher blocks to LSB parts of registers */
        ciphBytes0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 4 * bytesPerLoad512);
        ciphBytes1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 5 * bytesPerLoad512);
        ciphBytes2 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 6 * bytesPerLoad512);
        ciphBytes3 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 7 * bytesPerLoad512);

        /* prepare InputBlocks for decryption */
        ciphBytes0 = getInputBlocks(&currentState, &ciphBytes0, blocksCompressMask);
        ciphBytes1 = getInputBlocks(&currentState, &ciphBytes1, blocksCompressMask);
        ciphBytes2 = getInputBlocks(&currentState, &ciphBytes2, blocksCompressMask);
        ciphBytes3 = getInputBlocks(&currentState, &ciphBytes3, blocksCompressMask);

        ciphBytes0 = _mm512_shuffle_epi8(ciphBytes0, M512(swapBytes));
        ciphBytes1 = _mm512_shuffle_epi8(ciphBytes1, M512(swapBytes));
        ciphBytes2 = _mm512_shuffle_epi8(ciphBytes2, M512(swapBytes));
        ciphBytes3 = _mm512_shuffle_epi8(ciphBytes3, M512(swapBytes));
        /* clang-format off */
        TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], ciphBytes0, ciphBytes1, ciphBytes2, ciphBytes3);
        /* clang-format on */

        /* load cipher blocks to LSB parts of registers */
        ciphBytes0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 8 * bytesPerLoad512);
        ciphBytes1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 9 * bytesPerLoad512);
        ciphBytes2 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 10 * bytesPerLoad512);
        ciphBytes3 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 11 * bytesPerLoad512);

        /* prepare InputBlocks for decryption */
        ciphBytes0 = getInputBlocks(&currentState, &ciphBytes0, blocksCompressMask);
        ciphBytes1 = getInputBlocks(&currentState, &ciphBytes1, blocksCompressMask);
        ciphBytes2 = getInputBlocks(&currentState, &ciphBytes2, blocksCompressMask);
        ciphBytes3 = getInputBlocks(&currentState, &ciphBytes3, blocksCompressMask);

        ciphBytes0 = _mm512_shuffle_epi8(ciphBytes0, M512(swapBytes));
        ciphBytes1 = _mm512_shuffle_epi8(ciphBytes1, M512(swapBytes));
        ciphBytes2 = _mm512_shuffle_epi8(ciphBytes2, M512(swapBytes));
        ciphBytes3 = _mm512_shuffle_epi8(ciphBytes3, M512(swapBytes));
        TRANSPOSE_INP_512(TMP[8],
                          TMP[9],
                          TMP[10],
                          TMP[11],
                          ciphBytes0,
                          ciphBytes1,
                          ciphBytes2,
                          ciphBytes3);

        /* load cipher blocks to LSB parts of registers */
        ciphBytes0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 12 * bytesPerLoad512);
        ciphBytes1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 13 * bytesPerLoad512);
        ciphBytes2 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 14 * bytesPerLoad512);
        ciphBytes3 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 15 * bytesPerLoad512);

        /* prepare InputBlocks for decryption */
        ciphBytes0 = getInputBlocks(&currentState, &ciphBytes0, blocksCompressMask);
        ciphBytes1 = getInputBlocks(&currentState, &ciphBytes1, blocksCompressMask);
        ciphBytes2 = getInputBlocks(&currentState, &ciphBytes2, blocksCompressMask);
        ciphBytes3 = getInputBlocks(&currentState, &ciphBytes3, blocksCompressMask);

        ciphBytes0 = _mm512_shuffle_epi8(ciphBytes0, M512(swapBytes));
        ciphBytes1 = _mm512_shuffle_epi8(ciphBytes1, M512(swapBytes));
        ciphBytes2 = _mm512_shuffle_epi8(ciphBytes2, M512(swapBytes));
        ciphBytes3 = _mm512_shuffle_epi8(ciphBytes3, M512(swapBytes));
        TRANSPOSE_INP_512(TMP[12],
                          TMP[13],
                          TMP[14],
                          TMP[15],
                          ciphBytes0,
                          ciphBytes1,
                          ciphBytes2,
                          ciphBytes3);

        int itr;
        for (itr = 0; itr < 8; itr++, pRKey += 4) {
            /* initial xors */
            TMP[19] = TMP[18] = TMP[17] = TMP[16] = _mm512_set1_epi32((Ipp32s)pRKey[0]);
            TMP[16]                               = _mm512_xor_si512(TMP[16], TMP[1]);
            TMP[16]                               = _mm512_xor_si512(TMP[16], TMP[2]);
            TMP[16]                               = _mm512_xor_si512(TMP[16], TMP[3]);
            TMP[17]                               = _mm512_xor_si512(TMP[17], TMP[5]);
            TMP[17]                               = _mm512_xor_si512(TMP[17], TMP[6]);
            TMP[17]                               = _mm512_xor_si512(TMP[17], TMP[7]);
            TMP[18]                               = _mm512_xor_si512(TMP[18], TMP[9]);
            TMP[18]                               = _mm512_xor_si512(TMP[18], TMP[10]);
            TMP[18]                               = _mm512_xor_si512(TMP[18], TMP[11]);
            TMP[19]                               = _mm512_xor_si512(TMP[19], TMP[13]);
            TMP[19]                               = _mm512_xor_si512(TMP[19], TMP[14]);
            TMP[19]                               = _mm512_xor_si512(TMP[19], TMP[15]);
            /* Sbox */
            TMP[16] = sBox512(TMP[16]);
            TMP[17] = sBox512(TMP[17]);
            TMP[18] = sBox512(TMP[18]);
            TMP[19] = sBox512(TMP[19]);
            /* Sbox done, now L */
            TMP[0]  = _mm512_xor_si512(_mm512_xor_si512(TMP[0], TMP[16]), L512(TMP[16]));
            TMP[4]  = _mm512_xor_si512(_mm512_xor_si512(TMP[4], TMP[17]), L512(TMP[17]));
            TMP[8]  = _mm512_xor_si512(_mm512_xor_si512(TMP[8], TMP[18]), L512(TMP[18]));
            TMP[12] = _mm512_xor_si512(_mm512_xor_si512(TMP[12], TMP[19]), L512(TMP[19]));

            /* initial xors */
            TMP[19] = TMP[18] = TMP[17] = TMP[16] = _mm512_set1_epi32((Ipp32s)pRKey[1]);
            TMP[16]                               = _mm512_xor_si512(TMP[16], TMP[2]);
            TMP[16]                               = _mm512_xor_si512(TMP[16], TMP[3]);
            TMP[16]                               = _mm512_xor_si512(TMP[16], TMP[0]);
            TMP[17]                               = _mm512_xor_si512(TMP[17], TMP[6]);
            TMP[17]                               = _mm512_xor_si512(TMP[17], TMP[7]);
            TMP[17]                               = _mm512_xor_si512(TMP[17], TMP[4]);
            TMP[18]                               = _mm512_xor_si512(TMP[18], TMP[10]);
            TMP[18]                               = _mm512_xor_si512(TMP[18], TMP[11]);
            TMP[18]                               = _mm512_xor_si512(TMP[18], TMP[8]);
            TMP[19]                               = _mm512_xor_si512(TMP[19], TMP[14]);
            TMP[19]                               = _mm512_xor_si512(TMP[19], TMP[15]);
            TMP[19]                               = _mm512_xor_si512(TMP[19], TMP[12]);
            /* Sbox */
            TMP[16] = sBox512(TMP[16]);
            TMP[17] = sBox512(TMP[17]);
            TMP[18] = sBox512(TMP[18]);
            TMP[19] = sBox512(TMP[19]);
            /* Sbox done, now L */
            TMP[1]  = _mm512_xor_si512(_mm512_xor_si512(TMP[1], TMP[16]), L512(TMP[16]));
            TMP[5]  = _mm512_xor_si512(_mm512_xor_si512(TMP[5], TMP[17]), L512(TMP[17]));
            TMP[9]  = _mm512_xor_si512(_mm512_xor_si512(TMP[9], TMP[18]), L512(TMP[18]));
            TMP[13] = _mm512_xor_si512(_mm512_xor_si512(TMP[13], TMP[19]), L512(TMP[19]));

            /* initial xors */
            TMP[19] = TMP[18] = TMP[17] = TMP[16] = _mm512_set1_epi32((Ipp32s)pRKey[2]);
            TMP[16]                               = _mm512_xor_si512(TMP[16], TMP[3]);
            TMP[16]                               = _mm512_xor_si512(TMP[16], TMP[0]);
            TMP[16]                               = _mm512_xor_si512(TMP[16], TMP[1]);
            TMP[17]                               = _mm512_xor_si512(TMP[17], TMP[7]);
            TMP[17]                               = _mm512_xor_si512(TMP[17], TMP[4]);
            TMP[17]                               = _mm512_xor_si512(TMP[17], TMP[5]);
            TMP[18]                               = _mm512_xor_si512(TMP[18], TMP[11]);
            TMP[18]                               = _mm512_xor_si512(TMP[18], TMP[8]);
            TMP[18]                               = _mm512_xor_si512(TMP[18], TMP[9]);
            TMP[19]                               = _mm512_xor_si512(TMP[19], TMP[15]);
            TMP[19]                               = _mm512_xor_si512(TMP[19], TMP[12]);
            TMP[19]                               = _mm512_xor_si512(TMP[19], TMP[13]);
            /* Sbox */
            TMP[16] = sBox512(TMP[16]);
            TMP[17] = sBox512(TMP[17]);
            TMP[18] = sBox512(TMP[18]);
            TMP[19] = sBox512(TMP[19]);
            /* Sbox done, now L */
            TMP[2]  = _mm512_xor_si512(_mm512_xor_si512(TMP[2], TMP[16]), L512(TMP[16]));
            TMP[6]  = _mm512_xor_si512(_mm512_xor_si512(TMP[6], TMP[17]), L512(TMP[17]));
            TMP[10] = _mm512_xor_si512(_mm512_xor_si512(TMP[10], TMP[18]), L512(TMP[18]));
            TMP[14] = _mm512_xor_si512(_mm512_xor_si512(TMP[14], TMP[19]), L512(TMP[19]));

            /* initial xors */
            TMP[19] = TMP[18] = TMP[17] = TMP[16] = _mm512_set1_epi32((Ipp32s)pRKey[3]);
            TMP[16]                               = _mm512_xor_si512(TMP[16], TMP[0]);
            TMP[16]                               = _mm512_xor_si512(TMP[16], TMP[1]);
            TMP[16]                               = _mm512_xor_si512(TMP[16], TMP[2]);
            TMP[17]                               = _mm512_xor_si512(TMP[17], TMP[4]);
            TMP[17]                               = _mm512_xor_si512(TMP[17], TMP[5]);
            TMP[17]                               = _mm512_xor_si512(TMP[17], TMP[6]);
            TMP[18]                               = _mm512_xor_si512(TMP[18], TMP[8]);
            TMP[18]                               = _mm512_xor_si512(TMP[18], TMP[9]);
            TMP[18]                               = _mm512_xor_si512(TMP[18], TMP[10]);
            TMP[19]                               = _mm512_xor_si512(TMP[19], TMP[12]);
            TMP[19]                               = _mm512_xor_si512(TMP[19], TMP[13]);
            TMP[19]                               = _mm512_xor_si512(TMP[19], TMP[14]);
            /* Sbox */
            TMP[16] = sBox512(TMP[16]);
            TMP[17] = sBox512(TMP[17]);
            TMP[18] = sBox512(TMP[18]);
            TMP[19] = sBox512(TMP[19]);
            /* Sbox done, now L */
            TMP[3]  = _mm512_xor_si512(_mm512_xor_si512(TMP[3], TMP[16]), L512(TMP[16]));
            TMP[7]  = _mm512_xor_si512(_mm512_xor_si512(TMP[7], TMP[17]), L512(TMP[17]));
            TMP[11] = _mm512_xor_si512(_mm512_xor_si512(TMP[11], TMP[18]), L512(TMP[18]));
            TMP[15] = _mm512_xor_si512(_mm512_xor_si512(TMP[15], TMP[19]), L512(TMP[19]));
        }

        pRKey -= 32;

        TRANSPOSE_OUT_512(TMP[20], TMP[21], TMP[22], TMP[23], TMP[0], TMP[1], TMP[2], TMP[3]);
        TMP[20] = _mm512_shuffle_epi8(TMP[20], M512(swapBytes));
        TMP[21] = _mm512_shuffle_epi8(TMP[21], M512(swapBytes));
        TMP[22] = _mm512_shuffle_epi8(TMP[22], M512(swapBytes));
        TMP[23] = _mm512_shuffle_epi8(TMP[23], M512(swapBytes));

        /* load cipher blocks to MSB parts of registers */
        ciphBytes0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 0 * bytesPerLoad512);
        ciphBytes1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 1 * bytesPerLoad512);
        ciphBytes2 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 2 * bytesPerLoad512);
        ciphBytes3 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 3 * bytesPerLoad512);

        TMP[20] = _mm512_xor_si512(ciphBytes0, TMP[20]);
        TMP[21] = _mm512_xor_si512(ciphBytes1, TMP[21]);
        TMP[22] = _mm512_xor_si512(ciphBytes2, TMP[22]);
        TMP[23] = _mm512_xor_si512(ciphBytes3, TMP[23]);

        _mm512_mask_compressstoreu_epi8(pDst + 0 * bytesPerLoad512, kMsbMask64, TMP[20]);
        _mm512_mask_compressstoreu_epi8(pDst + 1 * bytesPerLoad512, kMsbMask64, TMP[21]);
        _mm512_mask_compressstoreu_epi8(pDst + 2 * bytesPerLoad512, kMsbMask64, TMP[22]);
        _mm512_mask_compressstoreu_epi8(pDst + 3 * bytesPerLoad512, kMsbMask64, TMP[23]);

        TRANSPOSE_OUT_512(TMP[20], TMP[21], TMP[22], TMP[23], TMP[4], TMP[5], TMP[6], TMP[7]);
        TMP[20] = _mm512_shuffle_epi8(TMP[20], M512(swapBytes));
        TMP[21] = _mm512_shuffle_epi8(TMP[21], M512(swapBytes));
        TMP[22] = _mm512_shuffle_epi8(TMP[22], M512(swapBytes));
        TMP[23] = _mm512_shuffle_epi8(TMP[23], M512(swapBytes));

        /* load cipher blocks to MSB parts of registers */
        ciphBytes0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 4 * bytesPerLoad512);
        ciphBytes1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 5 * bytesPerLoad512);
        ciphBytes2 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 6 * bytesPerLoad512);
        ciphBytes3 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 7 * bytesPerLoad512);

        TMP[20] = _mm512_xor_si512(ciphBytes0, TMP[20]);
        TMP[21] = _mm512_xor_si512(ciphBytes1, TMP[21]);
        TMP[22] = _mm512_xor_si512(ciphBytes2, TMP[22]);
        TMP[23] = _mm512_xor_si512(ciphBytes3, TMP[23]);

        _mm512_mask_compressstoreu_epi8(pDst + 4 * bytesPerLoad512, kMsbMask64, TMP[20]);
        _mm512_mask_compressstoreu_epi8(pDst + 5 * bytesPerLoad512, kMsbMask64, TMP[21]);
        _mm512_mask_compressstoreu_epi8(pDst + 6 * bytesPerLoad512, kMsbMask64, TMP[22]);
        _mm512_mask_compressstoreu_epi8(pDst + 7 * bytesPerLoad512, kMsbMask64, TMP[23]);

        TRANSPOSE_OUT_512(TMP[20], TMP[21], TMP[22], TMP[23], TMP[8], TMP[9], TMP[10], TMP[11]);
        TMP[20] = _mm512_shuffle_epi8(TMP[20], M512(swapBytes));
        TMP[21] = _mm512_shuffle_epi8(TMP[21], M512(swapBytes));
        TMP[22] = _mm512_shuffle_epi8(TMP[22], M512(swapBytes));
        TMP[23] = _mm512_shuffle_epi8(TMP[23], M512(swapBytes));

        /* load cipher blocks to MSB parts of registers */
        ciphBytes0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 8 * bytesPerLoad512);
        ciphBytes1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 9 * bytesPerLoad512);
        ciphBytes2 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 10 * bytesPerLoad512);
        ciphBytes3 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 11 * bytesPerLoad512);

        TMP[20] = _mm512_xor_si512(ciphBytes0, TMP[20]);
        TMP[21] = _mm512_xor_si512(ciphBytes1, TMP[21]);
        TMP[22] = _mm512_xor_si512(ciphBytes2, TMP[22]);
        TMP[23] = _mm512_xor_si512(ciphBytes3, TMP[23]);

        _mm512_mask_compressstoreu_epi8(pDst + 8 * bytesPerLoad512, kMsbMask64, TMP[20]);
        _mm512_mask_compressstoreu_epi8(pDst + 9 * bytesPerLoad512, kMsbMask64, TMP[21]);
        _mm512_mask_compressstoreu_epi8(pDst + 10 * bytesPerLoad512, kMsbMask64, TMP[22]);
        _mm512_mask_compressstoreu_epi8(pDst + 11 * bytesPerLoad512, kMsbMask64, TMP[23]);

        TRANSPOSE_OUT_512(TMP[20], TMP[21], TMP[22], TMP[23], TMP[12], TMP[13], TMP[14], TMP[15]);
        TMP[20] = _mm512_shuffle_epi8(TMP[20], M512(swapBytes));
        TMP[21] = _mm512_shuffle_epi8(TMP[21], M512(swapBytes));
        TMP[22] = _mm512_shuffle_epi8(TMP[22], M512(swapBytes));
        TMP[23] = _mm512_shuffle_epi8(TMP[23], M512(swapBytes));

        /* load cipher blocks to MSB parts of registers */
        ciphBytes0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 12 * bytesPerLoad512);
        ciphBytes1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 13 * bytesPerLoad512);
        ciphBytes2 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 14 * bytesPerLoad512);
        ciphBytes3 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 15 * bytesPerLoad512);

        TMP[20] = _mm512_xor_si512(ciphBytes0, TMP[20]);
        TMP[21] = _mm512_xor_si512(ciphBytes1, TMP[21]);
        TMP[22] = _mm512_xor_si512(ciphBytes2, TMP[22]);
        TMP[23] = _mm512_xor_si512(ciphBytes3, TMP[23]);

        _mm512_mask_compressstoreu_epi8(pDst + 12 * bytesPerLoad512, kMsbMask64, TMP[20]);
        _mm512_mask_compressstoreu_epi8(pDst + 13 * bytesPerLoad512, kMsbMask64, TMP[21]);
        _mm512_mask_compressstoreu_epi8(pDst + 14 * bytesPerLoad512, kMsbMask64, TMP[22]);
        _mm512_mask_compressstoreu_epi8(pDst + 15 * bytesPerLoad512, kMsbMask64, TMP[23]);

        pSrc += 16 * bytesPerLoad512;
        pDst += 16 * bytesPerLoad512;
    }

    /* clear secret data */
    PurgeBlock(TMP, sizeof(TMP));

    len -= processedLen;

    if (len) {
        _mm_storeu_si128((__m128i*)(pIV), currentState);
        cpSMS4_CFB_dec_gfni512x48(pDst, pSrc, len, cfbBlkSize, pRKey, pIV);
    }
}

/*
// 48*cfbBlkSize bytes processing
*/

static void cpSMS4_CFB_dec_gfni512x48(Ipp8u* pDst,
                                      const Ipp8u* pSrc,
                                      int len,
                                      int cfbBlkSize,
                                      const Ipp32u* pRKey,
                                      Ipp8u* pIV)
{
    __ALIGN16 __m512i TMP[19];

    /*
   // TMP[0..11]  - decrypted text blocks
   // TMP[12..14] - key
   // TMP[15..18] - temp out
   */

    const int bytesPerLoad512       = 4 * cfbBlkSize;
    const Ipp16u blocksCompressMask = (Ipp16u)(0xFFFF << cfbBlkSize);

    int processedLen = len - (len % (48 * cfbBlkSize));
    int blocks;

    /* load masks; allows to load 4 source blocks of cfbBlkSize each (in bytes) to LSB parts of 128-bit lanes in 512-bit register */
    __mmask64 kLsbMask64 = broadcast_16to64((Ipp16u)(0xFFFF << (16 - cfbBlkSize)));
    /* same mask to load in MSB parts */
    __mmask64 kMsbMask64 = broadcast_16to64((Ipp16u)~blocksCompressMask);

    /* load IV */
    __m128i currentState = _mm_maskz_loadu_epi64(0x03 /* load 128-bit */, pIV);

    for (blocks = len / cfbBlkSize; blocks >= (48); blocks -= (48)) {
        /* load cipher blocks to LSB parts of registers */
        __m512i ciphBytes0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 0 * bytesPerLoad512);
        __m512i ciphBytes1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 1 * bytesPerLoad512);
        __m512i ciphBytes2 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 2 * bytesPerLoad512);
        __m512i ciphBytes3 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 3 * bytesPerLoad512);

        /* prepare InputBlocks for decryption */
        ciphBytes0 = getInputBlocks(&currentState, &ciphBytes0, blocksCompressMask);
        ciphBytes1 = getInputBlocks(&currentState, &ciphBytes1, blocksCompressMask);
        ciphBytes2 = getInputBlocks(&currentState, &ciphBytes2, blocksCompressMask);
        ciphBytes3 = getInputBlocks(&currentState, &ciphBytes3, blocksCompressMask);

        ciphBytes0 = _mm512_shuffle_epi8(ciphBytes0, M512(swapBytes));
        ciphBytes1 = _mm512_shuffle_epi8(ciphBytes1, M512(swapBytes));
        ciphBytes2 = _mm512_shuffle_epi8(ciphBytes2, M512(swapBytes));
        ciphBytes3 = _mm512_shuffle_epi8(ciphBytes3, M512(swapBytes));
        /* clang-format off */
        TRANSPOSE_INP_512(TMP[0], TMP[1], TMP[2], TMP[3], ciphBytes0, ciphBytes1, ciphBytes2, ciphBytes3);
        /* clang-format on */

        /* load cipher blocks to LSB parts of registers */
        ciphBytes0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 4 * bytesPerLoad512);
        ciphBytes1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 5 * bytesPerLoad512);
        ciphBytes2 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 6 * bytesPerLoad512);
        ciphBytes3 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 7 * bytesPerLoad512);

        /* prepare InputBlocks for decryption */
        ciphBytes0 = getInputBlocks(&currentState, &ciphBytes0, blocksCompressMask);
        ciphBytes1 = getInputBlocks(&currentState, &ciphBytes1, blocksCompressMask);
        ciphBytes2 = getInputBlocks(&currentState, &ciphBytes2, blocksCompressMask);
        ciphBytes3 = getInputBlocks(&currentState, &ciphBytes3, blocksCompressMask);

        ciphBytes0 = _mm512_shuffle_epi8(ciphBytes0, M512(swapBytes));
        ciphBytes1 = _mm512_shuffle_epi8(ciphBytes1, M512(swapBytes));
        ciphBytes2 = _mm512_shuffle_epi8(ciphBytes2, M512(swapBytes));
        ciphBytes3 = _mm512_shuffle_epi8(ciphBytes3, M512(swapBytes));
        TRANSPOSE_INP_512(TMP[4],
                          TMP[5],
                          TMP[6],
                          TMP[7],
                          ciphBytes0,
                          ciphBytes1,
                          ciphBytes2,
                          ciphBytes3);

        /* load cipher blocks to LSB parts of registers */
        ciphBytes0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 8 * bytesPerLoad512);
        ciphBytes1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 9 * bytesPerLoad512);
        ciphBytes2 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 10 * bytesPerLoad512);
        ciphBytes3 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 11 * bytesPerLoad512);

        /* prepare InputBlocks for decryption */
        ciphBytes0 = getInputBlocks(&currentState, &ciphBytes0, blocksCompressMask);
        ciphBytes1 = getInputBlocks(&currentState, &ciphBytes1, blocksCompressMask);
        ciphBytes2 = getInputBlocks(&currentState, &ciphBytes2, blocksCompressMask);
        ciphBytes3 = getInputBlocks(&currentState, &ciphBytes3, blocksCompressMask);

        ciphBytes0 = _mm512_shuffle_epi8(ciphBytes0, M512(swapBytes));
        ciphBytes1 = _mm512_shuffle_epi8(ciphBytes1, M512(swapBytes));
        ciphBytes2 = _mm512_shuffle_epi8(ciphBytes2, M512(swapBytes));
        ciphBytes3 = _mm512_shuffle_epi8(ciphBytes3, M512(swapBytes));
        TRANSPOSE_INP_512(TMP[8],
                          TMP[9],
                          TMP[10],
                          TMP[11],
                          ciphBytes0,
                          ciphBytes1,
                          ciphBytes2,
                          ciphBytes3);

        int itr;
        for (itr = 0; itr < 8; itr++, pRKey += 4) {
            /* initial xors */
            TMP[14] = TMP[13] = TMP[12] = _mm512_set1_epi32((Ipp32s)pRKey[0]);
            TMP[12]                     = _mm512_xor_si512(TMP[12], TMP[1]);
            TMP[12]                     = _mm512_xor_si512(TMP[12], TMP[2]);
            TMP[12]                     = _mm512_xor_si512(TMP[12], TMP[3]);
            TMP[13]                     = _mm512_xor_si512(TMP[13], TMP[5]);
            TMP[13]                     = _mm512_xor_si512(TMP[13], TMP[6]);
            TMP[13]                     = _mm512_xor_si512(TMP[13], TMP[7]);
            TMP[14]                     = _mm512_xor_si512(TMP[14], TMP[9]);
            TMP[14]                     = _mm512_xor_si512(TMP[14], TMP[10]);
            TMP[14]                     = _mm512_xor_si512(TMP[14], TMP[11]);
            /* Sbox */
            TMP[12] = sBox512(TMP[12]);
            TMP[13] = sBox512(TMP[13]);
            TMP[14] = sBox512(TMP[14]);
            /* Sbox done, now L */
            TMP[0] = _mm512_xor_si512(_mm512_xor_si512(TMP[0], TMP[12]), L512(TMP[12]));
            TMP[4] = _mm512_xor_si512(_mm512_xor_si512(TMP[4], TMP[13]), L512(TMP[13]));
            TMP[8] = _mm512_xor_si512(_mm512_xor_si512(TMP[8], TMP[14]), L512(TMP[14]));

            /* initial xors */
            TMP[14] = TMP[13] = TMP[12] = _mm512_set1_epi32((Ipp32s)pRKey[1]);
            TMP[12]                     = _mm512_xor_si512(TMP[12], TMP[2]);
            TMP[12]                     = _mm512_xor_si512(TMP[12], TMP[3]);
            TMP[12]                     = _mm512_xor_si512(TMP[12], TMP[0]);
            TMP[13]                     = _mm512_xor_si512(TMP[13], TMP[6]);
            TMP[13]                     = _mm512_xor_si512(TMP[13], TMP[7]);
            TMP[13]                     = _mm512_xor_si512(TMP[13], TMP[4]);
            TMP[14]                     = _mm512_xor_si512(TMP[14], TMP[10]);
            TMP[14]                     = _mm512_xor_si512(TMP[14], TMP[11]);
            TMP[14]                     = _mm512_xor_si512(TMP[14], TMP[8]);
            /* Sbox */
            TMP[12] = sBox512(TMP[12]);
            TMP[13] = sBox512(TMP[13]);
            TMP[14] = sBox512(TMP[14]);
            /* Sbox done, now L */
            TMP[1] = _mm512_xor_si512(_mm512_xor_si512(TMP[1], TMP[12]), L512(TMP[12]));
            TMP[5] = _mm512_xor_si512(_mm512_xor_si512(TMP[5], TMP[13]), L512(TMP[13]));
            TMP[9] = _mm512_xor_si512(_mm512_xor_si512(TMP[9], TMP[14]), L512(TMP[14]));

            /* initial xors */
            TMP[14] = TMP[13] = TMP[12] = _mm512_set1_epi32((Ipp32s)pRKey[2]);
            TMP[12]                     = _mm512_xor_si512(TMP[12], TMP[3]);
            TMP[12]                     = _mm512_xor_si512(TMP[12], TMP[0]);
            TMP[12]                     = _mm512_xor_si512(TMP[12], TMP[1]);
            TMP[13]                     = _mm512_xor_si512(TMP[13], TMP[7]);
            TMP[13]                     = _mm512_xor_si512(TMP[13], TMP[4]);
            TMP[13]                     = _mm512_xor_si512(TMP[13], TMP[5]);
            TMP[14]                     = _mm512_xor_si512(TMP[14], TMP[11]);
            TMP[14]                     = _mm512_xor_si512(TMP[14], TMP[8]);
            TMP[14]                     = _mm512_xor_si512(TMP[14], TMP[9]);
            /* Sbox */
            TMP[12] = sBox512(TMP[12]);
            TMP[13] = sBox512(TMP[13]);
            TMP[14] = sBox512(TMP[14]);
            /* Sbox done, now L */
            TMP[2]  = _mm512_xor_si512(_mm512_xor_si512(TMP[2], TMP[12]), L512(TMP[12]));
            TMP[6]  = _mm512_xor_si512(_mm512_xor_si512(TMP[6], TMP[13]), L512(TMP[13]));
            TMP[10] = _mm512_xor_si512(_mm512_xor_si512(TMP[10], TMP[14]), L512(TMP[14]));

            /* initial xors */
            TMP[14] = TMP[13] = TMP[12] = _mm512_set1_epi32((Ipp32s)pRKey[3]);
            TMP[12]                     = _mm512_xor_si512(TMP[12], TMP[0]);
            TMP[12]                     = _mm512_xor_si512(TMP[12], TMP[1]);
            TMP[12]                     = _mm512_xor_si512(TMP[12], TMP[2]);
            TMP[13]                     = _mm512_xor_si512(TMP[13], TMP[4]);
            TMP[13]                     = _mm512_xor_si512(TMP[13], TMP[5]);
            TMP[13]                     = _mm512_xor_si512(TMP[13], TMP[6]);
            TMP[14]                     = _mm512_xor_si512(TMP[14], TMP[8]);
            TMP[14]                     = _mm512_xor_si512(TMP[14], TMP[9]);
            TMP[14]                     = _mm512_xor_si512(TMP[14], TMP[10]);
            /* Sbox */
            TMP[12] = sBox512(TMP[12]);
            TMP[13] = sBox512(TMP[13]);
            TMP[14] = sBox512(TMP[14]);
            /* Sbox done, now L */
            TMP[3]  = _mm512_xor_si512(_mm512_xor_si512(TMP[3], TMP[12]), L512(TMP[12]));
            TMP[7]  = _mm512_xor_si512(_mm512_xor_si512(TMP[7], TMP[13]), L512(TMP[13]));
            TMP[11] = _mm512_xor_si512(_mm512_xor_si512(TMP[11], TMP[14]), L512(TMP[14]));
        }
        pRKey -= 32;

        TRANSPOSE_OUT_512(TMP[15], TMP[16], TMP[17], TMP[18], TMP[0], TMP[1], TMP[2], TMP[3]);
        TMP[15] = _mm512_shuffle_epi8(TMP[15], M512(swapBytes));
        TMP[16] = _mm512_shuffle_epi8(TMP[16], M512(swapBytes));
        TMP[17] = _mm512_shuffle_epi8(TMP[17], M512(swapBytes));
        TMP[18] = _mm512_shuffle_epi8(TMP[18], M512(swapBytes));

        /* load cipher blocks to MSB parts of registers */
        ciphBytes0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 0 * bytesPerLoad512);
        ciphBytes1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 1 * bytesPerLoad512);
        ciphBytes2 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 2 * bytesPerLoad512);
        ciphBytes3 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 3 * bytesPerLoad512);

        TMP[15] = _mm512_xor_si512(ciphBytes0, TMP[15]);
        TMP[16] = _mm512_xor_si512(ciphBytes1, TMP[16]);
        TMP[17] = _mm512_xor_si512(ciphBytes2, TMP[17]);
        TMP[18] = _mm512_xor_si512(ciphBytes3, TMP[18]);

        _mm512_mask_compressstoreu_epi8(pDst + 0 * bytesPerLoad512, kMsbMask64, TMP[15]);
        _mm512_mask_compressstoreu_epi8(pDst + 1 * bytesPerLoad512, kMsbMask64, TMP[16]);
        _mm512_mask_compressstoreu_epi8(pDst + 2 * bytesPerLoad512, kMsbMask64, TMP[17]);
        _mm512_mask_compressstoreu_epi8(pDst + 3 * bytesPerLoad512, kMsbMask64, TMP[18]);

        TRANSPOSE_OUT_512(TMP[15], TMP[16], TMP[17], TMP[18], TMP[4], TMP[5], TMP[6], TMP[7]);
        TMP[15] = _mm512_shuffle_epi8(TMP[15], M512(swapBytes));
        TMP[16] = _mm512_shuffle_epi8(TMP[16], M512(swapBytes));
        TMP[17] = _mm512_shuffle_epi8(TMP[17], M512(swapBytes));
        TMP[18] = _mm512_shuffle_epi8(TMP[18], M512(swapBytes));

        /* load cipher blocks to MSB parts of registers */
        ciphBytes0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 4 * bytesPerLoad512);
        ciphBytes1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 5 * bytesPerLoad512);
        ciphBytes2 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 6 * bytesPerLoad512);
        ciphBytes3 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 7 * bytesPerLoad512);

        TMP[15] = _mm512_xor_si512(ciphBytes0, TMP[15]);
        TMP[16] = _mm512_xor_si512(ciphBytes1, TMP[16]);
        TMP[17] = _mm512_xor_si512(ciphBytes2, TMP[17]);
        TMP[18] = _mm512_xor_si512(ciphBytes3, TMP[18]);

        _mm512_mask_compressstoreu_epi8(pDst + 4 * bytesPerLoad512, kMsbMask64, TMP[15]);
        _mm512_mask_compressstoreu_epi8(pDst + 5 * bytesPerLoad512, kMsbMask64, TMP[16]);
        _mm512_mask_compressstoreu_epi8(pDst + 6 * bytesPerLoad512, kMsbMask64, TMP[17]);
        _mm512_mask_compressstoreu_epi8(pDst + 7 * bytesPerLoad512, kMsbMask64, TMP[18]);

        TRANSPOSE_OUT_512(TMP[15], TMP[16], TMP[17], TMP[18], TMP[8], TMP[9], TMP[10], TMP[11]);
        TMP[15] = _mm512_shuffle_epi8(TMP[15], M512(swapBytes));
        TMP[16] = _mm512_shuffle_epi8(TMP[16], M512(swapBytes));
        TMP[17] = _mm512_shuffle_epi8(TMP[17], M512(swapBytes));
        TMP[18] = _mm512_shuffle_epi8(TMP[18], M512(swapBytes));

        /* load cipher blocks to MSB parts of registers */
        ciphBytes0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 8 * bytesPerLoad512);
        ciphBytes1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 9 * bytesPerLoad512);
        ciphBytes2 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 10 * bytesPerLoad512);
        ciphBytes3 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 11 * bytesPerLoad512);

        TMP[15] = _mm512_xor_si512(ciphBytes0, TMP[15]);
        TMP[16] = _mm512_xor_si512(ciphBytes1, TMP[16]);
        TMP[17] = _mm512_xor_si512(ciphBytes2, TMP[17]);
        TMP[18] = _mm512_xor_si512(ciphBytes3, TMP[18]);

        _mm512_mask_compressstoreu_epi8(pDst + 8 * bytesPerLoad512, kMsbMask64, TMP[15]);
        _mm512_mask_compressstoreu_epi8(pDst + 9 * bytesPerLoad512, kMsbMask64, TMP[16]);
        _mm512_mask_compressstoreu_epi8(pDst + 10 * bytesPerLoad512, kMsbMask64, TMP[17]);
        _mm512_mask_compressstoreu_epi8(pDst + 11 * bytesPerLoad512, kMsbMask64, TMP[18]);

        pSrc += 12 * bytesPerLoad512;
        pDst += 12 * bytesPerLoad512;
    }

    /* clear secret data */
    PurgeBlock(TMP, sizeof(TMP));

    len -= processedLen;

    if (len) {
        _mm_storeu_si128((__m128i*)(pIV), currentState);
        cpSMS4_CFB_dec_gfni512x32(pDst, pSrc, len, cfbBlkSize, pRKey, pIV);
    }
}

/*
// 32*cfbBlkSize bytes processing
*/

static void cpSMS4_CFB_dec_gfni512x32(Ipp8u* pDst,
                                      const Ipp8u* pSrc,
                                      int len,
                                      int cfbBlkSize,
                                      const Ipp32u* pRKey,
                                      Ipp8u* pIV)
{
    __ALIGN16 __m512i TMP[15];

    /*
   // TMP[0..7]   - decrypted text blocks
   // TMP[8..9]   - key
   // TMP[10..14] - temp out
   */

    const int bytesPerLoad512       = 4 * cfbBlkSize;
    const Ipp16u blocksCompressMask = (Ipp16u)(0xFFFF << cfbBlkSize);

    int processedLen = len - (len % (32 * cfbBlkSize));
    int blocks;

    /* load masks; allows to load 4 source blocks of cfbBlkSize each (in bytes) to LSB parts of 128-bit lanes in 512-bit register */
    __mmask64 kLsbMask64 = broadcast_16to64((Ipp16u)(0xFFFF << (16 - cfbBlkSize)));
    /* same mask to load in MSB parts */
    __mmask64 kMsbMask64 = broadcast_16to64((Ipp16u)~blocksCompressMask);

    /* load IV */
    __m128i currentState = _mm_maskz_loadu_epi64(0x03 /* load 128-bit */, pIV);

    for (blocks = len / cfbBlkSize; blocks >= (32); blocks -= (32)) {
        /* load cipher blocks to LSB parts of registers */
        __m512i ciphBytes0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 0 * bytesPerLoad512);
        __m512i ciphBytes1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 1 * bytesPerLoad512);
        __m512i ciphBytes2 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 2 * bytesPerLoad512);
        __m512i ciphBytes3 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 3 * bytesPerLoad512);

        /* prepare InputBlocks for decryption */
        ciphBytes0 = getInputBlocks(&currentState, &ciphBytes0, blocksCompressMask);
        ciphBytes1 = getInputBlocks(&currentState, &ciphBytes1, blocksCompressMask);
        ciphBytes2 = getInputBlocks(&currentState, &ciphBytes2, blocksCompressMask);
        ciphBytes3 = getInputBlocks(&currentState, &ciphBytes3, blocksCompressMask);

        ciphBytes0 = _mm512_shuffle_epi8(ciphBytes0, M512(swapBytes));
        ciphBytes1 = _mm512_shuffle_epi8(ciphBytes1, M512(swapBytes));
        ciphBytes2 = _mm512_shuffle_epi8(ciphBytes2, M512(swapBytes));
        ciphBytes3 = _mm512_shuffle_epi8(ciphBytes3, M512(swapBytes));
        /* clang-format off */
        TRANSPOSE_INP_512(TMP[0], TMP[1], TMP[2], TMP[3], ciphBytes0, ciphBytes1, ciphBytes2, ciphBytes3);
        /* clang-format on */

        /* load cipher blocks to LSB parts of registers */
        ciphBytes0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 4 * bytesPerLoad512);
        ciphBytes1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 5 * bytesPerLoad512);
        ciphBytes2 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 6 * bytesPerLoad512);
        ciphBytes3 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 7 * bytesPerLoad512);

        /* prepare InputBlocks for decryption */
        ciphBytes0 = getInputBlocks(&currentState, &ciphBytes0, blocksCompressMask);
        ciphBytes1 = getInputBlocks(&currentState, &ciphBytes1, blocksCompressMask);
        ciphBytes2 = getInputBlocks(&currentState, &ciphBytes2, blocksCompressMask);
        ciphBytes3 = getInputBlocks(&currentState, &ciphBytes3, blocksCompressMask);

        ciphBytes0 = _mm512_shuffle_epi8(ciphBytes0, M512(swapBytes));
        ciphBytes1 = _mm512_shuffle_epi8(ciphBytes1, M512(swapBytes));
        ciphBytes2 = _mm512_shuffle_epi8(ciphBytes2, M512(swapBytes));
        ciphBytes3 = _mm512_shuffle_epi8(ciphBytes3, M512(swapBytes));
        TRANSPOSE_INP_512(TMP[4],
                          TMP[5],
                          TMP[6],
                          TMP[7],
                          ciphBytes0,
                          ciphBytes1,
                          ciphBytes2,
                          ciphBytes3);

        int itr;
        for (itr = 0; itr < 8; itr++, pRKey += 4) {
            /* initial xors */
            TMP[9] = TMP[8] = _mm512_set1_epi32((Ipp32s)pRKey[0]);
            TMP[8]          = _mm512_xor_si512(TMP[8], TMP[1]);
            TMP[8]          = _mm512_xor_si512(TMP[8], TMP[2]);
            TMP[8]          = _mm512_xor_si512(TMP[8], TMP[3]);
            TMP[9]          = _mm512_xor_si512(TMP[9], TMP[5]);
            TMP[9]          = _mm512_xor_si512(TMP[9], TMP[6]);
            TMP[9]          = _mm512_xor_si512(TMP[9], TMP[7]);
            /* Sbox */
            TMP[8] = sBox512(TMP[8]);
            TMP[9] = sBox512(TMP[9]);
            /* Sbox done, now L */
            TMP[0] = _mm512_xor_si512(_mm512_xor_si512(TMP[0], TMP[8]), L512(TMP[8]));
            TMP[4] = _mm512_xor_si512(_mm512_xor_si512(TMP[4], TMP[9]), L512(TMP[9]));

            /* initial xors */
            TMP[9] = TMP[8] = _mm512_set1_epi32((Ipp32s)pRKey[1]);
            TMP[8]          = _mm512_xor_si512(TMP[8], TMP[2]);
            TMP[8]          = _mm512_xor_si512(TMP[8], TMP[3]);
            TMP[8]          = _mm512_xor_si512(TMP[8], TMP[0]);
            TMP[9]          = _mm512_xor_si512(TMP[9], TMP[6]);
            TMP[9]          = _mm512_xor_si512(TMP[9], TMP[7]);
            TMP[9]          = _mm512_xor_si512(TMP[9], TMP[4]);
            /* Sbox */
            TMP[8] = sBox512(TMP[8]);
            TMP[9] = sBox512(TMP[9]);
            /* Sbox done, now L */
            TMP[1] = _mm512_xor_si512(_mm512_xor_si512(TMP[1], TMP[8]), L512(TMP[8]));
            TMP[5] = _mm512_xor_si512(_mm512_xor_si512(TMP[5], TMP[9]), L512(TMP[9]));

            /* initial xors */
            TMP[9] = TMP[8] = _mm512_set1_epi32((Ipp32s)pRKey[2]);
            TMP[8]          = _mm512_xor_si512(TMP[8], TMP[3]);
            TMP[8]          = _mm512_xor_si512(TMP[8], TMP[0]);
            TMP[8]          = _mm512_xor_si512(TMP[8], TMP[1]);
            TMP[9]          = _mm512_xor_si512(TMP[9], TMP[7]);
            TMP[9]          = _mm512_xor_si512(TMP[9], TMP[4]);
            TMP[9]          = _mm512_xor_si512(TMP[9], TMP[5]);
            /* Sbox */
            TMP[8] = sBox512(TMP[8]);
            TMP[9] = sBox512(TMP[9]);
            /* Sbox done, now L */
            TMP[2] = _mm512_xor_si512(_mm512_xor_si512(TMP[2], TMP[8]), L512(TMP[8]));
            TMP[6] = _mm512_xor_si512(_mm512_xor_si512(TMP[6], TMP[9]), L512(TMP[9]));

            /* initial xors */
            TMP[9] = TMP[8] = _mm512_set1_epi32((Ipp32s)pRKey[3]);
            TMP[8]          = _mm512_xor_si512(TMP[8], TMP[0]);
            TMP[8]          = _mm512_xor_si512(TMP[8], TMP[1]);
            TMP[8]          = _mm512_xor_si512(TMP[8], TMP[2]);
            TMP[9]          = _mm512_xor_si512(TMP[9], TMP[4]);
            TMP[9]          = _mm512_xor_si512(TMP[9], TMP[5]);
            TMP[9]          = _mm512_xor_si512(TMP[9], TMP[6]);
            /* Sbox */
            TMP[8] = sBox512(TMP[8]);
            TMP[9] = sBox512(TMP[9]);
            /* Sbox done, now L */
            TMP[3] = _mm512_xor_si512(_mm512_xor_si512(TMP[3], TMP[8]), L512(TMP[8]));
            TMP[7] = _mm512_xor_si512(_mm512_xor_si512(TMP[7], TMP[9]), L512(TMP[9]));
        }

        pRKey -= 32;

        TRANSPOSE_OUT_512(TMP[10], TMP[11], TMP[12], TMP[13], TMP[0], TMP[1], TMP[2], TMP[3]);
        TMP[10] = _mm512_shuffle_epi8(TMP[10], M512(swapBytes));
        TMP[11] = _mm512_shuffle_epi8(TMP[11], M512(swapBytes));
        TMP[12] = _mm512_shuffle_epi8(TMP[12], M512(swapBytes));
        TMP[13] = _mm512_shuffle_epi8(TMP[13], M512(swapBytes));

        /* load cipher blocks to MSB parts of registers */
        ciphBytes0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 0 * bytesPerLoad512);
        ciphBytes1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 1 * bytesPerLoad512);
        ciphBytes2 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 2 * bytesPerLoad512);
        ciphBytes3 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 3 * bytesPerLoad512);

        TMP[10] = _mm512_xor_si512(ciphBytes0, TMP[10]);
        TMP[11] = _mm512_xor_si512(ciphBytes1, TMP[11]);
        TMP[12] = _mm512_xor_si512(ciphBytes2, TMP[12]);
        TMP[13] = _mm512_xor_si512(ciphBytes3, TMP[13]);

        _mm512_mask_compressstoreu_epi8(pDst + 0 * bytesPerLoad512, kMsbMask64, TMP[10]);
        _mm512_mask_compressstoreu_epi8(pDst + 1 * bytesPerLoad512, kMsbMask64, TMP[11]);
        _mm512_mask_compressstoreu_epi8(pDst + 2 * bytesPerLoad512, kMsbMask64, TMP[12]);
        _mm512_mask_compressstoreu_epi8(pDst + 3 * bytesPerLoad512, kMsbMask64, TMP[13]);

        TRANSPOSE_OUT_512(TMP[10], TMP[11], TMP[12], TMP[13], TMP[4], TMP[5], TMP[6], TMP[7]);
        TMP[10] = _mm512_shuffle_epi8(TMP[10], M512(swapBytes));
        TMP[11] = _mm512_shuffle_epi8(TMP[11], M512(swapBytes));
        TMP[12] = _mm512_shuffle_epi8(TMP[12], M512(swapBytes));
        TMP[13] = _mm512_shuffle_epi8(TMP[13], M512(swapBytes));

        /* load cipher blocks to MSB parts of registers */
        ciphBytes0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 4 * bytesPerLoad512);
        ciphBytes1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 5 * bytesPerLoad512);
        ciphBytes2 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 6 * bytesPerLoad512);
        ciphBytes3 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 7 * bytesPerLoad512);

        TMP[10] = _mm512_xor_si512(ciphBytes0, TMP[10]);
        TMP[11] = _mm512_xor_si512(ciphBytes1, TMP[11]);
        TMP[12] = _mm512_xor_si512(ciphBytes2, TMP[12]);
        TMP[13] = _mm512_xor_si512(ciphBytes3, TMP[13]);

        _mm512_mask_compressstoreu_epi8(pDst + 4 * bytesPerLoad512, kMsbMask64, TMP[10]);
        _mm512_mask_compressstoreu_epi8(pDst + 5 * bytesPerLoad512, kMsbMask64, TMP[11]);
        _mm512_mask_compressstoreu_epi8(pDst + 6 * bytesPerLoad512, kMsbMask64, TMP[12]);
        _mm512_mask_compressstoreu_epi8(pDst + 7 * bytesPerLoad512, kMsbMask64, TMP[13]);

        pSrc += 8 * bytesPerLoad512;
        pDst += 8 * bytesPerLoad512;
    }

    /* clear secret data */
    PurgeBlock(TMP, sizeof(TMP));

    len -= processedLen;

    if (len) {
        _mm_storeu_si128((__m128i*)(pIV), currentState);
        cpSMS4_CFB_dec_gfni512x16(pDst, pSrc, len, cfbBlkSize, pRKey, pIV);
    }
}

/*
// 16*cfbBlkSize bytes processing
*/

static void cpSMS4_CFB_dec_gfni512x16(Ipp8u* pDst,
                                      const Ipp8u* pSrc,
                                      int len,
                                      int cfbBlkSize,
                                      const Ipp32u* pRKey,
                                      Ipp8u* pIV)
{
    __ALIGN16 __m512i TMP[9];

    /*
   // TMP[0..3] - decrypted text blocks
   // TMP[4]    - key
   // TMP[5..8] - temp out
   */

    const int bytesPerLoad512       = 4 * cfbBlkSize;
    const Ipp16u blocksCompressMask = (Ipp16u)(0xFFFF << cfbBlkSize);

    int processedLen = len - (len % (16 * cfbBlkSize));
    int blocks;

    /* load masks; allows to load 4 source blocks of cfbBlkSize each (in bytes) to LSB parts of 128-bit lanes in 512-bit register */
    __mmask64 kLsbMask64 = broadcast_16to64((Ipp16u)(0xFFFF << (16 - cfbBlkSize)));
    /* same mask to load in MSB parts */
    __mmask64 kMsbMask64 = broadcast_16to64((Ipp16u)~blocksCompressMask);

    /* load IV */
    __m128i currentState = _mm_maskz_loadu_epi64(0x03 /* load 128-bit */, pIV);

    for (blocks = len / cfbBlkSize; blocks >= (16); blocks -= (16)) {
        /* load cipher blocks to LSB parts of registers */
        __m512i ciphBytes0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 0 * bytesPerLoad512);
        __m512i ciphBytes1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 1 * bytesPerLoad512);
        __m512i ciphBytes2 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 2 * bytesPerLoad512);
        __m512i ciphBytes3 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 3 * bytesPerLoad512);

        /* prepare InputBlocks for decryption */
        ciphBytes0 = getInputBlocks(&currentState, &ciphBytes0, blocksCompressMask);
        ciphBytes1 = getInputBlocks(&currentState, &ciphBytes1, blocksCompressMask);
        ciphBytes2 = getInputBlocks(&currentState, &ciphBytes2, blocksCompressMask);
        ciphBytes3 = getInputBlocks(&currentState, &ciphBytes3, blocksCompressMask);

        ciphBytes0 = _mm512_shuffle_epi8(ciphBytes0, M512(swapBytes));
        ciphBytes1 = _mm512_shuffle_epi8(ciphBytes1, M512(swapBytes));
        ciphBytes2 = _mm512_shuffle_epi8(ciphBytes2, M512(swapBytes));
        ciphBytes3 = _mm512_shuffle_epi8(ciphBytes3, M512(swapBytes));
        /* clang-format off */
        TRANSPOSE_INP_512(TMP[0], TMP[1], TMP[2], TMP[3], ciphBytes0, ciphBytes1, ciphBytes2, ciphBytes3);
        /* clang-format on */

        int itr;
        for (itr = 0; itr < 8; itr++, pRKey += 4) {
            /* initial xors */
            TMP[4] = _mm512_set1_epi32((Ipp32s)pRKey[0]);
            TMP[4] = _mm512_xor_si512(TMP[4], TMP[1]);
            TMP[4] = _mm512_xor_si512(TMP[4], TMP[2]);
            TMP[4] = _mm512_xor_si512(TMP[4], TMP[3]);
            /* Sbox */
            TMP[4] = sBox512(TMP[4]);
            /* Sbox done, now L */
            TMP[0] = _mm512_xor_si512(_mm512_xor_si512(TMP[0], TMP[4]), L512(TMP[4]));

            /* initial xors */
            TMP[4] = _mm512_set1_epi32((Ipp32s)pRKey[1]);
            TMP[4] = _mm512_xor_si512(TMP[4], TMP[2]);
            TMP[4] = _mm512_xor_si512(TMP[4], TMP[3]);
            TMP[4] = _mm512_xor_si512(TMP[4], TMP[0]);
            /* Sbox */
            TMP[4] = sBox512(TMP[4]);
            /* Sbox done, now L */
            TMP[1] = _mm512_xor_si512(_mm512_xor_si512(TMP[1], TMP[4]), L512(TMP[4]));

            /* initial xors */
            TMP[4] = _mm512_set1_epi32((Ipp32s)pRKey[2]);
            TMP[4] = _mm512_xor_si512(TMP[4], TMP[3]);
            TMP[4] = _mm512_xor_si512(TMP[4], TMP[0]);
            TMP[4] = _mm512_xor_si512(TMP[4], TMP[1]);
            /* Sbox */
            TMP[4] = sBox512(TMP[4]);
            /* Sbox done, now L */
            TMP[2] = _mm512_xor_si512(_mm512_xor_si512(TMP[2], TMP[4]), L512(TMP[4]));

            /* initial xors */
            TMP[4] = _mm512_set1_epi32((Ipp32s)pRKey[3]);
            TMP[4] = _mm512_xor_si512(TMP[4], TMP[0]);
            TMP[4] = _mm512_xor_si512(TMP[4], TMP[1]);
            TMP[4] = _mm512_xor_si512(TMP[4], TMP[2]);
            /* Sbox */
            TMP[4] = sBox512(TMP[4]);
            /* Sbox done, now L */
            TMP[3] = _mm512_xor_si512(_mm512_xor_si512(TMP[3], TMP[4]), L512(TMP[4]));
        }

        pRKey -= 32;

        TRANSPOSE_OUT_512(TMP[5], TMP[6], TMP[7], TMP[8], TMP[0], TMP[1], TMP[2], TMP[3]);
        TMP[5] = _mm512_shuffle_epi8(TMP[5], M512(swapBytes));
        TMP[6] = _mm512_shuffle_epi8(TMP[6], M512(swapBytes));
        TMP[7] = _mm512_shuffle_epi8(TMP[7], M512(swapBytes));
        TMP[8] = _mm512_shuffle_epi8(TMP[8], M512(swapBytes));

        /* load cipher blocks to MSB parts of registers */
        ciphBytes0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 0 * bytesPerLoad512);
        ciphBytes1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 1 * bytesPerLoad512);
        ciphBytes2 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 2 * bytesPerLoad512);
        ciphBytes3 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 3 * bytesPerLoad512);

        TMP[5] = _mm512_xor_si512(ciphBytes0, TMP[5]);
        TMP[6] = _mm512_xor_si512(ciphBytes1, TMP[6]);
        TMP[7] = _mm512_xor_si512(ciphBytes2, TMP[7]);
        TMP[8] = _mm512_xor_si512(ciphBytes3, TMP[8]);

        _mm512_mask_compressstoreu_epi8(pDst + 0 * bytesPerLoad512, kMsbMask64, TMP[5]);
        _mm512_mask_compressstoreu_epi8(pDst + 1 * bytesPerLoad512, kMsbMask64, TMP[6]);
        _mm512_mask_compressstoreu_epi8(pDst + 2 * bytesPerLoad512, kMsbMask64, TMP[7]);
        _mm512_mask_compressstoreu_epi8(pDst + 3 * bytesPerLoad512, kMsbMask64, TMP[8]);

        pSrc += 4 * bytesPerLoad512;
        pDst += 4 * bytesPerLoad512;
    }

    /* clear secret data */
    PurgeBlock(TMP, sizeof(TMP));

    len -= processedLen;

    if (len) {
        _mm_storeu_si128((__m128i*)(pIV), currentState);
        cpSMS4_CFB_dec_gfni128x12(pDst, pSrc, len, cfbBlkSize, pRKey, pIV);
    }
}

/*
// 12*cfbBlkSize bytes processing
*/

static void cpSMS4_CFB_dec_gfni128x12(Ipp8u* pDst,
                                      const Ipp8u* pSrc,
                                      int len,
                                      int cfbBlkSize,
                                      const Ipp32u* pRKey,
                                      Ipp8u* pIV)
{
    __ALIGN16 __m128i TMP[15];

    /*
   // TMP[0..11]  - decrypted text blocks
   // TMP[12..14] - key
   */

    const __mmask16 blocksCompressMask = (__mmask16)(0xFFFF << cfbBlkSize);

    int processedLen = len - (len % (12 * cfbBlkSize));
    int blocks;

    /* load masks; allows to load 4 source blocks of cfbBlkSize each (in bytes) to LSB parts of 128-bit lanes in 512-bit register */
    __mmask16 kLsbMask16 = (__mmask16)(0xFFFF << (16 - cfbBlkSize));
    /* same mask to load in MSB parts */
    __mmask16 kMsbMask16 = (__mmask16)~blocksCompressMask;

    /* load IV */
    __m128i IV128 = _mm_maskz_loadu_epi64(0x03 /* load 128-bit */, pIV);

    __m128i currentState = IV128;

    for (blocks = len / cfbBlkSize; blocks >= (12); blocks -= (12)) {

        /* load cipher blocks to LSB parts of registers */
        __m128i ciphBytes0 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 0 * cfbBlkSize);
        __m128i ciphBytes1 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 1 * cfbBlkSize);
        __m128i ciphBytes2 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 2 * cfbBlkSize);
        __m128i ciphBytes3 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 3 * cfbBlkSize);

        /*  InpBlk_j = LSB_(b-s)(InpBlk_(j-1)) | C_(j-1); b = 128 bit, s = 8*cfbBlkSize bit */
        TMP[0] = currentState;
        /* drop MSB bits (size = cfbBlkSize) from the inpBlk_i (by mask) and append c_(i-1) to LSB bits */
        TMP[1] = _mm_mask_compress_epi8(ciphBytes0, blocksCompressMask, TMP[0]);
        TMP[2] = _mm_mask_compress_epi8(ciphBytes1, blocksCompressMask, TMP[1]);
        TMP[3] = _mm_mask_compress_epi8(ciphBytes2, blocksCompressMask, TMP[2]);

        /* next InputBlock ready */
        currentState = _mm_mask_compress_epi8(ciphBytes3, blocksCompressMask, TMP[3]);

        TMP[0] = _mm_shuffle_epi8(TMP[0], M128(swapBytes));
        TMP[1] = _mm_shuffle_epi8(TMP[1], M128(swapBytes));
        TMP[2] = _mm_shuffle_epi8(TMP[2], M128(swapBytes));
        TMP[3] = _mm_shuffle_epi8(TMP[3], M128(swapBytes));
        TRANSPOSE_INP_128(TMP[0], TMP[1], TMP[2], TMP[3], TMP[12]); /* TMP[12] - buffer */

        /* load cipher blocks to LSB parts of registers */
        ciphBytes0 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 4 * cfbBlkSize);
        ciphBytes1 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 5 * cfbBlkSize);
        ciphBytes2 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 6 * cfbBlkSize);
        ciphBytes3 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 7 * cfbBlkSize);

        /*  InpBlk_j = LSB_(b-s)(InpBlk_(j-1)) | C_(j-1); b = 128 bit, s = 8*cfbBlkSize bit */
        TMP[4] = currentState;
        /* drop MSB bits (size = cfbBlkSize) from the inpBlk_i (by mask) and append c_(i-1) to LSB bits */
        TMP[5] = _mm_mask_compress_epi8(ciphBytes0, blocksCompressMask, TMP[4]);
        TMP[6] = _mm_mask_compress_epi8(ciphBytes1, blocksCompressMask, TMP[5]);
        TMP[7] = _mm_mask_compress_epi8(ciphBytes2, blocksCompressMask, TMP[6]);

        /* next InputBlock ready */
        currentState = _mm_mask_compress_epi8(ciphBytes3, blocksCompressMask, TMP[7]);

        TMP[4] = _mm_shuffle_epi8(TMP[4], M128(swapBytes));
        TMP[5] = _mm_shuffle_epi8(TMP[5], M128(swapBytes));
        TMP[6] = _mm_shuffle_epi8(TMP[6], M128(swapBytes));
        TMP[7] = _mm_shuffle_epi8(TMP[7], M128(swapBytes));
        TRANSPOSE_INP_128(TMP[4], TMP[5], TMP[6], TMP[7], TMP[12]); /* TMP[12] - buffer */

        /* load cipher blocks to LSB parts of registers */
        ciphBytes0 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 8 * cfbBlkSize);
        ciphBytes1 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 9 * cfbBlkSize);
        ciphBytes2 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 10 * cfbBlkSize);
        ciphBytes3 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 11 * cfbBlkSize);

        /*  InpBlk_j = LSB_(b-s)(InpBlk_(j-1)) | C_(j-1); b = 128 bit, s = 8*cfbBlkSize bit */
        TMP[8] = currentState;
        /* drop MSB bits (size = cfbBlkSize) from the inpBlk_i (by mask) and append c_(i-1) to LSB bits */
        TMP[9]  = _mm_mask_compress_epi8(ciphBytes0, blocksCompressMask, TMP[8]);
        TMP[10] = _mm_mask_compress_epi8(ciphBytes1, blocksCompressMask, TMP[9]);
        TMP[11] = _mm_mask_compress_epi8(ciphBytes2, blocksCompressMask, TMP[10]);

        /* next InputBlock ready */
        currentState = _mm_mask_compress_epi8(ciphBytes3, blocksCompressMask, TMP[11]);

        TMP[8]  = _mm_shuffle_epi8(TMP[8], M128(swapBytes));
        TMP[9]  = _mm_shuffle_epi8(TMP[9], M128(swapBytes));
        TMP[10] = _mm_shuffle_epi8(TMP[10], M128(swapBytes));
        TMP[11] = _mm_shuffle_epi8(TMP[11], M128(swapBytes));
        TRANSPOSE_INP_128(TMP[8], TMP[9], TMP[10], TMP[11], TMP[12]); /* TMP[12] - buffer */


        int itr;
        for (itr = 0; itr < 8; itr++, pRKey += 4) {
            /* initial xors */
            TMP[13] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[0]), 0);
            TMP[14] = TMP[13];
            TMP[12] = TMP[13];
            TMP[12] = _mm_xor_si128(TMP[12], TMP[1]);
            TMP[12] = _mm_xor_si128(TMP[12], TMP[2]);
            TMP[12] = _mm_xor_si128(TMP[12], TMP[3]);
            TMP[13] = _mm_xor_si128(TMP[13], TMP[5]);
            TMP[13] = _mm_xor_si128(TMP[13], TMP[6]);
            TMP[13] = _mm_xor_si128(TMP[13], TMP[7]);
            TMP[14] = _mm_xor_si128(TMP[14], TMP[9]);
            TMP[14] = _mm_xor_si128(TMP[14], TMP[10]);
            TMP[14] = _mm_xor_si128(TMP[14], TMP[11]);
            /* Sbox */
            TMP[12] = sBox128(TMP[12]);
            TMP[13] = sBox128(TMP[13]);
            TMP[14] = sBox128(TMP[14]);
            /* Sbox done, now L */
            TMP[0] = _mm_xor_si128(_mm_xor_si128(TMP[0], TMP[12]), L128(TMP[12]));
            TMP[4] = _mm_xor_si128(_mm_xor_si128(TMP[4], TMP[13]), L128(TMP[13]));
            TMP[8] = _mm_xor_si128(_mm_xor_si128(TMP[8], TMP[14]), L128(TMP[14]));

            /* initial xors */
            TMP[13] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[1]), 0);
            TMP[14] = TMP[13];
            TMP[12] = TMP[13];
            TMP[12] = _mm_xor_si128(TMP[12], TMP[2]);
            TMP[12] = _mm_xor_si128(TMP[12], TMP[3]);
            TMP[12] = _mm_xor_si128(TMP[12], TMP[0]);
            TMP[13] = _mm_xor_si128(TMP[13], TMP[6]);
            TMP[13] = _mm_xor_si128(TMP[13], TMP[7]);
            TMP[13] = _mm_xor_si128(TMP[13], TMP[4]);
            TMP[14] = _mm_xor_si128(TMP[14], TMP[10]);
            TMP[14] = _mm_xor_si128(TMP[14], TMP[11]);
            TMP[14] = _mm_xor_si128(TMP[14], TMP[8]);
            /* Sbox */
            TMP[12] = sBox128(TMP[12]);
            TMP[13] = sBox128(TMP[13]);
            TMP[14] = sBox128(TMP[14]);
            /* Sbox done, now L */
            TMP[1] = _mm_xor_si128(_mm_xor_si128(TMP[1], TMP[12]), L128(TMP[12]));
            TMP[5] = _mm_xor_si128(_mm_xor_si128(TMP[5], TMP[13]), L128(TMP[13]));
            TMP[9] = _mm_xor_si128(_mm_xor_si128(TMP[9], TMP[14]), L128(TMP[14]));

            /* initial xors */
            TMP[13] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[2]), 0);
            TMP[14] = TMP[13];
            TMP[12] = TMP[13];
            TMP[12] = _mm_xor_si128(TMP[12], TMP[3]);
            TMP[12] = _mm_xor_si128(TMP[12], TMP[0]);
            TMP[12] = _mm_xor_si128(TMP[12], TMP[1]);
            TMP[13] = _mm_xor_si128(TMP[13], TMP[7]);
            TMP[13] = _mm_xor_si128(TMP[13], TMP[4]);
            TMP[13] = _mm_xor_si128(TMP[13], TMP[5]);
            TMP[14] = _mm_xor_si128(TMP[14], TMP[11]);
            TMP[14] = _mm_xor_si128(TMP[14], TMP[8]);
            TMP[14] = _mm_xor_si128(TMP[14], TMP[9]);
            /* Sbox */
            TMP[12] = sBox128(TMP[12]);
            TMP[13] = sBox128(TMP[13]);
            TMP[14] = sBox128(TMP[14]);
            /* Sbox done, now L */
            TMP[2]  = _mm_xor_si128(_mm_xor_si128(TMP[2], TMP[12]), L128(TMP[12]));
            TMP[6]  = _mm_xor_si128(_mm_xor_si128(TMP[6], TMP[13]), L128(TMP[13]));
            TMP[10] = _mm_xor_si128(_mm_xor_si128(TMP[10], TMP[14]), L128(TMP[14]));

            /* initial xors */
            TMP[13] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[3]), 0);
            TMP[14] = TMP[13];
            TMP[12] = TMP[13];
            TMP[12] = _mm_xor_si128(TMP[12], TMP[0]);
            TMP[12] = _mm_xor_si128(TMP[12], TMP[1]);
            TMP[12] = _mm_xor_si128(TMP[12], TMP[2]);
            TMP[13] = _mm_xor_si128(TMP[13], TMP[4]);
            TMP[13] = _mm_xor_si128(TMP[13], TMP[5]);
            TMP[13] = _mm_xor_si128(TMP[13], TMP[6]);
            TMP[14] = _mm_xor_si128(TMP[14], TMP[8]);
            TMP[14] = _mm_xor_si128(TMP[14], TMP[9]);
            TMP[14] = _mm_xor_si128(TMP[14], TMP[10]);
            /* Sbox */
            TMP[12] = sBox128(TMP[12]);
            TMP[13] = sBox128(TMP[13]);
            TMP[14] = sBox128(TMP[14]);
            /* Sbox done, now L */
            TMP[3]  = _mm_xor_si128(_mm_xor_si128(TMP[3], TMP[12]), L128(TMP[12]));
            TMP[7]  = _mm_xor_si128(_mm_xor_si128(TMP[7], TMP[13]), L128(TMP[13]));
            TMP[11] = _mm_xor_si128(_mm_xor_si128(TMP[11], TMP[14]), L128(TMP[14]));
        }

        pRKey -= 32;

        TRANSPOSE_OUT_128(TMP[0], TMP[1], TMP[2], TMP[3], TMP[12]); /* TMP[12] - buffer */
        /* Order of blocks is inverted */

        TMP[3] = _mm_shuffle_epi8(TMP[3], M128(swapBytes));
        TMP[2] = _mm_shuffle_epi8(TMP[2], M128(swapBytes));
        TMP[1] = _mm_shuffle_epi8(TMP[1], M128(swapBytes));
        TMP[0] = _mm_shuffle_epi8(TMP[0], M128(swapBytes));

        ciphBytes0 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 0 * cfbBlkSize);
        ciphBytes1 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 1 * cfbBlkSize);
        ciphBytes2 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 2 * cfbBlkSize);
        ciphBytes3 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 3 * cfbBlkSize);

        TMP[3] = _mm_xor_si128(ciphBytes0, TMP[3]);
        TMP[2] = _mm_xor_si128(ciphBytes1, TMP[2]);
        TMP[1] = _mm_xor_si128(ciphBytes2, TMP[1]);
        TMP[0] = _mm_xor_si128(ciphBytes3, TMP[0]);

        _mm_mask_compressstoreu_epi8(pDst + 0 * cfbBlkSize, kMsbMask16, TMP[3]);
        _mm_mask_compressstoreu_epi8(pDst + 1 * cfbBlkSize, kMsbMask16, TMP[2]);
        _mm_mask_compressstoreu_epi8(pDst + 2 * cfbBlkSize, kMsbMask16, TMP[1]);
        _mm_mask_compressstoreu_epi8(pDst + 3 * cfbBlkSize, kMsbMask16, TMP[0]);

        TRANSPOSE_OUT_128(TMP[4], TMP[5], TMP[6], TMP[7], TMP[12]); /* TMP[12] - buffer */
        /* Order of blocks is inverted */

        TMP[7] = _mm_shuffle_epi8(TMP[7], M128(swapBytes));
        TMP[6] = _mm_shuffle_epi8(TMP[6], M128(swapBytes));
        TMP[5] = _mm_shuffle_epi8(TMP[5], M128(swapBytes));
        TMP[4] = _mm_shuffle_epi8(TMP[4], M128(swapBytes));

        ciphBytes0 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 4 * cfbBlkSize);
        ciphBytes1 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 5 * cfbBlkSize);
        ciphBytes2 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 6 * cfbBlkSize);
        ciphBytes3 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 7 * cfbBlkSize);

        TMP[7] = _mm_xor_si128(ciphBytes0, TMP[7]);
        TMP[6] = _mm_xor_si128(ciphBytes1, TMP[6]);
        TMP[5] = _mm_xor_si128(ciphBytes2, TMP[5]);
        TMP[4] = _mm_xor_si128(ciphBytes3, TMP[4]);

        _mm_mask_compressstoreu_epi8(pDst + 4 * cfbBlkSize, kMsbMask16, TMP[7]);
        _mm_mask_compressstoreu_epi8(pDst + 5 * cfbBlkSize, kMsbMask16, TMP[6]);
        _mm_mask_compressstoreu_epi8(pDst + 6 * cfbBlkSize, kMsbMask16, TMP[5]);
        _mm_mask_compressstoreu_epi8(pDst + 7 * cfbBlkSize, kMsbMask16, TMP[4]);

        TRANSPOSE_OUT_128(TMP[8], TMP[9], TMP[10], TMP[11], TMP[12]); /* TMP[12] - buffer */
        /* Order of blocks is inverted */

        TMP[11] = _mm_shuffle_epi8(TMP[11], M128(swapBytes));
        TMP[10] = _mm_shuffle_epi8(TMP[10], M128(swapBytes));
        TMP[9]  = _mm_shuffle_epi8(TMP[9], M128(swapBytes));
        TMP[8]  = _mm_shuffle_epi8(TMP[8], M128(swapBytes));

        ciphBytes0 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 8 * cfbBlkSize);
        ciphBytes1 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 9 * cfbBlkSize);
        ciphBytes2 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 10 * cfbBlkSize);
        ciphBytes3 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 11 * cfbBlkSize);

        TMP[11] = _mm_xor_si128(ciphBytes0, TMP[11]);
        TMP[10] = _mm_xor_si128(ciphBytes1, TMP[10]);
        TMP[9]  = _mm_xor_si128(ciphBytes2, TMP[9]);
        TMP[8]  = _mm_xor_si128(ciphBytes3, TMP[8]);

        _mm_mask_compressstoreu_epi8(pDst + 8 * cfbBlkSize, kMsbMask16, TMP[11]);
        _mm_mask_compressstoreu_epi8(pDst + 9 * cfbBlkSize, kMsbMask16, TMP[10]);
        _mm_mask_compressstoreu_epi8(pDst + 10 * cfbBlkSize, kMsbMask16, TMP[9]);
        _mm_mask_compressstoreu_epi8(pDst + 11 * cfbBlkSize, kMsbMask16, TMP[8]);

        pSrc += 12 * cfbBlkSize;
        pDst += 12 * cfbBlkSize;
    }

    /* clear secret data */
    PurgeBlock(TMP, sizeof(TMP));

    len -= processedLen;

    if (len) {
        _mm_storeu_si128((__m128i*)(pIV), currentState);
        cpSMS4_CFB_dec_gfni128x8(pDst, pSrc, len, cfbBlkSize, pRKey, pIV);
    }
}

/*
// 8*cfbBlkSize bytes processing
*/

static void cpSMS4_CFB_dec_gfni128x8(Ipp8u* pDst,
                                     const Ipp8u* pSrc,
                                     int len,
                                     int cfbBlkSize,
                                     const Ipp32u* pRKey,
                                     Ipp8u* pIV)
{

    __ALIGN16 __m128i TMP[10];

    /*
   // TMP[0..7] - decrypted text blocks
   // TMP[8..9] - key
   */

    const __mmask16 blocksCompressMask = (__mmask16)(0xFFFF << cfbBlkSize);

    int processedLen = len - (len % (8 * cfbBlkSize));
    int blocks;

    /* load masks; allows to load 4 source blocks of cfbBlkSize each (in bytes) to LSB parts of 128-bit lanes in 512-bit register */
    __mmask16 kLsbMask16 = (__mmask16)(0xFFFF << (16 - cfbBlkSize));
    /* same mask to load in MSB parts */
    __mmask16 kMsbMask16 = (__mmask16)~blocksCompressMask;

    /* load IV */
    __m128i currentState = _mm_maskz_loadu_epi64(0x03 /* load 128-bit */, pIV);

    for (blocks = len / cfbBlkSize; blocks >= (8); blocks -= (8)) {

        /* load cipher blocks to LSB parts of registers */
        __m128i ciphBytes0 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 0 * cfbBlkSize);
        __m128i ciphBytes1 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 1 * cfbBlkSize);
        __m128i ciphBytes2 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 2 * cfbBlkSize);
        __m128i ciphBytes3 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 3 * cfbBlkSize);

        /*  InpBlk_j = LSB_(b-s)(InpBlk_(j-1)) | C_(j-1); b = 128 bit, s = 8*cfbBlkSize bit */
        TMP[0] = currentState;
        /* drop MSB bits (size = cfbBlkSize) from the inpBlk_i (by mask) and append c_(i-1) to LSB bits */
        TMP[1] = _mm_mask_compress_epi8(ciphBytes0, blocksCompressMask, TMP[0]);
        TMP[2] = _mm_mask_compress_epi8(ciphBytes1, blocksCompressMask, TMP[1]);
        TMP[3] = _mm_mask_compress_epi8(ciphBytes2, blocksCompressMask, TMP[2]);

        /* next InputBlock ready */
        currentState = _mm_mask_compress_epi8(ciphBytes3, blocksCompressMask, TMP[3]);

        TMP[0] = _mm_shuffle_epi8(TMP[0], M128(swapBytes));
        TMP[1] = _mm_shuffle_epi8(TMP[1], M128(swapBytes));
        TMP[2] = _mm_shuffle_epi8(TMP[2], M128(swapBytes));
        TMP[3] = _mm_shuffle_epi8(TMP[3], M128(swapBytes));
        TRANSPOSE_INP_128(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8]); /* TMP[8] - buffer */

        /* load cipher blocks to LSB parts of registers */
        ciphBytes0 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 4 * cfbBlkSize);
        ciphBytes1 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 5 * cfbBlkSize);
        ciphBytes2 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 6 * cfbBlkSize);
        ciphBytes3 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 7 * cfbBlkSize);

        /*  InpBlk_j = LSB_(b-s)(InpBlk_(j-1)) | C_(j-1); b = 128 bit, s = 8*cfbBlkSize bit */
        TMP[4] = currentState;
        /* drop MSB bits (size = cfbBlkSize) from the inpBlk_i (by mask) and append c_(i-1) to LSB bits */
        TMP[5] = _mm_mask_compress_epi8(ciphBytes0, blocksCompressMask, TMP[4]);
        TMP[6] = _mm_mask_compress_epi8(ciphBytes1, blocksCompressMask, TMP[5]);
        TMP[7] = _mm_mask_compress_epi8(ciphBytes2, blocksCompressMask, TMP[6]);

        /* next InputBlock ready */
        currentState = _mm_mask_compress_epi8(ciphBytes3, blocksCompressMask, TMP[7]);

        TMP[4] = _mm_shuffle_epi8(TMP[4], M128(swapBytes));
        TMP[5] = _mm_shuffle_epi8(TMP[5], M128(swapBytes));
        TMP[6] = _mm_shuffle_epi8(TMP[6], M128(swapBytes));
        TMP[7] = _mm_shuffle_epi8(TMP[7], M128(swapBytes));
        TRANSPOSE_INP_128(TMP[4], TMP[5], TMP[6], TMP[7], TMP[8]); /* TMP[8] - buffer */

        int itr;
        for (itr = 0; itr < 8; itr++, pRKey += 4) {
            /* initial xors */
            TMP[9] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[0]), 0);
            TMP[8] = TMP[9];
            TMP[8] = _mm_xor_si128(TMP[8], TMP[1]);
            TMP[8] = _mm_xor_si128(TMP[8], TMP[2]);
            TMP[8] = _mm_xor_si128(TMP[8], TMP[3]);
            TMP[9] = _mm_xor_si128(TMP[9], TMP[5]);
            TMP[9] = _mm_xor_si128(TMP[9], TMP[6]);
            TMP[9] = _mm_xor_si128(TMP[9], TMP[7]);
            /* Sbox */
            TMP[8] = sBox128(TMP[8]);
            TMP[9] = sBox128(TMP[9]);
            /* Sbox done, now L */
            TMP[0] = _mm_xor_si128(_mm_xor_si128(TMP[0], TMP[8]), L128(TMP[8]));
            TMP[4] = _mm_xor_si128(_mm_xor_si128(TMP[4], TMP[9]), L128(TMP[9]));

            /* initial xors */
            TMP[9] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[1]), 0);
            TMP[8] = TMP[9];
            TMP[8] = _mm_xor_si128(TMP[8], TMP[2]);
            TMP[8] = _mm_xor_si128(TMP[8], TMP[3]);
            TMP[8] = _mm_xor_si128(TMP[8], TMP[0]);
            TMP[9] = _mm_xor_si128(TMP[9], TMP[6]);
            TMP[9] = _mm_xor_si128(TMP[9], TMP[7]);
            TMP[9] = _mm_xor_si128(TMP[9], TMP[4]);
            /* Sbox */
            TMP[8] = sBox128(TMP[8]);
            TMP[9] = sBox128(TMP[9]);
            /* Sbox done, now L */
            TMP[1] = _mm_xor_si128(_mm_xor_si128(TMP[1], TMP[8]), L128(TMP[8]));
            TMP[5] = _mm_xor_si128(_mm_xor_si128(TMP[5], TMP[9]), L128(TMP[9]));

            /* initial xors */
            TMP[9] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[2]), 0);
            TMP[8] = TMP[9];
            TMP[8] = _mm_xor_si128(TMP[8], TMP[3]);
            TMP[8] = _mm_xor_si128(TMP[8], TMP[0]);
            TMP[8] = _mm_xor_si128(TMP[8], TMP[1]);
            TMP[9] = _mm_xor_si128(TMP[9], TMP[7]);
            TMP[9] = _mm_xor_si128(TMP[9], TMP[4]);
            TMP[9] = _mm_xor_si128(TMP[9], TMP[5]);
            /* Sbox */
            TMP[8] = sBox128(TMP[8]);
            TMP[9] = sBox128(TMP[9]);
            /* Sbox done, now L */
            TMP[2] = _mm_xor_si128(_mm_xor_si128(TMP[2], TMP[8]), L128(TMP[8]));
            TMP[6] = _mm_xor_si128(_mm_xor_si128(TMP[6], TMP[9]), L128(TMP[9]));

            /* initial xors */
            TMP[9] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[3]), 0);
            TMP[8] = TMP[9];
            TMP[8] = _mm_xor_si128(TMP[8], TMP[0]);
            TMP[8] = _mm_xor_si128(TMP[8], TMP[1]);
            TMP[8] = _mm_xor_si128(TMP[8], TMP[2]);
            TMP[9] = _mm_xor_si128(TMP[9], TMP[4]);
            TMP[9] = _mm_xor_si128(TMP[9], TMP[5]);
            TMP[9] = _mm_xor_si128(TMP[9], TMP[6]);
            /* Sbox */
            TMP[8] = sBox128(TMP[8]);
            TMP[9] = sBox128(TMP[9]);
            /* Sbox done, now L */
            TMP[3] = _mm_xor_si128(_mm_xor_si128(TMP[3], TMP[8]), L128(TMP[8]));
            TMP[7] = _mm_xor_si128(_mm_xor_si128(TMP[7], TMP[9]), L128(TMP[9]));
        }

        pRKey -= 32;

        TRANSPOSE_OUT_128(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8]); /* TMP[8] - buffer */
        /* Order of blocks is inverted */

        TMP[3] = _mm_shuffle_epi8(TMP[3], M128(swapBytes));
        TMP[2] = _mm_shuffle_epi8(TMP[2], M128(swapBytes));
        TMP[1] = _mm_shuffle_epi8(TMP[1], M128(swapBytes));
        TMP[0] = _mm_shuffle_epi8(TMP[0], M128(swapBytes));

        ciphBytes0 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 0 * cfbBlkSize);
        ciphBytes1 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 1 * cfbBlkSize);
        ciphBytes2 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 2 * cfbBlkSize);
        ciphBytes3 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 3 * cfbBlkSize);

        TMP[3] = _mm_xor_si128(ciphBytes0, TMP[3]);
        TMP[2] = _mm_xor_si128(ciphBytes1, TMP[2]);
        TMP[1] = _mm_xor_si128(ciphBytes2, TMP[1]);
        TMP[0] = _mm_xor_si128(ciphBytes3, TMP[0]);

        _mm_mask_compressstoreu_epi8(pDst + 0 * cfbBlkSize, kMsbMask16, TMP[3]);
        _mm_mask_compressstoreu_epi8(pDst + 1 * cfbBlkSize, kMsbMask16, TMP[2]);
        _mm_mask_compressstoreu_epi8(pDst + 2 * cfbBlkSize, kMsbMask16, TMP[1]);
        _mm_mask_compressstoreu_epi8(pDst + 3 * cfbBlkSize, kMsbMask16, TMP[0]);

        TRANSPOSE_OUT_128(TMP[4], TMP[5], TMP[6], TMP[7], TMP[8]); /* TMP[8] - buffer */
        /* Order of blocks is inverted */

        TMP[7] = _mm_shuffle_epi8(TMP[7], M128(swapBytes));
        TMP[6] = _mm_shuffle_epi8(TMP[6], M128(swapBytes));
        TMP[5] = _mm_shuffle_epi8(TMP[5], M128(swapBytes));
        TMP[4] = _mm_shuffle_epi8(TMP[4], M128(swapBytes));

        ciphBytes0 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 4 * cfbBlkSize);
        ciphBytes1 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 5 * cfbBlkSize);
        ciphBytes2 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 6 * cfbBlkSize);
        ciphBytes3 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 7 * cfbBlkSize);

        TMP[7] = _mm_xor_si128(ciphBytes0, TMP[7]);
        TMP[6] = _mm_xor_si128(ciphBytes1, TMP[6]);
        TMP[5] = _mm_xor_si128(ciphBytes2, TMP[5]);
        TMP[4] = _mm_xor_si128(ciphBytes3, TMP[4]);

        _mm_mask_compressstoreu_epi8(pDst + 4 * cfbBlkSize, kMsbMask16, TMP[7]);
        _mm_mask_compressstoreu_epi8(pDst + 5 * cfbBlkSize, kMsbMask16, TMP[6]);
        _mm_mask_compressstoreu_epi8(pDst + 6 * cfbBlkSize, kMsbMask16, TMP[5]);
        _mm_mask_compressstoreu_epi8(pDst + 7 * cfbBlkSize, kMsbMask16, TMP[4]);

        pSrc += 8 * cfbBlkSize;
        pDst += 8 * cfbBlkSize;
    }

    /* clear secret data */
    PurgeBlock(TMP, sizeof(TMP));

    len -= processedLen;

    if (len) {
        _mm_storeu_si128((__m128i*)(pIV), currentState);
        cpSMS4_CFB_dec_gfni128x4(pDst, pSrc, len, cfbBlkSize, pRKey, pIV);
    }
}

/*
// 4*cfbBlkSize bytes processing
*/

static void cpSMS4_CFB_dec_gfni128x4(Ipp8u* pDst,
                                     const Ipp8u* pSrc,
                                     int len,
                                     int cfbBlkSize,
                                     const Ipp32u* pRKey,
                                     Ipp8u* pIV)
{
    __ALIGN16 __m128i TMP[5];

    /*
   // TMP[0..3] - decrypted text blocks
   // TMP[4]    - key
   */

    const __mmask16 blocksCompressMask = (__mmask16)(0xFFFF << cfbBlkSize);

    int processedLen = len - (len % (4 * cfbBlkSize));
    int blocks;

    /* load masks; allows to load 4 source blocks of cfbBlkSize each (in bytes) to LSB parts of 128-bit lanes in 512-bit register */
    __mmask16 kLsbMask16 = (__mmask16)(0xFFFF << (16 - cfbBlkSize));
    /* same mask to load in MSB parts */
    __mmask16 kMsbMask16 = (__mmask16)~blocksCompressMask;

    /* load IV */
    __m128i currentState = _mm_maskz_loadu_epi64(0x03 /* load 128-bit */, pIV);

    for (blocks = len / cfbBlkSize; blocks >= (4); blocks -= (4)) {

        /* load cipher blocks to LSB parts of registers */
        __m128i ciphBytes0 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 0 * cfbBlkSize);
        __m128i ciphBytes1 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 1 * cfbBlkSize);
        __m128i ciphBytes2 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 2 * cfbBlkSize);
        __m128i ciphBytes3 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 3 * cfbBlkSize);

        /*  InpBlk_j = LSB_(b-s)(InpBlk_(j-1)) | C_(j-1); b = 128 bit, s = 8*cfbBlkSize bit */
        TMP[0] = currentState;
        /* drop MSB bits (size = cfbBlkSize) from the inpBlk_i (by mask) and append c_(i-1) to LSB bits */
        TMP[1] = _mm_mask_compress_epi8(ciphBytes0, blocksCompressMask, TMP[0]);
        TMP[2] = _mm_mask_compress_epi8(ciphBytes1, blocksCompressMask, TMP[1]);
        TMP[3] = _mm_mask_compress_epi8(ciphBytes2, blocksCompressMask, TMP[2]);

        /* next InputBlock ready */
        currentState = _mm_mask_compress_epi8(ciphBytes3, blocksCompressMask, TMP[3]);

        TMP[0] = _mm_shuffle_epi8(TMP[0], M128(swapBytes));
        TMP[1] = _mm_shuffle_epi8(TMP[1], M128(swapBytes));
        TMP[2] = _mm_shuffle_epi8(TMP[2], M128(swapBytes));
        TMP[3] = _mm_shuffle_epi8(TMP[3], M128(swapBytes));
        TRANSPOSE_INP_128(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4]); /* TMP[4] - buffer */

        int itr;
        for (itr = 0; itr < 8; itr++, pRKey += 4) {
            /* initial xors */
            TMP[4] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[0]), 0);
            TMP[4] = _mm_xor_si128(TMP[4], TMP[1]);
            TMP[4] = _mm_xor_si128(TMP[4], TMP[2]);
            TMP[4] = _mm_xor_si128(TMP[4], TMP[3]);
            /* Sbox */
            TMP[4] = sBox128(TMP[4]);
            /* Sbox done, now L */
            TMP[0] = _mm_xor_si128(_mm_xor_si128(TMP[0], TMP[4]), L128(TMP[4]));

            /* initial xors */
            TMP[4] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[1]), 0);
            TMP[4] = _mm_xor_si128(TMP[4], TMP[2]);
            TMP[4] = _mm_xor_si128(TMP[4], TMP[3]);
            TMP[4] = _mm_xor_si128(TMP[4], TMP[0]);
            /* Sbox */
            TMP[4] = sBox128(TMP[4]);
            /* Sbox done, now L */
            TMP[1] = _mm_xor_si128(_mm_xor_si128(TMP[1], TMP[4]), L128(TMP[4]));

            /* initial xors */
            TMP[4] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[2]), 0);
            TMP[4] = _mm_xor_si128(TMP[4], TMP[3]);
            TMP[4] = _mm_xor_si128(TMP[4], TMP[0]);
            TMP[4] = _mm_xor_si128(TMP[4], TMP[1]);
            /* Sbox */
            TMP[4] = sBox128(TMP[4]);
            /* Sbox done, now L */
            TMP[2] = _mm_xor_si128(_mm_xor_si128(TMP[2], TMP[4]), L128(TMP[4]));

            /* initial xors */
            TMP[4] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[3]), 0);
            TMP[4] = _mm_xor_si128(TMP[4], TMP[0]);
            TMP[4] = _mm_xor_si128(TMP[4], TMP[1]);
            TMP[4] = _mm_xor_si128(TMP[4], TMP[2]);
            /* Sbox */
            TMP[4] = sBox128(TMP[4]);
            /* Sbox done, now L */
            TMP[3] = _mm_xor_si128(_mm_xor_si128(TMP[3], TMP[4]), L128(TMP[4]));
        }

        pRKey -= 32;

        TRANSPOSE_OUT_128(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4]); /* TMP[4] - buffer */
        /* Order of blocks is inverted */

        TMP[3] = _mm_shuffle_epi8(TMP[3], M128(swapBytes));
        TMP[2] = _mm_shuffle_epi8(TMP[2], M128(swapBytes));
        TMP[1] = _mm_shuffle_epi8(TMP[1], M128(swapBytes));
        TMP[0] = _mm_shuffle_epi8(TMP[0], M128(swapBytes));

        ciphBytes0 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 0 * cfbBlkSize);
        ciphBytes1 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 1 * cfbBlkSize);
        ciphBytes2 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 2 * cfbBlkSize);
        ciphBytes3 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 3 * cfbBlkSize);

        TMP[3] = _mm_xor_si128(ciphBytes0, TMP[3]);
        TMP[2] = _mm_xor_si128(ciphBytes1, TMP[2]);
        TMP[1] = _mm_xor_si128(ciphBytes2, TMP[1]);
        TMP[0] = _mm_xor_si128(ciphBytes3, TMP[0]);

        _mm_mask_compressstoreu_epi8(pDst + 0 * cfbBlkSize, kMsbMask16, TMP[3]);
        _mm_mask_compressstoreu_epi8(pDst + 1 * cfbBlkSize, kMsbMask16, TMP[2]);
        _mm_mask_compressstoreu_epi8(pDst + 2 * cfbBlkSize, kMsbMask16, TMP[1]);
        _mm_mask_compressstoreu_epi8(pDst + 3 * cfbBlkSize, kMsbMask16, TMP[0]);

        pSrc += 4 * cfbBlkSize;
        pDst += 4 * cfbBlkSize;
    }

    /* clear secret data */
    PurgeBlock(TMP, sizeof(TMP));

    len -= processedLen;

    if (len) {
        _mm_storeu_si128((__m128i*)(pIV), currentState);
    }
}

#endif /* #if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) */
#endif /* _IPP32E>=_IPP32E_K1 */
