/*************************************************************************
* Copyright (C) 2002 Intel Corporation
*
* Licensed under the Apache License,  Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* 	http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law  or agreed  to  in  writing,  software
* distributed under  the License  is  distributed  on  an  "AS IS"  BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the  specific  language  governing  permissions  and
* limitations under the License.
*************************************************************************/

#include <internal/ecnist/ifma_arith_p384.h>
#include <internal/common/memory_clear.h>

#if (_MBX >= _MBX_K1)

/* Constants */

/*
// p384 = 2^384 - 2^128 - 2^96 + 2^32 - 1
// in 2^52 radix
*/
/* clang-format off */
__ALIGN64 static const int64u p384_mb[P384_LEN52][sizeof(U64)/sizeof(int64u)] = {
    { REP8_DECL(0x00000000ffffffff) },
    { REP8_DECL(0x000ff00000000000) },
    { REP8_DECL(0x000ffffffeffffff) },
    { REP8_DECL(0x000fffffffffffff) },
    { REP8_DECL(0x000fffffffffffff) },
    { REP8_DECL(0x000fffffffffffff) },
    { REP8_DECL(0x000fffffffffffff) },
    { REP8_DECL(0x00000000000fffff) }
};

/* 2*p384 */
__ALIGN64 static const int64u p384_x2[P384_LEN52][sizeof(U64)/sizeof(int64u)] = {
    { REP8_DECL(0x00000001fffffffe) },
    { REP8_DECL(0x000fe00000000000) },
    { REP8_DECL(0x000ffffffdffffff) },
    { REP8_DECL(0x000fffffffffffff) },
    { REP8_DECL(0x000fffffffffffff) },
    { REP8_DECL(0x000fffffffffffff) },
    { REP8_DECL(0x000fffffffffffff) },
    { REP8_DECL(0x00000000001fffff) }
};

/*
// Note that
// k0 = -(1/p384 mod 2^DIGIT_SIZE)  equals 1
// The implementation takes this fact into account
*/

/* to Montgomery conversion constant
// rr = 2^((P384_LEN52*DIGIT_SIZE)*2) mod p384
*/
__ALIGN64 static const int64u p384_rr_mb[P384_LEN52][sizeof(U64)/sizeof(int64u)] = {
    { REP8_DECL(0x0000000000000000) },
    { REP8_DECL(0x000fe00000001000) },
    { REP8_DECL(0x0000000000ffffff) },
    { REP8_DECL(0x0000000000000020) },
    { REP8_DECL(0x0000fffffffe0000) },
    { REP8_DECL(0x0000000020000000) },
    { REP8_DECL(0x0000000000000100) },
    { REP8_DECL(0x0000000000000000) }
};
/* clang-format on */


/*=====================================================================

 Specialized operations over p384:  sqr & mul

=====================================================================*/

#define ROUND_MUL(I, J, R_LO, R_HI)       \
    (R_LO) = fma52lo(R_LO, va[I], vb[J]); \
    (R_HI) = fma52hi(R_HI, va[I], vb[J]);

#define P384_REDUCTION_ROUND(u, r0, r1, r2, r3, r4, r5, r6, r7, r8, modulus)            \
    {                                                                                   \
        U64 t  = sub64(get_zero64(), u);                                                \
        U64 lo = and64(t, loadu64(VMASK52));                                            \
        U64 hi = add64(u, srai64(t, 63));                                               \
                                                                                        \
        (r0) = fma52lo(r0, u, (modulus)[0]);                                            \
        (r1) = fma52lo(r1, u, (modulus)[1]);                                            \
        (r2) = fma52lo(r2, u, (modulus)[2]);                                            \
        (r3) = add64(r3, lo);                     /*r3 = fma52lo(r3, u, modulus[3]); */ \
        (r4) = add64(r4, lo);                     /*r4 = fma52lo(r4, u, modulus[4]); */ \
        (r5) = add64(r5, lo);                     /*r5 = fma52lo(r5, u, modulus[5]); */ \
        (r6) = add64(r6, lo);                     /*r6 = fma52lo(r6, u, modulus[6]); */ \
        (r7) = fma52lo(r7, u, (modulus)[7]);                                            \
                                                                                        \
        (r1) = add64(r1, srli64(r0, DIGIT_SIZE)); /* carry propagation */               \
                                                                                        \
        (r1) = fma52hi(r1, u, (modulus)[0]);                                            \
        (r2) = fma52hi(r2, u, (modulus)[1]);                                            \
        (r3) = fma52hi(r3, u, (modulus)[2]);                                            \
        (r8) = fma52hi(r8, u, (modulus)[7]);                                            \
        (u)  = and64(add64(r1, slli64(r1, 32)),                                         \
                    loadu64(VMASK52)); /* update u = r1*k, k = (2^32 +1) */            \
        (r4) = add64(r4, hi);           /*r4 = fma52hi(r4, u, modulus[3]); */           \
        (r5) = add64(r5, hi);           /*r5 = fma52hi(r5, u, modulus[4]); */           \
        (r6) = add64(r6, hi);           /*r6 = fma52hi(r6, u, modulus[5]); */           \
        (r7) = add64(r7, hi);           /*r7 = fma52hi(r7, u, modulus[6]); */           \
    }

void MB_FUNC_NAME(ifma_amm52_p384_)(U64 r[], const U64 va[], const U64 vb[])
{
    U64 r0, r1, r2, r3, r4, r5, r6, r7;
    int itr;

    r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = get_zero64();

    for (itr = 0; itr < P384_LEN52; itr++) {
        U64 Yi, T, LO, HI;
        U64 Bi = loadu64(vb);
        vb++;

        fma52lo_mem(r0, r0, Bi, va, SIMD_BYTES * 0);
        Yi = and64(add64(r0, slli64(r0, 32)), loadu64(VMASK52)); /* k = (2^32 +1) */
        T  = sub64(get_zero64(), Yi);
        LO = and64(T, loadu64(VMASK52));
        HI = add64(Yi, srai64(T, 63));
        fma52lo_mem(r1, r1, Bi, va, SIMD_BYTES * 1);
        fma52lo_mem(r2, r2, Bi, va, SIMD_BYTES * 2);
        fma52lo_mem(r3, r3, Bi, va, SIMD_BYTES * 3);
        fma52lo_mem(r4, r4, Bi, va, SIMD_BYTES * 4);
        fma52lo_mem(r5, r5, Bi, va, SIMD_BYTES * 5);
        fma52lo_mem(r6, r6, Bi, va, SIMD_BYTES * 6);
        fma52lo_mem(r7, r7, Bi, va, SIMD_BYTES * 7);

        fma52lo_mem(r0, r0, Yi, p384_mb, SIMD_BYTES * 0);
        fma52lo_mem(r1, r1, Yi, p384_mb, SIMD_BYTES * 1);
        r1 = add64(r1, srli64(r0, DIGIT_SIZE)); /* carry propagation */
        fma52lo_mem(r2, r2, Yi, p384_mb, SIMD_BYTES * 2);
        r3 = add64(r3, LO); /* fma52lo_mem(r3, r3, Yi, p384_mb, SIMD_BYTES * 3); */
        r4 = add64(r4, LO); /* fma52lo_mem(r4, r4, Yi, p384_mb, SIMD_BYTES * 4); */
        r5 = add64(r5, LO); /* fma52lo_mem(r5, r5, Yi, p384_mb, SIMD_BYTES * 5); */
        r6 = add64(r6, LO); /* fma52lo_mem(r6, r6, Yi, p384_mb, SIMD_BYTES * 6); */
        fma52lo_mem(r7, r7, Yi, p384_mb, SIMD_BYTES * 7);

        fma52hi_mem(r0, r1, Bi, va, SIMD_BYTES * 0);
        fma52hi_mem(r1, r2, Bi, va, SIMD_BYTES * 1);
        fma52hi_mem(r2, r3, Bi, va, SIMD_BYTES * 2);
        fma52hi_mem(r3, r4, Bi, va, SIMD_BYTES * 3);
        fma52hi_mem(r4, r5, Bi, va, SIMD_BYTES * 4);
        fma52hi_mem(r5, r6, Bi, va, SIMD_BYTES * 5);
        fma52hi_mem(r6, r7, Bi, va, SIMD_BYTES * 6);
        fma52hi_mem(r7, get_zero64(), Bi, va, SIMD_BYTES * 7);

        fma52hi_mem(r0, r0, Yi, p384_mb, SIMD_BYTES * 0);
        fma52hi_mem(r1, r1, Yi, p384_mb, SIMD_BYTES * 1);
        fma52hi_mem(r2, r2, Yi, p384_mb, SIMD_BYTES * 2);
        r3 = add64(r3, HI); /* fma52hi_mem(r3, r3, Yi, p384_mb, SIMD_BYTES * 3); */
        r4 = add64(r4, HI); /* fma52hi_mem(r4, r4, Yi, p384_mb, SIMD_BYTES * 4); */
        r5 = add64(r5, HI); /* fma52hi_mem(r5, r5, Yi, p384_mb, SIMD_BYTES * 5); */
        r6 = add64(r6, HI); /* fma52hi_mem(r6, r6, Yi, p384_mb, SIMD_BYTES * 6); */
        fma52hi_mem(r7, r7, Yi, p384_mb, SIMD_BYTES * 7);
    }

    // normalization
    NORM_LSHIFTR(r, 0, 1)
    NORM_LSHIFTR(r, 1, 2)
    NORM_LSHIFTR(r, 2, 3)
    NORM_LSHIFTR(r, 3, 4)
    NORM_LSHIFTR(r, 4, 5)
    NORM_LSHIFTR(r, 5, 6)
    NORM_LSHIFTR(r, 6, 7)

    r[0] = r0;
    r[1] = r1;
    r[2] = r2;
    r[3] = r3;
    r[4] = r4;
    r[5] = r5;
    r[6] = r6;
    r[7] = r7;
}

void MB_FUNC_NAME(ifma_ams52_p384_)(U64 r[], const U64 va[])
{
    const U64* vb = va;
    U64* modulus  = (U64*)p384_mb;

    U64 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15;
    r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = r10 = r11 = r12 = r13 = r14 = r15 =
        get_zero64();

    // full square
    ROUND_MUL(0, 1, r1, r2)
    ROUND_MUL(0, 2, r2, r3)
    ROUND_MUL(0, 3, r3, r4)
    ROUND_MUL(0, 4, r4, r5)
    ROUND_MUL(0, 5, r5, r6)
    ROUND_MUL(0, 6, r6, r7)
    ROUND_MUL(0, 7, r7, r8)

    ROUND_MUL(1, 2, r3, r4)
    ROUND_MUL(1, 3, r4, r5)
    ROUND_MUL(1, 4, r5, r6)
    ROUND_MUL(1, 5, r6, r7)
    ROUND_MUL(1, 6, r7, r8)
    ROUND_MUL(1, 7, r8, r9)

    ROUND_MUL(2, 3, r5, r6)
    ROUND_MUL(2, 4, r6, r7)
    ROUND_MUL(2, 5, r7, r8)
    ROUND_MUL(2, 6, r8, r9)
    ROUND_MUL(2, 7, r9, r10)

    ROUND_MUL(3, 4, r7, r8)
    ROUND_MUL(3, 5, r8, r9)
    ROUND_MUL(3, 6, r9, r10)
    ROUND_MUL(3, 7, r10, r11)

    ROUND_MUL(4, 5, r9, r10)
    ROUND_MUL(4, 6, r10, r11)
    ROUND_MUL(4, 7, r11, r12)

    ROUND_MUL(5, 6, r11, r12)
    ROUND_MUL(5, 7, r12, r13)

    ROUND_MUL(6, 7, r13, r14)

    r1  = add64(r1, r1);
    r2  = add64(r2, r2);
    r3  = add64(r3, r3);
    r4  = add64(r4, r4);
    r5  = add64(r5, r5);
    r6  = add64(r6, r6);
    r7  = add64(r7, r7);
    r8  = add64(r8, r8);
    r9  = add64(r9, r9);
    r10 = add64(r10, r10);
    r11 = add64(r11, r11);
    r12 = add64(r12, r12);
    r13 = add64(r13, r13);
    r14 = add64(r14, r14);

    U64 u;
    ROUND_MUL(0, 0, r0, r1)
    u = and64(add64(r0, slli64(r0, 32)), loadu64(VMASK52));
    ROUND_MUL(1, 1, r2, r3)
    ROUND_MUL(2, 2, r4, r5)
    ROUND_MUL(3, 3, r6, r7)
    ROUND_MUL(4, 4, r8, r9)
    ROUND_MUL(5, 5, r10, r11)
    ROUND_MUL(6, 6, r12, r13)
    ROUND_MUL(7, 7, r14, r15)

    // reduction
    P384_REDUCTION_ROUND(u, r0, r1, r2, r3, r4, r5, r6, r7, r8, modulus);
    P384_REDUCTION_ROUND(u, r1, r2, r3, r4, r5, r6, r7, r8, r9, modulus);
    P384_REDUCTION_ROUND(u, r2, r3, r4, r5, r6, r7, r8, r9, r10, modulus);
    P384_REDUCTION_ROUND(u, r3, r4, r5, r6, r7, r8, r9, r10, r11, modulus);
    P384_REDUCTION_ROUND(u, r4, r5, r6, r7, r8, r9, r10, r11, r12, modulus);
    P384_REDUCTION_ROUND(u, r5, r6, r7, r8, r9, r10, r11, r12, r13, modulus);
    P384_REDUCTION_ROUND(u, r6, r7, r8, r9, r10, r11, r12, r13, r14, modulus);
    P384_REDUCTION_ROUND(u, r7, r8, r9, r10, r11, r12, r13, r14, r15, modulus);

    // normalization
    NORM_LSHIFTR(r, 8, 9)
    NORM_LSHIFTR(r, 9, 10)
    NORM_LSHIFTR(r, 10, 11)
    NORM_LSHIFTR(r, 11, 12)
    NORM_LSHIFTR(r, 12, 13)
    NORM_LSHIFTR(r, 13, 14)
    NORM_LSHIFTR(r, 14, 15)

    r[0] = r8;
    r[1] = r9;
    r[2] = r10;
    r[3] = r11;
    r[4] = r12;
    r[5] = r13;
    r[6] = r14;
    r[7] = r15;
}

void MB_FUNC_NAME(ifma_reduce52_p384_)(U64 R[], const U64 A[])
{
    /* r = a - p384_mb */
    U64 r0 = sub64(A[0], ((U64*)(p384_mb))[0]);
    U64 r1 = sub64(A[1], ((U64*)(p384_mb))[1]);
    U64 r2 = sub64(A[2], ((U64*)(p384_mb))[2]);
    U64 r3 = sub64(A[3], ((U64*)(p384_mb))[3]);
    U64 r4 = sub64(A[4], ((U64*)(p384_mb))[4]);
    U64 r5 = sub64(A[5], ((U64*)(p384_mb))[5]);
    U64 r6 = sub64(A[6], ((U64*)(p384_mb))[6]);
    U64 r7 = sub64(A[7], ((U64*)(p384_mb))[7]);

    /* normalize r0 - r7 */
    NORM_ASHIFTR(r, 0, 1)
    NORM_ASHIFTR(r, 1, 2)
    NORM_ASHIFTR(r, 2, 3)
    NORM_ASHIFTR(r, 3, 4)
    NORM_ASHIFTR(r, 4, 5)
    NORM_ASHIFTR(r, 5, 6)
    NORM_ASHIFTR(r, 6, 7)

    /* r = a<p384_mb? a : r */
    __mb_mask lt = cmp64_mask(r7, get_zero64(), _MM_CMPINT_LT);
    R[0]         = mask_mov64(r0, lt, A[0]);
    R[1]         = mask_mov64(r1, lt, A[1]);
    R[2]         = mask_mov64(r2, lt, A[2]);
    R[3]         = mask_mov64(r3, lt, A[3]);
    R[4]         = mask_mov64(r4, lt, A[4]);
    R[5]         = mask_mov64(r5, lt, A[5]);
    R[6]         = mask_mov64(r6, lt, A[6]);
    R[7]         = mask_mov64(r7, lt, A[7]);
}

void MB_FUNC_NAME(ifma_tomont52_p384_)(U64 r[], const U64 a[])
{
    MB_FUNC_NAME(ifma_amm52_p384_)(r, a, (U64*)p384_rr_mb);
}

void MB_FUNC_NAME(ifma_frommont52_p384_)(U64 r[], const U64 a[])
{
    MB_FUNC_NAME(ifma_amm52_p384_)(r, a, (U64*)ones);
    MB_FUNC_NAME(ifma_reduce52_p384_)(r, r);
}

/*
// computes r = 1/z = z^(p384-2) mod p384
//       => r = z^(0xFFFFFFFFFFFFFFFF FFFFFFFFFFFFFFFF FFFFFFFFFFFFFFFF FFFFFFFFFFFFFFFE FFFFFFFF00000000 00000000FFFFFFFD)
//
// note: z in in Montgomery domain (as soon mul() and sqr() below are amm-functions
//       r in Montgomery domain too
*/
#define fe52_sqr MB_FUNC_NAME(ifma_ams52_p384_)
#define fe52_mul MB_FUNC_NAME(ifma_amm52_p384_)

/* r = base^(2^n) */
__MBX_INLINE void fe52_sqr_pwr(U64 r[], const U64 base[], int n)
{
    if (r != base)
        MB_FUNC_NAME(mov_FE384_)(r, base);
    for (; n > 0; n--)
        fe52_sqr(r, r);
}

void MB_FUNC_NAME(ifma_aminv52_p384_)(U64 r[], const U64 z[])
{
    __ALIGN64 U64 u[P384_LEN52];
    __ALIGN64 U64 v[P384_LEN52];
    __ALIGN64 U64 zD[P384_LEN52];
    __ALIGN64 U64 zE[P384_LEN52];
    __ALIGN64 U64 zF[P384_LEN52];

    fe52_sqr(u, z);          /* u = z^2 */
    fe52_mul(v, u, z);       /* v = z^2 * z     = z^3  */
    fe52_sqr_pwr(zF, v, 2);  /* zF= (z^3)^(2^2) = z^12 */

    fe52_mul(zD, zF, z);     /* zD = z^12 * z    = z^xD */
    fe52_mul(zE, zF, u);     /* zE = z^12 * z^2  = z^xE */
    fe52_mul(zF, zF, v);     /* zF = z^12 * z^3  = z^xF */

    fe52_sqr_pwr(u, zF, 4);  /* u  = (z^xF)^(2^4)  = z^xF0 */
    fe52_mul(zD, u, zD);     /* zD = z^xF0 * z^xD  = z^xFD */
    fe52_mul(zE, u, zE);     /* zE = z^xF0 * z^xE  = z^xFE */
    fe52_mul(zF, u, zF);     /* zF = z^xF0 * z^xF  = z^xFF */

    fe52_sqr_pwr(u, zF, 8);  /* u = (z^xFF)^(2^8)    = z^xFF00 */
    fe52_mul(zD, u, zD);     /* zD = z^xFF00 * z^xFD = z^xFFFD */
    fe52_mul(zE, u, zE);     /* zE = z^xFF00 * z^xFE = z^xFFFE */
    fe52_mul(zF, u, zF);     /* zF = z^xFF00 * z^xFF = z^xFFFF */

    fe52_sqr_pwr(u, zF, 16); /* u = (z^xFFFF)^(2^16)       = z^xFFFF0000 */
    fe52_mul(zD, u, zD);     /* zD = z^xFFFF0000 * z^xFFFD = z^xFFFFFFFD */
    fe52_mul(zE, u, zE);     /* zE = z^xFFFF0000 * z^xFFFE = z^xFFFFFFFE */
    fe52_mul(zF, u, zF);     /* zF = z^xFFFF0000 * z^xFFFF = z^xFFFFFFFF */

    fe52_sqr_pwr(u, zF, 32); /* u = (z^xFFFFFFFF)^(2^32)               = z^xFFFFFFFF00000000 */
    fe52_mul(zE, u, zE);     /* zE = z^xFFFFFFFF00000000 * z^xFFFFFFFE = z^xFFFFFFFFFFFFFFFE */
    fe52_mul(zF, u, zF);     /* zF = z^xFFFFFFFF00000000 * z^xFFFFFFFF = z^xFFFFFFFFFFFFFFFF */

    /* v =  z^xFFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFF
        = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF */
    fe52_sqr_pwr(v, zF, 64);
    fe52_mul(v, v, zF);
    /* v =  z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFF
          = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFFF */
    fe52_sqr_pwr(v, v, 64);
    fe52_mul(v, v, zF);
    /* v =  z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFE
        = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE */
    fe52_sqr_pwr(v, v, 64);
    fe52_mul(v, v, zE);
    /* v =  z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE.0000000000000000 * z^xFFFFFFFF00000000
        = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE.FFFFFFFF00000000 */
    fe52_sqr_pwr(v, v, 64);
    fe52_mul(v, v, u);
    /* r =  z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE.FFFFFFFF00000000.0000000000000000 * z^xFFFFFFFD
        = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE.FFFFFFFF00000000.00000000FFFFFFFD */
    fe52_sqr_pwr(v, v, 64);
    fe52_mul(r, v, zD);

    /* Clear the temporary buffers with the powers of the input z (z can be a secret value) */
    MB_FUNC_NAME(zero_)((int64u(*)[MB_WIDTH])u, sizeof(u) / sizeof(U64));
    MB_FUNC_NAME(zero_)((int64u(*)[MB_WIDTH])v, sizeof(v) / sizeof(U64));
    MB_FUNC_NAME(zero_)((int64u(*)[MB_WIDTH])zD, sizeof(zD) / sizeof(U64));
    MB_FUNC_NAME(zero_)((int64u(*)[MB_WIDTH])zE, sizeof(zE) / sizeof(U64));
    MB_FUNC_NAME(zero_)((int64u(*)[MB_WIDTH])zF, sizeof(zF) / sizeof(U64));
}

/*=====================================================================

 Specialized single operations over p384:  add, sub, neg

=====================================================================*/
__MBX_INLINE __mb_mask MB_FUNC_NAME(lt_mbx_digit_)(const U64 a,
                                                   const U64 b,
                                                   const __mb_mask lt_mask)
{
    U64 d = mask_sub64(sub64(a, b), lt_mask, sub64(a, b), set1(1));
    return cmp64_mask(d, get_zero64(), _MM_CMPINT_LT);
}

void MB_FUNC_NAME(ifma_add52_p384_)(U64 R[], const U64 A[], const U64 B[])
{
    /* r = a + b */
    U64 r0 = add64(A[0], B[0]);
    U64 r1 = add64(A[1], B[1]);
    U64 r2 = add64(A[2], B[2]);
    U64 r3 = add64(A[3], B[3]);
    U64 r4 = add64(A[4], B[4]);
    U64 r5 = add64(A[5], B[5]);
    U64 r6 = add64(A[6], B[6]);
    U64 r7 = add64(A[7], B[7]);

    /* lt = {r0 - r7} < 2*p */
    __mb_mask lt = MB_FUNC_NAME(lt_mbx_digit_)(r0, ((U64*)(p384_x2))[0], 0);
    lt           = MB_FUNC_NAME(lt_mbx_digit_)(r1, ((U64*)(p384_x2))[1], lt);
    lt           = MB_FUNC_NAME(lt_mbx_digit_)(r2, ((U64*)(p384_x2))[2], lt);
    lt           = MB_FUNC_NAME(lt_mbx_digit_)(r3, ((U64*)(p384_x2))[3], lt);
    lt           = MB_FUNC_NAME(lt_mbx_digit_)(r4, ((U64*)(p384_x2))[4], lt);
    lt           = MB_FUNC_NAME(lt_mbx_digit_)(r5, ((U64*)(p384_x2))[5], lt);
    lt           = MB_FUNC_NAME(lt_mbx_digit_)(r6, ((U64*)(p384_x2))[6], lt);
    lt           = MB_FUNC_NAME(lt_mbx_digit_)(r7, ((U64*)(p384_x2))[7], lt);

    /* {r0 - r7} -= 2*p */
    r0 = mask_sub64(r0, (__mb_mask)~lt, r0, ((U64*)(p384_x2))[0]);
    r1 = mask_sub64(r1, (__mb_mask)~lt, r1, ((U64*)(p384_x2))[1]);
    r2 = mask_sub64(r2, (__mb_mask)~lt, r2, ((U64*)(p384_x2))[2]);
    r3 = mask_sub64(r3, (__mb_mask)~lt, r3, ((U64*)(p384_x2))[3]);
    r4 = mask_sub64(r4, (__mb_mask)~lt, r4, ((U64*)(p384_x2))[4]);
    r5 = mask_sub64(r5, (__mb_mask)~lt, r5, ((U64*)(p384_x2))[5]);
    r6 = mask_sub64(r6, (__mb_mask)~lt, r6, ((U64*)(p384_x2))[6]);
    r7 = mask_sub64(r7, (__mb_mask)~lt, r7, ((U64*)(p384_x2))[7]);

    /* normalize r0 - r7 */
    NORM_ASHIFTR(r, 0, 1)
    NORM_ASHIFTR(r, 1, 2)
    NORM_ASHIFTR(r, 2, 3)
    NORM_ASHIFTR(r, 3, 4)
    NORM_ASHIFTR(r, 4, 5)
    NORM_ASHIFTR(r, 5, 6)
    NORM_ASHIFTR(r, 6, 7)

    R[0] = r0;
    R[1] = r1;
    R[2] = r2;
    R[3] = r3;
    R[4] = r4;
    R[5] = r5;
    R[6] = r6;
    R[7] = r7;
}

void MB_FUNC_NAME(ifma_sub52_p384_)(U64 R[], const U64 A[], const U64 B[])
{
    /* r = a - b */
    U64 r0 = sub64(A[0], B[0]);
    U64 r1 = sub64(A[1], B[1]);
    U64 r2 = sub64(A[2], B[2]);
    U64 r3 = sub64(A[3], B[3]);
    U64 r4 = sub64(A[4], B[4]);
    U64 r5 = sub64(A[5], B[5]);
    U64 r6 = sub64(A[6], B[6]);
    U64 r7 = sub64(A[7], B[7]);

    /* lt = {r0 - r7} < 0 */
    __mb_mask lt = MB_FUNC_NAME(lt_mbx_digit_)(r0, get_zero64(), 0);
    lt           = MB_FUNC_NAME(lt_mbx_digit_)(r1, get_zero64(), lt);
    lt           = MB_FUNC_NAME(lt_mbx_digit_)(r2, get_zero64(), lt);
    lt           = MB_FUNC_NAME(lt_mbx_digit_)(r3, get_zero64(), lt);
    lt           = MB_FUNC_NAME(lt_mbx_digit_)(r4, get_zero64(), lt);
    lt           = MB_FUNC_NAME(lt_mbx_digit_)(r5, get_zero64(), lt);
    lt           = MB_FUNC_NAME(lt_mbx_digit_)(r6, get_zero64(), lt);
    lt           = MB_FUNC_NAME(lt_mbx_digit_)(r7, get_zero64(), lt);

    r0 = mask_add64(r0, lt, r0, ((U64*)(p384_x2))[0]);
    r1 = mask_add64(r1, lt, r1, ((U64*)(p384_x2))[1]);
    r2 = mask_add64(r2, lt, r2, ((U64*)(p384_x2))[2]);
    r3 = mask_add64(r3, lt, r3, ((U64*)(p384_x2))[3]);
    r4 = mask_add64(r4, lt, r4, ((U64*)(p384_x2))[4]);
    r5 = mask_add64(r5, lt, r5, ((U64*)(p384_x2))[5]);
    r6 = mask_add64(r6, lt, r6, ((U64*)(p384_x2))[6]);
    r7 = mask_add64(r7, lt, r7, ((U64*)(p384_x2))[7]);

    /* normalize r0 - r7 */
    NORM_ASHIFTR(r, 0, 1)
    NORM_ASHIFTR(r, 1, 2)
    NORM_ASHIFTR(r, 2, 3)
    NORM_ASHIFTR(r, 3, 4)
    NORM_ASHIFTR(r, 4, 5)
    NORM_ASHIFTR(r, 5, 6)
    NORM_ASHIFTR(r, 6, 7)

    R[0] = r0;
    R[1] = r1;
    R[2] = r2;
    R[3] = r3;
    R[4] = r4;
    R[5] = r5;
    R[6] = r6;
    R[7] = r7;
}

void MB_FUNC_NAME(ifma_neg52_p384_)(U64 R[], const U64 A[])
{
    __mb_mask nz_mask = (__mb_mask)(~MB_FUNC_NAME(is_zero_FE384_)(A));

    /* {r0 - r7} = 2*p - A */
    U64 r0 = mask_sub64(A[0], nz_mask, ((U64*)(p384_x2))[0], A[0]);
    U64 r1 = mask_sub64(A[0], nz_mask, ((U64*)(p384_x2))[1], A[1]);
    U64 r2 = mask_sub64(A[0], nz_mask, ((U64*)(p384_x2))[2], A[2]);
    U64 r3 = mask_sub64(A[0], nz_mask, ((U64*)(p384_x2))[3], A[3]);
    U64 r4 = mask_sub64(A[0], nz_mask, ((U64*)(p384_x2))[4], A[4]);
    U64 r5 = mask_sub64(A[0], nz_mask, ((U64*)(p384_x2))[5], A[5]);
    U64 r6 = mask_sub64(A[0], nz_mask, ((U64*)(p384_x2))[6], A[6]);
    U64 r7 = mask_sub64(A[0], nz_mask, ((U64*)(p384_x2))[7], A[7]);

    /* normalize r0 - r7 */
    NORM_ASHIFTR(r, 0, 1)
    NORM_ASHIFTR(r, 1, 2)
    NORM_ASHIFTR(r, 2, 3)
    NORM_ASHIFTR(r, 3, 4)
    NORM_ASHIFTR(r, 4, 5)
    NORM_ASHIFTR(r, 5, 6)
    NORM_ASHIFTR(r, 6, 7)

    R[0] = r0;
    R[1] = r1;
    R[2] = r2;
    R[3] = r3;
    R[4] = r4;
    R[5] = r5;
    R[6] = r6;
    R[7] = r7;
}

void MB_FUNC_NAME(ifma_double52_p384_)(U64 R[], const U64 A[])
{
    MB_FUNC_NAME(ifma_add52_p384_)(R, A, A);
}

void MB_FUNC_NAME(ifma_tripple52_p384_)(U64 R[], const U64 A[])
{
    U64 T[P384_LEN52];
    MB_FUNC_NAME(ifma_add52_p384_)(T, A, A);
    MB_FUNC_NAME(ifma_add52_p384_)(R, T, A);
}

void MB_FUNC_NAME(ifma_half52_p384_)(U64 R[], const U64 A[])
{
    U64 one  = set1(1);
    U64 base = set1(DIGIT_BASE);

    /* res = a + is_odd(a)? p384 : 0 */
    U64 mask = sub64(get_zero64(), and64(A[0], one));
    U64 t0   = add64(A[0], and64(((U64*)p384_mb)[0], mask));
    U64 t1   = add64(A[1], and64(((U64*)p384_mb)[1], mask));
    U64 t2   = add64(A[2], and64(((U64*)p384_mb)[2], mask));
    U64 t3   = add64(A[3], and64(((U64*)p384_mb)[3], mask));
    U64 t4   = add64(A[4], and64(((U64*)p384_mb)[4], mask));
    U64 t5   = add64(A[5], and64(((U64*)p384_mb)[5], mask));
    U64 t6   = add64(A[6], and64(((U64*)p384_mb)[6], mask));
    U64 t7   = add64(A[7], and64(((U64*)p384_mb)[7], mask));

    /* t =>> 1 */
    mask = sub64(get_zero64(), and64(t1, one));
    t0   = add64(t0, and64(base, mask));
    t0   = srli64(t0, 1);

    mask = sub64(get_zero64(), and64(t2, one));
    t1   = add64(t1, and64(base, mask));
    t1   = srli64(t1, 1);

    mask = sub64(get_zero64(), and64(t3, one));
    t2   = add64(t2, and64(base, mask));
    t2   = srli64(t2, 1);

    mask = sub64(get_zero64(), and64(t4, one));
    t3   = add64(t3, and64(base, mask));
    t3   = srli64(t3, 1);

    mask = sub64(get_zero64(), and64(t5, one));
    t4   = add64(t4, and64(base, mask));
    t4   = srli64(t4, 1);

    mask = sub64(get_zero64(), and64(t6, one));
    t5   = add64(t5, and64(base, mask));
    t5   = srli64(t5, 1);

    mask = sub64(get_zero64(), and64(t7, one));
    t6   = add64(t6, and64(base, mask));
    t6   = srli64(t6, 1);

    t7 = srli64(t7, 1);

    /* normalize t0, t1, t2, t3, t4 */
    NORM_LSHIFTR(t, 0, 1)
    NORM_LSHIFTR(t, 1, 2)
    NORM_LSHIFTR(t, 2, 3)
    NORM_LSHIFTR(t, 3, 4)
    NORM_LSHIFTR(t, 4, 5)
    NORM_LSHIFTR(t, 5, 6)
    NORM_LSHIFTR(t, 6, 7)

    R[0] = t0;
    R[1] = t1;
    R[2] = t2;
    R[3] = t3;
    R[4] = t4;
    R[5] = t5;
    R[6] = t6;
    R[7] = t7;
}

__mb_mask MB_FUNC_NAME(ifma_cmp_lt_p384_)(const U64 a[])
{
    return MB_FUNC_NAME(cmp_lt_FE384_)(a, (const U64(*))p384_mb);
}

__mb_mask MB_FUNC_NAME(ifma_check_range_p384_)(const U64 A[])
{
    __mb_mask mask = MB_FUNC_NAME(is_zero_FE384_)(A);
    mask |= (__mb_mask)~MB_FUNC_NAME(ifma_cmp_lt_p384_)(A);

    return mask;
}

#endif /* #if (_MBX>=_MBX_K1) */
