[PATCH] Fix ChaCha SIMD feed-forward carry (GH #1362, PR #1363)

The NEON, SSE2 and Altivec backends applied the per-block counter offset with a 64-bit add after the rounds. A carry from word 12 could therefore alter word 13 of the keystream.

Precompute each block's initial state, then use 32-bit feed-forward adds. Add regression vectors covering ChaCha8/12/20 and counter offsets 1, 2 and 3.

This matches the earlier AVX2 fix in GH #1069.

Gbp-Pq: Name fix_ChaCha_SIMD_feed-forward_carry.patch

diff --git a/TestVectors/chacha.txt b/TestVectors/chacha.txt
index 71abe2ec12ae4b6bd66225db233f0ca55e7cf06c..0ab64f600633cf8a91af0b042a59a8cde50337d4 100644 (file)
--- a/TestVectors/chacha.txt
+++ b/TestVectors/chacha.txt
@@ -773,4 +773,60 @@ Ciphertext: \
     F13C280D8CC49925E4A6A5922EC80E13A4CDFA840C70A1427A3CB699166991A5 \\r
     ACE4CD09E294D1912D4AD205D06F95D9C2F2BFCF453E8753F128765B62215F4D \\r
     92C74F2F626C6A640C0B1284D839EC81F1696281DAFC3E684593937023B58B1D\r
-Test: Encrypt
\ No newline at end of file
+Test: Encrypt\r
+\r
+AlgorithmType: SymmetricCipher\r
+Name: ChaCha\r
+Source: https://cr.yp.to/streamciphers/timings/estreambench/submissions/salsa20/chacha8/ref/chacha.c\r
+#\r
+# Regression vectors for GH #1362.\r
+# Each vector triggers a carry from word12 into word13 in one block of the\r
+# four-block SIMD path. Generated with Bernstein's chacha-ref.c.\r
+#\r
+Comment: GH 1362 feed-forward carry, block 1, rounds 8\r
+Key: 0x000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F\r
+IV: 0x17CFDB3A00000000\r
+Rounds: 8\r
+Plaintext: r256 00\r
+Ciphertext: \\r
+    F4BEF5FF324505F5AC995FF0069314F3584A2392F450B93A4D7B049A3D0BE914 \\r
+    8C0963D502E3585E1FC230F156F6A1C4DE685737CB4677BDEC137E96229EEB1C \\r
+    ADCCD48A78E6A066472F3FF5D668DA0DDB96B1906A61BB2FD39D933EB8BBFC16 \\r
+    DC43F5E461864B7E69B8B0CFDA572692000000001E9D7EC5DAE0D935AB4D1569 \\r
+    43EBE13C1F8E80C607D57380BEACA4A70A89876354A21C188E1493A5B3C9CAAA \\r
+    F06598AAF5F0F844E3C9BA3292855FCB510020DD88BA474DC697053E30AF25F7 \\r
+    E3B91414E4EFF8EC67456F048789C5609A15A93B7E565955937297228B3F1E16 \\r
+    8D1210ABBBAF12654CDF142B9D91A090F7FD8C0846774EABCF50500EE6735054\r
+Test: Encrypt\r
+#\r
+Comment: GH 1362 feed-forward carry, block 2, rounds 12\r
+Key: 0x000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F\r
+IV: 0xC163C31C00000000\r
+Rounds: 12\r
+Plaintext: r256 00\r
+Ciphertext: \\r
+    51433D526D00E44E9905838B1A69048CD0BAD2E1748057A9DF92BFCEF09FBD24 \\r
+    502A694CAD0490134A2E182284ABBB4AD413A9B0616E26C1C1798A3416402A4D \\r
+    DBC63B7FEFAA4ABF5A0D2CAE19E735FD21447051A2CEAFB4E99D540D2D47B5A8 \\r
+    C1D7D09D0222765E56DD7FC9F51D140456647FA84DAB10F13C3D64EC36AC97F9 \\r
+    444CEF5185EFF5039B00CEC268892179EC831C853C8D1BF5A0A1489690D21889 \\r
+    8363791430868E1BDB16E89776478B1D010000007BD1C25701F3FDEE17147D82 \\r
+    48E4B36903B2213832A5471546C53FD1F094DBC76A88FBED62F045A8AD87A2B2 \\r
+    596922216D5B6B5E4802ECA9658B2366FAE44C6F2AEC2A1DE1B25E033796624A\r
+Test: Encrypt\r
+#\r
+Comment: GH 1362 feed-forward carry, block 3, rounds 20\r
+Key: 0x000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F\r
+IV: 0x8BE87FDB00000000\r
+Rounds: 20\r
+Plaintext: r256 00\r
+Ciphertext: \\r
+    0BD19BFEB4F50A08BDB20235C159586875AD7770EAB9C3A29D7F218513C00FB7 \\r
+    89CFAD34E144B330B2992F9624E36BD05EC35ED8EBBE6D1C5EB6D999748DBC9F \\r
+    EEB98EE4A409A86CA038AA893C6F959848963522BF4B1A15A5FEDCE5EFE9C479 \\r
+    C00B46796769060892EBAC40853671E7EDED26BF14AFB3BB5FE307CE10E191C0 \\r
+    0D1209F28AA847D0226A587A555629B70615AC920119BA611E32533183E0CCA9 \\r
+    684EC26D1518279651EDA7D3F6BCA87EA5BDF5E75D1F5327E0B59DE58CA27EBE \\r
+    CC09DC27064200159FB5B0BAC0569CF9947F0C7793F77389E51176CA98698BDA \\r
+    65A3C0AD915B7D89C5D7820330F0576A02000000AC54DC348ED44704F81B12EA\r
+Test: Encrypt\r

diff --git a/chacha_simd.cpp b/chacha_simd.cpp
index 556070e52b1853c9d428bbc96dafeb016467c8fc..f92d0240d596f3d7c3c509604ed23220f178ab8b 100644 (file)
--- a/chacha_simd.cpp
+++ b/chacha_simd.cpp
@@ -309,25 +309,32 @@ void ChaCha_OperateKeystream_NEON(const word32 *state, const byte* input, byte *
         vld1q_u32(w+0), vld1q_u32(w+4), vld1q_u32(w+8)\r
     };\r
 \r
+    // Precompute each block's initial counter state.\r
+    // Feed-forward uses 32-bit word adds.\r
+    const uint32x4_t state3_0 = state3;\r
+    const uint32x4_t state3_1 = Add64(state3, CTRS[0]);\r
+    const uint32x4_t state3_2 = Add64(state3, CTRS[1]);\r
+    const uint32x4_t state3_3 = Add64(state3, CTRS[2]);\r
+\r
     uint32x4_t r0_0 = state0;\r
     uint32x4_t r0_1 = state1;\r
     uint32x4_t r0_2 = state2;\r
-    uint32x4_t r0_3 = state3;\r
+    uint32x4_t r0_3 = state3_0;\r
 \r
     uint32x4_t r1_0 = state0;\r
     uint32x4_t r1_1 = state1;\r
     uint32x4_t r1_2 = state2;\r
-    uint32x4_t r1_3 = Add64(r0_3, CTRS[0]);\r
+    uint32x4_t r1_3 = state3_1;\r
 \r
     uint32x4_t r2_0 = state0;\r
     uint32x4_t r2_1 = state1;\r
     uint32x4_t r2_2 = state2;\r
-    uint32x4_t r2_3 = Add64(r0_3, CTRS[1]);\r
+    uint32x4_t r2_3 = state3_2;\r
 \r
     uint32x4_t r3_0 = state0;\r
     uint32x4_t r3_1 = state1;\r
     uint32x4_t r3_2 = state2;\r
-    uint32x4_t r3_3 = Add64(r0_3, CTRS[2]);\r
+    uint32x4_t r3_3 = state3_3;\r
 \r
     for (int i = static_cast<int>(rounds); i > 0; i -= 2)\r
     {\r
@@ -487,25 +494,22 @@ void ChaCha_OperateKeystream_NEON(const word32 *state, const byte* input, byte *
     r0_0 = vaddq_u32(r0_0, state0);\r
     r0_1 = vaddq_u32(r0_1, state1);\r
     r0_2 = vaddq_u32(r0_2, state2);\r
-    r0_3 = vaddq_u32(r0_3, state3);\r
+    r0_3 = vaddq_u32(r0_3, state3_0);\r
 \r
     r1_0 = vaddq_u32(r1_0, state0);\r
     r1_1 = vaddq_u32(r1_1, state1);\r
     r1_2 = vaddq_u32(r1_2, state2);\r
-    r1_3 = vaddq_u32(r1_3, state3);\r
-    r1_3 = Add64(r1_3, CTRS[0]);\r
+    r1_3 = vaddq_u32(r1_3, state3_1);\r
 \r
     r2_0 = vaddq_u32(r2_0, state0);\r
     r2_1 = vaddq_u32(r2_1, state1);\r
     r2_2 = vaddq_u32(r2_2, state2);\r
-    r2_3 = vaddq_u32(r2_3, state3);\r
-    r2_3 = Add64(r2_3, CTRS[1]);\r
+    r2_3 = vaddq_u32(r2_3, state3_2);\r
 \r
     r3_0 = vaddq_u32(r3_0, state0);\r
     r3_1 = vaddq_u32(r3_1, state1);\r
     r3_2 = vaddq_u32(r3_2, state2);\r
-    r3_3 = vaddq_u32(r3_3, state3);\r
-    r3_3 = Add64(r3_3, CTRS[2]);\r
+    r3_3 = vaddq_u32(r3_3, state3_3);\r
 \r
     if (input)\r
     {\r
@@ -573,25 +577,32 @@ void ChaCha_OperateKeystream_SSE2(const word32 *state, const byte* input, byte *
     const __m128i state2 = _mm_load_si128(reinterpret_cast<const __m128i*>(state+2*4));\r
     const __m128i state3 = _mm_load_si128(reinterpret_cast<const __m128i*>(state+3*4));\r
 \r
+    // Precompute each block's initial counter state.\r
+    // Feed-forward uses 32-bit word adds.\r
+    const __m128i state3_0 = state3;\r
+    const __m128i state3_1 = _mm_add_epi64(state3, _mm_set_epi32(0, 0, 0, 1));\r
+    const __m128i state3_2 = _mm_add_epi64(state3, _mm_set_epi32(0, 0, 0, 2));\r
+    const __m128i state3_3 = _mm_add_epi64(state3, _mm_set_epi32(0, 0, 0, 3));\r
+\r
     __m128i r0_0 = state0;\r
     __m128i r0_1 = state1;\r
     __m128i r0_2 = state2;\r
-    __m128i r0_3 = state3;\r
+    __m128i r0_3 = state3_0;\r
 \r
     __m128i r1_0 = state0;\r
     __m128i r1_1 = state1;\r
     __m128i r1_2 = state2;\r
-    __m128i r1_3 = _mm_add_epi64(r0_3, _mm_set_epi32(0, 0, 0, 1));\r
+    __m128i r1_3 = state3_1;\r
 \r
     __m128i r2_0 = state0;\r
     __m128i r2_1 = state1;\r
     __m128i r2_2 = state2;\r
-    __m128i r2_3 = _mm_add_epi64(r0_3, _mm_set_epi32(0, 0, 0, 2));\r
+    __m128i r2_3 = state3_2;\r
 \r
     __m128i r3_0 = state0;\r
     __m128i r3_1 = state1;\r
     __m128i r3_2 = state2;\r
-    __m128i r3_3 = _mm_add_epi64(r0_3, _mm_set_epi32(0, 0, 0, 3));\r
+    __m128i r3_3 = state3_3;\r
 \r
     for (int i = static_cast<int>(rounds); i > 0; i -= 2)\r
     {\r
@@ -751,25 +762,22 @@ void ChaCha_OperateKeystream_SSE2(const word32 *state, const byte* input, byte *
     r0_0 = _mm_add_epi32(r0_0, state0);\r
     r0_1 = _mm_add_epi32(r0_1, state1);\r
     r0_2 = _mm_add_epi32(r0_2, state2);\r
-    r0_3 = _mm_add_epi32(r0_3, state3);\r
+    r0_3 = _mm_add_epi32(r0_3, state3_0);\r
 \r
     r1_0 = _mm_add_epi32(r1_0, state0);\r
     r1_1 = _mm_add_epi32(r1_1, state1);\r
     r1_2 = _mm_add_epi32(r1_2, state2);\r
-    r1_3 = _mm_add_epi32(r1_3, state3);\r
-    r1_3 = _mm_add_epi64(r1_3, _mm_set_epi32(0, 0, 0, 1));\r
+    r1_3 = _mm_add_epi32(r1_3, state3_1);\r
 \r
     r2_0 = _mm_add_epi32(r2_0, state0);\r
     r2_1 = _mm_add_epi32(r2_1, state1);\r
     r2_2 = _mm_add_epi32(r2_2, state2);\r
-    r2_3 = _mm_add_epi32(r2_3, state3);\r
-    r2_3 = _mm_add_epi64(r2_3, _mm_set_epi32(0, 0, 0, 2));\r
+    r2_3 = _mm_add_epi32(r2_3, state3_2);\r
 \r
     r3_0 = _mm_add_epi32(r3_0, state0);\r
     r3_1 = _mm_add_epi32(r3_1, state1);\r
     r3_2 = _mm_add_epi32(r3_2, state2);\r
-    r3_3 = _mm_add_epi32(r3_3, state3);\r
-    r3_3 = _mm_add_epi64(r3_3, _mm_set_epi32(0, 0, 0, 3));\r
+    r3_3 = _mm_add_epi32(r3_3, state3_3);\r
 \r
     if (input)\r
     {\r
@@ -844,25 +852,32 @@ inline void ChaCha_OperateKeystream_CORE(const word32 *state, const byte* input,
         {1,0,0,0}, {2,0,0,0}, {3,0,0,0}\r
     };\r
 \r
+    // Precompute each block's initial counter state.\r
+    // Feed-forward uses 32-bit word adds.\r
+    const uint32x4_p state3_0 = state3;\r
+    const uint32x4_p state3_1 = VecAdd64(state3, CTRS[0]);\r
+    const uint32x4_p state3_2 = VecAdd64(state3, CTRS[1]);\r
+    const uint32x4_p state3_3 = VecAdd64(state3, CTRS[2]);\r
+\r
     uint32x4_p r0_0 = state0;\r
     uint32x4_p r0_1 = state1;\r
     uint32x4_p r0_2 = state2;\r
-    uint32x4_p r0_3 = state3;\r
+    uint32x4_p r0_3 = state3_0;\r
 \r
     uint32x4_p r1_0 = state0;\r
     uint32x4_p r1_1 = state1;\r
     uint32x4_p r1_2 = state2;\r
-    uint32x4_p r1_3 = VecAdd64(r0_3, CTRS[0]);\r
+    uint32x4_p r1_3 = state3_1;\r
 \r
     uint32x4_p r2_0 = state0;\r
     uint32x4_p r2_1 = state1;\r
     uint32x4_p r2_2 = state2;\r
-    uint32x4_p r2_3 = VecAdd64(r0_3, CTRS[1]);\r
+    uint32x4_p r2_3 = state3_2;\r
 \r
     uint32x4_p r3_0 = state0;\r
     uint32x4_p r3_1 = state1;\r
     uint32x4_p r3_2 = state2;\r
-    uint32x4_p r3_3 = VecAdd64(r0_3, CTRS[2]);\r
+    uint32x4_p r3_3 = state3_3;\r
 \r
     for (int i = static_cast<int>(rounds); i > 0; i -= 2)\r
     {\r
@@ -1022,25 +1037,22 @@ inline void ChaCha_OperateKeystream_CORE(const word32 *state, const byte* input,
     r0_0 = VecAdd(r0_0, state0);\r
     r0_1 = VecAdd(r0_1, state1);\r
     r0_2 = VecAdd(r0_2, state2);\r
-    r0_3 = VecAdd(r0_3, state3);\r
+    r0_3 = VecAdd(r0_3, state3_0);\r
 \r
     r1_0 = VecAdd(r1_0, state0);\r
     r1_1 = VecAdd(r1_1, state1);\r
     r1_2 = VecAdd(r1_2, state2);\r
-    r1_3 = VecAdd(r1_3, state3);\r
-    r1_3 = VecAdd64(r1_3, CTRS[0]);\r
+    r1_3 = VecAdd(r1_3, state3_1);\r
 \r
     r2_0 = VecAdd(r2_0, state0);\r
     r2_1 = VecAdd(r2_1, state1);\r
     r2_2 = VecAdd(r2_2, state2);\r
-    r2_3 = VecAdd(r2_3, state3);\r
-    r2_3 = VecAdd64(r2_3, CTRS[1]);\r
+    r2_3 = VecAdd(r2_3, state3_2);\r
 \r
     r3_0 = VecAdd(r3_0, state0);\r
     r3_1 = VecAdd(r3_1, state1);\r
     r3_2 = VecAdd(r3_2, state2);\r
-    r3_3 = VecAdd(r3_3, state3);\r
-    r3_3 = VecAdd64(r3_3, CTRS[2]);\r
+    r3_3 = VecAdd(r3_3, state3_3);\r
 \r
     if (input)\r
     {\r