From: CoraleSoft <82213665+Coralesoft@users.noreply.github.com>
Date: Mon, 13 Jul 2026 13:28:33 +0000 (+0800)
Subject: [PATCH] Fix ChaCha SIMD feed-forward carry (GH #1362, PR #1363)
X-Git-Tag: archive/raspbian/8.9.0-3+rpi1^2~1
X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=0faabb691a8eda067a066ecb9c8faf0986ccd8b9;p=libcrypto%2B%2B.git

[PATCH] Fix ChaCha SIMD feed-forward carry (GH #1362, PR #1363)

The NEON, SSE2 and Altivec backends applied the per-block counter offset with a 64-bit add after the rounds. A carry from word 12 could therefore alter word 13 of the keystream.

Precompute each block's initial state, then use 32-bit feed-forward adds. Add regression vectors covering ChaCha8/12/20 and counter offsets 1, 2 and 3.

This matches the earlier AVX2 fix in GH #1069.

Gbp-Pq: Name fix_ChaCha_SIMD_feed-forward_carry.patch
---

diff --git a/TestVectors/chacha.txt b/TestVectors/chacha.txt
index 71abe2e..0ab64f6 100644
--- a/TestVectors/chacha.txt
+++ b/TestVectors/chacha.txt
@@ -773,4 +773,60 @@ Ciphertext: \
     F13C280D8CC49925E4A6A5922EC80E13A4CDFA840C70A1427A3CB699166991A5 \
     ACE4CD09E294D1912D4AD205D06F95D9C2F2BFCF453E8753F128765B62215F4D \
     92C74F2F626C6A640C0B1284D839EC81F1696281DAFC3E684593937023B58B1D
-Test: Encrypt
\ No newline at end of file
+Test: Encrypt
+
+AlgorithmType: SymmetricCipher
+Name: ChaCha
+Source: https://cr.yp.to/streamciphers/timings/estreambench/submissions/salsa20/chacha8/ref/chacha.c
+#
+# Regression vectors for GH #1362.
+# Each vector triggers a carry from word12 into word13 in one block of the
+# four-block SIMD path. Generated with Bernstein's chacha-ref.c.
+#
+Comment: GH 1362 feed-forward carry, block 1, rounds 8
+Key: 0x000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F
+IV: 0x17CFDB3A00000000
+Rounds: 8
+Plaintext: r256 00
+Ciphertext: \
+    F4BEF5FF324505F5AC995FF0069314F3584A2392F450B93A4D7B049A3D0BE914 \
+    8C0963D502E3585E1FC230F156F6A1C4DE685737CB4677BDEC137E96229EEB1C \
+    ADCCD48A78E6A066472F3FF5D668DA0DDB96B1906A61BB2FD39D933EB8BBFC16 \
+    DC43F5E461864B7E69B8B0CFDA572692000000001E9D7EC5DAE0D935AB4D1569 \
+    43EBE13C1F8E80C607D57380BEACA4A70A89876354A21C188E1493A5B3C9CAAA \
+    F06598AAF5F0F844E3C9BA3292855FCB510020DD88BA474DC697053E30AF25F7 \
+    E3B91414E4EFF8EC67456F048789C5609A15A93B7E565955937297228B3F1E16 \
+    8D1210ABBBAF12654CDF142B9D91A090F7FD8C0846774EABCF50500EE6735054
+Test: Encrypt
+#
+Comment: GH 1362 feed-forward carry, block 2, rounds 12
+Key: 0x000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F
+IV: 0xC163C31C00000000
+Rounds: 12
+Plaintext: r256 00
+Ciphertext: \
+    51433D526D00E44E9905838B1A69048CD0BAD2E1748057A9DF92BFCEF09FBD24 \
+    502A694CAD0490134A2E182284ABBB4AD413A9B0616E26C1C1798A3416402A4D \
+    DBC63B7FEFAA4ABF5A0D2CAE19E735FD21447051A2CEAFB4E99D540D2D47B5A8 \
+    C1D7D09D0222765E56DD7FC9F51D140456647FA84DAB10F13C3D64EC36AC97F9 \
+    444CEF5185EFF5039B00CEC268892179EC831C853C8D1BF5A0A1489690D21889 \
+    8363791430868E1BDB16E89776478B1D010000007BD1C25701F3FDEE17147D82 \
+    48E4B36903B2213832A5471546C53FD1F094DBC76A88FBED62F045A8AD87A2B2 \
+    596922216D5B6B5E4802ECA9658B2366FAE44C6F2AEC2A1DE1B25E033796624A
+Test: Encrypt
+#
+Comment: GH 1362 feed-forward carry, block 3, rounds 20
+Key: 0x000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F
+IV: 0x8BE87FDB00000000
+Rounds: 20
+Plaintext: r256 00
+Ciphertext: \
+    0BD19BFEB4F50A08BDB20235C159586875AD7770EAB9C3A29D7F218513C00FB7 \
+    89CFAD34E144B330B2992F9624E36BD05EC35ED8EBBE6D1C5EB6D999748DBC9F \
+    EEB98EE4A409A86CA038AA893C6F959848963522BF4B1A15A5FEDCE5EFE9C479 \
+    C00B46796769060892EBAC40853671E7EDED26BF14AFB3BB5FE307CE10E191C0 \
+    0D1209F28AA847D0226A587A555629B70615AC920119BA611E32533183E0CCA9 \
+    684EC26D1518279651EDA7D3F6BCA87EA5BDF5E75D1F5327E0B59DE58CA27EBE \
+    CC09DC27064200159FB5B0BAC0569CF9947F0C7793F77389E51176CA98698BDA \
+    65A3C0AD915B7D89C5D7820330F0576A02000000AC54DC348ED44704F81B12EA
+Test: Encrypt
diff --git a/chacha_simd.cpp b/chacha_simd.cpp
index 556070e..f92d024 100644
--- a/chacha_simd.cpp
+++ b/chacha_simd.cpp
@@ -309,25 +309,32 @@ void ChaCha_OperateKeystream_NEON(const word32 *state, const byte* input, byte *
         vld1q_u32(w+0), vld1q_u32(w+4), vld1q_u32(w+8)
     };
 
+    // Precompute each block's initial counter state.
+    // Feed-forward uses 32-bit word adds.
+    const uint32x4_t state3_0 = state3;
+    const uint32x4_t state3_1 = Add64(state3, CTRS[0]);
+    const uint32x4_t state3_2 = Add64(state3, CTRS[1]);
+    const uint32x4_t state3_3 = Add64(state3, CTRS[2]);
+
     uint32x4_t r0_0 = state0;
     uint32x4_t r0_1 = state1;
     uint32x4_t r0_2 = state2;
-    uint32x4_t r0_3 = state3;
+    uint32x4_t r0_3 = state3_0;
 
     uint32x4_t r1_0 = state0;
     uint32x4_t r1_1 = state1;
     uint32x4_t r1_2 = state2;
-    uint32x4_t r1_3 = Add64(r0_3, CTRS[0]);
+    uint32x4_t r1_3 = state3_1;
 
     uint32x4_t r2_0 = state0;
     uint32x4_t r2_1 = state1;
     uint32x4_t r2_2 = state2;
-    uint32x4_t r2_3 = Add64(r0_3, CTRS[1]);
+    uint32x4_t r2_3 = state3_2;
 
     uint32x4_t r3_0 = state0;
     uint32x4_t r3_1 = state1;
     uint32x4_t r3_2 = state2;
-    uint32x4_t r3_3 = Add64(r0_3, CTRS[2]);
+    uint32x4_t r3_3 = state3_3;
 
     for (int i = static_cast<int>(rounds); i > 0; i -= 2)
     {
@@ -487,25 +494,22 @@ void ChaCha_OperateKeystream_NEON(const word32 *state, const byte* input, byte *
     r0_0 = vaddq_u32(r0_0, state0);
     r0_1 = vaddq_u32(r0_1, state1);
     r0_2 = vaddq_u32(r0_2, state2);
-    r0_3 = vaddq_u32(r0_3, state3);
+    r0_3 = vaddq_u32(r0_3, state3_0);
 
     r1_0 = vaddq_u32(r1_0, state0);
     r1_1 = vaddq_u32(r1_1, state1);
     r1_2 = vaddq_u32(r1_2, state2);
-    r1_3 = vaddq_u32(r1_3, state3);
-    r1_3 = Add64(r1_3, CTRS[0]);
+    r1_3 = vaddq_u32(r1_3, state3_1);
 
     r2_0 = vaddq_u32(r2_0, state0);
     r2_1 = vaddq_u32(r2_1, state1);
     r2_2 = vaddq_u32(r2_2, state2);
-    r2_3 = vaddq_u32(r2_3, state3);
-    r2_3 = Add64(r2_3, CTRS[1]);
+    r2_3 = vaddq_u32(r2_3, state3_2);
 
     r3_0 = vaddq_u32(r3_0, state0);
     r3_1 = vaddq_u32(r3_1, state1);
     r3_2 = vaddq_u32(r3_2, state2);
-    r3_3 = vaddq_u32(r3_3, state3);
-    r3_3 = Add64(r3_3, CTRS[2]);
+    r3_3 = vaddq_u32(r3_3, state3_3);
 
     if (input)
     {
@@ -573,25 +577,32 @@ void ChaCha_OperateKeystream_SSE2(const word32 *state, const byte* input, byte *
     const __m128i state2 = _mm_load_si128(reinterpret_cast<const __m128i*>(state+2*4));
     const __m128i state3 = _mm_load_si128(reinterpret_cast<const __m128i*>(state+3*4));
 
+    // Precompute each block's initial counter state.
+    // Feed-forward uses 32-bit word adds.
+    const __m128i state3_0 = state3;
+    const __m128i state3_1 = _mm_add_epi64(state3, _mm_set_epi32(0, 0, 0, 1));
+    const __m128i state3_2 = _mm_add_epi64(state3, _mm_set_epi32(0, 0, 0, 2));
+    const __m128i state3_3 = _mm_add_epi64(state3, _mm_set_epi32(0, 0, 0, 3));
+
     __m128i r0_0 = state0;
     __m128i r0_1 = state1;
     __m128i r0_2 = state2;
-    __m128i r0_3 = state3;
+    __m128i r0_3 = state3_0;
 
     __m128i r1_0 = state0;
     __m128i r1_1 = state1;
     __m128i r1_2 = state2;
-    __m128i r1_3 = _mm_add_epi64(r0_3, _mm_set_epi32(0, 0, 0, 1));
+    __m128i r1_3 = state3_1;
 
     __m128i r2_0 = state0;
     __m128i r2_1 = state1;
     __m128i r2_2 = state2;
-    __m128i r2_3 = _mm_add_epi64(r0_3, _mm_set_epi32(0, 0, 0, 2));
+    __m128i r2_3 = state3_2;
 
     __m128i r3_0 = state0;
     __m128i r3_1 = state1;
     __m128i r3_2 = state2;
-    __m128i r3_3 = _mm_add_epi64(r0_3, _mm_set_epi32(0, 0, 0, 3));
+    __m128i r3_3 = state3_3;
 
     for (int i = static_cast<int>(rounds); i > 0; i -= 2)
     {
@@ -751,25 +762,22 @@ void ChaCha_OperateKeystream_SSE2(const word32 *state, const byte* input, byte *
     r0_0 = _mm_add_epi32(r0_0, state0);
     r0_1 = _mm_add_epi32(r0_1, state1);
     r0_2 = _mm_add_epi32(r0_2, state2);
-    r0_3 = _mm_add_epi32(r0_3, state3);
+    r0_3 = _mm_add_epi32(r0_3, state3_0);
 
     r1_0 = _mm_add_epi32(r1_0, state0);
     r1_1 = _mm_add_epi32(r1_1, state1);
     r1_2 = _mm_add_epi32(r1_2, state2);
-    r1_3 = _mm_add_epi32(r1_3, state3);
-    r1_3 = _mm_add_epi64(r1_3, _mm_set_epi32(0, 0, 0, 1));
+    r1_3 = _mm_add_epi32(r1_3, state3_1);
 
     r2_0 = _mm_add_epi32(r2_0, state0);
     r2_1 = _mm_add_epi32(r2_1, state1);
     r2_2 = _mm_add_epi32(r2_2, state2);
-    r2_3 = _mm_add_epi32(r2_3, state3);
-    r2_3 = _mm_add_epi64(r2_3, _mm_set_epi32(0, 0, 0, 2));
+    r2_3 = _mm_add_epi32(r2_3, state3_2);
 
     r3_0 = _mm_add_epi32(r3_0, state0);
     r3_1 = _mm_add_epi32(r3_1, state1);
     r3_2 = _mm_add_epi32(r3_2, state2);
-    r3_3 = _mm_add_epi32(r3_3, state3);
-    r3_3 = _mm_add_epi64(r3_3, _mm_set_epi32(0, 0, 0, 3));
+    r3_3 = _mm_add_epi32(r3_3, state3_3);
 
     if (input)
     {
@@ -844,25 +852,32 @@ inline void ChaCha_OperateKeystream_CORE(const word32 *state, const byte* input,
         {1,0,0,0}, {2,0,0,0}, {3,0,0,0}
     };
 
+    // Precompute each block's initial counter state.
+    // Feed-forward uses 32-bit word adds.
+    const uint32x4_p state3_0 = state3;
+    const uint32x4_p state3_1 = VecAdd64(state3, CTRS[0]);
+    const uint32x4_p state3_2 = VecAdd64(state3, CTRS[1]);
+    const uint32x4_p state3_3 = VecAdd64(state3, CTRS[2]);
+
     uint32x4_p r0_0 = state0;
     uint32x4_p r0_1 = state1;
     uint32x4_p r0_2 = state2;
-    uint32x4_p r0_3 = state3;
+    uint32x4_p r0_3 = state3_0;
 
     uint32x4_p r1_0 = state0;
     uint32x4_p r1_1 = state1;
     uint32x4_p r1_2 = state2;
-    uint32x4_p r1_3 = VecAdd64(r0_3, CTRS[0]);
+    uint32x4_p r1_3 = state3_1;
 
     uint32x4_p r2_0 = state0;
     uint32x4_p r2_1 = state1;
     uint32x4_p r2_2 = state2;
-    uint32x4_p r2_3 = VecAdd64(r0_3, CTRS[1]);
+    uint32x4_p r2_3 = state3_2;
 
     uint32x4_p r3_0 = state0;
     uint32x4_p r3_1 = state1;
     uint32x4_p r3_2 = state2;
-    uint32x4_p r3_3 = VecAdd64(r0_3, CTRS[2]);
+    uint32x4_p r3_3 = state3_3;
 
     for (int i = static_cast<int>(rounds); i > 0; i -= 2)
     {
@@ -1022,25 +1037,22 @@ inline void ChaCha_OperateKeystream_CORE(const word32 *state, const byte* input,
     r0_0 = VecAdd(r0_0, state0);
     r0_1 = VecAdd(r0_1, state1);
     r0_2 = VecAdd(r0_2, state2);
-    r0_3 = VecAdd(r0_3, state3);
+    r0_3 = VecAdd(r0_3, state3_0);
 
     r1_0 = VecAdd(r1_0, state0);
     r1_1 = VecAdd(r1_1, state1);
     r1_2 = VecAdd(r1_2, state2);
-    r1_3 = VecAdd(r1_3, state3);
-    r1_3 = VecAdd64(r1_3, CTRS[0]);
+    r1_3 = VecAdd(r1_3, state3_1);
 
     r2_0 = VecAdd(r2_0, state0);
     r2_1 = VecAdd(r2_1, state1);
     r2_2 = VecAdd(r2_2, state2);
-    r2_3 = VecAdd(r2_3, state3);
-    r2_3 = VecAdd64(r2_3, CTRS[1]);
+    r2_3 = VecAdd(r2_3, state3_2);
 
     r3_0 = VecAdd(r3_0, state0);
     r3_1 = VecAdd(r3_1, state1);
     r3_2 = VecAdd(r3_2, state2);
-    r3_3 = VecAdd(r3_3, state3);
-    r3_3 = VecAdd64(r3_3, CTRS[2]);
+    r3_3 = VecAdd(r3_3, state3_3);
 
     if (input)
     {