From: Carles Fernandez Date: Fri, 23 Sep 2016 17:16:27 +0000 (+0200) Subject: Add NEON, AVX and unaligned versions of SSE4.1 and SSE X-Git-Tag: archive/raspbian/1.3-2+rpi1^2~12 X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=d59cc6be4be0307a1f937a42213e3efd3d3b99d4;p=volk.git Add NEON, AVX and unaligned versions of SSE4.1 and SSE Gbp-Pq: Name 0006-Add-NEON-AVX-and-unaligned-versions-of-SSE4.1-and-SS.patch --- diff --git a/kernels/volk/volk_32f_index_max_32u.h b/kernels/volk/volk_32f_index_max_32u.h index 17b8f70..1888405 100644 --- a/kernels/volk/volk_32f_index_max_32u.h +++ b/kernels/volk/volk_32f_index_max_32u.h @@ -130,6 +130,69 @@ volk_32f_index_max_32u_a_sse4_1(uint32_t* target, const float* src0, uint32_t nu #endif /*LV_HAVE_SSE4_1*/ +#ifdef LV_HAVE_SSE4_1 +#include + +static inline void volk_32f_index_max_32u_u_sse4_1(uint32_t* target, const float* src0, uint32_t num_points) +{ + if(num_points > 0) + { + uint32_t number = 0; + const uint32_t quarterPoints = num_points / 4; + + float* inputPtr = (float*)src0; + + __m128 indexIncrementValues = _mm_set1_ps(4); + __m128 currentIndexes = _mm_set_ps(-1,-2,-3,-4); + + float max = src0[0]; + float index = 0; + __m128 maxValues = _mm_set1_ps(max); + __m128 maxValuesIndex = _mm_setzero_ps(); + __m128 compareResults; + __m128 currentValues; + + __VOLK_ATTR_ALIGNED(16) float maxValuesBuffer[4]; + __VOLK_ATTR_ALIGNED(16) float maxIndexesBuffer[4]; + + for(;number < quarterPoints; number++) + { + currentValues = _mm_loadu_ps(inputPtr); inputPtr += 4; + currentIndexes = _mm_add_ps(currentIndexes, indexIncrementValues); + compareResults = _mm_cmpgt_ps(maxValues, currentValues); + maxValuesIndex = _mm_blendv_ps(currentIndexes, maxValuesIndex, compareResults); + maxValues = _mm_blendv_ps(currentValues, maxValues, compareResults); + } + + // Calculate the largest value from the remaining 4 points + _mm_store_ps(maxValuesBuffer, maxValues); + _mm_store_ps(maxIndexesBuffer, maxValuesIndex); + + for(number = 0; number < 4; number++) + { + if(maxValuesBuffer[number] > max) + { + index = maxIndexesBuffer[number]; + max = maxValuesBuffer[number]; + } + } + + number = quarterPoints * 4; + for(;number < num_points; number++) + { + if(src0[number] > max) + { + index = number; + max = src0[number]; + } + } + target[0] = (uint32_t)index; + } +} + +#endif /*LV_HAVE_SSE4_1*/ + + #ifdef LV_HAVE_SSE #include @@ -193,6 +256,259 @@ volk_32f_index_max_32u_a_sse(uint32_t* target, const float* src0, uint32_t num_p #endif /*LV_HAVE_SSE*/ +#ifdef LV_HAVE_SSE +#include + +static inline void volk_32f_index_max_32u_u_sse(uint32_t* target, const float* src0, uint32_t num_points) +{ + if(num_points > 0) + { + uint32_t number = 0; + const uint32_t quarterPoints = num_points / 4; + + float* inputPtr = (float*)src0; + + __m128 indexIncrementValues = _mm_set1_ps(4); + __m128 currentIndexes = _mm_set_ps(-1,-2,-3,-4); + + float max = src0[0]; + float index = 0; + __m128 maxValues = _mm_set1_ps(max); + __m128 maxValuesIndex = _mm_setzero_ps(); + __m128 compareResults; + __m128 currentValues; + + __VOLK_ATTR_ALIGNED(16) float maxValuesBuffer[4]; + __VOLK_ATTR_ALIGNED(16) float maxIndexesBuffer[4]; + + for(;number < quarterPoints; number++) + { + currentValues = _mm_loadu_ps(inputPtr); inputPtr += 4; + currentIndexes = _mm_add_ps(currentIndexes, indexIncrementValues); + compareResults = _mm_cmpgt_ps(maxValues, currentValues); + maxValuesIndex = _mm_or_ps(_mm_and_ps(compareResults, maxValuesIndex) , _mm_andnot_ps(compareResults, currentIndexes)); + maxValues = _mm_or_ps(_mm_and_ps(compareResults, maxValues) , _mm_andnot_ps(compareResults, currentValues)); + } + + // Calculate the largest value from the remaining 4 points + _mm_store_ps(maxValuesBuffer, maxValues); + _mm_store_ps(maxIndexesBuffer, maxValuesIndex); + + for(number = 0; number < 4; number++) + { + if(maxValuesBuffer[number] > max) + { + index = maxIndexesBuffer[number]; + max = maxValuesBuffer[number]; + } + } + + number = quarterPoints * 4; + for(;number < num_points; number++) + { + if(src0[number] > max) + { + index = number; + max = src0[number]; + } + } + target[0] = (uint32_t)index; + } +} + +#endif /*LV_HAVE_SSE*/ + + +#ifdef LV_HAVE_AVX +#include + +static inline void volk_32f_index_max_32u_a_avx(uint32_t* target, const float* src0, uint32_t num_points) +{ + if(num_points > 0) + { + uint32_t number = 0; + const uint32_t quarterPoints = num_points / 8; + + float* inputPtr = (float*)src0; + + __m256 indexIncrementValues = _mm256_set1_ps(8); + __m256 currentIndexes = _mm256_set_ps(-1,-2,-3,-4,-5,-6,-7,-8); + + float max = src0[0]; + float index = 0; + __m256 maxValues = _mm256_set1_ps(max); + __m256 maxValuesIndex = _mm256_setzero_ps(); + __m256 compareResults; + __m256 currentValues; + + __VOLK_ATTR_ALIGNED(32) float maxValuesBuffer[8]; + __VOLK_ATTR_ALIGNED(32) float maxIndexesBuffer[8]; + + for(;number < quarterPoints; number++) + { + currentValues = _mm256_load_ps(inputPtr); inputPtr += 8; + currentIndexes = _mm256_add_ps(currentIndexes, indexIncrementValues); + compareResults = _mm256_cmp_ps(maxValues, currentValues, 0x1e); + maxValuesIndex = _mm256_blendv_ps(currentIndexes, maxValuesIndex, compareResults); + maxValues = _mm256_blendv_ps(currentValues, maxValues, compareResults); + } + + // Calculate the largest value from the remaining 8 points + _mm256_store_ps(maxValuesBuffer, maxValues); + _mm256_store_ps(maxIndexesBuffer, maxValuesIndex); + + for(number = 0; number < 8; number++) + { + if(maxValuesBuffer[number] > max) + { + index = maxIndexesBuffer[number]; + max = maxValuesBuffer[number]; + } + } + + number = quarterPoints * 8; + for(;number < num_points; number++) + { + if(src0[number] > max) + { + index = number; + max = src0[number]; + } + } + target[0] = (uint32_t)index; + } +} + +#endif /*LV_HAVE_AVX*/ + + +#ifdef LV_HAVE_AVX +#include + +static inline void volk_32f_index_max_32u_u_avx(uint32_t* target, const float* src0, uint32_t num_points) +{ + if(num_points > 0) + { + uint32_t number = 0; + const uint32_t quarterPoints = num_points / 8; + + float* inputPtr = (float*)src0; + + __m256 indexIncrementValues = _mm256_set1_ps(8); + __m256 currentIndexes = _mm256_set_ps(-1,-2,-3,-4,-5,-6,-7,-8); + + float max = src0[0]; + float index = 0; + __m256 maxValues = _mm256_set1_ps(max); + __m256 maxValuesIndex = _mm256_setzero_ps(); + __m256 compareResults; + __m256 currentValues; + + __VOLK_ATTR_ALIGNED(32) float maxValuesBuffer[8]; + __VOLK_ATTR_ALIGNED(32) float maxIndexesBuffer[8]; + + for(;number < quarterPoints; number++) + { + currentValues = _mm256_loadu_ps(inputPtr); inputPtr += 8; + currentIndexes = _mm256_add_ps(currentIndexes, indexIncrementValues); + compareResults = _mm256_cmp_ps(maxValues, currentValues, 0x1e); + maxValuesIndex = _mm256_blendv_ps(currentIndexes, maxValuesIndex, compareResults); + maxValues = _mm256_blendv_ps(currentValues, maxValues, compareResults); + } + + // Calculate the largest value from the remaining 8 points + _mm256_store_ps(maxValuesBuffer, maxValues); + _mm256_store_ps(maxIndexesBuffer, maxValuesIndex); + + for(number = 0; number < 8; number++) + { + if(maxValuesBuffer[number] > max) + { + index = maxIndexesBuffer[number]; + max = maxValuesBuffer[number]; + } + } + + number = quarterPoints * 8; + for(;number < num_points; number++) + { + if(src0[number] > max) + { + index = number; + max = src0[number]; + } + } + target[0] = (uint32_t)index; + } +} + +#endif /*LV_HAVE_AVX*/ + + +#ifdef LV_HAVE_NEON +#include + +static inline void volk_32f_index_max_32u_neon(uint32_t* target, const float* src0, uint32_t num_points) +{ + if(num_points > 0) + { + uint32_t number = 0; + const uint32_t quarterPoints = num_points / 4; + + float* inputPtr = (float*)src0; + float32x4_t indexIncrementValues = vdupq_n_f32(4); + __VOLK_ATTR_ALIGNED(16) float currentIndexes_float[4] = { -4.0f, -3.0f, -2.0f, -1.0f }; + float32x4_t currentIndexes = vld1q_f32(currentIndexes_float); + + float max = src0[0]; + float index = 0; + float32x4_t maxValues = vdupq_n_f32(max); + uint32x4_t maxValuesIndex = vmovq_n_u32(0); + uint32x4_t compareResults; + uint32x4_t currentIndexes_u; + float32x4_t currentValues; + + __VOLK_ATTR_ALIGNED(16) float maxValuesBuffer[4]; + __VOLK_ATTR_ALIGNED(16) float maxIndexesBuffer[4]; + + for(;number < quarterPoints; number++) + { + currentValues = vld1q_f32(inputPtr); inputPtr += 4; + currentIndexes = vaddq_f32(currentIndexes, indexIncrementValues); + currentIndexes_u = vcvtq_u32_f32(currentIndexes); + compareResults = vcgtq_f32( maxValues, currentValues); + maxValuesIndex = vorrq_u32( vandq_u32( compareResults, maxValuesIndex ), vbicq_u32(currentIndexes_u, compareResults) ); + maxValues = vmaxq_f32(currentValues, maxValues); + } + + // Calculate the largest value from the remaining 4 points + vst1q_f32(maxValuesBuffer, maxValues); + vst1q_f32(maxIndexesBuffer, vcvtq_f32_u32(maxValuesIndex)); + for(number = 0; number < 4; number++) + { + if(maxValuesBuffer[number] > max) + { + index = maxIndexesBuffer[number]; + max = maxValuesBuffer[number]; + } + } + + number = quarterPoints * 4; + for(;number < num_points; number++) + { + if(src0[number] > max) + { + index = number; + max = src0[number]; + } + } + target[0] = (uint32_t)index; + } +} + +#endif /*LV_HAVE_NEON*/ + + #ifdef LV_HAVE_GENERIC static inline void