From 1c89b9b7d3c71bb1a703caaf7c01c93bc9e2515f Mon Sep 17 00:00:00 2001 From: John Naylor Date: Tue, 9 Aug 2022 11:51:52 +0700 Subject: [PATCH v11 2/4] Add assert checking, run pgindent, comment changes --- src/include/port/pg_lfind.h | 78 ++++++++++++++++++++++++++----------- 1 file changed, 56 insertions(+), 22 deletions(-) diff --git a/src/include/port/pg_lfind.h b/src/include/port/pg_lfind.h index 24de544f63..fb125977b2 100644 --- a/src/include/port/pg_lfind.h +++ b/src/include/port/pg_lfind.h @@ -18,51 +18,85 @@ /* * pg_lfind32 * - * Returns true if there is an element in 'base' that equals 'key'. Otherwise, - * returns false. + * Return true if there is an element in 'base' that equals 'key', otherwise + * return false. */ static inline bool pg_lfind32(uint32 key, uint32 *base, uint32 nelem) { uint32 i = 0; - /* If possible, use SSE2 intrinsics to speed up the search. */ + /* Use SIMD intrinsics where available. */ #ifdef USE_SSE2 - const __m128i keys = _mm_set1_epi32(key); /* load 4 copies of key */ - uint32 iterations = nelem & ~0xF; /* round down to multiple of 16 */ - for (; i < iterations; i += 16) + /* + * A 16-byte register only has four 4-byte lanes. For better + * instruction-level parallelism, each loop iteration operates on a block + * of four registers. Testing has showed this is ~40% faster than using a + * block of two registers. + */ + const __m128i keys = _mm_set1_epi32(key); /* load 4 copies of key */ + uint32 iterations = nelem & ~0xF; /* round down to multiple of 16 */ + +#if defined(USE_ASSERT_CHECKING) + bool assert_result = false; + + /* pre-compute the result for assert checking */ + for (i = 0; i < nelem; i++) { - /* load the next 16 values into __m128i variables */ - const __m128i vals1 = _mm_loadu_si128((__m128i *) &base[i]); - const __m128i vals2 = _mm_loadu_si128((__m128i *) &base[i + 4]); - const __m128i vals3 = _mm_loadu_si128((__m128i *) &base[i + 8]); - const __m128i vals4 = _mm_loadu_si128((__m128i *) &base[i + 12]); + if (key == base[i]) + { + assert_result = true; + break; + } + } +#endif - /* perform the comparisons */ - const __m128i result1 = _mm_cmpeq_epi32(keys, vals1); - const __m128i result2 = _mm_cmpeq_epi32(keys, vals2); - const __m128i result3 = _mm_cmpeq_epi32(keys, vals3); - const __m128i result4 = _mm_cmpeq_epi32(keys, vals4); + for (i = 0; i < iterations; i += 16) + { + /* load the next block into 4 registers holding 4 values each */ + const __m128i vals1 = _mm_loadu_si128((__m128i *) & base[i]); + const __m128i vals2 = _mm_loadu_si128((__m128i *) & base[i + 4]); + const __m128i vals3 = _mm_loadu_si128((__m128i *) & base[i + 8]); + const __m128i vals4 = _mm_loadu_si128((__m128i *) & base[i + 12]); - /* shrink the results into a single variable */ - const __m128i tmp1 = _mm_or_si128(result1, result2); - const __m128i tmp2 = _mm_or_si128(result3, result4); - const __m128i result = _mm_or_si128(tmp1, tmp2); + /* compare each value to the key */ + const __m128i result1 = _mm_cmpeq_epi32(keys, vals1); + const __m128i result2 = _mm_cmpeq_epi32(keys, vals2); + const __m128i result3 = _mm_cmpeq_epi32(keys, vals3); + const __m128i result4 = _mm_cmpeq_epi32(keys, vals4); + + /* combine the results into a single variable */ + const __m128i tmp1 = _mm_or_si128(result1, result2); + const __m128i tmp2 = _mm_or_si128(result3, result4); + const __m128i result = _mm_or_si128(tmp1, tmp2); /* see if there was a match */ if (_mm_movemask_epi8(result) != 0) + { +#if defined(USE_ASSERT_CHECKING) + Assert(assert_result == true); +#endif return true; + } } -#endif +#endif /* USE_SSE2 */ - /* Process the remaining elements the slow way. */ + /* Process the remaining elements one at a time. */ for (; i < nelem; i++) { if (key == base[i]) + { +#if defined(USE_SSE2) && defined(USE_ASSERT_CHECKING) + Assert(assert_result == true); +#endif return true; + } } +#if defined(USE_SSE2) && defined(USE_ASSERT_CHECKING) + Assert(assert_result == false); +#endif return false; } -- 2.36.1