| /* |
| * Copyright (C) 2008 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| /* ---- includes ----------------------------------------------------------- */ |
| |
| #include "b_BasicEm/Basic.h" /* to disable some warnings in VC++ */ |
| |
| #if ( defined( WIN64 ) || defined( HW_SSE2 ) ) |
| |
| #include "emmintrin.h" |
| |
| /* disable warning "local variable 'x' used without having been initialized" */ |
| #pragma warning( disable : 4700 ) |
| |
| |
| /** Using half register (64-bit) in SSE2 to calculate dot product. |
| * This is a SSE2 reimplementation of bbs_dotProduct_intelMMX16 in Math.c. |
| * Dependencies: input vectors need to be 16-bit aligned |
| * Return Value: int32 containing resultL of dot product |
| */ |
| int32 bbs_dotProduct_64SSE2( const int16* vec1A, const int16* vec2A, uint32 sizeA ) |
| { |
| __m128i m_XMM0, m_XMM1, m_XMM2, m_XMM3, m_XMM4, m_XMM5, m_XMM6, m_XMM7, m_XMM8; |
| int16* vec1L = ( int16* )vec1A; |
| int16* vec2L = ( int16* )vec2A; |
| |
| int32 resultL = 0; |
| uint32 alignOffSetL = 0; |
| |
| /* initialize registers to 0 */ |
| m_XMM4 = _mm_xor_si128( m_XMM4, m_XMM4 ); |
| m_XMM6 = _mm_xor_si128( m_XMM6, m_XMM6 ); |
| m_XMM7 = _mm_xor_si128( m_XMM7, m_XMM7 ); |
| |
| alignOffSetL = sizeA % 16; |
| sizeA >>= 4; |
| |
| if( sizeA ) |
| { |
| while( sizeA > 0 ) |
| { |
| m_XMM0 = _mm_loadl_epi64( (__m128i *)&0[vec1L] ); |
| m_XMM7 = _mm_add_epi32( m_XMM7, m_XMM4 ); |
| |
| m_XMM1 = _mm_loadl_epi64( (__m128i *)&0[vec2L] ); |
| m_XMM7 = _mm_add_epi32( m_XMM7, m_XMM6 ); |
| |
| m_XMM2 = _mm_loadl_epi64( (__m128i *)&4[vec1L] ); |
| |
| m_XMM0 = _mm_madd_epi16( m_XMM0, m_XMM1 ); |
| |
| m_XMM3 = _mm_loadl_epi64( (__m128i *)&4[vec2L] ); |
| m_XMM4 = _mm_loadl_epi64( (__m128i *)&8[vec1L] ); |
| |
| m_XMM2 = _mm_madd_epi16( m_XMM2, m_XMM3 ); |
| |
| m_XMM5 = _mm_loadl_epi64( (__m128i *)&8[vec2L] ); |
| |
| m_XMM7 = _mm_add_epi32( m_XMM7, m_XMM0 ); |
| |
| m_XMM6 = _mm_loadl_epi64( (__m128i *)&12[vec1L] ); |
| |
| m_XMM4 = _mm_madd_epi16( m_XMM4, m_XMM5 ); |
| |
| m_XMM8 = _mm_loadl_epi64( (__m128i *)&12[vec2L] ); |
| m_XMM6 = _mm_madd_epi16( m_XMM6, m_XMM8 ); |
| |
| m_XMM7 = _mm_add_epi32( m_XMM7, m_XMM2 ); |
| |
| vec1L += 16; |
| vec2L += 16; |
| sizeA--; |
| } |
| |
| /* sum up accumulators */ |
| m_XMM7 = _mm_add_epi32( m_XMM7, m_XMM4 ); |
| |
| m_XMM7 = _mm_add_epi32( m_XMM7, m_XMM6 ); |
| |
| m_XMM0 = _mm_loadl_epi64( (__m128i *)&m_XMM7 ); |
| |
| m_XMM0 = _mm_srli_epi64( m_XMM0, 32 ); |
| |
| m_XMM7 = _mm_add_epi32( m_XMM7, m_XMM0 ); |
| |
| resultL = _mm_cvtsi128_si32( m_XMM7 ); |
| } |
| |
| /* switch statements produces faster code than loop */ |
| switch( alignOffSetL ) |
| { |
| case 15: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 14: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 13: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 12: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 11: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 10: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 9: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 8: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 7: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 6: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 5: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 4: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 3: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 2: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 1: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| } |
| |
| return resultL; |
| } |
| |
| /* ------------------------------------------------------------------------- */ |
| |
| /** Using full register (128-bit) in SSE2 to calculate dot Product. |
| * Dependencies: 16-bit aligned |
| * Return Value: int32 containing dot Product |
| */ |
| int32 bbs_dotProduct_128SSE2( const int16* vec1A, const int16* vec2A, uint32 sizeA ) |
| { |
| __m128i m_XMM0, m_XMM2, m_XMM3, m_XMM5, m_XMM6; |
| int16* vec1L = ( int16* )vec1A; |
| int16* vec2L = ( int16* )vec2A; |
| |
| int32 resultL = 0; |
| uint32 alignOffSetL = 0; |
| |
| m_XMM5 = _mm_xor_si128( m_XMM5, m_XMM5 ); |
| m_XMM6 = _mm_xor_si128( m_XMM6, m_XMM6 ); |
| |
| alignOffSetL = sizeA % 16; |
| sizeA >>= 4; |
| |
| if( sizeA ) |
| { |
| while( sizeA > 0 ) |
| { |
| m_XMM0 = _mm_load_si128( (__m128i *)&0[vec1L] ); |
| m_XMM5 = _mm_add_epi32( m_XMM5, m_XMM6 ); |
| |
| m_XMM2 = _mm_load_si128( (__m128i *)&0[vec2L] ); |
| |
| m_XMM6 = _mm_load_si128( (__m128i *)&8[vec1L] ); |
| |
| m_XMM0 = _mm_madd_epi16( m_XMM0, m_XMM2 ); |
| |
| m_XMM5 = _mm_add_epi32( m_XMM5, m_XMM0 ); |
| |
| m_XMM3 = _mm_load_si128( (__m128i *)&8[vec2L] ); |
| |
| m_XMM6 = _mm_madd_epi16( m_XMM6, m_XMM3 ); |
| |
| vec1L += 16; |
| vec2L += 16; |
| sizeA--; |
| } |
| |
| /* sum up accumulators */ |
| m_XMM5 = _mm_add_epi32( m_XMM5, m_XMM6 ); |
| |
| m_XMM0 = _mm_load_si128( (__m128i *)&m_XMM5 ); |
| |
| resultL = _mm_cvtsi128_si32( m_XMM0 ); /* 1st 32bits */ |
| |
| m_XMM0 = _mm_srli_si128( m_XMM0, 4 ); |
| |
| resultL += _mm_cvtsi128_si32( m_XMM0 ); /* 2nd 32bits */ |
| |
| m_XMM0 = _mm_srli_si128( m_XMM0, 4 ); |
| |
| resultL += _mm_cvtsi128_si32( m_XMM0 ); /* 3rd 32bits */ |
| |
| m_XMM0 = _mm_srli_si128( m_XMM0, 4 ); |
| |
| resultL += _mm_cvtsi128_si32( m_XMM0 ); /* 4th 32bits */ |
| } |
| |
| switch( alignOffSetL ) |
| { |
| case 15: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 14: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 13: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 12: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 11: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 10: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 9: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 8: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 7: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 6: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 5: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 4: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 3: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 2: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 1: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| } |
| |
| return resultL; |
| } |
| |
| /* ------------------------------------------------------------------------- */ |
| |
| |
| /** Using full register (128-bit) in SSE2 to calculate dot product (non aligned version). |
| * Dependencies: memory does not need to be 16-bit aligned |
| * Return Value: int32 containing dot product |
| */ |
| int32 bbs_dotProduct_u128SSE2( const int16* vec1A, const int16* vec2A, uint32 sizeA ) |
| { |
| __m128i m_XMM0, m_XMM2, m_XMM3, m_XMM5, m_XMM6; |
| int16* vec1L = ( int16* )vec1A; |
| int16* vec2L = ( int16* )vec2A; |
| int32 resultL = 0; |
| uint32 alignOffSetL = 0; |
| |
| /* initialize registers to 0 */ |
| m_XMM5 = _mm_xor_si128( m_XMM5, m_XMM5 ); |
| m_XMM6 = _mm_xor_si128( m_XMM6, m_XMM6 ); |
| |
| |
| alignOffSetL = sizeA % 16; |
| sizeA >>= 4; |
| |
| if( sizeA ) |
| { |
| while( sizeA > 0 ) |
| { |
| m_XMM0 = _mm_loadu_si128( (__m128i *)&0[vec1L] ); |
| m_XMM5 = _mm_add_epi32( m_XMM5, m_XMM6 ); |
| |
| m_XMM2 = _mm_loadu_si128( (__m128i *)&0[vec2L] ); |
| |
| m_XMM6 = _mm_loadu_si128( (__m128i *)&8[vec1L] ); |
| |
| m_XMM0 = _mm_madd_epi16( m_XMM0, m_XMM2 ); |
| |
| m_XMM5 = _mm_add_epi32( m_XMM5, m_XMM0 ); |
| |
| m_XMM3 = _mm_loadu_si128( (__m128i *)&8[vec2L] ); |
| |
| m_XMM6 = _mm_madd_epi16( m_XMM6, m_XMM3 ); |
| |
| vec1L += 16; |
| vec2L += 16; |
| sizeA--; |
| } |
| |
| /* sum up accumulators */ |
| m_XMM5 = _mm_add_epi32( m_XMM5, m_XMM6 ); |
| |
| m_XMM0 = _mm_loadu_si128( (__m128i *)&m_XMM5 ); |
| |
| resultL = _mm_cvtsi128_si32( m_XMM0 ); /* 1st 32bits */ |
| |
| m_XMM0 = _mm_srli_si128( m_XMM0, 4 ); |
| |
| resultL += _mm_cvtsi128_si32( m_XMM0 ); /* 2nd 32bits */ |
| |
| m_XMM0 = _mm_srli_si128( m_XMM0, 4 ); |
| |
| resultL += _mm_cvtsi128_si32( m_XMM0 ); /* 3rd 32bits */ |
| |
| m_XMM0 = _mm_srli_si128( m_XMM0, 4 ); |
| |
| resultL += _mm_cvtsi128_si32( m_XMM0 ); /* 4th 32bits */ |
| } |
| |
| |
| switch( alignOffSetL ) |
| { |
| case 15: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 14: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 13: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 12: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 11: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 10: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 9: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 8: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 7: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 6: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 5: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 4: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 3: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 2: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| case 1: |
| resultL += ( int32 )*vec1L++ * *vec2L++; |
| } |
| |
| return resultL; |
| } |
| |
| /* ------------------------------------------------------------------------- */ |
| |
| #endif /* HW_SSE2 */ |
