58 #ifndef INCLUDED_volk_32fc_x2_dot_prod_32fc_u_H 59 #define INCLUDED_volk_32fc_x2_dot_prod_32fc_u_H 67 #ifdef LV_HAVE_GENERIC 73 unsigned int num_points)
76 float* res = (
float*)result;
77 float* in = (
float*)input;
78 float* tp = (
float*)taps;
79 unsigned int n_2_ccomplex_blocks = num_points / 2;
81 float sum0[2] = { 0, 0 };
82 float sum1[2] = { 0, 0 };
85 for (
i = 0;
i < n_2_ccomplex_blocks; ++
i) {
86 sum0[0] += in[0] * tp[0] - in[1] * tp[1];
87 sum0[1] += in[0] * tp[1] + in[1] * tp[0];
88 sum1[0] += in[2] * tp[2] - in[3] * tp[3];
89 sum1[1] += in[2] * tp[3] + in[3] * tp[2];
95 res[0] = sum0[0] + sum1[0];
96 res[1] = sum0[1] + sum1[1];
100 *result += input[num_points - 1] * taps[num_points - 1];
107 #if LV_HAVE_SSE && LV_HAVE_64 109 static inline void volk_32fc_x2_dot_prod_32fc_u_sse_64(
lv_32fc_t* result,
112 unsigned int num_points)
115 const unsigned int num_bytes = num_points * 8;
116 unsigned int isodd = num_points & 1;
119 "# ccomplex_dotprod_generic (float* result, const float *input,\n\t" 120 "# const float *taps, unsigned num_bytes)\n\t" 121 "# float sum0 = 0;\n\t" 122 "# float sum1 = 0;\n\t" 123 "# float sum2 = 0;\n\t" 124 "# float sum3 = 0;\n\t" 126 "# sum0 += input[0] * taps[0] - input[1] * taps[1];\n\t" 127 "# sum1 += input[0] * taps[1] + input[1] * taps[0];\n\t" 128 "# sum2 += input[2] * taps[2] - input[3] * taps[3];\n\t" 129 "# sum3 += input[2] * taps[3] + input[3] * taps[2];\n\t" 132 "# } while (--n_2_ccomplex_blocks != 0);\n\t" 133 "# result[0] = sum0 + sum2;\n\t" 134 "# result[1] = sum1 + sum3;\n\t" 135 "# TODO: prefetch and better scheduling\n\t" 136 " xor %%r9, %%r9\n\t" 137 " xor %%r10, %%r10\n\t" 138 " movq %%rcx, %%rax\n\t" 139 " movq %%rcx, %%r8\n\t" 140 " movq %[rsi], %%r9\n\t" 141 " movq %[rdx], %%r10\n\t" 142 " xorps %%xmm6, %%xmm6 # zero accumulators\n\t" 143 " movups 0(%%r9), %%xmm0\n\t" 144 " xorps %%xmm7, %%xmm7 # zero accumulators\n\t" 145 " movups 0(%%r10), %%xmm2\n\t" 146 " shr $5, %%rax # rax = n_2_ccomplex_blocks / 2\n\t" 148 " jmp .%=L1_test\n\t" 149 " # 4 taps / loop\n\t" 150 " # something like ?? cycles / loop\n\t" 152 "# complex prod: C += A * B, w/ temp Z & Y (or B), xmmPN=$0x8000000080000000\n\t" 153 "# movups (%%r9), %%xmmA\n\t" 154 "# movups (%%r10), %%xmmB\n\t" 155 "# movups %%xmmA, %%xmmZ\n\t" 156 "# shufps $0xb1, %%xmmZ, %%xmmZ # swap internals\n\t" 157 "# mulps %%xmmB, %%xmmA\n\t" 158 "# mulps %%xmmZ, %%xmmB\n\t" 159 "# # SSE replacement for: pfpnacc %%xmmB, %%xmmA\n\t" 160 "# xorps %%xmmPN, %%xmmA\n\t" 161 "# movups %%xmmA, %%xmmZ\n\t" 162 "# unpcklps %%xmmB, %%xmmA\n\t" 163 "# unpckhps %%xmmB, %%xmmZ\n\t" 164 "# movups %%xmmZ, %%xmmY\n\t" 165 "# shufps $0x44, %%xmmA, %%xmmZ # b01000100\n\t" 166 "# shufps $0xee, %%xmmY, %%xmmA # b11101110\n\t" 167 "# addps %%xmmZ, %%xmmA\n\t" 168 "# addps %%xmmA, %%xmmC\n\t" 169 "# A=xmm0, B=xmm2, Z=xmm4\n\t" 170 "# A'=xmm1, B'=xmm3, Z'=xmm5\n\t" 171 " movups 16(%%r9), %%xmm1\n\t" 172 " movups %%xmm0, %%xmm4\n\t" 173 " mulps %%xmm2, %%xmm0\n\t" 174 " shufps $0xb1, %%xmm4, %%xmm4 # swap internals\n\t" 175 " movups 16(%%r10), %%xmm3\n\t" 176 " movups %%xmm1, %%xmm5\n\t" 177 " addps %%xmm0, %%xmm6\n\t" 178 " mulps %%xmm3, %%xmm1\n\t" 179 " shufps $0xb1, %%xmm5, %%xmm5 # swap internals\n\t" 180 " addps %%xmm1, %%xmm6\n\t" 181 " mulps %%xmm4, %%xmm2\n\t" 182 " movups 32(%%r9), %%xmm0\n\t" 183 " addps %%xmm2, %%xmm7\n\t" 184 " mulps %%xmm5, %%xmm3\n\t" 186 " movups 32(%%r10), %%xmm2\n\t" 187 " addps %%xmm3, %%xmm7\n\t" 188 " add $32, %%r10\n\t" 192 " # We've handled the bulk of multiplies up to here.\n\t" 193 " # Let's sse if original n_2_ccomplex_blocks was odd.\n\t" 194 " # If so, we've got 2 more taps to do.\n\t" 197 " # The count was odd, do 2 more taps.\n\t" 198 " # Note that we've already got mm0/mm2 preloaded\n\t" 199 " # from the main loop.\n\t" 200 " movups %%xmm0, %%xmm4\n\t" 201 " mulps %%xmm2, %%xmm0\n\t" 202 " shufps $0xb1, %%xmm4, %%xmm4 # swap internals\n\t" 203 " addps %%xmm0, %%xmm6\n\t" 204 " mulps %%xmm4, %%xmm2\n\t" 205 " addps %%xmm2, %%xmm7\n\t" 207 " # neg inversor\n\t" 208 " xorps %%xmm1, %%xmm1\n\t" 209 " mov $0x80000000, %%r9\n\t" 210 " movd %%r9, %%xmm1\n\t" 211 " shufps $0x11, %%xmm1, %%xmm1 # b00010001 # 0 -0 0 -0\n\t" 213 " xorps %%xmm1, %%xmm6\n\t" 214 " movups %%xmm6, %%xmm2\n\t" 215 " unpcklps %%xmm7, %%xmm6\n\t" 216 " unpckhps %%xmm7, %%xmm2\n\t" 217 " movups %%xmm2, %%xmm3\n\t" 218 " shufps $0x44, %%xmm6, %%xmm2 # b01000100\n\t" 219 " shufps $0xee, %%xmm3, %%xmm6 # b11101110\n\t" 220 " addps %%xmm2, %%xmm6\n\t" 221 " # xmm6 = r1 i2 r3 i4\n\t" 222 " movhlps %%xmm6, %%xmm4 # xmm4 = r3 i4 ?? ??\n\t" 223 " addps %%xmm4, %%xmm6 # xmm6 = r1+r3 i2+i4 ?? ??\n\t" 224 " movlps %%xmm6, (%[rdi]) # store low 2x32 bits (complex) " 227 : [rsi]
"r"(input), [rdx]
"r"(taps),
"c"(num_bytes), [rdi]
"r"(result)
228 :
"rax",
"r8",
"r9",
"r10");
232 *result += input[num_points - 1] * taps[num_points - 1];
243 #include <pmmintrin.h> 248 unsigned int num_points)
252 memset(&dotProduct, 0x0, 2 *
sizeof(
float));
254 unsigned int number = 0;
255 const unsigned int halfPoints = num_points / 2;
256 unsigned int isodd = num_points & 1;
258 __m128 x, y, yl, yh, z, tmp1, tmp2, dotProdVal;
263 dotProdVal = _mm_setzero_ps();
265 for (; number < halfPoints; number++) {
267 x = _mm_loadu_ps((
float*)a);
268 y = _mm_loadu_ps((
float*)b);
270 yl = _mm_moveldup_ps(y);
271 yh = _mm_movehdup_ps(y);
273 tmp1 = _mm_mul_ps(x, yl);
275 x = _mm_shuffle_ps(x, x, 0xB1);
277 tmp2 = _mm_mul_ps(x, yh);
279 z = _mm_addsub_ps(tmp1,
283 _mm_add_ps(dotProdVal, z);
291 _mm_storeu_ps((
float*)dotProductVector,
294 dotProduct += (dotProductVector[0] + dotProductVector[1]);
297 dotProduct += input[num_points - 1] * taps[num_points - 1];
300 *result = dotProduct;
391 #include <immintrin.h> 396 unsigned int num_points)
399 unsigned int isodd = num_points & 3;
402 memset(&dotProduct, 0x0, 2 *
sizeof(
float));
404 unsigned int number = 0;
405 const unsigned int quarterPoints = num_points / 4;
407 __m256 x, y, yl, yh, z, tmp1, tmp2, dotProdVal;
412 dotProdVal = _mm256_setzero_ps();
414 for (; number < quarterPoints; number++) {
415 x = _mm256_loadu_ps((
float*)a);
416 y = _mm256_loadu_ps((
float*)b);
418 yl = _mm256_moveldup_ps(y);
419 yh = _mm256_movehdup_ps(y);
421 tmp1 = _mm256_mul_ps(x, yl);
423 x = _mm256_shuffle_ps(x, x, 0xB1);
425 tmp2 = _mm256_mul_ps(x, yh);
427 z = _mm256_addsub_ps(tmp1,
430 dotProdVal = _mm256_add_ps(dotProdVal,
439 _mm256_storeu_ps((
float*)dotProductVector,
442 dotProduct += (dotProductVector[0] + dotProductVector[1] + dotProductVector[2] +
443 dotProductVector[3]);
445 for (
i = num_points - isodd;
i < num_points;
i++) {
446 dotProduct += input[
i] * taps[
i];
449 *result = dotProduct;
454 #if LV_HAVE_AVX && LV_HAVE_FMA 455 #include <immintrin.h> 457 static inline void volk_32fc_x2_dot_prod_32fc_u_avx_fma(
lv_32fc_t* result,
460 unsigned int num_points)
463 unsigned int isodd = num_points & 3;
466 memset(&dotProduct, 0x0, 2 *
sizeof(
float));
468 unsigned int number = 0;
469 const unsigned int quarterPoints = num_points / 4;
471 __m256 x, y, yl, yh, z, tmp1, tmp2, dotProdVal;
476 dotProdVal = _mm256_setzero_ps();
478 for (; number < quarterPoints; number++) {
480 x = _mm256_loadu_ps((
float*)a);
481 y = _mm256_loadu_ps((
float*)b);
483 yl = _mm256_moveldup_ps(y);
484 yh = _mm256_movehdup_ps(y);
488 x = _mm256_shuffle_ps(x, x, 0xB1);
490 tmp2 = _mm256_mul_ps(x, yh);
492 z = _mm256_fmaddsub_ps(
495 dotProdVal = _mm256_add_ps(dotProdVal,
504 _mm256_storeu_ps((
float*)dotProductVector,
507 dotProduct += (dotProductVector[0] + dotProductVector[1] + dotProductVector[2] +
508 dotProductVector[3]);
510 for (
i = num_points - isodd;
i < num_points;
i++) {
511 dotProduct += input[
i] * taps[
i];
514 *result = dotProduct;
521 #ifndef INCLUDED_volk_32fc_x2_dot_prod_32fc_a_H 522 #define INCLUDED_volk_32fc_x2_dot_prod_32fc_a_H 530 #ifdef LV_HAVE_GENERIC 536 unsigned int num_points)
539 const unsigned int num_bytes = num_points * 8;
541 float* res = (
float*)result;
542 float* in = (
float*)input;
543 float* tp = (
float*)taps;
544 unsigned int n_2_ccomplex_blocks = num_bytes >> 4;
546 float sum0[2] = { 0, 0 };
547 float sum1[2] = { 0, 0 };
550 for (
i = 0;
i < n_2_ccomplex_blocks; ++
i) {
551 sum0[0] += in[0] * tp[0] - in[1] * tp[1];
552 sum0[1] += in[0] * tp[1] + in[1] * tp[0];
553 sum1[0] += in[2] * tp[2] - in[3] * tp[3];
554 sum1[1] += in[2] * tp[3] + in[3] * tp[2];
560 res[0] = sum0[0] + sum1[0];
561 res[1] = sum0[1] + sum1[1];
563 if (num_points & 1) {
564 *result += input[num_points - 1] * taps[num_points - 1];
571 #if LV_HAVE_SSE && LV_HAVE_64 574 static inline void volk_32fc_x2_dot_prod_32fc_a_sse_64(
lv_32fc_t* result,
577 unsigned int num_points)
580 const unsigned int num_bytes = num_points * 8;
581 unsigned int isodd = num_points & 1;
584 "# ccomplex_dotprod_generic (float* result, const float *input,\n\t" 585 "# const float *taps, unsigned num_bytes)\n\t" 586 "# float sum0 = 0;\n\t" 587 "# float sum1 = 0;\n\t" 588 "# float sum2 = 0;\n\t" 589 "# float sum3 = 0;\n\t" 591 "# sum0 += input[0] * taps[0] - input[1] * taps[1];\n\t" 592 "# sum1 += input[0] * taps[1] + input[1] * taps[0];\n\t" 593 "# sum2 += input[2] * taps[2] - input[3] * taps[3];\n\t" 594 "# sum3 += input[2] * taps[3] + input[3] * taps[2];\n\t" 597 "# } while (--n_2_ccomplex_blocks != 0);\n\t" 598 "# result[0] = sum0 + sum2;\n\t" 599 "# result[1] = sum1 + sum3;\n\t" 600 "# TODO: prefetch and better scheduling\n\t" 601 " xor %%r9, %%r9\n\t" 602 " xor %%r10, %%r10\n\t" 603 " movq %%rcx, %%rax\n\t" 604 " movq %%rcx, %%r8\n\t" 605 " movq %[rsi], %%r9\n\t" 606 " movq %[rdx], %%r10\n\t" 607 " xorps %%xmm6, %%xmm6 # zero accumulators\n\t" 608 " movaps 0(%%r9), %%xmm0\n\t" 609 " xorps %%xmm7, %%xmm7 # zero accumulators\n\t" 610 " movaps 0(%%r10), %%xmm2\n\t" 611 " shr $5, %%rax # rax = n_2_ccomplex_blocks / 2\n\t" 613 " jmp .%=L1_test\n\t" 614 " # 4 taps / loop\n\t" 615 " # something like ?? cycles / loop\n\t" 617 "# complex prod: C += A * B, w/ temp Z & Y (or B), xmmPN=$0x8000000080000000\n\t" 618 "# movaps (%%r9), %%xmmA\n\t" 619 "# movaps (%%r10), %%xmmB\n\t" 620 "# movaps %%xmmA, %%xmmZ\n\t" 621 "# shufps $0xb1, %%xmmZ, %%xmmZ # swap internals\n\t" 622 "# mulps %%xmmB, %%xmmA\n\t" 623 "# mulps %%xmmZ, %%xmmB\n\t" 624 "# # SSE replacement for: pfpnacc %%xmmB, %%xmmA\n\t" 625 "# xorps %%xmmPN, %%xmmA\n\t" 626 "# movaps %%xmmA, %%xmmZ\n\t" 627 "# unpcklps %%xmmB, %%xmmA\n\t" 628 "# unpckhps %%xmmB, %%xmmZ\n\t" 629 "# movaps %%xmmZ, %%xmmY\n\t" 630 "# shufps $0x44, %%xmmA, %%xmmZ # b01000100\n\t" 631 "# shufps $0xee, %%xmmY, %%xmmA # b11101110\n\t" 632 "# addps %%xmmZ, %%xmmA\n\t" 633 "# addps %%xmmA, %%xmmC\n\t" 634 "# A=xmm0, B=xmm2, Z=xmm4\n\t" 635 "# A'=xmm1, B'=xmm3, Z'=xmm5\n\t" 636 " movaps 16(%%r9), %%xmm1\n\t" 637 " movaps %%xmm0, %%xmm4\n\t" 638 " mulps %%xmm2, %%xmm0\n\t" 639 " shufps $0xb1, %%xmm4, %%xmm4 # swap internals\n\t" 640 " movaps 16(%%r10), %%xmm3\n\t" 641 " movaps %%xmm1, %%xmm5\n\t" 642 " addps %%xmm0, %%xmm6\n\t" 643 " mulps %%xmm3, %%xmm1\n\t" 644 " shufps $0xb1, %%xmm5, %%xmm5 # swap internals\n\t" 645 " addps %%xmm1, %%xmm6\n\t" 646 " mulps %%xmm4, %%xmm2\n\t" 647 " movaps 32(%%r9), %%xmm0\n\t" 648 " addps %%xmm2, %%xmm7\n\t" 649 " mulps %%xmm5, %%xmm3\n\t" 651 " movaps 32(%%r10), %%xmm2\n\t" 652 " addps %%xmm3, %%xmm7\n\t" 653 " add $32, %%r10\n\t" 657 " # We've handled the bulk of multiplies up to here.\n\t" 658 " # Let's sse if original n_2_ccomplex_blocks was odd.\n\t" 659 " # If so, we've got 2 more taps to do.\n\t" 662 " # The count was odd, do 2 more taps.\n\t" 663 " # Note that we've already got mm0/mm2 preloaded\n\t" 664 " # from the main loop.\n\t" 665 " movaps %%xmm0, %%xmm4\n\t" 666 " mulps %%xmm2, %%xmm0\n\t" 667 " shufps $0xb1, %%xmm4, %%xmm4 # swap internals\n\t" 668 " addps %%xmm0, %%xmm6\n\t" 669 " mulps %%xmm4, %%xmm2\n\t" 670 " addps %%xmm2, %%xmm7\n\t" 672 " # neg inversor\n\t" 673 " xorps %%xmm1, %%xmm1\n\t" 674 " mov $0x80000000, %%r9\n\t" 675 " movd %%r9, %%xmm1\n\t" 676 " shufps $0x11, %%xmm1, %%xmm1 # b00010001 # 0 -0 0 -0\n\t" 678 " xorps %%xmm1, %%xmm6\n\t" 679 " movaps %%xmm6, %%xmm2\n\t" 680 " unpcklps %%xmm7, %%xmm6\n\t" 681 " unpckhps %%xmm7, %%xmm2\n\t" 682 " movaps %%xmm2, %%xmm3\n\t" 683 " shufps $0x44, %%xmm6, %%xmm2 # b01000100\n\t" 684 " shufps $0xee, %%xmm3, %%xmm6 # b11101110\n\t" 685 " addps %%xmm2, %%xmm6\n\t" 686 " # xmm6 = r1 i2 r3 i4\n\t" 687 " movhlps %%xmm6, %%xmm4 # xmm4 = r3 i4 ?? ??\n\t" 688 " addps %%xmm4, %%xmm6 # xmm6 = r1+r3 i2+i4 ?? ??\n\t" 689 " movlps %%xmm6, (%[rdi]) # store low 2x32 bits (complex) " 692 : [rsi]
"r"(input), [rdx]
"r"(taps),
"c"(num_bytes), [rdi]
"r"(result)
693 :
"rax",
"r8",
"r9",
"r10");
697 *result += input[num_points - 1] * taps[num_points - 1];
705 #if LV_HAVE_SSE && LV_HAVE_32 707 static inline void volk_32fc_x2_dot_prod_32fc_a_sse_32(
lv_32fc_t* result,
710 unsigned int num_points)
716 const unsigned int num_bytes = num_points*8;
717 unsigned int isodd = num_points & 1;
722 " #movl %%esp, %%ebp\n\t" 723 " movl 12(%%ebp), %%eax # input\n\t" 724 " movl 16(%%ebp), %%edx # taps\n\t" 725 " movl 20(%%ebp), %%ecx # n_bytes\n\t" 726 " xorps %%xmm6, %%xmm6 # zero accumulators\n\t" 727 " movaps 0(%%eax), %%xmm0\n\t" 728 " xorps %%xmm7, %%xmm7 # zero accumulators\n\t" 729 " movaps 0(%%edx), %%xmm2\n\t" 730 " shrl $5, %%ecx # ecx = n_2_ccomplex_blocks / 2\n\t" 731 " jmp .%=L1_test\n\t" 732 " # 4 taps / loop\n\t" 733 " # something like ?? cycles / loop\n\t" 735 "# complex prod: C += A * B, w/ temp Z & Y (or B), xmmPN=$0x8000000080000000\n\t" 736 "# movaps (%%eax), %%xmmA\n\t" 737 "# movaps (%%edx), %%xmmB\n\t" 738 "# movaps %%xmmA, %%xmmZ\n\t" 739 "# shufps $0xb1, %%xmmZ, %%xmmZ # swap internals\n\t" 740 "# mulps %%xmmB, %%xmmA\n\t" 741 "# mulps %%xmmZ, %%xmmB\n\t" 742 "# # SSE replacement for: pfpnacc %%xmmB, %%xmmA\n\t" 743 "# xorps %%xmmPN, %%xmmA\n\t" 744 "# movaps %%xmmA, %%xmmZ\n\t" 745 "# unpcklps %%xmmB, %%xmmA\n\t" 746 "# unpckhps %%xmmB, %%xmmZ\n\t" 747 "# movaps %%xmmZ, %%xmmY\n\t" 748 "# shufps $0x44, %%xmmA, %%xmmZ # b01000100\n\t" 749 "# shufps $0xee, %%xmmY, %%xmmA # b11101110\n\t" 750 "# addps %%xmmZ, %%xmmA\n\t" 751 "# addps %%xmmA, %%xmmC\n\t" 752 "# A=xmm0, B=xmm2, Z=xmm4\n\t" 753 "# A'=xmm1, B'=xmm3, Z'=xmm5\n\t" 754 " movaps 16(%%eax), %%xmm1\n\t" 755 " movaps %%xmm0, %%xmm4\n\t" 756 " mulps %%xmm2, %%xmm0\n\t" 757 " shufps $0xb1, %%xmm4, %%xmm4 # swap internals\n\t" 758 " movaps 16(%%edx), %%xmm3\n\t" 759 " movaps %%xmm1, %%xmm5\n\t" 760 " addps %%xmm0, %%xmm6\n\t" 761 " mulps %%xmm3, %%xmm1\n\t" 762 " shufps $0xb1, %%xmm5, %%xmm5 # swap internals\n\t" 763 " addps %%xmm1, %%xmm6\n\t" 764 " mulps %%xmm4, %%xmm2\n\t" 765 " movaps 32(%%eax), %%xmm0\n\t" 766 " addps %%xmm2, %%xmm7\n\t" 767 " mulps %%xmm5, %%xmm3\n\t" 768 " addl $32, %%eax\n\t" 769 " movaps 32(%%edx), %%xmm2\n\t" 770 " addps %%xmm3, %%xmm7\n\t" 771 " addl $32, %%edx\n\t" 775 " # We've handled the bulk of multiplies up to here.\n\t" 776 " # Let's sse if original n_2_ccomplex_blocks was odd.\n\t" 777 " # If so, we've got 2 more taps to do.\n\t" 778 " movl 20(%%ebp), %%ecx # n_2_ccomplex_blocks\n\t" 779 " shrl $4, %%ecx\n\t" 780 " andl $1, %%ecx\n\t" 782 " # The count was odd, do 2 more taps.\n\t" 783 " # Note that we've already got mm0/mm2 preloaded\n\t" 784 " # from the main loop.\n\t" 785 " movaps %%xmm0, %%xmm4\n\t" 786 " mulps %%xmm2, %%xmm0\n\t" 787 " shufps $0xb1, %%xmm4, %%xmm4 # swap internals\n\t" 788 " addps %%xmm0, %%xmm6\n\t" 789 " mulps %%xmm4, %%xmm2\n\t" 790 " addps %%xmm2, %%xmm7\n\t" 792 " # neg inversor\n\t" 793 " movl 8(%%ebp), %%eax \n\t" 794 " xorps %%xmm1, %%xmm1\n\t" 795 " movl $0x80000000, (%%eax)\n\t" 796 " movss (%%eax), %%xmm1\n\t" 797 " shufps $0x11, %%xmm1, %%xmm1 # b00010001 # 0 -0 0 -0\n\t" 799 " xorps %%xmm1, %%xmm6\n\t" 800 " movaps %%xmm6, %%xmm2\n\t" 801 " unpcklps %%xmm7, %%xmm6\n\t" 802 " unpckhps %%xmm7, %%xmm2\n\t" 803 " movaps %%xmm2, %%xmm3\n\t" 804 " shufps $0x44, %%xmm6, %%xmm2 # b01000100\n\t" 805 " shufps $0xee, %%xmm3, %%xmm6 # b11101110\n\t" 806 " addps %%xmm2, %%xmm6\n\t" 807 " # xmm6 = r1 i2 r3 i4\n\t" 808 " #movl 8(%%ebp), %%eax # @result\n\t" 809 " movhlps %%xmm6, %%xmm4 # xmm4 = r3 i4 ?? ??\n\t" 810 " addps %%xmm4, %%xmm6 # xmm6 = r1+r3 i2+i4 ?? ??\n\t" 811 " movlps %%xmm6, (%%eax) # store low 2x32 bits (complex) to memory\n\t" 815 :
"eax",
"ecx",
"edx" 819 int getem = num_bytes % 16;
822 *result += (input[num_points - 1] * taps[num_points - 1]);
833 #include <pmmintrin.h> 838 unsigned int num_points)
841 const unsigned int num_bytes = num_points * 8;
842 unsigned int isodd = num_points & 1;
845 memset(&dotProduct, 0x0, 2 *
sizeof(
float));
847 unsigned int number = 0;
848 const unsigned int halfPoints = num_bytes >> 4;
850 __m128 x, y, yl, yh, z, tmp1, tmp2, dotProdVal;
855 dotProdVal = _mm_setzero_ps();
857 for (; number < halfPoints; number++) {
859 x = _mm_load_ps((
float*)a);
860 y = _mm_load_ps((
float*)b);
862 yl = _mm_moveldup_ps(y);
863 yh = _mm_movehdup_ps(y);
865 tmp1 = _mm_mul_ps(x, yl);
867 x = _mm_shuffle_ps(x, x, 0xB1);
869 tmp2 = _mm_mul_ps(x, yh);
871 z = _mm_addsub_ps(tmp1,
875 _mm_add_ps(dotProdVal, z);
883 _mm_store_ps((
float*)dotProductVector,
886 dotProduct += (dotProductVector[0] + dotProductVector[1]);
889 dotProduct += input[num_points - 1] * taps[num_points - 1];
892 *result = dotProduct;
983 #include <arm_neon.h> 988 unsigned int num_points)
991 unsigned int quarter_points = num_points / 4;
998 float32x4x2_t a_val, b_val, c_val, accumulator;
999 float32x4x2_t tmp_real, tmp_imag;
1000 accumulator.val[0] = vdupq_n_f32(0);
1001 accumulator.val[1] = vdupq_n_f32(0);
1003 for (number = 0; number < quarter_points; ++number) {
1004 a_val = vld2q_f32((
float*)a_ptr);
1005 b_val = vld2q_f32((
float*)b_ptr);
1011 tmp_real.val[0] = vmulq_f32(a_val.val[0], b_val.val[0]);
1013 tmp_real.val[1] = vmulq_f32(a_val.val[1], b_val.val[1]);
1017 tmp_imag.val[0] = vmulq_f32(a_val.val[0], b_val.val[1]);
1019 tmp_imag.val[1] = vmulq_f32(a_val.val[1], b_val.val[0]);
1021 c_val.val[0] = vsubq_f32(tmp_real.val[0], tmp_real.val[1]);
1022 c_val.val[1] = vaddq_f32(tmp_imag.val[0], tmp_imag.val[1]);
1024 accumulator.val[0] = vaddq_f32(accumulator.val[0], c_val.val[0]);
1025 accumulator.val[1] = vaddq_f32(accumulator.val[1], c_val.val[1]);
1031 vst2q_f32((
float*)accum_result, accumulator);
1032 *result = accum_result[0] + accum_result[1] + accum_result[2] + accum_result[3];
1035 for (number = quarter_points * 4; number < num_points; ++number) {
1036 *result += (*a_ptr++) * (*b_ptr++);
1042 #include <arm_neon.h> 1046 unsigned int num_points)
1049 unsigned int quarter_points = num_points / 4;
1050 unsigned int number;
1056 float32x4x2_t a_val, b_val, accumulator;
1057 float32x4x2_t tmp_imag;
1058 accumulator.val[0] = vdupq_n_f32(0);
1059 accumulator.val[1] = vdupq_n_f32(0);
1061 for (number = 0; number < quarter_points; ++number) {
1062 a_val = vld2q_f32((
float*)a_ptr);
1063 b_val = vld2q_f32((
float*)b_ptr);
1068 tmp_imag.val[1] = vmulq_f32(a_val.val[1], b_val.val[0]);
1069 tmp_imag.val[0] = vmulq_f32(a_val.val[0], b_val.val[0]);
1072 tmp_imag.val[1] = vmlaq_f32(tmp_imag.val[1], a_val.val[0], b_val.val[1]);
1073 tmp_imag.val[0] = vmlsq_f32(tmp_imag.val[0], a_val.val[1], b_val.val[1]);
1075 accumulator.val[0] = vaddq_f32(accumulator.val[0], tmp_imag.val[0]);
1076 accumulator.val[1] = vaddq_f32(accumulator.val[1], tmp_imag.val[1]);
1083 vst2q_f32((
float*)accum_result, accumulator);
1084 *result = accum_result[0] + accum_result[1] + accum_result[2] + accum_result[3];
1087 for (number = quarter_points * 4; number < num_points; ++number) {
1088 *result += (*a_ptr++) * (*b_ptr++);
1097 unsigned int num_points)
1100 unsigned int quarter_points = num_points / 4;
1101 unsigned int number;
1107 float32x4x2_t a_val, b_val, accumulator1, accumulator2;
1108 accumulator1.val[0] = vdupq_n_f32(0);
1109 accumulator1.val[1] = vdupq_n_f32(0);
1110 accumulator2.val[0] = vdupq_n_f32(0);
1111 accumulator2.val[1] = vdupq_n_f32(0);
1113 for (number = 0; number < quarter_points; ++number) {
1114 a_val = vld2q_f32((
float*)a_ptr);
1115 b_val = vld2q_f32((
float*)b_ptr);
1120 accumulator1.val[0] = vmlaq_f32(accumulator1.val[0], a_val.val[0], b_val.val[0]);
1121 accumulator1.val[1] = vmlaq_f32(accumulator1.val[1], a_val.val[0], b_val.val[1]);
1122 accumulator2.val[0] = vmlsq_f32(accumulator2.val[0], a_val.val[1], b_val.val[1]);
1123 accumulator2.val[1] = vmlaq_f32(accumulator2.val[1], a_val.val[1], b_val.val[0]);
1128 accumulator1.val[0] = vaddq_f32(accumulator1.val[0], accumulator2.val[0]);
1129 accumulator1.val[1] = vaddq_f32(accumulator1.val[1], accumulator2.val[1]);
1131 vst2q_f32((
float*)accum_result, accumulator1);
1132 *result = accum_result[0] + accum_result[1] + accum_result[2] + accum_result[3];
1135 for (number = quarter_points * 4; number < num_points; ++number) {
1136 *result += (*a_ptr++) * (*b_ptr++);
1145 unsigned int num_points)
1150 unsigned int quarter_points = num_points / 8;
1151 unsigned int number;
1157 float32x4x4_t a_val, b_val, accumulator1, accumulator2;
1158 float32x4x2_t reduced_accumulator;
1159 accumulator1.val[0] = vdupq_n_f32(0);
1160 accumulator1.val[1] = vdupq_n_f32(0);
1161 accumulator1.val[2] = vdupq_n_f32(0);
1162 accumulator1.val[3] = vdupq_n_f32(0);
1163 accumulator2.val[0] = vdupq_n_f32(0);
1164 accumulator2.val[1] = vdupq_n_f32(0);
1165 accumulator2.val[2] = vdupq_n_f32(0);
1166 accumulator2.val[3] = vdupq_n_f32(0);
1169 for (number = 0; number < quarter_points; ++number) {
1170 a_val = vld4q_f32((
float*)a_ptr);
1171 b_val = vld4q_f32((
float*)b_ptr);
1176 accumulator1.val[0] = vmlaq_f32(accumulator1.val[0], a_val.val[0], b_val.val[0]);
1177 accumulator1.val[1] = vmlaq_f32(accumulator1.val[1], a_val.val[0], b_val.val[1]);
1179 accumulator1.val[2] = vmlaq_f32(accumulator1.val[2], a_val.val[2], b_val.val[2]);
1180 accumulator1.val[3] = vmlaq_f32(accumulator1.val[3], a_val.val[2], b_val.val[3]);
1182 accumulator2.val[0] = vmlsq_f32(accumulator2.val[0], a_val.val[1], b_val.val[1]);
1183 accumulator2.val[1] = vmlaq_f32(accumulator2.val[1], a_val.val[1], b_val.val[0]);
1185 accumulator2.val[2] = vmlsq_f32(accumulator2.val[2], a_val.val[3], b_val.val[3]);
1186 accumulator2.val[3] = vmlaq_f32(accumulator2.val[3], a_val.val[3], b_val.val[2]);
1192 accumulator1.val[0] = vaddq_f32(accumulator1.val[0], accumulator1.val[2]);
1193 accumulator1.val[1] = vaddq_f32(accumulator1.val[1], accumulator1.val[3]);
1194 accumulator2.val[0] = vaddq_f32(accumulator2.val[0], accumulator2.val[2]);
1195 accumulator2.val[1] = vaddq_f32(accumulator2.val[1], accumulator2.val[3]);
1196 reduced_accumulator.val[0] = vaddq_f32(accumulator1.val[0], accumulator2.val[0]);
1197 reduced_accumulator.val[1] = vaddq_f32(accumulator1.val[1], accumulator2.val[1]);
1200 vst2q_f32((
float*)accum_result, reduced_accumulator);
1201 *result = accum_result[0] + accum_result[1] + accum_result[2] + accum_result[3];
1204 for (number = quarter_points * 8; number < num_points; ++number) {
1205 *result += (*a_ptr++) * (*b_ptr++);
1213 #include <immintrin.h> 1218 unsigned int num_points)
1221 unsigned int isodd = num_points & 3;
1224 memset(&dotProduct, 0x0, 2 *
sizeof(
float));
1226 unsigned int number = 0;
1227 const unsigned int quarterPoints = num_points / 4;
1229 __m256 x, y, yl, yh, z, tmp1, tmp2, dotProdVal;
1234 dotProdVal = _mm256_setzero_ps();
1236 for (; number < quarterPoints; number++) {
1238 x = _mm256_load_ps((
float*)a);
1239 y = _mm256_load_ps((
float*)b);
1241 yl = _mm256_moveldup_ps(y);
1242 yh = _mm256_movehdup_ps(y);
1244 tmp1 = _mm256_mul_ps(x, yl);
1246 x = _mm256_shuffle_ps(x, x, 0xB1);
1248 tmp2 = _mm256_mul_ps(x, yh);
1250 z = _mm256_addsub_ps(tmp1,
1253 dotProdVal = _mm256_add_ps(dotProdVal,
1262 _mm256_store_ps((
float*)dotProductVector,
1265 dotProduct += (dotProductVector[0] + dotProductVector[1] + dotProductVector[2] +
1266 dotProductVector[3]);
1268 for (
i = num_points - isodd;
i < num_points;
i++) {
1269 dotProduct += input[
i] * taps[
i];
1272 *result = dotProduct;
1277 #if LV_HAVE_AVX && LV_HAVE_FMA 1278 #include <immintrin.h> 1280 static inline void volk_32fc_x2_dot_prod_32fc_a_avx_fma(
lv_32fc_t* result,
1283 unsigned int num_points)
1286 unsigned int isodd = num_points & 3;
1289 memset(&dotProduct, 0x0, 2 *
sizeof(
float));
1291 unsigned int number = 0;
1292 const unsigned int quarterPoints = num_points / 4;
1294 __m256 x, y, yl, yh, z, tmp1, tmp2, dotProdVal;
1299 dotProdVal = _mm256_setzero_ps();
1301 for (; number < quarterPoints; number++) {
1303 x = _mm256_load_ps((
float*)a);
1304 y = _mm256_load_ps((
float*)b);
1306 yl = _mm256_moveldup_ps(y);
1307 yh = _mm256_movehdup_ps(y);
1311 x = _mm256_shuffle_ps(x, x, 0xB1);
1313 tmp2 = _mm256_mul_ps(x, yh);
1315 z = _mm256_fmaddsub_ps(
1318 dotProdVal = _mm256_add_ps(dotProdVal,
1327 _mm256_store_ps((
float*)dotProductVector,
1330 dotProduct += (dotProductVector[0] + dotProductVector[1] + dotProductVector[2] +
1331 dotProductVector[3]);
1333 for (
i = num_points - isodd;
i < num_points;
i++) {
1334 dotProduct += input[
i] * taps[
i];
1337 *result = dotProduct;
static void volk_32fc_x2_dot_prod_32fc_generic(lv_32fc_t *result, const lv_32fc_t *input, const lv_32fc_t *taps, unsigned int num_points)
Definition: volk_32fc_x2_dot_prod_32fc.h:70
static void volk_32fc_x2_dot_prod_32fc_u_sse3(lv_32fc_t *result, const lv_32fc_t *input, const lv_32fc_t *taps, unsigned int num_points)
Definition: volk_32fc_x2_dot_prod_32fc.h:245
#define __VOLK_ASM
Definition: volk_common.h:63
static void volk_32fc_x2_dot_prod_32fc_a_sse3(lv_32fc_t *result, const lv_32fc_t *input, const lv_32fc_t *taps, unsigned int num_points)
Definition: volk_32fc_x2_dot_prod_32fc.h:835
#define __VOLK_VOLATILE
Definition: volk_common.h:64
static void volk_32fc_x2_dot_prod_32fc_neon_optfma(lv_32fc_t *result, const lv_32fc_t *input, const lv_32fc_t *taps, unsigned int num_points)
Definition: volk_32fc_x2_dot_prod_32fc.h:1094
static void volk_32fc_x2_dot_prod_32fc_a_generic(lv_32fc_t *result, const lv_32fc_t *input, const lv_32fc_t *taps, unsigned int num_points)
Definition: volk_32fc_x2_dot_prod_32fc.h:533
static void volk_32fc_x2_dot_prod_32fc_neon(lv_32fc_t *result, const lv_32fc_t *input, const lv_32fc_t *taps, unsigned int num_points)
Definition: volk_32fc_x2_dot_prod_32fc.h:985
#define __VOLK_PREFETCH(addr)
Definition: volk_common.h:62
for i
Definition: volk_config_fixed.tmpl.h:25
#define __VOLK_ATTR_ALIGNED(x)
Definition: volk_common.h:56
static void volk_32fc_x2_dot_prod_32fc_u_avx(lv_32fc_t *result, const lv_32fc_t *input, const lv_32fc_t *taps, unsigned int num_points)
Definition: volk_32fc_x2_dot_prod_32fc.h:393
float complex lv_32fc_t
Definition: volk_complex.h:70
static void volk_32fc_x2_dot_prod_32fc_neon_optfmaunroll(lv_32fc_t *result, const lv_32fc_t *input, const lv_32fc_t *taps, unsigned int num_points)
Definition: volk_32fc_x2_dot_prod_32fc.h:1142
static void volk_32fc_x2_dot_prod_32fc_a_avx(lv_32fc_t *result, const lv_32fc_t *input, const lv_32fc_t *taps, unsigned int num_points)
Definition: volk_32fc_x2_dot_prod_32fc.h:1215
static void volk_32fc_x2_dot_prod_32fc_neon_opttests(lv_32fc_t *result, const lv_32fc_t *input, const lv_32fc_t *taps, unsigned int num_points)
Definition: volk_32fc_x2_dot_prod_32fc.h:1043