Updated the math functions for better alignment handling

benchmarks/BM_mathfuns.cpp

@@ -1,150 +1,141 @@
+#include "../sources/SIMDHelpers.h"
+#include "absl/types/span.h"
 #include <benchmark/benchmark.h>
-#include <random>
-#include <numeric>
-#include <vector>
 #include <cmath>
 #include <iostream>
-#include "../sources/SIMDHelpers.h"
+#include <numeric>
+#include <random>
+#include <vector>
 
 class MyFixture : public benchmark::Fixture {
 public:
-  void SetUp(const ::benchmark::State& state) {
+    void SetUp(const ::benchmark::State& state)
-    std::random_device rd { };
+    {
-    std::mt19937 gen { rd() };
+        std::random_device rd {};
-    std::uniform_real_distribution<float> dist { 0.1, 1 };
+        std::mt19937 gen { rd() };
-    source = std::vector<float>(state.range(0));
+        std::uniform_real_distribution<float> dist { 0.1, 1 };
-    result = std::vector<float>(state.range(0));
+        source = std::vector<float>(state.range(0));
-    std::generate(source.begin(), source.end(), [&]() { return dist(gen); });
+        result = std::vector<float>(state.range(0));
-  }
+        std::generate(source.begin(), source.end(), [&]() { return dist(gen); });
+    }
 
-  void TearDown(const ::benchmark::State& state [[maybe_unused]]) {
+    void TearDown(const ::benchmark::State& state [[maybe_unused]])
-  }
+    {
+    }
 
-  std::vector<float> source;
+    std::vector<float> source;
-  std::vector<float> result;
+    std::vector<float> result;
 };
 
-
+BENCHMARK_DEFINE_F(MyFixture, Dummy)
-BENCHMARK_DEFINE_F(MyFixture, Dummy)(benchmark::State& state) {
+(benchmark::State& state)
-    for (auto _ : state)
+{
-    {
+    for (auto _ : state) {
-         for (int i = 0; i < state.range(0); ++i)
+        for (int i = 0; i < state.range(0); ++i)
             result[i] = source[i];
         benchmark::DoNotOptimize(result);
     }
 }
 
-BENCHMARK_DEFINE_F(MyFixture, StdExp)(benchmark::State& state) {
+BENCHMARK_DEFINE_F(MyFixture, ScalarExp)
-    for (auto _ : state)
+(benchmark::State& state)
-    {
+{
-        for (int i = 0; i < state.range(0); ++i)
+    for (auto _ : state) {
-            result[i] = std::exp(source[i]);
-        benchmark::DoNotOptimize(result);
-    }
-}
-
-BENCHMARK_DEFINE_F(MyFixture, ScalarExp)(benchmark::State& state) {
-    for (auto _ : state)
-    {
         exp<float, false>(source, absl::MakeSpan(result));
         benchmark::DoNotOptimize(result);
     }
 }
 
-BENCHMARK_DEFINE_F(MyFixture, SIMDExp)(benchmark::State& state) {
+BENCHMARK_DEFINE_F(MyFixture, SIMDExp)
-    for (auto _ : state)
+(benchmark::State& state)
-    {
+{
+    for (auto _ : state) {
         exp<float, true>(source, absl::MakeSpan(result));
         benchmark::DoNotOptimize(result);
     }
 }
 
-BENCHMARK_DEFINE_F(MyFixture, StdLog)(benchmark::State& state) {
+BENCHMARK_DEFINE_F(MyFixture, ScalarExp_Unaligned)
-    for (auto _ : state)
+(benchmark::State& state)
-    {
+{
-        for (int i = 0; i < state.range(0); ++i)
+    for (auto _ : state) {
-            result[i] = std::log(source[i]);
+        exp<float, false>(absl::MakeSpan(source).subspan(1), absl::MakeSpan(result).subspan(1));
         benchmark::DoNotOptimize(result);
     }
 }
 
-BENCHMARK_DEFINE_F(MyFixture, ScalarLog)(benchmark::State& state) {
+BENCHMARK_DEFINE_F(MyFixture, SIMDExp_Unaligned)
-    for (auto _ : state)
+(benchmark::State& state)
-    {
+{
+    for (auto _ : state) {
+        exp<float, true>(absl::MakeSpan(source).subspan(1), absl::MakeSpan(result).subspan(1));
+        benchmark::DoNotOptimize(result);
+    }
+}
+
+BENCHMARK_DEFINE_F(MyFixture, ScalarLog)
+(benchmark::State& state)
+{
+    for (auto _ : state) {
         log<float, false>(source, absl::MakeSpan(result));
         benchmark::DoNotOptimize(result);
     }
 }
 
-BENCHMARK_DEFINE_F(MyFixture, SIMDLog)(benchmark::State& state) {
+BENCHMARK_DEFINE_F(MyFixture, SIMDLog)
-    for (auto _ : state)
+(benchmark::State& state)
-    {
+{
+    for (auto _ : state) {
         log<float, true>(source, absl::MakeSpan(result));
         benchmark::DoNotOptimize(result);
     }
 }
 
-BENCHMARK_DEFINE_F(MyFixture, StdSin)(benchmark::State& state) {
+BENCHMARK_DEFINE_F(MyFixture, ScalarSin)
-    for (auto _ : state)
+(benchmark::State& state)
-    {
+{
-        for (int i = 0; i < state.range(0); ++i)
+    for (auto _ : state) {
-            result[i] = std::sin(source[i]);
-        benchmark::DoNotOptimize(result);
-    }
-}
-
-BENCHMARK_DEFINE_F(MyFixture, ScalarSin)(benchmark::State& state) {
-    for (auto _ : state)
-    {
         sin<float, false>(source, absl::MakeSpan(result));
         benchmark::DoNotOptimize(result);
     }
 }
 
-BENCHMARK_DEFINE_F(MyFixture, SIMDSin)(benchmark::State& state) {
+BENCHMARK_DEFINE_F(MyFixture, SIMDSin)
-    for (auto _ : state)
+(benchmark::State& state)
-    {
+{
+    for (auto _ : state) {
         sin<float, true>(source, absl::MakeSpan(result));
         benchmark::DoNotOptimize(result);
     }
 }
 
-BENCHMARK_DEFINE_F(MyFixture, StdCos)(benchmark::State& state) {
+BENCHMARK_DEFINE_F(MyFixture, ScalarCos)
-    for (auto _ : state)
+(benchmark::State& state)
-    {
+{
-        for (int i = 0; i < state.range(0); ++i)
+    for (auto _ : state) {
-            result[i] = std::cos(source[i]);
-        benchmark::DoNotOptimize(result);
-    }
-}
-
-BENCHMARK_DEFINE_F(MyFixture, ScalarCos)(benchmark::State& state) {
-    for (auto _ : state)
-    {
         cos<float, false>(source, absl::MakeSpan(result));
         benchmark::DoNotOptimize(result);
     }
 }
 
-BENCHMARK_DEFINE_F(MyFixture, SIMDCos)(benchmark::State& state) {
+BENCHMARK_DEFINE_F(MyFixture, SIMDCos)
-    for (auto _ : state)
+(benchmark::State& state)
-    {
+{
+    for (auto _ : state) {
         cos<float, true>(source, absl::MakeSpan(result));
         benchmark::DoNotOptimize(result);
     }
 }
 
 BENCHMARK_REGISTER_F(MyFixture, Dummy)->RangeMultiplier(4)->Range(1 << 6, 1 << 10);
-BENCHMARK_REGISTER_F(MyFixture, StdExp)->RangeMultiplier(4)->Range(1 << 6, 1 << 10);
 BENCHMARK_REGISTER_F(MyFixture, ScalarExp)->RangeMultiplier(4)->Range(1 << 6, 1 << 10);
 BENCHMARK_REGISTER_F(MyFixture, SIMDExp)->RangeMultiplier(4)->Range(1 << 6, 1 << 10);
-BENCHMARK_REGISTER_F(MyFixture, StdLog)->RangeMultiplier(4)->Range(1 << 6, 1 << 10);
+BENCHMARK_REGISTER_F(MyFixture, ScalarExp_Unaligned)->RangeMultiplier(4)->Range(1 << 6, 1 << 10);
+BENCHMARK_REGISTER_F(MyFixture, SIMDExp_Unaligned)->RangeMultiplier(4)->Range(1 << 6, 1 << 10);
 BENCHMARK_REGISTER_F(MyFixture, ScalarLog)->RangeMultiplier(4)->Range(1 << 6, 1 << 10);
 BENCHMARK_REGISTER_F(MyFixture, SIMDLog)->RangeMultiplier(4)->Range(1 << 6, 1 << 10);
-BENCHMARK_REGISTER_F(MyFixture, StdSin)->RangeMultiplier(4)->Range(1 << 6, 1 << 10);
 BENCHMARK_REGISTER_F(MyFixture, ScalarSin)->RangeMultiplier(4)->Range(1 << 6, 1 << 10);
 BENCHMARK_REGISTER_F(MyFixture, SIMDSin)->RangeMultiplier(4)->Range(1 << 6, 1 << 10);
-BENCHMARK_REGISTER_F(MyFixture, StdCos)->RangeMultiplier(4)->Range(1 << 6, 1 << 10);
 BENCHMARK_REGISTER_F(MyFixture, ScalarCos)->RangeMultiplier(4)->Range(1 << 6, 1 << 10);
 BENCHMARK_REGISTER_F(MyFixture, SIMDCos)->RangeMultiplier(4)->Range(1 << 6, 1 << 10);
 

sources/SIMDSSE.cpp

@@ -1,5 +1,5 @@
-#include "SIMDHelpers.h"
 #include "Helpers.h"
+#include "SIMDHelpers.h"
 
 #if HAVE_X86INTRIN_H
 #include <x86intrin.h>
@@ -13,21 +13,23 @@
 
 using Type = float;
 [[maybe_unused]] constexpr uintptr_t TypeAlignment { 4 };
-[[maybe_unused]] constexpr uintptr_t TypeAlignmentMask { TypeAlignment - 1 } ;
+[[maybe_unused]] constexpr uintptr_t TypeAlignmentMask { TypeAlignment - 1 };
 [[maybe_unused]] constexpr uintptr_t ByteAlignment { TypeAlignment * sizeof(Type) };
 [[maybe_unused]] constexpr uintptr_t ByteAlignmentMask { ByteAlignment - 1 };
 
-
+struct AlignmentSentinels {
-struct AlignmentSentinels { float* nextAligned; float* lastAligned; };
+    float* nextAligned;
+    float* lastAligned;
+};
 
 float* nextAligned(const float* ptr)
 {
-    return reinterpret_cast<float*>( (reinterpret_cast<uintptr_t>(ptr) + ByteAlignmentMask) & (~ByteAlignmentMask) );
+    return reinterpret_cast<float*>((reinterpret_cast<uintptr_t>(ptr) + ByteAlignmentMask) & (~ByteAlignmentMask));
 }
 
 float* prevAligned(const float* ptr)
 {
-    return reinterpret_cast<float*>( reinterpret_cast<uintptr_t>(ptr) & (~ByteAlignmentMask) );
+    return reinterpret_cast<float*>(reinterpret_cast<uintptr_t>(ptr) & (~ByteAlignmentMask));
 }
 
 bool unaligned(const float* ptr)
@@ -50,7 +52,7 @@ bool unaligned(const float* ptr1, const float* ptr2, const float* ptr3, const fl
     return unaligned(ptr1) || unaligned(ptr2) || unaligned(ptr3) || unaligned(ptr4);
 }
 
-template<>
+template <>
 void readInterleaved<float, true>(absl::Span<const float> input, absl::Span<float> outputLeft, absl::Span<float> outputRight) noexcept
 {
     // The size of the outputs is not big enough for the input...
@@ -63,14 +65,13 @@ void readInterleaved<float, true>(absl::Span<const float> input, absl::Span<floa
     auto* lOut = outputLeft.begin();
     auto* rOut = outputRight.begin();
 
-    const auto size = std::min(input.size(), std::min(outputLeft.size() * 2, outputRight.size() * 2 ));
+    const auto size = std::min(input.size(), std::min(outputLeft.size() * 2, outputRight.size() * 2));
     const auto* lastAligned = prevAligned(input.begin() + size - TypeAlignment);
 
     while (unaligned(in, lOut, rOut) && in < lastAligned)
         snippetRead<float>(in, lOut, rOut);
 
-    while (in < lastAligned )
+    while (in < lastAligned) {
-    {
         auto register0 = _mm_load_ps(in);
         in += TypeAlignment;
         auto register1 = _mm_load_ps(in);
@@ -91,7 +92,7 @@ void readInterleaved<float, true>(absl::Span<const float> input, absl::Span<floa
         snippetRead<float>(in, lOut, rOut);
 }
 
-template<>
+template <>
 void writeInterleaved<float, true>(absl::Span<const float> inputLeft, absl::Span<const float> inputRight, absl::Span<float> output) noexcept
 {
     // The size of the output is not big enough for the inputs...
@@ -108,8 +109,7 @@ void writeInterleaved<float, true>(absl::Span<const float> inputLeft, absl::Span
     while (unaligned(out, rIn, lIn) && out < lastAligned)
         snippetWrite<float>(out, lIn, rIn);
 
-    while (out < lastAligned)
+    while (out < lastAligned) {
-    {
         const auto lInRegister = _mm_load_ps(lIn);
         const auto rInRegister = _mm_load_ps(rIn);
 
@@ -129,8 +129,7 @@ void writeInterleaved<float, true>(absl::Span<const float> inputLeft, absl::Span
         snippetWrite<float>(out, lIn, rIn);
 }
 
-
+template <>
-template<>
 void fill<float, true>(absl::Span<float> output, float value) noexcept
 {
     const auto mmValue = _mm_set_ps1(value);
@@ -150,67 +149,95 @@ void fill<float, true>(absl::Span<float> output, float value) noexcept
         *out++ = value;
 }
 
-template<>
+template <>
 void exp<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept
 {
     ASSERT(output.size() >= input.size());
     auto* in = input.begin();
     auto* out = output.begin();
     auto* sentinel = in + std::min(input.size(), output.size());
-    while (in < sentinel)
+    const auto* lastAligned = prevAligned(sentinel);
-    {
+
-        _mm_storeu_ps(out, exp_ps(_mm_loadu_ps(in)));
+    while (unaligned(in, out) && in < lastAligned)
+        *out++ = std::exp(*in++);
+
+    while (in < lastAligned) {
+        _mm_store_ps(out, exp_ps(_mm_load_ps(in)));
         out += TypeAlignment;
         in += TypeAlignment;
     }
+
+    while (in < sentinel)
+        *out++ = std::exp(*in++);
 }
 
-template<>
+template <>
 void cos<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept
 {
     ASSERT(output.size() >= input.size());
     auto* in = input.begin();
     auto* out = output.begin();
     auto* sentinel = in + std::min(input.size(), output.size());
-    while (in < sentinel)
+    const auto* lastAligned = prevAligned(sentinel);
-    {
+
-        _mm_storeu_ps(out, cos_ps(_mm_loadu_ps(in)));
+    while (unaligned(in, out) && in < lastAligned)
+        *out++ = std::exp(*in++);
+
+    while (in < lastAligned) {
+        _mm_store_ps(out, cos_ps(_mm_load_ps(in)));
         out += TypeAlignment;
         in += TypeAlignment;
     }
+
+    while (in < sentinel)
+        *out++ = std::exp(*in++);
 }
 
-template<>
+template <>
 void log<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept
 {
     ASSERT(output.size() >= input.size());
     auto* in = input.begin();
     auto* out = output.begin();
     auto* sentinel = in + std::min(input.size(), output.size());
-    while (in < sentinel)
+    const auto* lastAligned = prevAligned(sentinel);
-    {
+
-        _mm_storeu_ps(out, log_ps(_mm_loadu_ps(in)));
+    while (unaligned(in, out) && in < lastAligned)
+        *out++ = std::exp(*in++);
+
+    while (in < lastAligned) {
+        _mm_store_ps(out, log_ps(_mm_load_ps(in)));
         out += TypeAlignment;
         in += TypeAlignment;
     }
+
+    while (in < sentinel)
+        *out++ = std::exp(*in++);
 }
 
-template<>
+template <>
 void sin<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept
 {
     ASSERT(output.size() >= input.size());
     auto* in = input.begin();
     auto* out = output.begin();
     auto* sentinel = in + std::min(input.size(), output.size());
-    while (in < sentinel)
+    const auto* lastAligned = prevAligned(sentinel);
-    {
+
-        _mm_storeu_ps(out, sin_ps(_mm_loadu_ps(in)));
+    while (unaligned(in, out) && in < lastAligned)
+        *out++ = std::exp(*in++);
+
+    while (in < lastAligned) {
+        _mm_store_ps(out, sin_ps(_mm_load_ps(in)));
         out += TypeAlignment;
         in += TypeAlignment;
     }
+
+    while (in < sentinel)
+        *out++ = std::exp(*in++);
 }
 
-template<>
+template <>
 void applyGain<float, true>(float gain, absl::Span<const float> input, absl::Span<float> output) noexcept
 {
     auto* in = input.begin();
@@ -222,8 +249,7 @@ void applyGain<float, true>(float gain, absl::Span<const float> input, absl::Spa
     while (unaligned(out, in) && out < lastAligned)
         *out++ = gain * (*in++);
 
-    while (out < lastAligned)
+    while (out < lastAligned) {
-    {
         _mm_store_ps(out, _mm_mul_ps(mmGain, _mm_load_ps(in)));
         in += TypeAlignment;
         out += TypeAlignment;
@@ -233,7 +259,7 @@ void applyGain<float, true>(float gain, absl::Span<const float> input, absl::Spa
         *out++ = gain * (*in++);
 }
 
-template<>
+template <>
 void applyGain<float, true>(absl::Span<const float> gain, absl::Span<const float> input, absl::Span<float> output) noexcept
 {
     auto* in = input.begin();
@@ -245,8 +271,7 @@ void applyGain<float, true>(absl::Span<const float> gain, absl::Span<const float
     while (unaligned(out, in, g) && out < lastAligned)
         snippetGainSpan<float>(g, in, out);
 
-    while (out < lastAligned)
+    while (out < lastAligned) {
-    {
         _mm_store_ps(out, _mm_mul_ps(_mm_load_ps(g), _mm_load_ps(in)));
         g += TypeAlignment;
         in += TypeAlignment;
@@ -257,8 +282,14 @@ void applyGain<float, true>(absl::Span<const float> gain, absl::Span<const float
         snippetGainSpan<float>(g, in, out);
 }
 
-template<>
+template <>
-void loopingSFZIndex<float, true>(absl::Span<const float> jumps, absl::Span<float> leftCoeffs, absl::Span<float> rightCoeffs, absl::Span<int> indices, float floatIndex, float loopEnd, float loopStart) noexcept
+void loopingSFZIndex<float, true>(  absl::Span<const float> jumps,
+                                    absl::Span<float> leftCoeffs, 
+                                    absl::Span<float> rightCoeffs, 
+                                    absl::Span<int> indices, 
+                                    float floatIndex, 
+                                    float loopEnd, 
+                                    float loopStart) noexcept
 {
     ASSERT(indices.size() >= jumps.size());
     ASSERT(indices.size() == leftCoeffs.size());
@@ -276,10 +307,9 @@ void loopingSFZIndex<float, true>(absl::Span<const float> jumps, absl::Span<floa
         snippetLoopingIndex<float>(jump, leftCoeff, rightCoeff, index, floatIndex, loopEnd, loopStart);
 
     auto mmFloatIndex = _mm_set_ps1(floatIndex);
-    const auto mmJumpBack   = _mm_set1_ps(loopEnd - loopStart);
+    const auto mmJumpBack = _mm_set1_ps(loopEnd - loopStart);
     const auto mmLoopEnd = _mm_set1_ps(loopEnd);
-    while (jump < alignedEnd)
+    while (jump < alignedEnd) {
-    {
         auto mmOffset = _mm_load_ps(jump);
         mmOffset = _mm_add_ps(mmOffset, _mm_castsi128_ps(_mm_slli_si128(_mm_castps_si128(mmOffset), 4)));
         mmOffset = _mm_add_ps(mmOffset, _mm_shuffle_ps(_mm_setzero_ps(), mmOffset, 0x40));
@@ -313,20 +343,19 @@ void loopingSFZIndex<float, true>(absl::Span<const float> jumps, absl::Span<floa
         snippetLoopingIndex<float>(jump, leftCoeff, rightCoeff, index, floatIndex, loopEnd, loopStart);
 }
 
-template<>
+template <>
 float linearRamp<float, true>(absl::Span<float> output, float value, float step) noexcept
 {
     auto* out = output.begin();
     const auto* lastAligned = prevAligned(output.end());
 
-    while(unaligned(out) && out < lastAligned)
+    while (unaligned(out) && out < lastAligned)
         snippetRampLinear<float>(out, value, step);
 
     auto mmValue = _mm_set1_ps(value);
-    auto mmStep = _mm_set_ps(step+step+step+step, step+step+step, step+step, step);
+    auto mmStep = _mm_set_ps(step + step + step + step, step + step + step, step + step, step);
 
-    while (out < lastAligned)
+    while (out < lastAligned) {
-    {
         mmValue = _mm_add_ps(mmValue, mmStep);
         _mm_store_ps(out, mmValue);
         mmValue = _mm_shuffle_ps(mmValue, mmValue, _MM_SHUFFLE(3, 3, 3, 3));
@@ -334,25 +363,24 @@ float linearRamp<float, true>(absl::Span<float> output, float value, float step)
     }
 
     value = _mm_cvtss_f32(mmValue);
-    while(out < output.end())
+    while (out < output.end())
         snippetRampLinear<float>(out, value, step);
     return value;
 }
 
-template<>
+template <>
 float multiplicativeRamp<float, true>(absl::Span<float> output, float value, float step) noexcept
 {
     auto* out = output.begin();
     const auto* lastAligned = prevAligned(output.end());
 
-    while(unaligned(out) && out < lastAligned)
+    while (unaligned(out) && out < lastAligned)
         snippetRampMultiplicative<float>(out, value, step);
 
     auto mmValue = _mm_set1_ps(value);
-    auto mmStep = _mm_set_ps(step*step*step*step, step*step*step, step*step, step);
+    auto mmStep = _mm_set_ps(step * step * step * step, step * step * step, step * step, step);
 
-    while (out < lastAligned)
+    while (out < lastAligned) {
-    {
         mmValue = _mm_mul_ps(mmValue, mmStep);
         _mm_store_ps(out, mmValue);
         mmValue = _mm_shuffle_ps(mmValue, mmValue, _MM_SHUFFLE(3, 3, 3, 3));
@@ -360,12 +388,12 @@ float multiplicativeRamp<float, true>(absl::Span<float> output, float value, flo
     }
 
     value = _mm_cvtss_f32(mmValue);
-    while(out < output.end())
+    while (out < output.end())
         snippetRampMultiplicative<float>(out, value, step);
     return value;
 }
 
-template<>
+template <>
 void add<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept
 {
     ASSERT(output.size() >= input.size());
@@ -374,16 +402,15 @@ void add<float, true>(absl::Span<const float> input, absl::Span<float> output) n
     auto* sentinel = out + min(input.size(), output.size());
     const auto* lastAligned = prevAligned(sentinel);
 
-    while(unaligned(in, out) && out < lastAligned)
+    while (unaligned(in, out) && out < lastAligned)
         snippetAdd<float>(in, out);
 
-    while(out < lastAligned)
+    while (out < lastAligned) {
-    {
         _mm_store_ps(out, _mm_add_ps(_mm_load_ps(in), _mm_load_ps(out)));
         out += TypeAlignment;
         in += TypeAlignment;
     }
 
-    while(out < sentinel)
+    while (out < sentinel)
         snippetAdd<float>(in, out);
 }