diff --git a/benchmarks/BM_mathfuns.cpp b/benchmarks/BM_mathfuns.cpp index 28e8fe39..f15ef79d 100644 --- a/benchmarks/BM_mathfuns.cpp +++ b/benchmarks/BM_mathfuns.cpp @@ -1,150 +1,141 @@ +#include "../sources/SIMDHelpers.h" +#include "absl/types/span.h" #include -#include -#include -#include #include #include -#include "../sources/SIMDHelpers.h" +#include +#include +#include class MyFixture : public benchmark::Fixture { public: - void SetUp(const ::benchmark::State& state) { - std::random_device rd { }; - std::mt19937 gen { rd() }; - std::uniform_real_distribution dist { 0.1, 1 }; - source = std::vector(state.range(0)); - result = std::vector(state.range(0)); - std::generate(source.begin(), source.end(), [&]() { return dist(gen); }); - } + void SetUp(const ::benchmark::State& state) + { + std::random_device rd {}; + std::mt19937 gen { rd() }; + std::uniform_real_distribution dist { 0.1, 1 }; + source = std::vector(state.range(0)); + result = std::vector(state.range(0)); + std::generate(source.begin(), source.end(), [&]() { return dist(gen); }); + } - void TearDown(const ::benchmark::State& state [[maybe_unused]]) { - } + void TearDown(const ::benchmark::State& state [[maybe_unused]]) + { + } - std::vector source; - std::vector result; + std::vector source; + std::vector result; }; - -BENCHMARK_DEFINE_F(MyFixture, Dummy)(benchmark::State& state) { - for (auto _ : state) - { - for (int i = 0; i < state.range(0); ++i) +BENCHMARK_DEFINE_F(MyFixture, Dummy) +(benchmark::State& state) +{ + for (auto _ : state) { + for (int i = 0; i < state.range(0); ++i) result[i] = source[i]; benchmark::DoNotOptimize(result); } } -BENCHMARK_DEFINE_F(MyFixture, StdExp)(benchmark::State& state) { - for (auto _ : state) - { - for (int i = 0; i < state.range(0); ++i) - result[i] = std::exp(source[i]); - benchmark::DoNotOptimize(result); - } -} - -BENCHMARK_DEFINE_F(MyFixture, ScalarExp)(benchmark::State& state) { - for (auto _ : state) - { +BENCHMARK_DEFINE_F(MyFixture, ScalarExp) +(benchmark::State& state) +{ + for (auto _ : state) { exp(source, absl::MakeSpan(result)); benchmark::DoNotOptimize(result); } } -BENCHMARK_DEFINE_F(MyFixture, SIMDExp)(benchmark::State& state) { - for (auto _ : state) - { +BENCHMARK_DEFINE_F(MyFixture, SIMDExp) +(benchmark::State& state) +{ + for (auto _ : state) { exp(source, absl::MakeSpan(result)); benchmark::DoNotOptimize(result); } } -BENCHMARK_DEFINE_F(MyFixture, StdLog)(benchmark::State& state) { - for (auto _ : state) - { - for (int i = 0; i < state.range(0); ++i) - result[i] = std::log(source[i]); +BENCHMARK_DEFINE_F(MyFixture, ScalarExp_Unaligned) +(benchmark::State& state) +{ + for (auto _ : state) { + exp(absl::MakeSpan(source).subspan(1), absl::MakeSpan(result).subspan(1)); benchmark::DoNotOptimize(result); } } -BENCHMARK_DEFINE_F(MyFixture, ScalarLog)(benchmark::State& state) { - for (auto _ : state) - { +BENCHMARK_DEFINE_F(MyFixture, SIMDExp_Unaligned) +(benchmark::State& state) +{ + for (auto _ : state) { + exp(absl::MakeSpan(source).subspan(1), absl::MakeSpan(result).subspan(1)); + benchmark::DoNotOptimize(result); + } +} + +BENCHMARK_DEFINE_F(MyFixture, ScalarLog) +(benchmark::State& state) +{ + for (auto _ : state) { log(source, absl::MakeSpan(result)); benchmark::DoNotOptimize(result); } } -BENCHMARK_DEFINE_F(MyFixture, SIMDLog)(benchmark::State& state) { - for (auto _ : state) - { +BENCHMARK_DEFINE_F(MyFixture, SIMDLog) +(benchmark::State& state) +{ + for (auto _ : state) { log(source, absl::MakeSpan(result)); benchmark::DoNotOptimize(result); } } -BENCHMARK_DEFINE_F(MyFixture, StdSin)(benchmark::State& state) { - for (auto _ : state) - { - for (int i = 0; i < state.range(0); ++i) - result[i] = std::sin(source[i]); - benchmark::DoNotOptimize(result); - } -} - -BENCHMARK_DEFINE_F(MyFixture, ScalarSin)(benchmark::State& state) { - for (auto _ : state) - { +BENCHMARK_DEFINE_F(MyFixture, ScalarSin) +(benchmark::State& state) +{ + for (auto _ : state) { sin(source, absl::MakeSpan(result)); benchmark::DoNotOptimize(result); } } -BENCHMARK_DEFINE_F(MyFixture, SIMDSin)(benchmark::State& state) { - for (auto _ : state) - { +BENCHMARK_DEFINE_F(MyFixture, SIMDSin) +(benchmark::State& state) +{ + for (auto _ : state) { sin(source, absl::MakeSpan(result)); benchmark::DoNotOptimize(result); } } -BENCHMARK_DEFINE_F(MyFixture, StdCos)(benchmark::State& state) { - for (auto _ : state) - { - for (int i = 0; i < state.range(0); ++i) - result[i] = std::cos(source[i]); - benchmark::DoNotOptimize(result); - } -} - -BENCHMARK_DEFINE_F(MyFixture, ScalarCos)(benchmark::State& state) { - for (auto _ : state) - { +BENCHMARK_DEFINE_F(MyFixture, ScalarCos) +(benchmark::State& state) +{ + for (auto _ : state) { cos(source, absl::MakeSpan(result)); benchmark::DoNotOptimize(result); } } -BENCHMARK_DEFINE_F(MyFixture, SIMDCos)(benchmark::State& state) { - for (auto _ : state) - { +BENCHMARK_DEFINE_F(MyFixture, SIMDCos) +(benchmark::State& state) +{ + for (auto _ : state) { cos(source, absl::MakeSpan(result)); benchmark::DoNotOptimize(result); } } BENCHMARK_REGISTER_F(MyFixture, Dummy)->RangeMultiplier(4)->Range(1 << 6, 1 << 10); -BENCHMARK_REGISTER_F(MyFixture, StdExp)->RangeMultiplier(4)->Range(1 << 6, 1 << 10); BENCHMARK_REGISTER_F(MyFixture, ScalarExp)->RangeMultiplier(4)->Range(1 << 6, 1 << 10); BENCHMARK_REGISTER_F(MyFixture, SIMDExp)->RangeMultiplier(4)->Range(1 << 6, 1 << 10); -BENCHMARK_REGISTER_F(MyFixture, StdLog)->RangeMultiplier(4)->Range(1 << 6, 1 << 10); +BENCHMARK_REGISTER_F(MyFixture, ScalarExp_Unaligned)->RangeMultiplier(4)->Range(1 << 6, 1 << 10); +BENCHMARK_REGISTER_F(MyFixture, SIMDExp_Unaligned)->RangeMultiplier(4)->Range(1 << 6, 1 << 10); BENCHMARK_REGISTER_F(MyFixture, ScalarLog)->RangeMultiplier(4)->Range(1 << 6, 1 << 10); BENCHMARK_REGISTER_F(MyFixture, SIMDLog)->RangeMultiplier(4)->Range(1 << 6, 1 << 10); -BENCHMARK_REGISTER_F(MyFixture, StdSin)->RangeMultiplier(4)->Range(1 << 6, 1 << 10); BENCHMARK_REGISTER_F(MyFixture, ScalarSin)->RangeMultiplier(4)->Range(1 << 6, 1 << 10); BENCHMARK_REGISTER_F(MyFixture, SIMDSin)->RangeMultiplier(4)->Range(1 << 6, 1 << 10); -BENCHMARK_REGISTER_F(MyFixture, StdCos)->RangeMultiplier(4)->Range(1 << 6, 1 << 10); BENCHMARK_REGISTER_F(MyFixture, ScalarCos)->RangeMultiplier(4)->Range(1 << 6, 1 << 10); BENCHMARK_REGISTER_F(MyFixture, SIMDCos)->RangeMultiplier(4)->Range(1 << 6, 1 << 10); diff --git a/sources/SIMDSSE.cpp b/sources/SIMDSSE.cpp index 134ba28f..93ae8fc8 100644 --- a/sources/SIMDSSE.cpp +++ b/sources/SIMDSSE.cpp @@ -1,5 +1,5 @@ -#include "SIMDHelpers.h" #include "Helpers.h" +#include "SIMDHelpers.h" #if HAVE_X86INTRIN_H #include @@ -13,21 +13,23 @@ using Type = float; [[maybe_unused]] constexpr uintptr_t TypeAlignment { 4 }; -[[maybe_unused]] constexpr uintptr_t TypeAlignmentMask { TypeAlignment - 1 } ; +[[maybe_unused]] constexpr uintptr_t TypeAlignmentMask { TypeAlignment - 1 }; [[maybe_unused]] constexpr uintptr_t ByteAlignment { TypeAlignment * sizeof(Type) }; [[maybe_unused]] constexpr uintptr_t ByteAlignmentMask { ByteAlignment - 1 }; - -struct AlignmentSentinels { float* nextAligned; float* lastAligned; }; +struct AlignmentSentinels { + float* nextAligned; + float* lastAligned; +}; float* nextAligned(const float* ptr) { - return reinterpret_cast( (reinterpret_cast(ptr) + ByteAlignmentMask) & (~ByteAlignmentMask) ); + return reinterpret_cast((reinterpret_cast(ptr) + ByteAlignmentMask) & (~ByteAlignmentMask)); } float* prevAligned(const float* ptr) { - return reinterpret_cast( reinterpret_cast(ptr) & (~ByteAlignmentMask) ); + return reinterpret_cast(reinterpret_cast(ptr) & (~ByteAlignmentMask)); } bool unaligned(const float* ptr) @@ -50,7 +52,7 @@ bool unaligned(const float* ptr1, const float* ptr2, const float* ptr3, const fl return unaligned(ptr1) || unaligned(ptr2) || unaligned(ptr3) || unaligned(ptr4); } -template<> +template <> void readInterleaved(absl::Span input, absl::Span outputLeft, absl::Span outputRight) noexcept { // The size of the outputs is not big enough for the input... @@ -63,14 +65,13 @@ void readInterleaved(absl::Span input, absl::Span(in, lOut, rOut); - while (in < lastAligned ) - { + while (in < lastAligned) { auto register0 = _mm_load_ps(in); in += TypeAlignment; auto register1 = _mm_load_ps(in); @@ -86,12 +87,12 @@ void readInterleaved(absl::Span input, absl::Span(in, lOut, rOut); } -template<> +template <> void writeInterleaved(absl::Span inputLeft, absl::Span inputRight, absl::Span output) noexcept { // The size of the output is not big enough for the inputs... @@ -108,8 +109,7 @@ void writeInterleaved(absl::Span inputLeft, absl::Span while (unaligned(out, rIn, lIn) && out < lastAligned) snippetWrite(out, lIn, rIn); - while (out < lastAligned) - { + while (out < lastAligned) { const auto lInRegister = _mm_load_ps(lIn); const auto rInRegister = _mm_load_ps(rIn); @@ -129,14 +129,13 @@ void writeInterleaved(absl::Span inputLeft, absl::Span snippetWrite(out, lIn, rIn); } - -template<> +template <> void fill(absl::Span output, float value) noexcept { const auto mmValue = _mm_set_ps1(value); auto* out = output.begin(); const auto* lastAligned = prevAligned(output.end()); - + while (unaligned(out) && out < lastAligned) *out++ = value; @@ -145,72 +144,100 @@ void fill(absl::Span output, float value) noexcept _mm_store_ps(out, mmValue); out += TypeAlignment; } - + while (out < output.end()) *out++ = value; } -template<> +template <> void exp(absl::Span input, absl::Span output) noexcept { ASSERT(output.size() >= input.size()); auto* in = input.begin(); auto* out = output.begin(); auto* sentinel = in + std::min(input.size(), output.size()); - while (in < sentinel) - { - _mm_storeu_ps(out, exp_ps(_mm_loadu_ps(in))); + const auto* lastAligned = prevAligned(sentinel); + + while (unaligned(in, out) && in < lastAligned) + *out++ = std::exp(*in++); + + while (in < lastAligned) { + _mm_store_ps(out, exp_ps(_mm_load_ps(in))); out += TypeAlignment; in += TypeAlignment; } + + while (in < sentinel) + *out++ = std::exp(*in++); } -template<> +template <> void cos(absl::Span input, absl::Span output) noexcept { ASSERT(output.size() >= input.size()); auto* in = input.begin(); auto* out = output.begin(); auto* sentinel = in + std::min(input.size(), output.size()); - while (in < sentinel) - { - _mm_storeu_ps(out, cos_ps(_mm_loadu_ps(in))); + const auto* lastAligned = prevAligned(sentinel); + + while (unaligned(in, out) && in < lastAligned) + *out++ = std::exp(*in++); + + while (in < lastAligned) { + _mm_store_ps(out, cos_ps(_mm_load_ps(in))); out += TypeAlignment; in += TypeAlignment; } + + while (in < sentinel) + *out++ = std::exp(*in++); } -template<> +template <> void log(absl::Span input, absl::Span output) noexcept { ASSERT(output.size() >= input.size()); auto* in = input.begin(); auto* out = output.begin(); auto* sentinel = in + std::min(input.size(), output.size()); - while (in < sentinel) - { - _mm_storeu_ps(out, log_ps(_mm_loadu_ps(in))); + const auto* lastAligned = prevAligned(sentinel); + + while (unaligned(in, out) && in < lastAligned) + *out++ = std::exp(*in++); + + while (in < lastAligned) { + _mm_store_ps(out, log_ps(_mm_load_ps(in))); out += TypeAlignment; in += TypeAlignment; } + + while (in < sentinel) + *out++ = std::exp(*in++); } -template<> +template <> void sin(absl::Span input, absl::Span output) noexcept { ASSERT(output.size() >= input.size()); auto* in = input.begin(); auto* out = output.begin(); auto* sentinel = in + std::min(input.size(), output.size()); - while (in < sentinel) - { - _mm_storeu_ps(out, sin_ps(_mm_loadu_ps(in))); + const auto* lastAligned = prevAligned(sentinel); + + while (unaligned(in, out) && in < lastAligned) + *out++ = std::exp(*in++); + + while (in < lastAligned) { + _mm_store_ps(out, sin_ps(_mm_load_ps(in))); out += TypeAlignment; in += TypeAlignment; } + + while (in < sentinel) + *out++ = std::exp(*in++); } -template<> +template <> void applyGain(float gain, absl::Span input, absl::Span output) noexcept { auto* in = input.begin(); @@ -221,9 +248,8 @@ void applyGain(float gain, absl::Span input, absl::Spa while (unaligned(out, in) && out < lastAligned) *out++ = gain * (*in++); - - while (out < lastAligned) - { + + while (out < lastAligned) { _mm_store_ps(out, _mm_mul_ps(mmGain, _mm_load_ps(in))); in += TypeAlignment; out += TypeAlignment; @@ -233,7 +259,7 @@ void applyGain(float gain, absl::Span input, absl::Spa *out++ = gain * (*in++); } -template<> +template <> void applyGain(absl::Span gain, absl::Span input, absl::Span output) noexcept { auto* in = input.begin(); @@ -245,8 +271,7 @@ void applyGain(absl::Span gain, absl::Span(g, in, out); - while (out < lastAligned) - { + while (out < lastAligned) { _mm_store_ps(out, _mm_mul_ps(_mm_load_ps(g), _mm_load_ps(in))); g += TypeAlignment; in += TypeAlignment; @@ -257,8 +282,14 @@ void applyGain(absl::Span gain, absl::Span(g, in, out); } -template<> -void loopingSFZIndex(absl::Span jumps, absl::Span leftCoeffs, absl::Span rightCoeffs, absl::Span indices, float floatIndex, float loopEnd, float loopStart) noexcept +template <> +void loopingSFZIndex( absl::Span jumps, + absl::Span leftCoeffs, + absl::Span rightCoeffs, + absl::Span indices, + float floatIndex, + float loopEnd, + float loopStart) noexcept { ASSERT(indices.size() >= jumps.size()); ASSERT(indices.size() == leftCoeffs.size()); @@ -276,12 +307,11 @@ void loopingSFZIndex(absl::Span jumps, absl::Span(jump, leftCoeff, rightCoeff, index, floatIndex, loopEnd, loopStart); auto mmFloatIndex = _mm_set_ps1(floatIndex); - const auto mmJumpBack = _mm_set1_ps(loopEnd - loopStart); + const auto mmJumpBack = _mm_set1_ps(loopEnd - loopStart); const auto mmLoopEnd = _mm_set1_ps(loopEnd); - while (jump < alignedEnd) - { + while (jump < alignedEnd) { auto mmOffset = _mm_load_ps(jump); - mmOffset = _mm_add_ps(mmOffset, _mm_castsi128_ps(_mm_slli_si128(_mm_castps_si128(mmOffset), 4))); + mmOffset = _mm_add_ps(mmOffset, _mm_castsi128_ps(_mm_slli_si128(_mm_castps_si128(mmOffset), 4))); mmOffset = _mm_add_ps(mmOffset, _mm_shuffle_ps(_mm_setzero_ps(), mmOffset, 0x40)); mmFloatIndex = _mm_add_ps(mmFloatIndex, mmOffset); @@ -293,7 +323,7 @@ void loopingSFZIndex(absl::Span jumps, absl::Span(index), mmIndices); - + auto mmRight = _mm_sub_ps(mmFloatIndex, _mm_cvtepi32_ps(mmIndices)); auto mmLeft = _mm_sub_ps(_mm_set_ps1(1.0f), mmRight); _mm_store_ps(leftCoeff, mmLeft); @@ -313,20 +343,19 @@ void loopingSFZIndex(absl::Span jumps, absl::Span(jump, leftCoeff, rightCoeff, index, floatIndex, loopEnd, loopStart); } -template<> +template <> float linearRamp(absl::Span output, float value, float step) noexcept { auto* out = output.begin(); const auto* lastAligned = prevAligned(output.end()); - while(unaligned(out) && out < lastAligned) + while (unaligned(out) && out < lastAligned) snippetRampLinear(out, value, step); auto mmValue = _mm_set1_ps(value); - auto mmStep = _mm_set_ps(step+step+step+step, step+step+step, step+step, step); + auto mmStep = _mm_set_ps(step + step + step + step, step + step + step, step + step, step); - while (out < lastAligned) - { + while (out < lastAligned) { mmValue = _mm_add_ps(mmValue, mmStep); _mm_store_ps(out, mmValue); mmValue = _mm_shuffle_ps(mmValue, mmValue, _MM_SHUFFLE(3, 3, 3, 3)); @@ -334,25 +363,24 @@ float linearRamp(absl::Span output, float value, float step) } value = _mm_cvtss_f32(mmValue); - while(out < output.end()) + while (out < output.end()) snippetRampLinear(out, value, step); return value; } -template<> +template <> float multiplicativeRamp(absl::Span output, float value, float step) noexcept { auto* out = output.begin(); const auto* lastAligned = prevAligned(output.end()); - while(unaligned(out) && out < lastAligned) + while (unaligned(out) && out < lastAligned) snippetRampMultiplicative(out, value, step); auto mmValue = _mm_set1_ps(value); - auto mmStep = _mm_set_ps(step*step*step*step, step*step*step, step*step, step); + auto mmStep = _mm_set_ps(step * step * step * step, step * step * step, step * step, step); - while (out < lastAligned) - { + while (out < lastAligned) { mmValue = _mm_mul_ps(mmValue, mmStep); _mm_store_ps(out, mmValue); mmValue = _mm_shuffle_ps(mmValue, mmValue, _MM_SHUFFLE(3, 3, 3, 3)); @@ -360,12 +388,12 @@ float multiplicativeRamp(absl::Span output, float value, flo } value = _mm_cvtss_f32(mmValue); - while(out < output.end()) + while (out < output.end()) snippetRampMultiplicative(out, value, step); return value; } -template<> +template <> void add(absl::Span input, absl::Span output) noexcept { ASSERT(output.size() >= input.size()); @@ -373,17 +401,16 @@ void add(absl::Span input, absl::Span output) n auto* out = output.begin(); auto* sentinel = out + min(input.size(), output.size()); const auto* lastAligned = prevAligned(sentinel); - - while(unaligned(in, out) && out < lastAligned) + + while (unaligned(in, out) && out < lastAligned) snippetAdd(in, out); - while(out < lastAligned) - { + while (out < lastAligned) { _mm_store_ps(out, _mm_add_ps(_mm_load_ps(in), _mm_load_ps(out))); out += TypeAlignment; in += TypeAlignment; } - while(out < sentinel) + while (out < sentinel) snippetAdd(in, out); } \ No newline at end of file