From c014a34385671110102c9b00778cf5e86d773505 Mon Sep 17 00:00:00 2001 From: Paul Ferrand Date: Wed, 31 Jul 2019 01:45:58 +0200 Subject: [PATCH] WIP on vector operations --- benchmarks/AudioBuffers.cpp | 70 +++++++-- sources/AudioBuffer.h | 106 ++++++++++++- sources/Buffer.h | 4 +- sources/Globals.h | 4 +- tests/AudioBuffer.cpp | 297 ++++++++++++++++++++++++++++++++++++ 5 files changed, 460 insertions(+), 21 deletions(-) diff --git a/benchmarks/AudioBuffers.cpp b/benchmarks/AudioBuffers.cpp index 1c2c9fba..bd3a766b 100644 --- a/benchmarks/AudioBuffers.cpp +++ b/benchmarks/AudioBuffers.cpp @@ -1,9 +1,10 @@ #include #include "../sources/AudioBuffer.h" #include +#include -static void AB_Joint_Fill_float(benchmark::State& state) { - AudioBuffer buffer (65535); +static void Joint_Fill_float(benchmark::State& state) { + AudioBuffer buffer (100001); float fillValue = 0.0f; for (auto _ : state) { buffer.fill(fillValue); @@ -11,20 +12,43 @@ static void AB_Joint_Fill_float(benchmark::State& state) { } } // Register the function as a benchmark -BENCHMARK(AB_Joint_Fill_float); +BENCHMARK(Joint_Fill_float); -static void AB_Split_Fill_float(benchmark::State& state) { - SplitAudioBuffer buffer (65535); +static void Split_Fill_float(benchmark::State& state) { + SplitAudioBuffer buffer (100001); float fillValue = 0.0f; for (auto _ : state) { buffer.fill(fillValue); fillValue += 1.0f; } } -BENCHMARK(AB_Split_Fill_float); +BENCHMARK(Split_Fill_float); -static void AB_Joint_Fill_double(benchmark::State& state) { - AudioBuffer buffer (65535); +static void Joint_Fill_float_SSE(benchmark::State& state) { + AudioBuffer buffer (100001); + float fillValue = 0.0f; + for (auto _ : state) { + buffer.fill(fillValue); + fillValue += 1.0f; + } +} +// Register the function as a benchmark +BENCHMARK(Joint_Fill_float_SSE); + + +static void Split_Fill_float_SSE(benchmark::State& state) { + SplitAudioBuffer buffer (100001); + float fillValue = 0.0f; + for (auto _ : state) { + buffer.fill(fillValue); + fillValue += 1.0f; + } +} + +BENCHMARK(Split_Fill_float_SSE); + +static void Joint_Fill_double(benchmark::State& state) { + AudioBuffer buffer (100001); double fillValue = 0.0; for (auto _ : state) { buffer.fill(fillValue); @@ -32,15 +56,37 @@ static void AB_Joint_Fill_double(benchmark::State& state) { } } // Register the function as a benchmark -BENCHMARK(AB_Joint_Fill_double); +BENCHMARK(Joint_Fill_double); -static void AB_Split_Fill_double(benchmark::State& state) { - SplitAudioBuffer buffer (65535); +static void Split_Fill_double(benchmark::State& state) { + SplitAudioBuffer buffer (100001); double fillValue = 0.0; for (auto _ : state) { buffer.fill(fillValue); fillValue += 1.0; } } -BENCHMARK(AB_Split_Fill_double); + +BENCHMARK(Split_Fill_double); + +constexpr int size { 1039247 }; +static void Interleaved_Read(benchmark::State& state) { + AudioBuffer buffer (size); + std::array input; + std::iota(input.begin(), input.end(), 1.0f); + for (auto _ : state) { + buffer.readInterleaved(input.data(), size); + } +} +BENCHMARK(Interleaved_Read); + +static void Interleaved_Read_SSE(benchmark::State& state) { + AudioBuffer buffer (size); + std::array input; + std::iota(input.begin(), input.end(), 1.0f); + for (auto _ : state) { + buffer.readInterleaved(input.data(), size); + } +} +BENCHMARK(Interleaved_Read_SSE); BENCHMARK_MAIN(); \ No newline at end of file diff --git a/sources/AudioBuffer.h b/sources/AudioBuffer.h index 0c33a175..373d36f9 100644 --- a/sources/AudioBuffer.h +++ b/sources/AudioBuffer.h @@ -2,6 +2,8 @@ #include "Buffer.h" #include "Helpers.h" #include "Globals.h" +#include +#include template @@ -35,14 +37,19 @@ public: { ASSERT(channelIndex >= 0); ASSERT(sampleIndex >= 0); - return *(buffer.data() + numFrames * channelIndex + sampleIndex); + return *(channels[channelIndex] + sampleIndex); } + template void fill(Type value) noexcept { - if constexpr (config::vectorOperation == config::VectorOperations::sse) + if constexpr (op == VectorOperations::sse && std::is_same::value) { - + static_assert(Alignment == 16, "Wrong alignment"); + const __m128 mmValue = _mm_set_ps1(value); + for(auto i = 0; i < NumChannels; ++i) + for (auto mm = (__m128*)alignedBegin(i); mm < (__m128*)alignedEnd(i); mm++) + _mm_store_ps((float*)mm, mmValue); } else { @@ -51,6 +58,65 @@ public: } } + template + void readInterleaved(float* input, int numFrames) noexcept + { + ASSERT(this->numFrames >= numFrames); + if constexpr (op == VectorOperations::sse && std::is_same::value && NumChannels == 2) + { + static_assert(Alignment == 16, "Wrong alignment"); + const int residualFrames = numFrames & (2 * TypeAlignment - 1); + const int lastAligned = numFrames - residualFrames; + float* in = input; + float* out0 = getChannel(0); + float* out1 = getChannel(1); + const float* end = input + 2 * lastAligned; + while (in < end) + { + auto input0 = _mm_load_ps(in); + in += 4; + auto input1 = _mm_load_ps(in); + in += 4; + auto intermediate0 = _mm_unpacklo_ps(input0, input1); + auto intermediate1 = _mm_unpackhi_ps(input0, input1); + auto output0 = _mm_unpacklo_ps(intermediate0, intermediate1); + auto output1 = _mm_unpackhi_ps(intermediate0, intermediate1); + _mm_store_ps(out0, output0); + _mm_store_ps(out1, output1); + out0 += 4; + out1 += 4; + } + + for (auto chanIdx = 0; chanIdx < NumChannels; chanIdx++) + { + auto* _in = input + 2 * lastAligned + chanIdx; + auto* _end = input + numFrames * NumChannels; + auto* _out = getChannel(chanIdx) + lastAligned; + while (_in < _end) + { + *_out = *_in; + _in += 2; + _out += 1; + } + } + } + else + { + for (auto chanIdx = 0; chanIdx < NumChannels; chanIdx++) + { + auto* _in = input + chanIdx; + auto* _end = input + numFrames * NumChannels; + auto* _out = getChannel(chanIdx); + while (_in < _end) + { + *_out = *_in; + _in += NumChannels; + _out += 1; + } + } + } + } + Type* getChannel(int channelIndex) noexcept { return channels[channelIndex]; @@ -58,12 +124,22 @@ public: Type* begin(int channelIndex) noexcept { - return buffer.data() + numFrames * channelIndex; + return channels[channelIndex]; } Type* end(int channelIndex) noexcept { - return buffer.data() + numFrames * (channelIndex + 1); + return buffer.data() + numFrames * (channelIndex + 1) + padding * channelIndex; + } + + Type* alignedBegin(int channelIndex) noexcept + { + return begin(channelIndex); + } + + Type* alignedEnd(int channelIndex) noexcept + { + return buffer.data() + (channelIndex + 1) * (numFrames + padding); } Type& operator()(int channelIndex, int sampleIndex) noexcept @@ -84,6 +160,7 @@ private: int padding { 0 }; std::array channels; Buffer buffer {}; + Buffer tempBuffer {2 * Alignment}; }; template @@ -113,11 +190,16 @@ public: return resizedOK; } + template void fill(Type value) noexcept { - if constexpr (config::vectorOperation == config::VectorOperations::sse) + if constexpr (op == VectorOperations::sse) { - + static_assert(Alignment == 16, "Wrong alignment"); + const __m128 mmValue = _mm_set_ps1(value); + for(auto i = 0; i < NumChannels; ++i) + for (auto mm = (__m128*)alignedBegin(i); mm < (__m128*)alignedEnd(i); mm++) + _mm_store_ps((float*)mm, mmValue); } else { @@ -153,6 +235,16 @@ public: return buffers[channelIndex].end(); } + Type* alignedBegin(int channelIndex) noexcept + { + return begin(channelIndex); + } + + Type* alignedEnd(int channelIndex) noexcept + { + return buffers[channelIndex].alignedEnd(); + } + int getNumFrames() const noexcept { return numFrames; } int getNumChannels() const noexcept { return NumChannels; } bool empty() const noexcept { return numFrames == 0; } diff --git a/sources/Buffer.h b/sources/Buffer.h index b5ffaa69..7cd0a812 100644 --- a/sources/Buffer.h +++ b/sources/Buffer.h @@ -9,6 +9,7 @@ class Buffer { static_assert(std::is_arithmetic::value, "Type should be arithmetic"); static_assert(Alignment == 0 || Alignment == 4 || Alignment == 8 || Alignment == 16, "Bad alignment value"); +static constexpr auto AlignmentMask { Alignment - 1 }; public: Buffer() { } Buffer(size_t size) @@ -23,7 +24,7 @@ public: return true; } - auto tempSize = newSize + Alignment; + auto tempSize = newSize + 2 * AlignmentMask; // To ensure that we have leeway at the beginning and at the end auto* newData = largerData != nullptr ? std::realloc(largerData, tempSize * sizeof(Type)) : std::malloc(tempSize * sizeof(Type)); if (newData == nullptr) return false; @@ -54,6 +55,7 @@ public: Type* begin() noexcept { return data(); } Type* end() noexcept { return data() + alignedSize; } + Type* alignedEnd() noexcept { return data() + alignedSize; } // const Type* cbegin() const noexcept { return data(); } // const Type* cend() const noexcept { return data() + alignedSize; } private: diff --git a/sources/Globals.h b/sources/Globals.h index 6ef6b00c..ff8e188e 100644 --- a/sources/Globals.h +++ b/sources/Globals.h @@ -26,4 +26,6 @@ namespace config inline constexpr unsigned int defaultAlignment { 16 }; inline constexpr VectorOperations vectorOperation { VectorOperations::standard }; -} // namespace config \ No newline at end of file +} // namespace config + +#include \ No newline at end of file diff --git a/tests/AudioBuffer.cpp b/tests/AudioBuffer.cpp index 774e057c..708e99bb 100644 --- a/tests/AudioBuffer.cpp +++ b/tests/AudioBuffer.cpp @@ -215,4 +215,301 @@ TEST_CASE("[AudioBuffer/SplitBuffer] Channel alignments (floats)") channelAlignmentTest(65537); channelAlignmentTest(65536); channelAlignmentTest(65535); +} + +TEST_CASE("[AudioBuffer/SplitBuffer] Channel alignments (doubles)") +{ + channelAlignmentTest(4); + channelAlignmentTest(5); + channelAlignmentTest(8); + channelAlignmentTest(256); + channelAlignmentTest(257); + channelAlignmentTest(1023); + channelAlignmentTest(1024); + channelAlignmentTest(65537); + channelAlignmentTest(65536); + channelAlignmentTest(65535); + + channelAlignmentTest(4); + channelAlignmentTest(5); + channelAlignmentTest(8); + channelAlignmentTest(256); + channelAlignmentTest(257); + channelAlignmentTest(1023); + channelAlignmentTest(1024); + channelAlignmentTest(65537); + channelAlignmentTest(65536); + channelAlignmentTest(65535); + + channelAlignmentTest(4); + channelAlignmentTest(5); + channelAlignmentTest(8); + channelAlignmentTest(256); + channelAlignmentTest(257); + channelAlignmentTest(1023); + channelAlignmentTest(1024); + channelAlignmentTest(65537); + channelAlignmentTest(65536); + channelAlignmentTest(65535); + + channelAlignmentTest(4); + channelAlignmentTest(5); + channelAlignmentTest(8); + channelAlignmentTest(256); + channelAlignmentTest(257); + channelAlignmentTest(1023); + channelAlignmentTest(1024); + channelAlignmentTest(65537); + channelAlignmentTest(65536); + channelAlignmentTest(65535); + + channelAlignmentTest(4); + channelAlignmentTest(5); + channelAlignmentTest(8); + channelAlignmentTest(256); + channelAlignmentTest(257); + channelAlignmentTest(1023); + channelAlignmentTest(1024); + channelAlignmentTest(65537); + channelAlignmentTest(65536); + channelAlignmentTest(65535); + + channelAlignmentTest(4); + channelAlignmentTest(5); + channelAlignmentTest(8); + channelAlignmentTest(256); + channelAlignmentTest(257); + channelAlignmentTest(1023); + channelAlignmentTest(1024); + channelAlignmentTest(65537); + channelAlignmentTest(65536); + channelAlignmentTest(65535); +} + +TEST_CASE("[AudioBuffer] fills") +{ + SECTION("Floats - 0.0") + { + AudioBuffer buffer(10); + buffer.fill(0.0f); + std::array expected; + std::fill(expected.begin(), expected.end(), 0.0f); + std::array real { 2.0f }; + for (auto chanIdx = 0; chanIdx < buffer.getNumChannels(); ++chanIdx) + { + for (auto frameIdx = 0; frameIdx < buffer.getNumFrames(); ++frameIdx) + real[frameIdx] = buffer(chanIdx, frameIdx); + REQUIRE( real == expected ); + } + } + + SECTION("Floats - 1.0") + { + AudioBuffer buffer(10); + buffer.fill(1.0f); + std::array expected; + std::fill(expected.begin(), expected.end(), 1.0f); + std::array real { 2.0f }; + for (auto chanIdx = 0; chanIdx < buffer.getNumChannels(); ++chanIdx) + { + for (auto frameIdx = 0; frameIdx < buffer.getNumFrames(); ++frameIdx) + real[frameIdx] = buffer(chanIdx, frameIdx); + REQUIRE( real == expected ); + } + } + + SECTION("Doubles - 0.0") + { + AudioBuffer buffer(10); + buffer.fill(0.0); + std::array expected; + std::fill(expected.begin(), expected.end(), 0.0); + std::array real { 2.0 }; + for (auto chanIdx = 0; chanIdx < buffer.getNumChannels(); ++chanIdx) + { + for (auto frameIdx = 0; frameIdx < buffer.getNumFrames(); ++frameIdx) + real[frameIdx] = buffer(chanIdx, frameIdx); + REQUIRE( real == expected ); + } + } + + SECTION("Doubles - 1.0") + { + AudioBuffer buffer(10); + buffer.fill(1.0); + std::array expected; + std::fill(expected.begin(), expected.end(), 1.0); + std::array real { 2.0 }; + for (auto chanIdx = 0; chanIdx < buffer.getNumChannels(); ++chanIdx) + { + for (auto frameIdx = 0; frameIdx < buffer.getNumFrames(); ++frameIdx) + real[frameIdx] = buffer(chanIdx, frameIdx); + REQUIRE( real == expected ); + } + } + + SECTION("Floats - 0.0 - SSE") + { + AudioBuffer buffer(10); + buffer.fill(0.0f); + std::array expected; + std::fill(expected.begin(), expected.end(), 0.0f); + std::array real { 2.0f }; + for (auto chanIdx = 0; chanIdx < buffer.getNumChannels(); ++chanIdx) + { + for (auto frameIdx = 0; frameIdx < buffer.getNumFrames(); ++frameIdx) + real[frameIdx] = buffer(chanIdx, frameIdx); + REQUIRE( real == expected ); + } + } + + SECTION("Floats - 1.0 - SSE") + { + AudioBuffer buffer(10); + buffer.fill(1.0f); + std::array expected { 1.0f }; + std::fill(expected.begin(), expected.end(), 1.0f); + std::array real { 2.0f }; + for (auto chanIdx = 0; chanIdx < buffer.getNumChannels(); ++chanIdx) + { + for (auto frameIdx = 0; frameIdx < buffer.getNumFrames(); ++frameIdx) + real[frameIdx] = buffer(chanIdx, frameIdx); + REQUIRE( real == expected ); + } + } +} + +TEST_CASE("[SplitAudioBuffer] fills") +{ + SECTION("Floats - 0.0") + { + SplitAudioBuffer buffer(10); + buffer.fill(0.0f); + std::array expected; + std::fill(expected.begin(), expected.end(), 0.0f); + std::array real { 2.0f }; + for (auto chanIdx = 0; chanIdx < buffer.getNumChannels(); ++chanIdx) + { + for (auto frameIdx = 0; frameIdx < buffer.getNumFrames(); ++frameIdx) + real[frameIdx] = buffer(chanIdx, frameIdx); + REQUIRE( real == expected ); + } + } + + SECTION("Floats - 1.0") + { + SplitAudioBuffer buffer(10); + buffer.fill(1.0f); + std::array expected; + std::fill(expected.begin(), expected.end(), 1.0f); + std::array real { 2.0f }; + for (auto chanIdx = 0; chanIdx < buffer.getNumChannels(); ++chanIdx) + { + for (auto frameIdx = 0; frameIdx < buffer.getNumFrames(); ++frameIdx) + real[frameIdx] = buffer(chanIdx, frameIdx); + REQUIRE( real == expected ); + } + } + + SECTION("Doubles - 0.0") + { + SplitAudioBuffer buffer(10); + buffer.fill(0.0); + std::array expected; + std::fill(expected.begin(), expected.end(), 0.0); + std::array real { 2.0 }; + for (auto chanIdx = 0; chanIdx < buffer.getNumChannels(); ++chanIdx) + { + for (auto frameIdx = 0; frameIdx < buffer.getNumFrames(); ++frameIdx) + real[frameIdx] = buffer(chanIdx, frameIdx); + REQUIRE( real == expected ); + } + } + + SECTION("Doubles - 1.0") + { + SplitAudioBuffer buffer(10); + buffer.fill(1.0); + std::array expected; + std::fill(expected.begin(), expected.end(), 1.0); + std::array real { 2.0 }; + for (auto chanIdx = 0; chanIdx < buffer.getNumChannels(); ++chanIdx) + { + for (auto frameIdx = 0; frameIdx < buffer.getNumFrames(); ++frameIdx) + real[frameIdx] = buffer(chanIdx, frameIdx); + REQUIRE( real == expected ); + } + } + + SECTION("Floats - 0.0 - SSE") + { + SplitAudioBuffer buffer(10); + buffer.fill(0.0f); + std::array expected; + std::fill(expected.begin(), expected.end(), 0.0f); + std::array real { 2.0f }; + for (auto chanIdx = 0; chanIdx < buffer.getNumChannels(); ++chanIdx) + { + for (auto frameIdx = 0; frameIdx < buffer.getNumFrames(); ++frameIdx) + real[frameIdx] = buffer(chanIdx, frameIdx); + REQUIRE( real == expected ); + } + } + + SECTION("Floats - 1.0 - SSE") + { + SplitAudioBuffer buffer(10); + buffer.fill(1.0f); + std::array expected { 1.0f }; + std::fill(expected.begin(), expected.end(), 1.0f); + std::array real { 2.0f }; + for (auto chanIdx = 0; chanIdx < buffer.getNumChannels(); ++chanIdx) + { + for (auto frameIdx = 0; frameIdx < buffer.getNumFrames(); ++frameIdx) + real[frameIdx] = buffer(chanIdx, frameIdx); + REQUIRE( real == expected ); + } + } +} + +TEST_CASE("[AudioBuffer] Interleave read") +{ + AudioBuffer buffer(8); + std::array input = { 0.0f, 10.0f, 1.0f, 11.0f, 2.0f, 12.0f, 3.0f, 13.0f, 4.0f, 14.0f, 5.0f, 15.0f, 6.0f, 16.0f, 7.0f, 17.0f}; + std::array expected = { 0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f, 16.0f, 17.0f }; + buffer.readInterleaved(input.data(), 8); + std::array real { 0.0f }; + auto realIdx = 0; + for (auto chanIdx = 0; chanIdx < buffer.getNumChannels(); ++chanIdx) + for (auto frameIdx = 0; frameIdx < buffer.getNumFrames(); ++frameIdx) + real[realIdx++] = buffer(chanIdx, frameIdx); + REQUIRE( real == expected ); +} + +TEST_CASE("[AudioBuffer] Interleave read -- SSE") +{ + AudioBuffer buffer(8); + std::array input = { 0.0f, 10.0f, 1.0f, 11.0f, 2.0f, 12.0f, 3.0f, 13.0f, 4.0f, 14.0f, 5.0f, 15.0f, 6.0f, 16.0f, 7.0f, 17.0f}; + std::array expected = { 0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f, 16.0f, 17.0f }; + buffer.readInterleaved(input.data(), 8); + std::array real { 0.0f }; + auto realIdx = 0; + for (auto chanIdx = 0; chanIdx < buffer.getNumChannels(); ++chanIdx) + for (auto frameIdx = 0; frameIdx < buffer.getNumFrames(); ++frameIdx) + real[realIdx++] = buffer(chanIdx, frameIdx); + REQUIRE( real == expected ); +} + +TEST_CASE("[AudioBuffer] Interleave read unaligned end -- SSE") +{ + AudioBuffer buffer(10); + std::array input = { 0.0f, 10.0f, 1.0f, 11.0f, 2.0f, 12.0f, 3.0f, 13.0f, 4.0f, 14.0f, 5.0f, 15.0f, 6.0f, 16.0f, 7.0f, 17.0f, 8.0f, 18.0f, 9.0f, 19.0f}; + std::array expected = { 0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f, 16.0f, 17.0f, 18.0f, 19.0f}; + buffer.readInterleaved(input.data(), 10); + std::array real { 0.0f }; + auto realIdx = 0; + for (auto chanIdx = 0; chanIdx < buffer.getNumChannels(); ++chanIdx) + for (auto frameIdx = 0; frameIdx < buffer.getNumFrames(); ++frameIdx) + real[realIdx++] = buffer(chanIdx, frameIdx); + REQUIRE( real == expected ); } \ No newline at end of file