sam/sfizz
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Reorganize the SIMD functions to "hide" part of it in a subnamespace

Browse source
This commit is contained in:
Paul Ferrand 2019-11-22 17:51:06 +01:00
parent 77c0f23369
commit 87e07d4e28
2 changed files with 201 additions and 175 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

308
sfizz/SIMDHelpers.h
View file

@ -29,13 +29,14 @@
#include <absl/types/span.h> #include <absl/types/span.h>
#include <cmath> #include <cmath>
namespace sfz namespace sfz {
{ namespace _internals {
template <class T> template <class T>
inline void snippetRead(const T*& input, T*& outputLeft, T*& outputRight) inline void snippetRead(const T*& input, T*& outputLeft, T*& outputRight)
{ {
*outputLeft++ = *input++; *outputLeft++ = *input++;
*outputRight++ = *input++; *outputRight++ = *input++;
}
} }
template <class T, bool SIMD = SIMDConfig::readInterleaved> template <class T, bool SIMD = SIMDConfig::readInterleaved>
@ -49,14 +50,16 @@ void readInterleaved(absl::Span<const T> input, absl::Span<T> outputLeft, absl::
auto* lOut = outputLeft.begin(); auto* lOut = outputLeft.begin();
auto* rOut = outputRight.begin(); auto* rOut = outputRight.begin();
while (in < (input.end() - 1) && lOut < outputLeft.end() && rOut < outputRight.end()) while (in < (input.end() - 1) && lOut < outputLeft.end() && rOut < outputRight.end())
snippetRead<T>(in, lOut, rOut); _internals::snippetRead<T>(in, lOut, rOut);
} }
template <class T> namespace _internals {
inline void snippetWrite(T*& output, const T*& inputLeft, const T*& inputRight) template <class T>
{ inline void snippetWrite(T*& output, const T*& inputLeft, const T*& inputRight)
*output++ = *inputLeft++; {
*output++ = *inputRight++; *output++ = *inputLeft++;
*output++ = *inputRight++;
}
} }
template <class T, bool SIMD = SIMDConfig::writeInterleaved> template <class T, bool SIMD = SIMDConfig::writeInterleaved>
@ -69,7 +72,7 @@ void writeInterleaved(absl::Span<const T> inputLeft, absl::Span<const T> inputRi
auto* rIn = inputRight.begin(); auto* rIn = inputRight.begin();
auto* out = output.begin(); auto* out = output.begin();
while (lIn < inputLeft.end() && rIn < inputRight.end() && out < (output.end() - 1)) while (lIn < inputLeft.end() && rIn < inputRight.end() && out < (output.end() - 1))
snippetWrite<T>(out, lIn, rIn); _internals::snippetWrite<T>(out, lIn, rIn);
} }
// Specializations // Specializations
@ -135,27 +138,26 @@ void cos(absl::Span<const Type> input, absl::Span<Type> output) noexcept
template <> template <>
void cos<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept; void cos<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept;
template <> namespace _internals {
void cos<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept; template <class T>
inline void snippetSaturatingIndex(const T*& jump, T*& leftCoeff, T*& rightCoeff, int*& index, T& floatIndex, T loopEnd)
template <class T> {
inline void snippetSaturatingIndex(const T*& jump, T*& leftCoeff, T*& rightCoeff, int*& index, T& floatIndex, T loopEnd) floatIndex += *jump;
{ if (floatIndex >= loopEnd) {
floatIndex += *jump; floatIndex = loopEnd;
if (floatIndex >= loopEnd) { *index = static_cast<int>(floatIndex) - 1;
floatIndex = loopEnd; *rightCoeff = static_cast<T>(1.0);
*index = static_cast<int>(floatIndex) - 1; *leftCoeff = static_cast<T>(0.0);
*rightCoeff = static_cast<T>(1.0); } else {
*leftCoeff = static_cast<T>(0.0); *index = static_cast<int>(floatIndex);
} else { *rightCoeff = floatIndex - *index;
*index = static_cast<int>(floatIndex); *leftCoeff = static_cast<T>(1.0) - *rightCoeff;
*rightCoeff = floatIndex - *index; }
*leftCoeff = static_cast<T>(1.0) - *rightCoeff; index++;
leftCoeff++;
rightCoeff++;
jump++;
} }
index++;
leftCoeff++;
rightCoeff++;
jump++;
} }
template <class T, bool SIMD = SIMDConfig::saturatingSFZIndex> template <class T, bool SIMD = SIMDConfig::saturatingSFZIndex>
@ -173,26 +175,28 @@ float saturatingSFZIndex(absl::Span<const T> jumps, absl::Span<T> leftCoeffs, ab
auto* sentinel = jumps.begin() + size; auto* sentinel = jumps.begin() + size;
while (jump < sentinel) while (jump < sentinel)
snippetSaturatingIndex<T>(jump, leftCoeff, rightCoeff, index, floatIndex, loopEnd); _internals::snippetSaturatingIndex<T>(jump, leftCoeff, rightCoeff, index, floatIndex, loopEnd);
return floatIndex; return floatIndex;
} }
template <> template <>
float saturatingSFZIndex<float, true>(absl::Span<const float> jumps, absl::Span<float> leftCoeffs, absl::Span<float> rightCoeffs, absl::Span<int> indices, float floatIndex, float loopEnd) noexcept; float saturatingSFZIndex<float, true>(absl::Span<const float> jumps, absl::Span<float> leftCoeffs, absl::Span<float> rightCoeffs, absl::Span<int> indices, float floatIndex, float loopEnd) noexcept;
template <class T> namespace _internals {
inline void snippetLoopingIndex(const T*& jump, T*& leftCoeff, T*& rightCoeff, int*& index, T& floatIndex, T loopEnd, T loopStart) template <class T>
{ inline void snippetLoopingIndex(const T*& jump, T*& leftCoeff, T*& rightCoeff, int*& index, T& floatIndex, T loopEnd, T loopStart)
floatIndex += *jump; {
if (floatIndex >= loopEnd) floatIndex += *jump;
floatIndex -= loopEnd - loopStart; if (floatIndex >= loopEnd)
*index = static_cast<int>(floatIndex); floatIndex -= loopEnd - loopStart;
*rightCoeff = floatIndex - *index; *index = static_cast<int>(floatIndex);
*leftCoeff = 1.0f - *rightCoeff; *rightCoeff = floatIndex - *index;
index++; *leftCoeff = 1.0f - *rightCoeff;
leftCoeff++; index++;
rightCoeff++; leftCoeff++;
jump++; rightCoeff++;
jump++;
}
} }
template <class T, bool SIMD = SIMDConfig::loopingSFZIndex> template <class T, bool SIMD = SIMDConfig::loopingSFZIndex>
@ -210,17 +214,19 @@ float loopingSFZIndex(absl::Span<const T> jumps, absl::Span<T> leftCoeffs, absl:
auto* sentinel = jumps.begin() + size; auto* sentinel = jumps.begin() + size;
while (jump < sentinel) while (jump < sentinel)
snippetLoopingIndex<T>(jump, leftCoeff, rightCoeff, index, floatIndex, loopEnd, loopStart); _internals::snippetLoopingIndex<T>(jump, leftCoeff, rightCoeff, index, floatIndex, loopEnd, loopStart);
return floatIndex; return floatIndex;
} }
template <> template <>
float loopingSFZIndex<float, true>(absl::Span<const float> jumps, absl::Span<float> leftCoeff, absl::Span<float> rightCoeff, absl::Span<int> indices, float floatIndex, float loopEnd, float loopStart) noexcept; float loopingSFZIndex<float, true>(absl::Span<const float> jumps, absl::Span<float> leftCoeff, absl::Span<float> rightCoeff, absl::Span<int> indices, float floatIndex, float loopEnd, float loopStart) noexcept;
template <class T> namespace _internals {
inline void snippetGain(T gain, const T*& input, T*& output) template <class T>
{ inline void snippetGain(T gain, const T*& input, T*& output)
*output++ = gain * (*input++); {
*output++ = gain * (*input++);
}
} }
template <class T, bool SIMD = SIMDConfig::gain> template <class T, bool SIMD = SIMDConfig::gain>
@ -231,13 +237,15 @@ void applyGain(T gain, absl::Span<const T> input, absl::Span<T> output) noexcept
auto* out = output.begin(); auto* out = output.begin();
auto* sentinel = out + std::min(output.size(), input.size()); auto* sentinel = out + std::min(output.size(), input.size());
while (out < sentinel) while (out < sentinel)
snippetGain<T>(gain, in, out); _internals::snippetGain<T>(gain, in, out);
} }
template <class T> namespace _internals {
inline void snippetGainSpan(const T*& gain, const T*& input, T*& output) template <class T>
{ inline void snippetGainSpan(const T*& gain, const T*& input, T*& output)
*output++ = (*gain++) * (*input++); {
*output++ = (*gain++) * (*input++);
}
} }
template <class T, bool SIMD = SIMDConfig::gain> template <class T, bool SIMD = SIMDConfig::gain>
@ -250,7 +258,7 @@ void applyGain(absl::Span<const T> gain, absl::Span<const T> input, absl::Span<T
auto* out = output.begin(); auto* out = output.begin();
auto* sentinel = out + std::min(gain.size(), std::min(output.size(), input.size())); auto* sentinel = out + std::min(gain.size(), std::min(output.size(), input.size()));
while (out < sentinel) while (out < sentinel)
snippetGainSpan<T>(g, in, out); _internals::snippetGainSpan<T>(g, in, out);
} }
template <class T, bool SIMD = SIMDConfig::gain> template <class T, bool SIMD = SIMDConfig::gain>
@ -271,10 +279,12 @@ void applyGain<float, true>(float gain, absl::Span<const float> input, absl::Spa
template <> template <>
void applyGain<float, true>(absl::Span<const float> gain, absl::Span<const float> input, absl::Span<float> output) noexcept; void applyGain<float, true>(absl::Span<const float> gain, absl::Span<const float> input, absl::Span<float> output) noexcept;
template <class T> namespace _internals {
inline void snippetMultiplyAdd(const T*& gain, const T*& input, T*& output) template <class T>
{ inline void snippetMultiplyAdd(const T*& gain, const T*& input, T*& output)
*output++ += (*gain++) * (*input++); {
*output++ += (*gain++) * (*input++);
}
} }
template <class T, bool SIMD = SIMDConfig::multiplyAdd> template <class T, bool SIMD = SIMDConfig::multiplyAdd>
@ -287,17 +297,19 @@ void multiplyAdd(absl::Span<const T> gain, absl::Span<const T> input, absl::Span
auto* out = output.begin(); auto* out = output.begin();
auto* sentinel = out + std::min(gain.size(), std::min(output.size(), input.size())); auto* sentinel = out + std::min(gain.size(), std::min(output.size(), input.size()));
while (out < sentinel) while (out < sentinel)
snippetMultiplyAdd<T>(g, in, out); _internals::snippetMultiplyAdd<T>(g, in, out);
} }
template <> template <>
void multiplyAdd<float, true>(absl::Span<const float> gain, absl::Span<const float> input, absl::Span<float> output) noexcept; void multiplyAdd<float, true>(absl::Span<const float> gain, absl::Span<const float> input, absl::Span<float> output) noexcept;
template <class T> namespace _internals {
inline void snippetRampLinear(T*& output, T& value, T step) template <class T>
{ inline void snippetRampLinear(T*& output, T& value, T step)
value += step; {
*output++ = value; value += step;
*output++ = value;
}
} }
template <class T, bool SIMD = SIMDConfig::linearRamp> template <class T, bool SIMD = SIMDConfig::linearRamp>
@ -305,15 +317,17 @@ T linearRamp(absl::Span<T> output, T start, T step) noexcept
{ {
auto* out = output.begin(); auto* out = output.begin();
while (out < output.end()) while (out < output.end())
snippetRampLinear<T>(out, start, step); _internals::snippetRampLinear<T>(out, start, step);
return start; return start;
} }
template <class T> namespace _internals {
inline void snippetRampMultiplicative(T*& output, T& value, T step) template <class T>
{ inline void snippetRampMultiplicative(T*& output, T& value, T step)
value *= step; {
*output++ = value; value *= step;
*output++ = value;
}
} }
template <class T, bool SIMD = SIMDConfig::multiplicativeRamp> template <class T, bool SIMD = SIMDConfig::multiplicativeRamp>
@ -321,7 +335,7 @@ T multiplicativeRamp(absl::Span<T> output, T start, T step) noexcept
{ {
auto* out = output.begin(); auto* out = output.begin();
while (out < output.end()) while (out < output.end())
snippetRampMultiplicative<T>(out, start, step); _internals::snippetRampMultiplicative<T>(out, start, step);
return start; return start;
} }
@ -331,10 +345,17 @@ float linearRamp<float, true>(absl::Span<float> output, float start, float step)
template <> template <>
float multiplicativeRamp<float, true>(absl::Span<float> output, float start, float step) noexcept; float multiplicativeRamp<float, true>(absl::Span<float> output, float start, float step) noexcept;
template <class T> namespace _internals {
inline void snippetAdd(const T*& input, T*& output) template <class T>
{ inline void snippetAdd(const T*& input, T*& output)
*output++ += *input++; {
*output++ += *input++;
}
template <class T>
inline void snippetAdd(const T value, T*& output)
{
*output++ += value;
}
} }
template <class T, bool SIMD = SIMDConfig::add> template <class T, bool SIMD = SIMDConfig::add>
@ -345,41 +366,36 @@ void add(absl::Span<const T> input, absl::Span<T> output) noexcept
auto* out = output.begin(); auto* out = output.begin();
auto* sentinel = out + min(input.size(), output.size()); auto* sentinel = out + min(input.size(), output.size());
while (out < sentinel) while (out < sentinel)
snippetAdd(in, out); _internals::snippetAdd(in, out);
} }
template <> template <>
void add<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept; void add<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept;
template <class T>
inline void snippetAdd(const T value, T*& output)
{
*output++ += value;
}
template <class T, bool SIMD = SIMDConfig::add> template <class T, bool SIMD = SIMDConfig::add>
void add(T value, absl::Span<T> output) noexcept void add(T value, absl::Span<T> output) noexcept
{ {
auto* out = output.begin(); auto* out = output.begin();
auto* sentinel = output.end(); auto* sentinel = output.end();
while (out < sentinel) while (out < sentinel)
snippetAdd(value, out); _internals::snippetAdd(value, out);
} }
template <> template <>
void add<float, true>(float value, absl::Span<float> output) noexcept; void add<float, true>(float value, absl::Span<float> output) noexcept;
namespace _internals {
template <class T>
inline void snippetSubtract(const T*& input, T*& output)
{
*output++ -= *input++;
}
template <class T> template <class T>
inline void snippetSubtract(const T*& input, T*& output) inline void snippetSubtract(const T value, T*& output)
{ {
*output++ -= *input++; *output++ -= value;
} }
template <class T>
inline void snippetSubtract(const T value, T*& output)
{
*output++ -= value;
} }
template <class T, bool SIMD = SIMDConfig::subtract> template <class T, bool SIMD = SIMDConfig::subtract>
@ -388,7 +404,7 @@ void subtract(const T value, absl::Span<T> output) noexcept
auto* out = output.begin(); auto* out = output.begin();
auto* sentinel = output.end(); auto* sentinel = output.end();
while (out < sentinel) while (out < sentinel)
snippetSubtract(value, out); _internals::snippetSubtract(value, out);
} }
template <class T, bool SIMD = SIMDConfig::subtract> template <class T, bool SIMD = SIMDConfig::subtract>
@ -399,7 +415,7 @@ void subtract(absl::Span<const T> input, absl::Span<T> output) noexcept
auto* out = output.begin(); auto* out = output.begin();
auto* sentinel = out + min(input.size(), output.size()); auto* sentinel = out + min(input.size(), output.size());
while (out < sentinel) while (out < sentinel)
snippetSubtract(in, out); _internals::snippetSubtract(in, out);
} }
template <> template <>
@ -408,10 +424,12 @@ void subtract<float, true>(absl::Span<const float> input, absl::Span<float> outp
template <> template <>
void subtract<float, true>(const float value, absl::Span<float> output) noexcept; void subtract<float, true>(const float value, absl::Span<float> output) noexcept;
template <class T> namespace _internals {
void snippetCopy(const T*& input, T*& output) template <class T>
{ void snippetCopy(const T*& input, T*& output)
*output++ = *input++; {
*output++ = *input++;
}
} }
template <class T, bool SIMD = SIMDConfig::copy> template <class T, bool SIMD = SIMDConfig::copy>
@ -422,18 +440,20 @@ void copy(absl::Span<const T> input, absl::Span<T> output) noexcept
auto* out = output.begin(); auto* out = output.begin();
auto* sentinel = out + min(input.size(), output.size()); auto* sentinel = out + min(input.size(), output.size());
while (out < sentinel) while (out < sentinel)
snippetCopy(in, out); _internals::snippetCopy(in, out);
} }
template <> template <>
void copy<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept; void copy<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept;
template <class T> namespace _internals {
inline void snippetPan(const T*& pan, T*& left, T*& right) template <class T>
{ inline void snippetPan(const T*& pan, T*& left, T*& right)
const auto circlePan = piFour<float> * (static_cast<T>(1.0) + *pan++); {
*left++ *= std::cos(circlePan); const auto circlePan = piFour<float> * (static_cast<T>(1.0) + *pan++);
*right++ *= std::sin(circlePan); *left++ *= std::cos(circlePan);
*right++ *= std::sin(circlePan);
}
} }
template <class T, bool SIMD = SIMDConfig::pan> template <class T, bool SIMD = SIMDConfig::pan>
@ -446,7 +466,7 @@ void pan(absl::Span<const T> panEnvelope, absl::Span<T> leftBuffer, absl::Span<T
auto* right = rightBuffer.begin(); auto* right = rightBuffer.begin();
auto* sentinel = pan + min(panEnvelope.size(), leftBuffer.size(), rightBuffer.size()); auto* sentinel = pan + min(panEnvelope.size(), leftBuffer.size(), rightBuffer.size());
while (pan < sentinel) while (pan < sentinel)
snippetPan(pan, left, right); _internals::snippetPan(pan, left, right);
} }
template <> template <>
@ -455,7 +475,7 @@ void pan<float, true>(absl::Span<const float> panEnvelope, absl::Span<float> lef
template <class T, bool SIMD = SIMDConfig::mean> template <class T, bool SIMD = SIMDConfig::mean>
T mean(absl::Span<const T> vector) noexcept T mean(absl::Span<const T> vector) noexcept
{ {
T result { 0.0 }; T result{ 0.0 };
if (vector.size() == 0) if (vector.size() == 0)
return result; return result;
@ -472,7 +492,7 @@ float mean<float, true>(absl::Span<const float> vector) noexcept;
template <class T, bool SIMD = SIMDConfig::meanSquared> template <class T, bool SIMD = SIMDConfig::meanSquared>
T meanSquared(absl::Span<const T> vector) noexcept T meanSquared(absl::Span<const T> vector) noexcept
{ {
T result { 0.0 }; T result{ 0.0 };
if (vector.size() == 0) if (vector.size() == 0)
return result; return result;
@ -488,11 +508,13 @@ T meanSquared(absl::Span<const T> vector) noexcept
template <> template <>
float meanSquared<float, true>(absl::Span<const float> vector) noexcept; float meanSquared<float, true>(absl::Span<const float> vector) noexcept;
template <class T> namespace _internals {
inline void snippetCumsum(const T*& input, T*& output) template <class T>
{ inline void snippetCumsum(const T*& input, T*& output)
*output = *(output - 1) + *input++; {
output++; *output = *(output - 1) + *input++;
output++;
}
} }
template <class T, bool SIMD = SIMDConfig::cumsum> template <class T, bool SIMD = SIMDConfig::cumsum>
@ -508,25 +530,27 @@ void cumsum(absl::Span<const T> input, absl::Span<T> output) noexcept
*out++ = *in++; *out++ = *in++;
while (in < sentinel) while (in < sentinel)
snippetCumsum(in, out); _internals::snippetCumsum(in, out);
} }
template <> template <>
void cumsum<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept; void cumsum<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept;
template<class T> namespace _internals {
void snippetSFZInterpolationCast(const T*& floatJump, int*& jump, T*& leftCoeff, T*& rightCoeff) template <class T>
{ void snippetSFZInterpolationCast(const T*& floatJump, int*& jump, T*& leftCoeff, T*& rightCoeff)
*jump = static_cast<int>(*floatJump); {
*rightCoeff = *floatJump - static_cast<float>(*jump); *jump = static_cast<int>(*floatJump);
*leftCoeff = static_cast<T>(1.0) - *rightCoeff; *rightCoeff = *floatJump - static_cast<float>(*jump);
leftCoeff++; *leftCoeff = static_cast<T>(1.0) - *rightCoeff;
jump++; leftCoeff++;
rightCoeff++; jump++;
floatJump++; rightCoeff++;
floatJump++;
}
} }
template<class T, bool SIMD = SIMDConfig::sfzInterpolationCast> template <class T, bool SIMD = SIMDConfig::sfzInterpolationCast>
void sfzInterpolationCast(absl::Span<const T> floatJumps, absl::Span<int> jumps, absl::Span<T> leftCoeffs, absl::Span<T> rightCoeffs) noexcept void sfzInterpolationCast(absl::Span<const T> floatJumps, absl::Span<int> jumps, absl::Span<T> leftCoeffs, absl::Span<T> rightCoeffs) noexcept
{ {
ASSERT(jumps.size() >= floatJumps.size()); ASSERT(jumps.size() >= floatJumps.size());
@ -540,18 +564,20 @@ void sfzInterpolationCast(absl::Span<const T> floatJumps, absl::Span<int> jumps,
const auto sentinel = floatJump + min(floatJumps.size(), jumps.size(), leftCoeffs.size(), rightCoeffs.size()); const auto sentinel = floatJump + min(floatJumps.size(), jumps.size(), leftCoeffs.size(), rightCoeffs.size());
while (floatJump < sentinel) while (floatJump < sentinel)
snippetSFZInterpolationCast(floatJump, jump, leftCoeff, rightCoeff); _internals::snippetSFZInterpolationCast(floatJump, jump, leftCoeff, rightCoeff);
} }
template<> template <>
void sfzInterpolationCast<float, true>(absl::Span<const float> floatJumps, absl::Span<int> jumps, absl::Span<float> leftCoeffs, absl::Span<float> rightCoeffs) noexcept; void sfzInterpolationCast<float, true>(absl::Span<const float> floatJumps, absl::Span<int> jumps, absl::Span<float> leftCoeffs, absl::Span<float> rightCoeffs) noexcept;
template <class T> namespace _internals {
inline void snippetDiff(const T*& input, T*& output) template <class T>
{ inline void snippetDiff(const T*& input, T*& output)
*output = *input - *(input - 1); {
output++; *output = *input - *(input - 1);
input++; output++;
input++;
}
} }
template <class T, bool SIMD = SIMDConfig::diff> template <class T, bool SIMD = SIMDConfig::diff>
@ -567,10 +593,10 @@ void diff(absl::Span<const T> input, absl::Span<T> output) noexcept
*out++ = *in++; *out++ = *in++;
while (in < sentinel) while (in < sentinel)
snippetDiff(in, out); _internals::snippetDiff(in, out);
} }
template <> template <>
void cumsum<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept; void diff<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept;
} // namespace sfz } // namespace sfz

68
sfizz/SIMDSSE.cpp
View file

@ -92,7 +92,7 @@ void sfz::readInterleaved<float, true>(absl::Span<const float> input, absl::Span
const auto* lastAligned = prevAligned(input.begin() + size - TypeAlignment); const auto* lastAligned = prevAligned(input.begin() + size - TypeAlignment);
while (unaligned(in, lOut, rOut) && in < lastAligned) while (unaligned(in, lOut, rOut) && in < lastAligned)
snippetRead<float>(in, lOut, rOut); _internals::snippetRead<float>(in, lOut, rOut);
while (in < lastAligned) { while (in < lastAligned) {
auto register0 = _mm_load_ps(in); auto register0 = _mm_load_ps(in);
@ -112,7 +112,7 @@ void sfz::readInterleaved<float, true>(absl::Span<const float> input, absl::Span
} }
while (in < input.end() - 1) while (in < input.end() - 1)
snippetRead<float>(in, lOut, rOut); _internals::snippetRead<float>(in, lOut, rOut);
} }
template <> template <>
@ -130,7 +130,7 @@ void sfz::writeInterleaved<float, true>(absl::Span<const float> inputLeft, absl:
const auto* lastAligned = prevAligned(output.begin() + size - TypeAlignment); const auto* lastAligned = prevAligned(output.begin() + size - TypeAlignment);
while (unaligned(out, rIn, lIn) && out < lastAligned) while (unaligned(out, rIn, lIn) && out < lastAligned)
snippetWrite<float>(out, lIn, rIn); _internals::snippetWrite<float>(out, lIn, rIn);
while (out < lastAligned) { while (out < lastAligned) {
const auto lInRegister = _mm_load_ps(lIn); const auto lInRegister = _mm_load_ps(lIn);
@ -149,7 +149,7 @@ void sfz::writeInterleaved<float, true>(absl::Span<const float> inputLeft, absl:
} }
while (out < output.end() - 1) while (out < output.end() - 1)
snippetWrite<float>(out, lIn, rIn); _internals::snippetWrite<float>(out, lIn, rIn);
} }
template <> template <>
@ -292,7 +292,7 @@ void sfz::applyGain<float, true>(absl::Span<const float> gain, absl::Span<const
const auto* lastAligned = prevAligned(output.begin() + size); const auto* lastAligned = prevAligned(output.begin() + size);
while (unaligned(out, in, g) && out < lastAligned) while (unaligned(out, in, g) && out < lastAligned)
snippetGainSpan<float>(g, in, out); _internals::snippetGainSpan<float>(g, in, out);
while (out < lastAligned) { while (out < lastAligned) {
_mm_store_ps(out, _mm_mul_ps(_mm_load_ps(g), _mm_load_ps(in))); _mm_store_ps(out, _mm_mul_ps(_mm_load_ps(g), _mm_load_ps(in)));
@ -302,7 +302,7 @@ void sfz::applyGain<float, true>(absl::Span<const float> gain, absl::Span<const
} }
while (out < output.end()) while (out < output.end())
snippetGainSpan<float>(g, in, out); _internals::snippetGainSpan<float>(g, in, out);
} }
template <> template <>
@ -315,7 +315,7 @@ void sfz::multiplyAdd<float, true>(absl::Span<const float> gain, absl::Span<cons
const auto* lastAligned = prevAligned(output.begin() + size); const auto* lastAligned = prevAligned(output.begin() + size);
while (unaligned(out, in, g) && out < lastAligned) while (unaligned(out, in, g) && out < lastAligned)
snippetMultiplyAdd<float>(g, in, out); _internals::snippetMultiplyAdd<float>(g, in, out);
while (out < lastAligned) { while (out < lastAligned) {
auto mmOut = _mm_load_ps(out); auto mmOut = _mm_load_ps(out);
@ -327,7 +327,7 @@ void sfz::multiplyAdd<float, true>(absl::Span<const float> gain, absl::Span<cons
} }
while (out < output.end()) while (out < output.end())
snippetMultiplyAdd<float>(g, in, out); _internals::snippetMultiplyAdd<float>(g, in, out);
} }
template <> template <>
@ -352,7 +352,7 @@ float sfz::loopingSFZIndex<float, true>(absl::Span<const float> jumps,
const auto* alignedEnd = prevAligned(sentinel); const auto* alignedEnd = prevAligned(sentinel);
while (unaligned(reinterpret_cast<float*>(index), leftCoeff, rightCoeff, jump) && jump < alignedEnd) while (unaligned(reinterpret_cast<float*>(index), leftCoeff, rightCoeff, jump) && jump < alignedEnd)
snippetLoopingIndex<float>(jump, leftCoeff, rightCoeff, index, floatIndex, loopEnd, loopStart); _internals::snippetLoopingIndex<float>(jump, leftCoeff, rightCoeff, index, floatIndex, loopEnd, loopStart);
auto mmFloatIndex = _mm_set_ps1(floatIndex); auto mmFloatIndex = _mm_set_ps1(floatIndex);
const auto mmJumpBack = _mm_set1_ps(loopEnd - loopStart); const auto mmJumpBack = _mm_set1_ps(loopEnd - loopStart);
@ -388,7 +388,7 @@ float sfz::loopingSFZIndex<float, true>(absl::Span<const float> jumps,
floatIndex = _mm_cvtss_f32(mmFloatIndex); floatIndex = _mm_cvtss_f32(mmFloatIndex);
while (jump < sentinel) while (jump < sentinel)
snippetLoopingIndex<float>(jump, leftCoeff, rightCoeff, index, floatIndex, loopEnd, loopStart); _internals::snippetLoopingIndex<float>(jump, leftCoeff, rightCoeff, index, floatIndex, loopEnd, loopStart);
return floatIndex; return floatIndex;
} }
@ -413,7 +413,7 @@ float sfz::saturatingSFZIndex<float, true>(absl::Span<const float> jumps,
const auto* alignedEnd = prevAligned(sentinel); const auto* alignedEnd = prevAligned(sentinel);
while (unaligned(reinterpret_cast<float*>(index), leftCoeff, rightCoeff, jump) && jump < alignedEnd) while (unaligned(reinterpret_cast<float*>(index), leftCoeff, rightCoeff, jump) && jump < alignedEnd)
snippetSaturatingIndex<float>(jump, leftCoeff, rightCoeff, index, floatIndex, loopEnd); _internals::snippetSaturatingIndex<float>(jump, leftCoeff, rightCoeff, index, floatIndex, loopEnd);
auto mmFloatIndex = _mm_set_ps1(floatIndex); auto mmFloatIndex = _mm_set_ps1(floatIndex);
const auto mmLoopEnd = _mm_set1_ps(loopEnd); const auto mmLoopEnd = _mm_set1_ps(loopEnd);
@ -446,7 +446,7 @@ float sfz::saturatingSFZIndex<float, true>(absl::Span<const float> jumps,
floatIndex = _mm_cvtss_f32(mmFloatIndex); floatIndex = _mm_cvtss_f32(mmFloatIndex);
while (jump < sentinel) while (jump < sentinel)
snippetSaturatingIndex<float>(jump, leftCoeff, rightCoeff, index, floatIndex, loopEnd); _internals::snippetSaturatingIndex<float>(jump, leftCoeff, rightCoeff, index, floatIndex, loopEnd);
return floatIndex; return floatIndex;
} }
@ -457,7 +457,7 @@ float sfz::linearRamp<float, true>(absl::Span<float> output, float value, float
const auto* lastAligned = prevAligned(output.end()); const auto* lastAligned = prevAligned(output.end());
while (unaligned(out) && out < lastAligned) while (unaligned(out) && out < lastAligned)
snippetRampLinear<float>(out, value, step); _internals::snippetRampLinear<float>(out, value, step);
auto mmValue = _mm_set1_ps(value); auto mmValue = _mm_set1_ps(value);
auto mmStep = _mm_set_ps(step + step + step + step, step + step + step, step + step, step); auto mmStep = _mm_set_ps(step + step + step + step, step + step + step, step + step, step);
@ -471,7 +471,7 @@ float sfz::linearRamp<float, true>(absl::Span<float> output, float value, float
value = _mm_cvtss_f32(mmValue); value = _mm_cvtss_f32(mmValue);
while (out < output.end()) while (out < output.end())
snippetRampLinear<float>(out, value, step); _internals::snippetRampLinear<float>(out, value, step);
return value; return value;
} }
@ -482,7 +482,7 @@ float sfz::multiplicativeRamp<float, true>(absl::Span<float> output, float value
const auto* lastAligned = prevAligned(output.end()); const auto* lastAligned = prevAligned(output.end());
while (unaligned(out) && out < lastAligned) while (unaligned(out) && out < lastAligned)
snippetRampMultiplicative<float>(out, value, step); _internals::snippetRampMultiplicative<float>(out, value, step);
auto mmValue = _mm_set1_ps(value); auto mmValue = _mm_set1_ps(value);
auto mmStep = _mm_set_ps(step * step * step * step, step * step * step, step * step, step); auto mmStep = _mm_set_ps(step * step * step * step, step * step * step, step * step, step);
@ -496,7 +496,7 @@ float sfz::multiplicativeRamp<float, true>(absl::Span<float> output, float value
value = _mm_cvtss_f32(mmValue); value = _mm_cvtss_f32(mmValue);
while (out < output.end()) while (out < output.end())
snippetRampMultiplicative<float>(out, value, step); _internals::snippetRampMultiplicative<float>(out, value, step);
return value; return value;
} }
@ -510,7 +510,7 @@ void sfz::add<float, true>(absl::Span<const float> input, absl::Span<float> outp
const auto* lastAligned = prevAligned(sentinel); const auto* lastAligned = prevAligned(sentinel);
while (unaligned(in, out) && out < lastAligned) while (unaligned(in, out) && out < lastAligned)
snippetAdd<float>(in, out); _internals::snippetAdd<float>(in, out);
while (out < lastAligned) { while (out < lastAligned) {
_mm_store_ps(out, _mm_add_ps(_mm_load_ps(in), _mm_load_ps(out))); _mm_store_ps(out, _mm_add_ps(_mm_load_ps(in), _mm_load_ps(out)));
@ -519,7 +519,7 @@ void sfz::add<float, true>(absl::Span<const float> input, absl::Span<float> outp
} }
while (out < sentinel) while (out < sentinel)
snippetAdd<float>(in, out); _internals::snippetAdd<float>(in, out);
} }
template <> template <>
@ -530,7 +530,7 @@ void sfz::add<float, true>(float value, absl::Span<float> output) noexcept
const auto* lastAligned = prevAligned(sentinel); const auto* lastAligned = prevAligned(sentinel);
while (unaligned(out) && out < lastAligned) while (unaligned(out) && out < lastAligned)
snippetAdd<float>(value, out); _internals::snippetAdd<float>(value, out);
auto mmValue = _mm_set_ps1(value); auto mmValue = _mm_set_ps1(value);
while (out < lastAligned) { while (out < lastAligned) {
@ -539,7 +539,7 @@ void sfz::add<float, true>(float value, absl::Span<float> output) noexcept
} }
while (out < sentinel) while (out < sentinel)
snippetAdd<float>(value, out); _internals::snippetAdd<float>(value, out);
} }
template <> template <>
@ -552,7 +552,7 @@ void sfz::subtract<float, true>(absl::Span<const float> input, absl::Span<float>
const auto* lastAligned = prevAligned(sentinel); const auto* lastAligned = prevAligned(sentinel);
while (unaligned(in, out) && out < lastAligned) while (unaligned(in, out) && out < lastAligned)
snippetSubtract<float>(in, out); _internals::snippetSubtract<float>(in, out);
while (out < lastAligned) { while (out < lastAligned) {
_mm_store_ps(out, _mm_sub_ps(_mm_load_ps(out), _mm_load_ps(in))); _mm_store_ps(out, _mm_sub_ps(_mm_load_ps(out), _mm_load_ps(in)));
@ -561,7 +561,7 @@ void sfz::subtract<float, true>(absl::Span<const float> input, absl::Span<float>
} }
while (out < sentinel) while (out < sentinel)
snippetSubtract<float>(in, out); _internals::snippetSubtract<float>(in, out);
} }
template <> template <>
@ -572,7 +572,7 @@ void sfz::subtract<float, true>(const float value, absl::Span<float> output) noe
const auto* lastAligned = prevAligned(sentinel); const auto* lastAligned = prevAligned(sentinel);
while (unaligned(out) && out < lastAligned) while (unaligned(out) && out < lastAligned)
snippetSubtract<float>(value, out); _internals::snippetSubtract<float>(value, out);
auto mmValue = _mm_set_ps1(value); auto mmValue = _mm_set_ps1(value);
while (out < lastAligned) { while (out < lastAligned) {
@ -581,7 +581,7 @@ void sfz::subtract<float, true>(const float value, absl::Span<float> output) noe
} }
while (out < sentinel) while (out < sentinel)
snippetSubtract<float>(value, out); _internals::snippetSubtract<float>(value, out);
} }
template <> template <>
@ -594,7 +594,7 @@ void sfz::copy<float, true>(absl::Span<const float> input, absl::Span<float> out
const auto* lastAligned = prevAligned(sentinel); const auto* lastAligned = prevAligned(sentinel);
while (unaligned(in, out) && out < lastAligned) while (unaligned(in, out) && out < lastAligned)
snippetCopy<float>(in, out); _internals::snippetCopy<float>(in, out);
while (out < lastAligned) { while (out < lastAligned) {
_mm_store_ps(out, _mm_load_ps(in)); _mm_store_ps(out, _mm_load_ps(in));
@ -603,7 +603,7 @@ void sfz::copy<float, true>(absl::Span<const float> input, absl::Span<float> out
} }
while (out < sentinel) while (out < sentinel)
snippetCopy<float>(in, out); _internals::snippetCopy<float>(in, out);
} }
template <> template <>
@ -618,7 +618,7 @@ void sfz::pan<float, true>(absl::Span<const float> panEnvelope, absl::Span<float
const auto* lastAligned = prevAligned(sentinel); const auto* lastAligned = prevAligned(sentinel);
while (unaligned(pan, left, right) && pan < lastAligned) while (unaligned(pan, left, right) && pan < lastAligned)
snippetPan(pan, left, right); _internals::snippetPan(pan, left, right);
const auto mmOne = _mm_set_ps1(1.0f); const auto mmOne = _mm_set_ps1(1.0f);
const auto mmPiFour = _mm_set_ps1(piFour<float>); const auto mmPiFour = _mm_set_ps1(piFour<float>);
@ -639,7 +639,7 @@ void sfz::pan<float, true>(absl::Span<const float> panEnvelope, absl::Span<float
} }
while (pan < sentinel) while (pan < sentinel)
snippetPan(pan, left, right); _internals::snippetPan(pan, left, right);
} }
template <> template <>
@ -725,7 +725,7 @@ void sfz::cumsum<float, true>(absl::Span<const float> input, absl::Span<float> o
*out++ = *in++; *out++ = *in++;
while (unaligned(in, out) && in < lastAligned) while (unaligned(in, out) && in < lastAligned)
snippetCumsum(in, out); _internals::snippetCumsum(in, out);
auto mmOutput = _mm_set_ps1(*(out - 1)); auto mmOutput = _mm_set_ps1(*(out - 1));
while (in < lastAligned) { while (in < lastAligned) {
@ -740,7 +740,7 @@ void sfz::cumsum<float, true>(absl::Span<const float> input, absl::Span<float> o
} }
while (in < sentinel) while (in < sentinel)
snippetCumsum(in, out); _internals::snippetCumsum(in, out);
} }
template <> template <>
@ -758,7 +758,7 @@ void sfz::sfzInterpolationCast<float, true>(absl::Span<const float> floatJumps,
const auto lastAligned = prevAligned(sentinel); const auto lastAligned = prevAligned(sentinel);
while (unaligned(floatJump, reinterpret_cast<float*>(jump), leftCoeff, rightCoeff) && floatJump < lastAligned) while (unaligned(floatJump, reinterpret_cast<float*>(jump), leftCoeff, rightCoeff) && floatJump < lastAligned)
snippetSFZInterpolationCast(floatJump, jump, leftCoeff, rightCoeff); _internals::snippetSFZInterpolationCast(floatJump, jump, leftCoeff, rightCoeff);
while (floatJump < lastAligned) { while (floatJump < lastAligned) {
auto mmFloatJumps = _mm_load_ps(floatJump); auto mmFloatJumps = _mm_load_ps(floatJump);
@ -776,7 +776,7 @@ void sfz::sfzInterpolationCast<float, true>(absl::Span<const float> floatJumps,
} }
while(floatJump < sentinel) while(floatJump < sentinel)
snippetSFZInterpolationCast(floatJump, jump, leftCoeff, rightCoeff); _internals::snippetSFZInterpolationCast(floatJump, jump, leftCoeff, rightCoeff);
} }
template <> template <>
@ -793,7 +793,7 @@ void sfz::diff<float, true>(absl::Span<const float> input, absl::Span<float> out
*out++ = *in++; *out++ = *in++;
while (unaligned(in, out) && in < lastAligned) while (unaligned(in, out) && in < lastAligned)
snippetDiff(in, out); _internals::snippetDiff(in, out);
auto mmBase = _mm_set_ps1(*(in - 1)); auto mmBase = _mm_set_ps1(*(in - 1));
while (in < lastAligned) { while (in < lastAligned) {
@ -808,5 +808,5 @@ void sfz::diff<float, true>(absl::Span<const float> input, absl::Span<float> out
} }
while (in < sentinel) while (in < sentinel)
snippetDiff(in, out); _internals::snippetDiff(in, out);
} }