Use rounding instead of truncating for a more natural behavior of the quantization at the edges of the relevant range
This commit is contained in:
parent
d60a851fc0
commit
e04bf7e9b8
1 changed files with 3 additions and 4 deletions
|
|
@ -136,9 +136,8 @@ void LinearEnvelope<Type>::getQuantizedBlock(absl::Span<Type> output, Type quant
|
||||||
ASSERT(quantizationStep != 0.0);
|
ASSERT(quantizationStep != 0.0);
|
||||||
int index { 0 };
|
int index { 0 };
|
||||||
|
|
||||||
const auto halfStep = quantizationStep / 2;
|
auto quantize = [quantizationStep](Type value) -> Type {
|
||||||
auto quantize = [quantizationStep, halfStep](Type value) -> Type {
|
return std::round(value / quantizationStep) * quantizationStep;
|
||||||
return static_cast<int>((value + halfStep) / quantizationStep) * quantizationStep;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const auto outputSize = static_cast<int>(output.size());
|
const auto outputSize = static_cast<int>(output.size());
|
||||||
|
|
@ -227,7 +226,7 @@ void MultiplicativeEnvelope<Type>::getQuantizedBlock(absl::Span<Type> output, Ty
|
||||||
// log q log q
|
// log q log q
|
||||||
// and log(b)\log(q) is between 0 and 1.
|
// and log(b)\log(q) is between 0 and 1.
|
||||||
auto quantize = [logStep](Type value) -> Type {
|
auto quantize = [logStep](Type value) -> Type {
|
||||||
return std::exp(logStep * static_cast<int>(std::log(value)/logStep));
|
return std::exp(logStep * std::round(std::log(value)/logStep));
|
||||||
};
|
};
|
||||||
|
|
||||||
const auto outputSize = static_cast<int>(output.size());
|
const auto outputSize = static_cast<int>(output.size());
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue