Improve compute_residual_from_qlp_coefficients datapath selection

This bases the selection of the datapath on the actual predictor
coefficients instead of on the predictor coefficient precision

4 files changed, 32 insertions, 2 deletions

diff --git a/src/libFLAC/include/private/lpc.h b/src/libFLAC/include/private/lpc.h
index f538e645..b1321bcf 100644
--- a/src/libFLAC/include/private/lpc.h
+++ b/src/libFLAC/include/private/lpc.h
@@ -96,6 +96,9 @@ void FLAC__lpc_compute_autocorrelation_intrin_neon_lag_14(const FLAC__real data[
 #endif
 #endif /* FLAC__NO_ASM */
 
+uint32_t FLAC__lpc_max_prediction_before_shift_bps(uint32_t subframe_bps, const FLAC__int32 qlp_coeff[], uint32_t order);
+uint32_t FLAC__lpc_max_residual_bps(uint32_t subframe_bps, const FLAC__int32 qlp_coeff[], uint32_t order, int lp_quantization);
+
 /*
  *	FLAC__lpc_compute_lp_coefficients()
  *	--------------------------------------------------------------------
diff --git a/src/libFLAC/lpc.c b/src/libFLAC/lpc.c
index 33157746..ba3ae272 100644
--- a/src/libFLAC/lpc.c
+++ b/src/libFLAC/lpc.c
@@ -35,6 +35,7 @@
 #endif
 
 #include <math.h>
+#include <stdlib.h>
 
 #include "FLAC/assert.h"
 #include "FLAC/format.h"
@@ -259,6 +260,31 @@ int FLAC__lpc_quantize_coefficients(const FLAC__real lp_coeff[], uint32_t order,
 	return 0;
 }
 
+
+uint32_t FLAC__lpc_max_prediction_before_shift_bps(uint32_t subframe_bps, const FLAC__int32 * flac_restrict qlp_coeff, uint32_t order)
+{
+	/* This used to be subframe_bps + qlp_coeff_precision + FLAC__bitmath_ilog2(order)
+	 * but that treats both the samples as well as the predictor as unknown. The
+	 * predictor is known however, so taking the log2 of the sum of the absolute values
+	 * of all coefficients is a more accurate representation of the predictor */
+	FLAC__int32 abs_sum_of_qlp_coeff = 0;
+	for(uint32_t i = 0; i < order; i++)
+		abs_sum_of_qlp_coeff += abs(qlp_coeff[i]);
+	if(abs_sum_of_qlp_coeff == 0)
+		abs_sum_of_qlp_coeff = 1;
+	return subframe_bps + FLAC__bitmath_silog2(abs_sum_of_qlp_coeff);
+}
+
+
+uint32_t FLAC__lpc_max_residual_bps(uint32_t subframe_bps, const FLAC__int32 * flac_restrict qlp_coeff, uint32_t order, int lp_quantization)
+{
+	FLAC__int32 predictor_sum_bps = FLAC__lpc_max_prediction_before_shift_bps(subframe_bps, qlp_coeff, order) - lp_quantization;
+	if((int)subframe_bps > predictor_sum_bps)
+		return subframe_bps + 1;
+	else
+		return predictor_sum_bps + 1;
+}
+
 #if defined(_MSC_VER)
 // silence MSVC warnings about __restrict modifier
 #pragma warning ( disable : 4028 )
diff --git a/src/libFLAC/stream_decoder.c b/src/libFLAC/stream_decoder.c
index cc583d7d..b6def9c9 100644
--- a/src/libFLAC/stream_decoder.c
+++ b/src/libFLAC/stream_decoder.c
@@ -2797,7 +2797,8 @@ FLAC__bool read_subframe_lpc_(FLAC__StreamDecoder *decoder, uint32_t channel, ui
 	/* decode the subframe */
 	if(do_full_decode) {
 		memcpy(decoder->private_->output[channel], subframe->warmup, sizeof(FLAC__int32) * order);
-		if(bps + subframe->qlp_coeff_precision + FLAC__bitmath_ilog2(order) <= 32)
+		if(FLAC__lpc_max_residual_bps(bps, subframe->qlp_coeff, order, subframe->quantization_level) <= 32 &&
+		   FLAC__lpc_max_prediction_before_shift_bps(bps, subframe->qlp_coeff, order) <= 32)
 			FLAC__lpc_restore_signal(decoder->private_->residual[channel], decoder->private_->frame.header.blocksize-order, subframe->qlp_coeff, order, subframe->quantization_level, decoder->private_->output[channel]+order);
 		else
 			FLAC__lpc_restore_signal_wide(decoder->private_->residual[channel], decoder->private_->frame.header.blocksize-order, subframe->qlp_coeff, order, subframe->quantization_level, decoder->private_->output[channel]+order);
diff --git a/src/libFLAC/stream_encoder.c b/src/libFLAC/stream_encoder.c
index 10bdaf21..f2daa2f6 100644
--- a/src/libFLAC/stream_encoder.c
+++ b/src/libFLAC/stream_encoder.c
@@ -3865,7 +3865,7 @@ uint32_t evaluate_lpc_subframe_(
 	if(ret != 0)
 		return 0; /* this is a hack to indicate to the caller that we can't do lp at this order on this subframe */
 
-	if(subframe_bps + qlp_coeff_precision + FLAC__bitmath_ilog2(order) <= 32)
+	if(FLAC__lpc_max_prediction_before_shift_bps(subframe_bps, qlp_coeff, order) <= 32)
 		if(subframe_bps <= 16 && qlp_coeff_precision <= 16)
 			encoder->private_->local_lpc_compute_residual_from_qlp_coefficients_16bit(signal+order, residual_samples, qlp_coeff, order, quantization, residual);
 		else