diff options
| author | Yaowu Xu <yaowu@google.com> | 2016-06-15 18:03:57 -0700 |
|---|---|---|
| committer | Yaowu Xu <yaowu@google.com> | 2016-06-16 21:45:48 +0000 |
| commit | e5e998a6eb0a51dcaa1be6984525d2263968cf52 (patch) | |
| tree | 19335953d663d97f149a63faca2f7aac14ab7886 /vpx_dsp/variance.c | |
| parent | 0c1da40ea86b08a702782bec667f8b358fc7100f (diff) | |
| download | libvpx-e5e998a6eb0a51dcaa1be6984525d2263968cf52.tar.gz | |
vpx_dsp/variance.c: change to use correct type
This commit change to use int64_t to represent the sum of pixel
differences, which can be negative.
This fixes a number of ubsan warnings.
BUG=webm:1219
Change-Id: I885f245ae895ab92ca5f3b9848d37024b07aac98
Diffstat (limited to 'vpx_dsp/variance.c')
| -rw-r--r-- | vpx_dsp/variance.c | 8 |
1 files changed, 4 insertions, 4 deletions
diff --git a/vpx_dsp/variance.c b/vpx_dsp/variance.c index e8bddb0a0..cdec0db55 100644 --- a/vpx_dsp/variance.c +++ b/vpx_dsp/variance.c @@ -275,7 +275,7 @@ void vpx_comp_avg_pred_c(uint8_t *comp_pred, const uint8_t *pred, #if CONFIG_VP9_HIGHBITDEPTH static void highbd_variance64(const uint8_t *a8, int a_stride, const uint8_t *b8, int b_stride, - int w, int h, uint64_t *sse, uint64_t *sum) { + int w, int h, uint64_t *sse, int64_t *sum) { int i, j; uint16_t *a = CONVERT_TO_SHORTPTR(a8); @@ -298,7 +298,7 @@ static void highbd_8_variance(const uint8_t *a8, int a_stride, const uint8_t *b8, int b_stride, int w, int h, uint32_t *sse, int *sum) { uint64_t sse_long = 0; - uint64_t sum_long = 0; + int64_t sum_long = 0; highbd_variance64(a8, a_stride, b8, b_stride, w, h, &sse_long, &sum_long); *sse = (uint32_t)sse_long; *sum = (int)sum_long; @@ -308,7 +308,7 @@ static void highbd_10_variance(const uint8_t *a8, int a_stride, const uint8_t *b8, int b_stride, int w, int h, uint32_t *sse, int *sum) { uint64_t sse_long = 0; - uint64_t sum_long = 0; + int64_t sum_long = 0; highbd_variance64(a8, a_stride, b8, b_stride, w, h, &sse_long, &sum_long); *sse = (uint32_t)ROUND_POWER_OF_TWO(sse_long, 4); *sum = (int)ROUND_POWER_OF_TWO(sum_long, 2); @@ -318,7 +318,7 @@ static void highbd_12_variance(const uint8_t *a8, int a_stride, const uint8_t *b8, int b_stride, int w, int h, uint32_t *sse, int *sum) { uint64_t sse_long = 0; - uint64_t sum_long = 0; + int64_t sum_long = 0; highbd_variance64(a8, a_stride, b8, b_stride, w, h, &sse_long, &sum_long); *sse = (uint32_t)ROUND_POWER_OF_TWO(sse_long, 8); *sum = (int)ROUND_POWER_OF_TWO(sum_long, 4); |