diff options
| author | Michael Niedermayer <michaelni@gmx.at> | 2014-01-12 02:27:00 +0100 |
|---|---|---|
| committer | Michael Niedermayer <michaelni@gmx.at> | 2014-01-14 15:58:59 +0100 |
| commit | 4b0cad6596bcbce5833251c9ba2a60617186a39c (patch) | |
| tree | 073328bd46a141a9d2ac0f3827e3ae2255b99173 /libavcodec/simple_idct_template.c | |
| parent | a7ea733b721e034447e4e5bb013e2dd25614f530 (diff) | |
| download | ffmpeg-4b0cad6596bcbce5833251c9ba2a60617186a39c.tar.gz | |
avcodec/simple_idct_template: fix rounding of the special DC case for 10bit
MSE doesnt change for the test as the code only triggers
for very sparse matrixes
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
Diffstat (limited to 'libavcodec/simple_idct_template.c')
| -rw-r--r-- | libavcodec/simple_idct_template.c | 8 |
1 files changed, 4 insertions, 4 deletions
diff --git a/libavcodec/simple_idct_template.c b/libavcodec/simple_idct_template.c index 838cea4951..ac04923d96 100644 --- a/libavcodec/simple_idct_template.c +++ b/libavcodec/simple_idct_template.c @@ -107,10 +107,10 @@ static inline void FUNC(idctRowCondDC)(int16_t *row, int extra_shift) #define ROW0_MASK (0xffffLL << 48 * HAVE_BIGENDIAN) if (((((uint64_t *)row)[0] & ~ROW0_MASK) | ((uint64_t *)row)[1]) == 0) { uint64_t temp; - if (DC_SHIFT - extra_shift > 0) { + if (DC_SHIFT - extra_shift >= 0) { temp = (row[0] << (DC_SHIFT - extra_shift)) & 0xffff; } else { - temp = (row[0] >> (extra_shift - DC_SHIFT)) & 0xffff; + temp = ((row[0] + (1<<(extra_shift - DC_SHIFT-1))) >> (extra_shift - DC_SHIFT)) & 0xffff; } temp += temp << 16; temp += temp << 32; @@ -124,10 +124,10 @@ static inline void FUNC(idctRowCondDC)(int16_t *row, int extra_shift) ((uint32_t*)row)[3] | row[1])) { uint32_t temp; - if (DC_SHIFT - extra_shift > 0) { + if (DC_SHIFT - extra_shift >= 0) { temp = (row[0] << (DC_SHIFT - extra_shift)) & 0xffff; } else { - temp = (row[0] >> (extra_shift - DC_SHIFT)) & 0xffff; + temp = ((row[0] + (1<<(extra_shift - DC_SHIFT-1))) >> (extra_shift - DC_SHIFT)) & 0xffff; } temp += temp << 16; ((uint32_t*)row)[0]=((uint32_t*)row)[1] = |