diff options
| author | Michael Niedermayer <michaelni@gmx.at> | 2015-07-02 18:42:05 +0200 |
|---|---|---|
| committer | Michael Niedermayer <michaelni@gmx.at> | 2015-07-02 18:43:03 +0200 |
| commit | 9dc0bac9719a7b4c926d0c928ea8684e005f979d (patch) | |
| tree | db2aa8e8636afba14a9086596656eb6e0c1c64e3 /libavcodec/motion_est_template.c | |
| parent | db89f45535aa3e99bceb5f6bf957c90e7ca39841 (diff) | |
| download | ffmpeg-9dc0bac9719a7b4c926d0c928ea8684e005f979d.tar.gz | |
avcodec/motion_est_template: Fix undefined shifts in CHECK_MV()
Fixes:asan_heap-oob_4d5bb0_2295_cov_3374722539_hotel_california_ra5.1_640x480_30s.rmvb
Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
Diffstat (limited to 'libavcodec/motion_est_template.c')
| -rw-r--r-- | libavcodec/motion_est_template.c | 6 |
1 files changed, 3 insertions, 3 deletions
diff --git a/libavcodec/motion_est_template.c b/libavcodec/motion_est_template.c index cb8780160b..eb84d5d84d 100644 --- a/libavcodec/motion_est_template.c +++ b/libavcodec/motion_est_template.c @@ -358,8 +358,8 @@ static int qpel_motion_search(MpegEncContext * s, #define CHECK_MV(x,y)\ {\ - const unsigned key = ((y)<<ME_MAP_MV_BITS) + (x) + map_generation;\ - const int index= (((y)<<ME_MAP_SHIFT) + (x))&(ME_MAP_SIZE-1);\ + const unsigned key = ((unsigned)(y)<<ME_MAP_MV_BITS) + (x) + map_generation;\ + const int index= (((unsigned)(y)<<ME_MAP_SHIFT) + (x))&(ME_MAP_SIZE-1);\ av_assert2((x) >= xmin);\ av_assert2((x) <= xmax);\ av_assert2((y) >= ymin);\ @@ -368,7 +368,7 @@ static int qpel_motion_search(MpegEncContext * s, d= cmp(s, x, y, 0, 0, size, h, ref_index, src_index, cmpf, chroma_cmpf, flags);\ map[index]= key;\ score_map[index]= d;\ - d += (mv_penalty[((x)<<shift)-pred_x] + mv_penalty[((y)<<shift)-pred_y])*penalty_factor;\ + d += (mv_penalty[((x)*(1<<shift))-pred_x] + mv_penalty[((y)*(1<<shift))-pred_y])*penalty_factor;\ COPY3_IF_LT(dmin, d, best[0], x, best[1], y)\ }\ } |