From: Michael Niedermayer <michaelni@gmx.at>
Date: Sun, 8 Mar 2015 00:45:53 +0000 (+0100)
Subject: avcodec/mpegvideo_motion: Fix off by 1 error in MV bounds checking in qpel, chroma_4m... 
X-Git-Tag: archive/raspbian/6%11.12-1_deb8u9+rpi1^2~8
X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=5521a1d75e607e4ef5af31e3021ba9dc4ea6b75a;p=libav.git

avcodec/mpegvideo_motion: Fix off by 1 error in MV bounds checking in qpel, chroma_4mv and 8x8

No testcase available

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

Gbp-Pq: Name CVE-2017-9987-2.patch
---

diff --git a/libavcodec/mpegvideo_motion.c b/libavcodec/mpegvideo_motion.c
index 0e39325..ae14040 100644
--- a/libavcodec/mpegvideo_motion.c
+++ b/libavcodec/mpegvideo_motion.c
@@ -533,8 +533,8 @@ static inline void qpel_motion(MpegEncContext *s,
     ptr_cb = ref_picture[1] + uvsrc_y * uvlinesize + uvsrc_x;
     ptr_cr = ref_picture[2] + uvsrc_y * uvlinesize + uvsrc_x;
 
-    if ((unsigned)src_x > FFMAX(s->h_edge_pos - (motion_x & 3) - 16, 0) ||
-        (unsigned)src_y > FFMAX(v_edge_pos - (motion_y & 3) - h, 0)) {
+    if ((unsigned)src_x > FFMAX(s->h_edge_pos - (motion_x & 3) - 15, 0) ||
+        (unsigned)src_y > FFMAX(v_edge_pos - (motion_y & 3) - h + 1, 0)) {
         s->vdsp.emulated_edge_mc(s->edge_emu_buffer, ptr_y,
                                  s->linesize, s->linesize,
                                  17, 17 + field_based,
@@ -616,8 +616,8 @@ static void chroma_4mv_motion(MpegEncContext *s,
 
     offset = src_y * s->uvlinesize + src_x;
     ptr    = ref_picture[1] + offset;
-    if ((unsigned)src_x > FFMAX((s->h_edge_pos >> 1) - (dxy & 1) - 8, 0) ||
-        (unsigned)src_y > FFMAX((s->v_edge_pos >> 1) - (dxy >> 1) - 8, 0)) {
+    if ((unsigned)src_x >= FFMAX((s->h_edge_pos >> 1) - (dxy  & 1) - 7, 0) ||
+        (unsigned)src_y >= FFMAX((s->v_edge_pos >> 1) - (dxy >> 1) - 7, 0)) {
         s->vdsp.emulated_edge_mc(s->edge_emu_buffer, ptr,
                                  s->uvlinesize, s->uvlinesize,
                                  9, 9, src_x, src_y,
@@ -774,8 +774,8 @@ static inline void apply_8x8(MpegEncContext *s,
                 dxy &= ~12;
 
             ptr = ref_picture[0] + (src_y * s->linesize) + (src_x);
-            if ((unsigned)src_x > FFMAX(s->h_edge_pos - (motion_x & 3) - 8, 0) ||
-                (unsigned)src_y > FFMAX(s->v_edge_pos - (motion_y & 3) - 8, 0)) {
+            if ((unsigned)src_x >= FFMAX(s->h_edge_pos - (motion_x & 3) - 7, 0) ||
+                (unsigned)src_y >= FFMAX(s->v_edge_pos - (motion_y & 3) - 7, 0)) {
                 s->vdsp.emulated_edge_mc(s->edge_emu_buffer, ptr,
                                          s->linesize, s->linesize,
                                          9, 9,