avfilter/vf_transpose: Fix regression with packed pixel formats

Regression since: c6939f65a116b1ffed345d29d8621ee4ffb32235
Found-by: Paul B Mahol <onemda@gmail.com>
Reviewed-by: Paul B Mahol <onemda@gmail.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
Gbp-Pq: Name CVE-2018-6392-1.patch

diff --git a/libavfilter/vf_transpose.c b/libavfilter/vf_transpose.c
index 07602b9086dd3ab46f7778d7f2de50065d830619..f2d8fe46e1446c162a7ddedacbd9a68b6b05a2dd 100644 (file)
--- a/libavfilter/vf_transpose.c
+++ b/libavfilter/vf_transpose.c
@@ -27,6 +27,7 @@
 
 #include <stdio.h>
 
+#include "libavutil/avassert.h"
 #include "libavutil/imgutils.h"
 #include "libavutil/internal.h"
 #include "libavutil/intreadwrite.h"
@@ -48,6 +49,7 @@ enum TransposeDir {
 typedef struct TransContext {
     const AVClass *class;
     int hsub, vsub;
+    int planes;
     int pixsteps[4];
 
     enum TransposeDir dir;
@@ -93,6 +95,9 @@ static int config_props_output(AVFilterLink *outlink)
 
     trans->hsub = desc_in->log2_chroma_w;
     trans->vsub = desc_in->log2_chroma_h;
+    trans->planes = desc_in->nb_components;
+
+    av_assert0(desc_in->nb_components == desc_out->nb_components);
 
     av_image_fill_max_pixsteps(trans->pixsteps, NULL, desc_out);
 
@@ -135,7 +140,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in)
         out->sample_aspect_ratio.den = in->sample_aspect_ratio.num;
     }
 
-    for (plane = 0; out->data[plane]; plane++) {
+    for (plane = 0; plane < trans->planes; plane++) {
         int hsub    = plane == 1 || plane == 2 ? trans->hsub : 0;
         int vsub    = plane == 1 || plane == 2 ? trans->vsub : 0;
         int pixstep = trans->pixsteps[plane];