CVE-2026-54240, CVE-2026-54241: fix pixel-accessor integer overflow

Origin: upstream, https://github.com/strukturag/libde265/commit/bdca87569b9c63c2a7054d90ae4462dbb78d159a
Bug: https://github.com/strukturag/libde265/security/advisories/GHSA-ccfw-29x7-rrx3
Bug: https://github.com/strukturag/libde265/security/advisories/GHSA-j2qq-x2xq-g9wr
Applied-Upstream: 1.1.1

The image pixel accessors computed xpos + ypos*stride in signed 32-bit
arithmetic; frames with width*height > INT32_MAX wrap the offset and the
returned pointer lands far outside the buffer (OOB read/write). Widen
stride/offset math to 64 bits and store validated SPS dimensions in
uint16_t.

Gbp-Pq: Name CVE-2026-54240_CVE-2026-54241.patch

diff --git a/libde265/de265.cc b/libde265/de265.cc
index 6ff0191802c1957de3677f089b03c330c7d69297..571ed06b3b853efa9bca197a3311755cb00e6b96 100644 (file)
--- a/libde265/de265.cc
+++ b/libde265/de265.cc
@@ -672,7 +672,7 @@ LIBDE265_API const uint8_t* de265_get_image_plane(const de265_image* img, int ch
 
   uint8_t* data = img->pixels_confwin[channel];
 
-  if (stride) *stride = img->get_image_stride(channel) * ((de265_get_bits_per_pixel(img, channel)+7) / 8);
+  if (stride) *stride = static_cast<int>(img->get_image_stride(channel) * ((de265_get_bits_per_pixel(img, channel)+7) / 8));
 
   return data;
 }

diff --git a/libde265/image-io.cc b/libde265/image-io.cc
index 4983783797236c4684ae1652408463a6b057623c..c6370bcd0455a8df3eac66d08e57777ce38eeb7d 100644 (file)
--- a/libde265/image-io.cc
+++ b/libde265/image-io.cc
@@ -73,7 +73,7 @@ de265_image* ImageSource_YUV::read_next_image()
   // --- load image ---
 
   uint8_t* p;
-  int stride;
+  ptrdiff_t stride;
 
   p = img->get_image_plane(0);  stride = img->get_image_stride(0);
   for (int y=0;y<height;y++) {
@@ -154,7 +154,7 @@ void ImageSink_YUV::send_image(const de265_image* img)
   // --- write image ---
 
   const uint8_t* p;
-  int stride;
+  ptrdiff_t stride;
 
   int width = img->get_width();
   int height= img->get_height();

diff --git a/libde265/image.cc b/libde265/image.cc
index 9ddfed2c3bfd6da123bf5cc02ad9a2fa74040df0..9beeb07ef7f695582f4161f8125e3738f2e60b32 100644 (file)
--- a/libde265/image.cc
+++ b/libde265/image.cc
@@ -183,7 +183,7 @@ de265_image_allocation de265_image::default_image_allocation = {
 };
 
 
-void de265_image::set_image_plane(int cIdx, uint8_t* mem, int stride, void *userdata)
+void de265_image::set_image_plane(int cIdx, uint8_t* mem, ptrdiff_t stride, void *userdata)
 {
   pixels[cIdx] = mem;
   plane_user_data[cIdx] = userdata;

diff --git a/libde265/image.h b/libde265/image.h
index 0a0c0e3225fe92463e8c1a4ff725e3f80c0e170e..c514429d4659b9bd1a5294496a35d0904bb4bd85 100644 (file)
--- a/libde265/image.h
+++ b/libde265/image.h
@@ -26,6 +26,7 @@
 #endif
 
 #include <assert.h>
+#include <stddef.h>
 #include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
@@ -254,11 +255,11 @@ struct de265_image {
   /* */ uint8_t* get_image_plane(int cIdx)       { return pixels[cIdx]; }
   const uint8_t* get_image_plane(int cIdx) const { return pixels[cIdx]; }
 
-  void set_image_plane(int cIdx, uint8_t* mem, int stride, void *userdata);
+  void set_image_plane(int cIdx, uint8_t* mem, ptrdiff_t stride, void *userdata);
 
   uint8_t* get_image_plane_at_pos(int cIdx, int xpos,int ypos)
   {
-    int stride = get_image_stride(cIdx);
+    ptrdiff_t stride = get_image_stride(cIdx);
     return pixels[cIdx] + xpos + ypos*stride;
   }
 
@@ -267,38 +268,38 @@ struct de265_image {
   template <class pixel_t>
   pixel_t* get_image_plane_at_pos_NEW(int cIdx, int xpos,int ypos)
   {
-    int stride = get_image_stride(cIdx);
+    ptrdiff_t stride = get_image_stride(cIdx);
     return (pixel_t*)(pixels[cIdx] + (xpos + ypos*stride)*sizeof(pixel_t));
   }
 
   const uint8_t* get_image_plane_at_pos(int cIdx, int xpos,int ypos) const
   {
-    int stride = get_image_stride(cIdx);
+    ptrdiff_t stride = get_image_stride(cIdx);
     return pixels[cIdx] + xpos + ypos*stride;
   }
 
   void* get_image_plane_at_pos_any_depth(int cIdx, int xpos,int ypos)
   {
-    int stride = get_image_stride(cIdx);
+    ptrdiff_t stride = get_image_stride(cIdx);
     return pixels[cIdx] + ((xpos + ypos*stride) << bpp_shift[cIdx]);
   }
 
   const void* get_image_plane_at_pos_any_depth(int cIdx, int xpos,int ypos) const
   {
-    int stride = get_image_stride(cIdx);
+    ptrdiff_t stride = get_image_stride(cIdx);
     return pixels[cIdx] + ((xpos + ypos*stride) << bpp_shift[cIdx]);
   }
 
   /* Number of pixels in one row (not number of bytes).
    */
-  int get_image_stride(int cIdx) const
+  ptrdiff_t get_image_stride(int cIdx) const
   {
     if (cIdx==0) return stride;
     else         return chroma_stride;
   }
 
-  int get_luma_stride() const { return stride; }
-  int get_chroma_stride() const { return chroma_stride; }
+  ptrdiff_t get_luma_stride() const { return stride; }
+  ptrdiff_t get_chroma_stride() const { return chroma_stride; }
 
   int get_width (int cIdx=0) const { return cIdx==0 ? width  : chroma_width;  }
   int get_height(int cIdx=0) const { return cIdx==0 ? height : chroma_height; }
@@ -352,8 +353,8 @@ private:
 
   int width, height;  // size in luma pixels
 
-  int chroma_width, chroma_height;
-  int stride, chroma_stride;
+  int chroma_width = 0, chroma_height = 0;
+  ptrdiff_t stride = 0, chroma_stride = 0;
 
 public:
   uint8_t BitDepth_Y, BitDepth_C;

diff --git a/libde265/intrapred.h b/libde265/intrapred.h
index 5c997908f894f719ae21af6f0137143908890090..2b2248793c1b90eab8b81163fec076ed1bb0627b 100644 (file)
--- a/libde265/intrapred.h
+++ b/libde265/intrapred.h
@@ -533,7 +533,7 @@ void intra_border_computer<pixel_t>::fill_from_image()
   assert(nT<=32);
 
   pixel_t* image;
-  int stride;
+  ptrdiff_t stride;
   image  = (pixel_t*)img->get_image_plane(cIdx);
   stride = img->get_image_stride(cIdx);
 

diff --git a/libde265/motion.cc b/libde265/motion.cc
index 0ddfaabd9f397ac8d19ed02954f3b8f29a0ef586..350300644966b13deb563bb61134ca7db401743a 100644 (file)
--- a/libde265/motion.cc
+++ b/libde265/motion.cc
@@ -50,7 +50,7 @@ void mc_luma(const base_context* ctx,
              const seq_parameter_set* sps, int mv_x, int mv_y,
              int xP,int yP,
              int16_t* out, int out_stride,
-             const pixel_t* ref, int ref_stride,
+             const pixel_t* ref, ptrdiff_t ref_stride,
              int nPbW, int nPbH, int bitDepth_L)
 {
   int xFracL = mv_x & 3;
@@ -129,7 +129,7 @@ void mc_luma(const base_context* ctx,
     pixel_t padbuf[(MAX_CU_SIZE+16)*(MAX_CU_SIZE+7)];
 
     const pixel_t* src_ptr;
-    int src_stride;
+    ptrdiff_t src_stride;
 
     if (-extra_left + xIntOffsL >= 0 &&
         -extra_top  + yIntOffsL >= 0 &&
@@ -176,7 +176,7 @@ void mc_chroma(const base_context* ctx,
                int mv_x, int mv_y,
                int xP,int yP,
                int16_t* out, int out_stride,
-               const pixel_t* ref, int ref_stride,
+               const pixel_t* ref, ptrdiff_t ref_stride,
                int nPbWC, int nPbHC, int bit_depth_C)
 {
   // chroma sample interpolation process (8.5.3.2.2.2)
@@ -222,7 +222,7 @@ void mc_chroma(const base_context* ctx,
     pixel_t padbuf[(MAX_CU_SIZE+16)*(MAX_CU_SIZE+3)];
 
     const pixel_t* src_ptr;
-    int src_stride;
+    ptrdiff_t src_stride;
 
     int extra_top  = 1;
     int extra_left = 1;

diff --git a/libde265/sao.cc b/libde265/sao.cc
index 1b88a1021ba198ca23581af6d1f022d1da741e51..92125db0932a5f93fba98400f072bb47f59df74f 100644 (file)
--- a/libde265/sao.cc
+++ b/libde265/sao.cc
@@ -28,8 +28,8 @@
 template <class pixel_t>
 void apply_sao_internal(de265_image* img, int xCtb,int yCtb,
                         const slice_segment_header* shdr, int cIdx, int nSW,int nSH,
-                        const pixel_t* in_img,  int in_stride,
-                        /* */ pixel_t* out_img, int out_stride)
+                        const pixel_t* in_img,  ptrdiff_t in_stride,
+                        /* */ pixel_t* out_img, ptrdiff_t out_stride)
 {
   const sao_info* saoinfo = img->get_sao_info(xCtb,yCtb);
 
@@ -77,7 +77,7 @@ void apply_sao_internal(de265_image* img, int xCtb,int yCtb,
 
   if (SaoTypeIdx==2) {
     int hPos[2], vPos[2];
-    int vPosStride[2]; // vPos[] multiplied by image stride
+    ptrdiff_t vPosStride[2]; // vPos[] multiplied by image stride
     int SaoEoClass = (saoinfo->SaoEoClass >> (2*cIdx)) & 0x3;
 
     switch (SaoEoClass) {
@@ -277,8 +277,8 @@ void apply_sao_internal(de265_image* img, int xCtb,int yCtb,
 template <class pixel_t>
 void apply_sao(de265_image* img, int xCtb,int yCtb,
                const slice_segment_header* shdr, int cIdx, int nSW,int nSH,
-               const pixel_t* in_img,  int in_stride,
-               /* */ pixel_t* out_img, int out_stride)
+               const pixel_t* in_img,  ptrdiff_t in_stride,
+               /* */ pixel_t* out_img, ptrdiff_t out_stride)
 {
   if (img->high_bit_depth(cIdx)) {
     apply_sao_internal<uint16_t>(img,xCtb,yCtb, shdr,cIdx,nSW,nSH,
@@ -343,8 +343,8 @@ void apply_sample_adaptive_offset_sequential(de265_image* img)
     return;
   }
 
-  int lumaImageSize   = img->get_image_stride(0) * img->get_height(0) * img->get_bytes_per_pixel(0);
-  int chromaImageSize = img->get_image_stride(1) * img->get_height(1) * img->get_bytes_per_pixel(1);
+  size_t lumaImageSize   = static_cast<size_t>(img->get_image_stride(0)) * img->get_height(0) * img->get_bytes_per_pixel(0);
+  size_t chromaImageSize = static_cast<size_t>(img->get_image_stride(1)) * img->get_height(1) * img->get_bytes_per_pixel(1);
 
   uint8_t* inputCopy = new uint8_t[ libde265_max(lumaImageSize, chromaImageSize) ];
   if (inputCopy == NULL) {
@@ -358,10 +358,10 @@ void apply_sample_adaptive_offset_sequential(de265_image* img)
 
   for (int cIdx=0;cIdx<nChannels;cIdx++) {
 
-    int stride = img->get_image_stride(cIdx);
+    ptrdiff_t stride = img->get_image_stride(cIdx);
     int height = img->get_height(cIdx);
 
-    memcpy(inputCopy, img->get_image_plane(cIdx), stride * height * img->get_bytes_per_pixel(cIdx));
+    memcpy(inputCopy, img->get_image_plane(cIdx), static_cast<size_t>(stride) * height * img->get_bytes_per_pixel(cIdx));
 
     for (int yCtb=0; yCtb<sps.PicHeightInCtbsY; yCtb++)
       for (int xCtb=0; xCtb<sps.PicWidthInCtbsY; xCtb++)

diff --git a/libde265/sei.cc b/libde265/sei.cc
index 978f7fb7f17e511f7f81506284e230a526d45079..38de33d67c17679f6b45afce99acff7885713be5 100644 (file)
--- a/libde265/sei.cc
+++ b/libde265/sei.cc
@@ -99,7 +99,7 @@ static void dump_sei_decoded_picture_hash(const sei_message* sei,
 class raw_hash_data
 {
 public:
-  raw_hash_data(int w, int stride);
+  raw_hash_data(int w, ptrdiff_t stride);
   ~raw_hash_data();
 
   struct data_chunk {
@@ -111,13 +111,14 @@ public:
   data_chunk prepare_16bit(const uint8_t* data,int y);
 
 private:
-  int mWidth, mStride;
+  int mWidth;
+  ptrdiff_t mStride;
 
   uint8_t* mMem;
 };
 
 
-raw_hash_data::raw_hash_data(int w, int stride)
+raw_hash_data::raw_hash_data(int w, ptrdiff_t stride)
 {
   mWidth=w;
   mStride=stride;
@@ -157,7 +158,7 @@ raw_hash_data::data_chunk raw_hash_data::prepare_16bit(const uint8_t* data,int y
 }
 
 
-static uint32_t compute_checksum_8bit(uint8_t* data,int w,int h,int stride, int bit_depth)
+static uint32_t compute_checksum(uint8_t* data,int w,int h,ptrdiff_t stride, int bit_depth)
 {
   uint32_t sum = 0;
 
@@ -169,6 +170,8 @@ static uint32_t compute_checksum_8bit(uint8_t* data,int w,int h,int stride, int
       }
   }
   else {
+    auto* data16 = reinterpret_cast<uint16_t*>(data);
+    ptrdiff_t stride16 = stride / 2;
     for (int y=0; y<h; y++)
       for(int x=0; x<w; x++) {
         uint8_t xorMask = ( x & 0xFF ) ^ ( y & 0xFF ) ^ ( x  >>  8 ) ^ ( y  >>  8 );
@@ -222,7 +225,7 @@ static inline uint16_t crc_process_byte_parallel(uint16_t crc, uint8_t byte)
           (t << 12)) & 0xFFFF;
 }
 
-static uint32_t compute_CRC_8bit_fast(const uint8_t* data,int w,int h,int stride, int bit_depth)
+static uint32_t compute_CRC_8bit_fast(const uint8_t* data,int w,int h,ptrdiff_t stride, int bit_depth)
 {
   raw_hash_data raw_data(w,stride);
 
@@ -248,7 +251,7 @@ static uint32_t compute_CRC_8bit_fast(const uint8_t* data,int w,int h,int stride
 }
 
 
-static void compute_MD5(uint8_t* data,int w,int h,int stride, uint8_t* result, int bit_depth)
+static void compute_MD5(uint8_t* data,int w,int h,ptrdiff_t stride, uint8_t* result, int bit_depth)
 {
   MD5_CTX md5;
   MD5_Init(&md5);
@@ -287,7 +290,8 @@ static de265_error process_sei_decoded_picture_hash(const sei_message* sei, de26
   int nHashes = img->get_sps().chroma_format_idc==0 ? 1 : 3;
   for (int i=0;i<nHashes;i++) {
     uint8_t* data;
-    int w,h,stride;
+    int w,h;
+    ptrdiff_t stride;
 
     w = img->get_width(i);
     h = img->get_height(i);
@@ -339,7 +343,7 @@ static de265_error process_sei_decoded_picture_hash(const sei_message* sei, de26
 
     case sei_decoded_picture_hash_type_checksum:
       {
-        uint32_t chksum = compute_checksum_8bit(data,w,h,stride, img->get_bit_depth(i));
+        uint32_t chksum = compute_checksum(data,w,h,stride, img->get_bit_depth(i));
 
         if (chksum != seihash->checksum[i]) {
 /*

diff --git a/libde265/sps.cc b/libde265/sps.cc
index 0652e4266abc27a3dfcb74e22e9f738055496aed..29409fc586e89a9517ef6fd7b7cadc9da54aa25f 100644 (file)
--- a/libde265/sps.cc
+++ b/libde265/sps.cc
@@ -521,7 +521,7 @@ de265_error seq_parameter_set::compute_derived_values(bool sanitize_values)
   PicHeightInCtbsY   = ceil_div(pic_height_in_luma_samples,CtbSizeY);
   PicSizeInMinCbsY   = PicWidthInMinCbsY * PicHeightInMinCbsY;
   PicSizeInCtbsY = PicWidthInCtbsY * PicHeightInCtbsY;
-  PicSizeInSamplesY = pic_width_in_luma_samples * pic_height_in_luma_samples;
+  PicSizeInSamplesY = static_cast<uint32_t>(pic_width_in_luma_samples) * pic_height_in_luma_samples;
 
   if (chroma_format_idc==0 || separate_colour_plane_flag) {
     CtbWidthC  = 0;
@@ -1276,7 +1276,7 @@ de265_error seq_parameter_set::write(error_queue* errqueue, CABAC_encoder& out)
   PicHeightInCtbsY   = ceil_div(pic_height_in_luma_samples,CtbSizeY);
   PicSizeInMinCbsY   = PicWidthInMinCbsY * PicHeightInMinCbsY;
   PicSizeInCtbsY = PicWidthInCtbsY * PicHeightInCtbsY;
-  PicSizeInSamplesY = pic_width_in_luma_samples * pic_height_in_luma_samples;
+  PicSizeInSamplesY = static_cast<uint32_t>(pic_width_in_luma_samples) * pic_height_in_luma_samples;
   if (chroma_format_idc==0 || separate_colour_plane_flag) {
     CtbWidthC  = 0;
     CtbHeightC = 0;

diff --git a/libde265/sps.h b/libde265/sps.h
index 9759bae0ad6d06d31c5acabfe6f5d93d99e66bb7..18be31b20925975f8b52f01651be6c4ad2ba8537 100644 (file)
--- a/libde265/sps.h
+++ b/libde265/sps.h
@@ -39,6 +39,13 @@ class error_queue;
 #define MAX_PICTURE_WIDTH  65535
 #define MAX_PICTURE_HEIGHT 65535
 
+// pic_width/height_in_luma_samples are stored as uint16_t and PicSizeInSamplesY as uint32_t,
+// so these limits must keep width/height in 16 bits and their product in 32 bits.
+static_assert(MAX_PICTURE_WIDTH  <= 0xFFFF, "picture width must fit in uint16_t");
+static_assert(MAX_PICTURE_HEIGHT <= 0xFFFF, "picture height must fit in uint16_t");
+static_assert((uint64_t)MAX_PICTURE_WIDTH * MAX_PICTURE_HEIGHT <= 0xFFFFFFFFu,
+              "total luma sample count must fit in uint32_t");
+
 enum {
   CHROMA_MONO = 0,
   CHROMA_420 = 1,
@@ -110,10 +117,10 @@ public:
   int seq_parameter_set_id;
   int chroma_format_idc;
 
-  char separate_colour_plane_flag;
-  int  pic_width_in_luma_samples;
-  int  pic_height_in_luma_samples;
-  char conformance_window_flag;
+  bool separate_colour_plane_flag;
+  uint16_t pic_width_in_luma_samples;   // <= MAX_PICTURE_WIDTH  (validated on parse)
+  uint16_t pic_height_in_luma_samples;  // <= MAX_PICTURE_HEIGHT (validated on parse)
+  bool conformance_window_flag;
 
   int conf_win_left_offset;
   int conf_win_right_offset;