An additional defense in the Vorbis codec.

source/patched-ffmpeg/libavcodec/vorbisdec.c

 /**
  * @file
  * Vorbis I decoder
  * @author Denes Balatoni  ( dbalatoni programozo hu )
  *
  * This file is part of FFmpeg.
  *
  * FFmpeg is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
@@ skipping 1253 matching lines @@
 }
 
 // Read and decode residue
 
 static av_always_inline int vorbis_residue_decode_internal(vorbis_context *vc,
                                                            vorbis_residue *vr,
                                                            unsigned ch,
                                                            uint8_t *do_not_decode,
                                                            float *vec,
                                                            unsigned vlen,
+                                                           unsigned ch_left,
                                                            int vr_type)
 {
     GetBitContext *gb = &vc->gb;
     unsigned c_p_c        = vc->codebooks[vr->classbook].dimensions;
     unsigned ptns_to_read = vr->ptns_to_read;
     uint8_t *classifs = vr->classifs;
     unsigned pass, ch_used, i, j, k, l;
+    unsigned max_output = (ch - 1) * vlen;
 
     if (vr_type == 2) {
         for (j = 1; j < ch; ++j)
             do_not_decode[0] &= do_not_decode[j];  // FIXME - clobbering input
         if (do_not_decode[0])
             return 0;
         ch_used = 1;
+        max_output += vr->end / ch;
     } else {
         ch_used = ch;
+        max_output += vr->end;
+    }
+
+    if (max_output > ch_left * vlen) {
+        av_log(vc->avccontext, AV_LOG_ERROR, "Insufficient output buffer\n");
+        return -1;
     }
 
     av_dlog(NULL, " residue type 0/1/2 decode begin, ch: %d  cpc %d  \n", ch, c_p_c);
 
     for (pass = 0; pass <= vr->maxpass; ++pass) { // FIXME OPTIMIZE?
         uint16_t voffset, partition_count, j_times_ptns_to_read;
 
         voffset = vr->begin;
         for(partition_count=0;partition_count<ptns_to_read && !(gb->buffer_exhausted && gb->index > gb->size_in_bits);) {  // SPEC        error
             if (!pass) {
@@ skipping 106 matching lines @@
                 voffset += vr->partition_size;
             }
         }
     }
     return 0;
 }
 
 static inline int vorbis_residue_decode(vorbis_context *vc, vorbis_residue *vr,
                                         unsigned ch,
                                         uint8_t *do_not_decode,
-                                        float *vec, unsigned vlen)
+                                        float *vec, unsigned vlen,
+                                        unsigned ch_left)
 {
     if (vr->type == 2)
-        return vorbis_residue_decode_internal(vc, vr, ch, do_not_decode, vec, vlen, 2);
+        return vorbis_residue_decode_internal(vc, vr, ch, do_not_decode, vec, vlen, ch_left, 2);
     else if (vr->type == 1)
-        return vorbis_residue_decode_internal(vc, vr, ch, do_not_decode, vec, vlen, 1);
+        return vorbis_residue_decode_internal(vc, vr, ch, do_not_decode, vec, vlen, ch_left, 1);
     else if (vr->type == 0)
-        return vorbis_residue_decode_internal(vc, vr, ch, do_not_decode, vec, vlen, 0);
+        return vorbis_residue_decode_internal(vc, vr, ch, do_not_decode, vec, vlen, ch_left, 0);
     else {
         av_log(vc->avccontext, AV_LOG_ERROR, " Invalid residue type while residue decode?! \n");
         return -1;
     }
 }
 
 void vorbis_inverse_coupling(float *mag, float *ang, int blocksize)
 {
     int i;
     for (i = 0;  i < blocksize;  i++) {
@@ skipping 27 matching lines @@
     unsigned mode_number, blockflag, blocksize;
     int i, j;
     uint8_t no_residue[255];
     uint8_t do_not_decode[255];
     vorbis_mapping *mapping;
     float *ch_res_ptr   = vc->channel_residues;
     float *ch_floor_ptr = vc->channel_floors;
     uint8_t res_chan[255];
     unsigned res_num = 0;
     int retlen  = 0;
-    int ch_left = vc->audio_channels;
+    unsigned ch_left = vc->audio_channels;
+    unsigned vlen;
 
     if (get_bits1(gb)) {
         av_log(vc->avccontext, AV_LOG_ERROR, "Not a Vorbis I audio packet.\n");
         return -1; // packet type not audio
     }
 
     if (vc->mode_count == 1) {
         mode_number = 0;
     } else {
         GET_VALIDATED_INDEX(mode_number, ilog(vc->mode_count-1), vc->mode_count)
     }
     vc->mode_number = mode_number;
     mapping = &vc->mappings[vc->modes[mode_number].mapping];
 
     av_dlog(NULL, " Mode number: %u , mapping: %d , blocktype %d\n", mode_number,
             vc->modes[mode_number].mapping, vc->modes[mode_number].blockflag);
 
     blockflag = vc->modes[mode_number].blockflag;
     blocksize = vc->blocksize[blockflag];
+    vlen = blocksize / 2;
     if (blockflag)
         skip_bits(gb, 2); // previous_window, next_window
 
-    memset(ch_res_ptr,   0, sizeof(float) * vc->audio_channels * blocksize / 2); //FIXME can this be removed ?
-    memset(ch_floor_ptr, 0, sizeof(float) * vc->audio_channels * blocksize / 2); //FIXME can this be removed ?
+    memset(ch_res_ptr,   0, sizeof(float) * vc->audio_channels * vlen); //FIXME can this be removed ?
+    memset(ch_floor_ptr, 0, sizeof(float) * vc->audio_channels * vlen); //FIXME can this be removed ?
 
 // Decode floor
 
     for (i = 0; i < vc->audio_channels; ++i) {
         vorbis_floor *floor;
         int ret;
         if (mapping->submaps > 1) {
             floor = &vc->floors[mapping->submap_floor[mapping->mux[i]]];
         } else {
             floor = &vc->floors[mapping->submap_floor[0]];
         }
 
         ret = floor->decode(vc, &floor->data, ch_floor_ptr);
 
         if (ret < 0) {
             av_log(vc->avccontext, AV_LOG_ERROR, "Invalid codebook in vorbis_floor_decode.\n");
             return -1;
         }
         no_residue[i] = ret;
-        ch_floor_ptr += blocksize / 2;
+        ch_floor_ptr += vlen;
     }
 
 // Nonzero vector propagate
 
     for (i = mapping->coupling_steps - 1; i >= 0; --i) {
         if (!(no_residue[mapping->magnitude[i]] & no_residue[mapping->angle[i]])) {
             no_residue[mapping->magnitude[i]] = 0;
             no_residue[mapping->angle[i]]     = 0;
         }
     }
 
 // Decode residue
 
     for (i = 0; i < mapping->submaps; ++i) {
         vorbis_residue *residue;
         unsigned ch = 0;
+        int ret;
 
         for (j = 0; j < vc->audio_channels; ++j) {
             if ((mapping->submaps == 1) || (i == mapping->mux[j])) {
                 res_chan[j] = res_num;
                 if (no_residue[j]) {
                     do_not_decode[ch] = 1;
                 } else {
                     do_not_decode[ch] = 0;
                 }
                 ++ch;
                 ++res_num;
             }
         }
         residue = &vc->residues[mapping->submap_residue[i]];
         if (ch_left < ch) {
             av_log(vc->avccontext, AV_LOG_ERROR, "Too many channels in vorbis_floor_decode.\n");
             return -1;
         }
-        vorbis_residue_decode(vc, residue, ch, do_not_decode, ch_res_ptr, blocksize/2);
+        ret = vorbis_residue_decode(vc, residue, ch, do_not_decode, ch_res_ptr, vlen, ch_left);
+        if (ret < 0)
+            return ret;
 
-        ch_res_ptr += ch * blocksize / 2;
+        ch_res_ptr += ch * vlen;
         ch_left -= ch;
     }
 
 // Inverse coupling
 
     for (i = mapping->coupling_steps - 1; i >= 0; --i) { //warning: i has to be signed
         float *mag, *ang;
 
         mag = vc->channel_residues+res_chan[mapping->magnitude[i]] * blocksize / 2;
         ang = vc->channel_residues+res_chan[mapping->angle[i]]     * blocksize / 2;
@@ skipping 117 matching lines @@
     NULL,
     vorbis_decode_close,
     vorbis_decode_frame,
     .long_name = NULL_IF_CONFIG_SMALL("Vorbis"),
     .channel_layouts = ff_vorbis_channel_layouts,
     .sample_fmts = (const enum AVSampleFormat[]) {
         AV_SAMPLE_FMT_FLT, AV_SAMPLE_FMT_S16, AV_SAMPLE_FMT_NONE
     },
 };