Fix vorbis decoder bug.

source/patched-ffmpeg/libavcodec/vorbisdec.c

 /**
  * @file
  * Vorbis I decoder
  * @author Denes Balatoni  ( dbalatoni programozo hu )
  *
  * This file is part of FFmpeg.
  *
  * FFmpeg is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
@@ skipping 642 matching lines @@
 
         res_setup->type = get_bits(gb, 16);
 
         av_dlog(NULL, " %u. residue type %d\n", i, res_setup->type);
 
         res_setup->begin          = get_bits(gb, 24);
         res_setup->end            = get_bits(gb, 24);
         res_setup->partition_size = get_bits(gb, 24) + 1;
         /* Validations to prevent a buffer overflow later. */
         if (res_setup->begin>res_setup->end ||
-            res_setup->end > vc->avccontext->channels * vc->blocksize[1] / 2 ||
+            res_setup->end > (res_setup->type == 2 ? vc->avccontext->channels : 1) * vc->blocksize[1] / 2 ||

[rbultje1, 2011/10/27 23:36:13]

I'm clearly not knowledgeable enough to understand whether this is correct for
vorbis or not. But if it works on sample files from e.g. xiph's conformance
suite, then it's fine with me. Upstream can figure out if the change is
technically 100% correct or can be finetuned.

             (res_setup->end-res_setup->begin) / res_setup->partition_size > V_MAX_PARTITIONS) {
             av_log(vc->avccontext, AV_LOG_ERROR,
                    "partition out of bounds: type, begin, end, size, blocksize: %"PRIu16", %"PRIu32", %"PRIu32", %u, %"PRIu32"\n",
                    res_setup->type, res_setup->begin, res_setup->end,
                    res_setup->partition_size, vc->blocksize[1] / 2);
             return -1;
         }
 
         res_setup->classifications = get_bits(gb, 6) + 1;
         GET_VALIDATED_INDEX(res_setup->classbook, 8, vc->codebook_count)
@@ skipping 787 matching lines @@
     unsigned mode_number, blockflag, blocksize;
     int i, j;
     uint8_t no_residue[255];
     uint8_t do_not_decode[255];
     vorbis_mapping *mapping;
     float *ch_res_ptr   = vc->channel_residues;
     float *ch_floor_ptr = vc->channel_floors;
     uint8_t res_chan[255];
     unsigned res_num = 0;
     int retlen  = 0;
+    int ch_left = vc->audio_channels;
 
     if (get_bits1(gb)) {
         av_log(vc->avccontext, AV_LOG_ERROR, "Not a Vorbis I audio packet.\n");
         return -1; // packet type not audio
     }
 
     if (vc->mode_count == 1) {
         mode_number = 0;
     } else {
         GET_VALIDATED_INDEX(mode_number, ilog(vc->mode_count-1), vc->mode_count)
@@ skipping 54 matching lines @@
                 if (no_residue[j]) {
                     do_not_decode[ch] = 1;
                 } else {
                     do_not_decode[ch] = 0;
                 }
                 ++ch;
                 ++res_num;
             }
         }
         residue = &vc->residues[mapping->submap_residue[i]];
+        if (ch_left < ch) {
+            av_log(vc->avccontext, AV_LOG_ERROR, "Too many channels in vorbis_floor_decode.\n");
+            return -1;
+        }
         vorbis_residue_decode(vc, residue, ch, do_not_decode, ch_res_ptr, blocksize/2);
 
         ch_res_ptr += ch * blocksize / 2;
+        ch_left -= ch;
     }
 
 // Inverse coupling
 
     for (i = mapping->coupling_steps - 1; i >= 0; --i) { //warning: i has to be signed
         float *mag, *ang;
 
         mag = vc->channel_residues+res_chan[mapping->magnitude[i]] * blocksize / 2;
         ang = vc->channel_residues+res_chan[mapping->angle[i]]     * blocksize / 2;
         vc->dsp.vorbis_inverse_coupling(mag, ang, blocksize / 2);
@@ skipping 116 matching lines @@
     NULL,
     vorbis_decode_close,
     vorbis_decode_frame,
     .long_name = NULL_IF_CONFIG_SMALL("Vorbis"),
     .channel_layouts = ff_vorbis_channel_layouts,
     .sample_fmts = (const enum AVSampleFormat[]) {
         AV_SAMPLE_FMT_FLT, AV_SAMPLE_FMT_S16, AV_SAMPLE_FMT_NONE
     },
 };