Fix vorbis decoder bug.

source/patched-ffmpeg/libavcodec/vorbisdec.c

Index: source/patched-ffmpeg/libavcodec/vorbisdec.c
===================================================================
--- source/patched-ffmpeg/libavcodec/vorbisdec.c	(revision 107637)
+++ source/patched-ffmpeg/libavcodec/vorbisdec.c	(working copy)
@@ -660,7 +660,7 @@
         res_setup->partition_size = get_bits(gb, 24) + 1;
         /* Validations to prevent a buffer overflow later. */
         if (res_setup->begin>res_setup->end ||
-            res_setup->end > vc->avccontext->channels * vc->blocksize[1] / 2 ||
+            res_setup->end > (res_setup->type == 2 ? vc->avccontext->channels : 1) * vc->blocksize[1] / 2 ||

[rbultje1, 2011/10/27 23:36:13]

I'm clearly not knowledgeable enough to understand whether this is correct for
vorbis or not. But if it works on sample files from e.g. xiph's conformance
suite, then it's fine with me. Upstream can figure out if the change is
technically 100% correct or can be finetuned.

             (res_setup->end-res_setup->begin) / res_setup->partition_size > V_MAX_PARTITIONS) {
             av_log(vc->avccontext, AV_LOG_ERROR,
                    "partition out of bounds: type, begin, end, size, blocksize: %"PRIu16", %"PRIu32", %"PRIu32", %u, %"PRIu32"\n",
@@ -1468,6 +1468,7 @@
     uint8_t res_chan[255];
     unsigned res_num = 0;
     int retlen  = 0;
+    int ch_left = vc->audio_channels;
 
     if (get_bits1(gb)) {
         av_log(vc->avccontext, AV_LOG_ERROR, "Not a Vorbis I audio packet.\n");
@@ -1542,9 +1543,14 @@
             }
         }
         residue = &vc->residues[mapping->submap_residue[i]];
+        if (ch_left < ch) {
+            av_log(vc->avccontext, AV_LOG_ERROR, "Too many channels in vorbis_floor_decode.\n");
+            return -1;
+        }
         vorbis_residue_decode(vc, residue, ch, do_not_decode, ch_res_ptr, blocksize/2);
 
         ch_res_ptr += ch * blocksize / 2;
+        ch_left -= ch;
     }
 
 // Inverse coupling