Bug#437916: patch for the fix?

[Kees Cook]

After spending some time digging around in the subversion logs, I
*think* the attached patch is the final fixes that were described in the
changelog.  I'm unclear on the floor1.c changes, but they don't seem to
hurt.

I will likely use this in Ubuntu's update, but I haven't finished
testing yet.

-- 
Kees Cook                                            @outflux.net

Index: res0.c
===================================================================
--- res0.c	(revision 7187)
+++ res0.c	(revision 13167)
@@ -512,7 +512,7 @@
 
 #ifdef TRAIN_RES
   for(i=0;i<ch;i++)
-    for(j=info->begin;j<info->end;j++){
+    for(j=info->begin;j<end;j++){
       if(in[i][j]>look->tmax)look->tmax=in[i][j];
       if(in[i][j]<look->tmin)look->tmin=in[i][j];
     }
@@ -617,8 +618,11 @@
   /* move all this setup out later */
   int samples_per_partition=info->grouping;
   int partitions_per_word=look->phrasebook->dim;
-  int n=info->end-info->begin;
+  int max=vb->pcmend>>1;
+  int end=(info->end<max?info->end:max);
+  int n=end-info->begin;
   
+  if(n>0){
   int partvals=n/samples_per_partition;
   int partwords=(partvals+partitions_per_word-1)/partitions_per_word;
   int ***partword=alloca(ch*sizeof(*partword));
@@ -655,7 +660,7 @@
 	}
     } 
   }
-  
+  }
  errout:
  eopbreak:
   return(0);
@@ -833,8 +838,11 @@
   /* move all this setup out later */
   int samples_per_partition=info->grouping;
   int partitions_per_word=look->phrasebook->dim;
-  int n=info->end-info->begin;
+  int max=(vb->pcmend*ch)>>1;
+  int end=(info->end<max?info->end:max);
+  int n=end-info->begin;
 
+  if(n>0){
   int partvals=n/samples_per_partition;
   int partwords=(partvals+partitions_per_word-1)/partitions_per_word;
   int **partword=_vorbis_block_alloc(vb,partwords*sizeof(*partword));
@@ -867,7 +875,7 @@
 	}
     } 
   }
-  
+  }
  errout:
  eopbreak:
   return(0);
Index: floor1.c
===================================================================
--- floor1.c	(revision 7187)
+++ floor1.c	(revision 13167)
@@ -358,7 +358,7 @@
   0.82788260F, 0.88168307F, 0.9389798F, 1.F, 
 };
 
-static void render_line(int x0,int x1,int y0,int y1,float *d){
+static void render_line(int n, int x0,int x1,int y0,int y1,float *d){
   int dy=y1-y0;
   int adx=x1-x0;
   int ady=abs(dy);
@@ -370,8 +370,12 @@
 
   ady-=abs(base*adx);
 
+  if(n>x1)n=x1;
+
+  if(x<n)
   d[x]*=FLOOR1_fromdB_LOOKUP[y];
-  while(++x<x1){
+
+  while(++x<n){
     err=err+ady;
     if(err>=adx){
       err-=adx;
@@ -1068,7 +1072,7 @@
 	hy*=info->mult;
 	hx=info->postlist[current];
 	
-	render_line(lx,hx,ly,hy,out);
+	render_line(n,lx,hx,ly,hy,out);
 	
 	lx=hx;
 	ly=hy;