Add proper guarding to cases where declared floor/residue decode size

is *larger* than the current blocksize.  Handle according to spec.



git-svn-id: https://svn.xiph.org/trunk/Tremor@13156 0101bb08-14d6-0310-b084-bc0e0c8e3800

2 files changed, 85 insertions, 76 deletions

diff --git a/floor1.c b/floor1.c
index 80c6dd2..276ed8d 100644
--- a/floor1.c
+++ b/floor1.c
@@ -282,7 +282,7 @@ static const ogg_int32_t FLOOR_fromdB_LOOKUP[256]={
   XdB(0x69f80e9a), XdB(0x70dafda8), XdB(0x78307d76), XdB(0x7fffffff),
 };
   
-static void render_line(int x0,int x1,int y0,int y1,ogg_int32_t *d){
+static void render_line(int n, int x0,int x1,int y0,int y1,ogg_int32_t *d){
   int dy=y1-y0;
   int adx=x1-x0;
   int ady=abs(dy);
@@ -292,11 +292,13 @@ static void render_line(int x0,int x1,int y0,int y1,ogg_int32_t *d){
   int y=y0;
   int err=0;
 
+  if(n>x1)n=x1;
   ady-=abs(base*adx);
 
-  d[x]= MULT31_SHIFT15(d[x],FLOOR_fromdB_LOOKUP[y]);
+  if(x<n)
+    d[x]= MULT31_SHIFT15(d[x],FLOOR_fromdB_LOOKUP[y]);
 
-  while(++x<x1){
+  while(++x<n){
     err=err+ady;
     if(err>=adx){
       err-=adx;
@@ -418,7 +420,7 @@ static int floor1_inverse2(vorbis_block *vb,vorbis_look_floor *in,void *memo,
 	hy*=info->mult;
 	hx=info->postlist[current];
 	
-	render_line(lx,hx,ly,hy,out);
+	render_line(n,lx,hx,ly,hy,out);
 	
 	lx=hx;
 	ly=hy;
diff --git a/res012.c b/res012.c
index e5d87e9..224b54e 100644
--- a/res012.c
+++ b/res012.c
@@ -187,45 +187,48 @@ static int _01inverse(vorbis_block *vb,vorbis_look_residue *vl,
   /* move all this setup out later */
   int samples_per_partition=info->grouping;
   int partitions_per_word=look->phrasebook->dim;
-  int n=info->end-info->begin;
-  
-  int partvals=n/samples_per_partition;
-  int partwords=(partvals+partitions_per_word-1)/partitions_per_word;
-  int ***partword=(int ***)alloca(ch*sizeof(*partword));
-
-  for(j=0;j<ch;j++)
-    partword[j]=(int **)_vorbis_block_alloc(vb,partwords*sizeof(*partword[j]));
-
-  for(s=0;s<look->stages;s++){
-
-    /* each loop decodes on partition codeword containing 
-       partitions_pre_word partitions */
-    for(i=0,l=0;i<partvals;l++){
-      if(s==0){
-	/* fetch the partition word for each channel */
-	for(j=0;j<ch;j++){
-	  int temp=vorbis_book_decode(look->phrasebook,&vb->opb);
-	  if(temp==-1)goto eopbreak;
-	  partword[j][l]=look->decodemap[temp];
-	  if(partword[j][l]==NULL)goto errout;
-	}
-      }
+  int max=vb->pcmend>>1;
+  int end=(info->end<max?info->end:max);
+  int n=end-info->begin;
+
+  if(n>0){
+    int partvals=n/samples_per_partition;
+    int partwords=(partvals+partitions_per_word-1)/partitions_per_word;
+    int ***partword=(int ***)alloca(ch*sizeof(*partword));
+    
+    for(j=0;j<ch;j++)
+      partword[j]=(int **)_vorbis_block_alloc(vb,partwords*sizeof(*partword[j]));
+    
+    for(s=0;s<look->stages;s++){
       
-      /* now we decode residual values for the partitions */
-      for(k=0;k<partitions_per_word && i<partvals;k++,i++)
-	for(j=0;j<ch;j++){
-	  long offset=info->begin+i*samples_per_partition;
-	  if(info->secondstages[partword[j][l][k]]&(1<<s)){
-	    codebook *stagebook=look->partbooks[partword[j][l][k]][s];
-	    if(stagebook){
-	      if(decodepart(stagebook,in[j]+offset,&vb->opb,
-			    samples_per_partition,-8)==-1)goto eopbreak;
-	    }
+      /* each loop decodes on partition codeword containing 
+	 partitions_pre_word partitions */
+      for(i=0,l=0;i<partvals;l++){
+	if(s==0){
+	  /* fetch the partition word for each channel */
+	  for(j=0;j<ch;j++){
+	    int temp=vorbis_book_decode(look->phrasebook,&vb->opb);
+	    if(temp==-1)goto eopbreak;
+	    partword[j][l]=look->decodemap[temp];
+	    if(partword[j][l]==NULL)goto errout;
 	  }
 	}
-    } 
+	
+	/* now we decode residual values for the partitions */
+	for(k=0;k<partitions_per_word && i<partvals;k++,i++)
+	  for(j=0;j<ch;j++){
+	    long offset=info->begin+i*samples_per_partition;
+	    if(info->secondstages[partword[j][l][k]]&(1<<s)){
+	      codebook *stagebook=look->partbooks[partword[j][l][k]][s];
+	      if(stagebook){
+		if(decodepart(stagebook,in[j]+offset,&vb->opb,
+			      samples_per_partition,-8)==-1)goto eopbreak;
+	      }
+	    }
+	  }
+      } 
+    }
   }
-  
  errout:
  eopbreak:
   return(0);
@@ -265,45 +268,49 @@ int res2_inverse(vorbis_block *vb,vorbis_look_residue *vl,
   /* move all this setup out later */
   int samples_per_partition=info->grouping;
   int partitions_per_word=look->phrasebook->dim;
-  int n=info->end-info->begin;
-
-  int partvals=n/samples_per_partition;
-  int partwords=(partvals+partitions_per_word-1)/partitions_per_word;
-  int **partword=(int **)_vorbis_block_alloc(vb,partwords*sizeof(*partword));
-  int beginoff=info->begin/ch;
-
-  for(i=0;i<ch;i++)if(nonzero[i])break;
-  if(i==ch)return(0); /* no nonzero vectors */
-
-  samples_per_partition/=ch;
-
-  for(s=0;s<look->stages;s++){
-    for(i=0,l=0;i<partvals;l++){
-
-      if(s==0){
-	/* fetch the partition word */
-	int temp=vorbis_book_decode(look->phrasebook,&vb->opb);
-	if(temp==-1)goto eopbreak;
-	partword[l]=look->decodemap[temp];
-	if(partword[l]==NULL)goto errout;
-      }
-
-      /* now we decode residual values for the partitions */
-      for(k=0;k<partitions_per_word && i<partvals;k++,i++)
-	if(info->secondstages[partword[l][k]]&(1<<s)){
-	  codebook *stagebook=look->partbooks[partword[l][k]][s];
-	  
-	  if(stagebook){
-	    if(vorbis_book_decodevv_add(stagebook,in,
-					i*samples_per_partition+beginoff,ch,
-					&vb->opb,
-					samples_per_partition,-8)==-1)
-	      goto eopbreak;
-	  }
+  int max=(vb->pcmend*ch)>>1;
+  int end=(info->end<max?info->end:max);
+  int n=end-info->begin;
+
+  if(n>0){
+    
+    int partvals=n/samples_per_partition;
+    int partwords=(partvals+partitions_per_word-1)/partitions_per_word;
+    int **partword=(int **)_vorbis_block_alloc(vb,partwords*sizeof(*partword));
+    int beginoff=info->begin/ch;
+    
+    for(i=0;i<ch;i++)if(nonzero[i])break;
+    if(i==ch)return(0); /* no nonzero vectors */
+    
+    samples_per_partition/=ch;
+    
+    for(s=0;s<look->stages;s++){
+      for(i=0,l=0;i<partvals;l++){
+	
+	if(s==0){
+	  /* fetch the partition word */
+	  int temp=vorbis_book_decode(look->phrasebook,&vb->opb);
+	  if(temp==-1)goto eopbreak;
+	  partword[l]=look->decodemap[temp];
+	  if(partword[l]==NULL)goto errout;
 	}
-    } 
+
+	/* now we decode residual values for the partitions */
+	for(k=0;k<partitions_per_word && i<partvals;k++,i++)
+	  if(info->secondstages[partword[l][k]]&(1<<s)){
+	    codebook *stagebook=look->partbooks[partword[l][k]][s];
+	    
+	    if(stagebook){
+	      if(vorbis_book_decodevv_add(stagebook,in,
+					  i*samples_per_partition+beginoff,ch,
+					  &vb->opb,
+					  samples_per_partition,-8)==-1)
+		goto eopbreak;
+	    }
+	  }
+      } 
+    }
   }
-  
  errout:
  eopbreak:
   return(0);