Add proper guarding to cases where declared floor/residue decode size is *larger* than the current blocksize. Handle according to spec. (lowmem branch)

git-svn-id: https://svn.xiph.org/branches/lowmem-branch/Tremor@13157 0101bb08-14d6-0310-b084-bc0e0c8e3800

2 files changed, 116 insertions, 100 deletions

diff --git a/floor1.c b/floor1.c
index 1bca579..4f53bf7 100644
--- a/floor1.c
+++ b/floor1.c
@@ -196,7 +196,7 @@ static int render_point(int x0,int x1,int y0,int y1,int x){
   }
 }
 
-static void render_line(int x0,int x1,int y0,int y1,ogg_int32_t *d){
+static void render_line(int n,int x0,int x1,int y0,int y1,ogg_int32_t *d){
   int dy=y1-y0;
   int adx=x1-x0;
   int ady=abs(dy);
@@ -206,11 +206,13 @@ static void render_line(int x0,int x1,int y0,int y1,ogg_int32_t *d){
   int y=y0;
   int err=0;
 
+  if(n>x1)n=x1;
   ady-=abs(base*adx);
 
-  d[x]= MULT31_SHIFT15(d[x],FLOOR_fromdB_LOOKUP[y]);
+  if(x<n)
+    d[x]= MULT31_SHIFT15(d[x],FLOOR_fromdB_LOOKUP[y]);
 
-  while(++x<x1){
+  while(++x<n){
     err=err+ady;
     if(err>=adx){
       err-=adx;
@@ -336,7 +338,7 @@ int floor1_inverse2(vorbis_dsp_state *vd,vorbis_info_floor *in,
 	hy*=info->mult;
 	hx=info->postlist[current];
 	
-	render_line(lx,hx,ly,hy,out);
+	render_line(n,lx,hx,ly,hy,out);
 	
 	lx=hx;
 	ly=hy;
diff --git a/res012.c b/res012.c
index 5844db9..8e69799 100644
--- a/res012.c
+++ b/res012.c
@@ -88,121 +88,135 @@ int res_inverse(vorbis_dsp_state *vd,vorbis_info_residue *info,
   codebook *phrasebook=ci->book_param+info->groupbook;
   int samples_per_partition=info->grouping;
   int partitions_per_word=phrasebook->dim;
-  int n=info->end-info->begin;
-  int partvals=n/samples_per_partition;
-  int partwords=(partvals+partitions_per_word-1)/partitions_per_word;
+  int pcmend=ci->blocksizes[vd->W];
 
   if(info->type<2){
-    for(i=0;i<ch;i++)
-      if(nonzero[i])
-	in[used++]=in[i];
-    ch=used;
-
-    if(used){
+    int max=pcmend>>1;
+    int end=(info->end<max?info->end:max);
+    int n=end-info->begin;
+    
+    if(n>0){
+      int partvals=n/samples_per_partition;
+      int partwords=(partvals+partitions_per_word-1)/partitions_per_word;
       
-      char **partword=(char **)alloca(ch*sizeof(*partword));
-      for(j=0;j<ch;j++)
-	partword[j]=(char *)alloca(partwords*partitions_per_word*
-				   sizeof(*partword[j]));
-
-      for(s=0;s<info->stages;s++){
+      for(i=0;i<ch;i++)
+	if(nonzero[i])
+	  in[used++]=in[i];
+      ch=used;
+      
+      if(used){
+	
+	char **partword=(char **)alloca(ch*sizeof(*partword));
+	for(j=0;j<ch;j++)
+	  partword[j]=(char *)alloca(partwords*partitions_per_word*
+				     sizeof(*partword[j]));
 	
+	for(s=0;s<info->stages;s++){
+	  
+	  for(i=0;i<partvals;){
+	    if(s==0){
+	      /* fetch the partition word for each channel */
+	      
+	      partword[0][i+partitions_per_word-1]=1;
+	      for(k=partitions_per_word-2;k>=0;k--)
+		partword[0][i+k]=partword[0][i+k+1]*info->partitions;
+	      
+	      for(j=1;j<ch;j++)
+		for(k=partitions_per_word-1;k>=0;k--)
+		  partword[j][i+k]=partword[j-1][i+k];
+	      
+	      for(j=0;j<ch;j++){
+		int temp=vorbis_book_decode(phrasebook,&vd->opb);
+		if(oggpack_eop(&vd->opb))goto eopbreak;
+		
+		/* this can be done quickly in assembly due to the quotient
+		   always being at most six bits */
+		for(k=0;k<partitions_per_word;k++){
+		  ogg_uint32_t div=partword[j][i+k];
+		  partword[j][i+k]=temp/div;
+		  temp-=partword[j][i+k]*div;
+		}
+		
+	      }
+	    }
+	    
+	    /* now we decode residual values for the partitions */
+	    for(k=0;k<partitions_per_word && i<partvals;k++,i++)
+	      for(j=0;j<ch;j++){
+		long offset=info->begin+i*samples_per_partition;
+		if(info->stagemasks[(int)partword[j][i]]&(1<<s)){
+		  codebook *stagebook=ci->book_param+
+		    info->stagebooks[(partword[j][i]<<3)+s];
+		  if(info->type){
+		    if(vorbis_book_decodev_add(stagebook,in[j]+offset,&vd->opb,
+					       samples_per_partition,-8)==-1)
+		      goto eopbreak;
+		  }else{
+		    if(vorbis_book_decodevs_add(stagebook,in[j]+offset,&vd->opb,
+						samples_per_partition,-8)==-1)
+		      goto eopbreak;
+		  }
+		}
+	      }
+	  }
+	} 
+      }
+    }
+  }else{
+    int max=(pcmend*ch)>>1;
+    int end=(info->end<max?info->end:max);
+    int n=end-info->begin;
+    
+    if(n>0){
+      int partvals=n/samples_per_partition;
+      int partwords=(partvals+partitions_per_word-1)/partitions_per_word;
+      
+      char *partword=
+	(char *)alloca(partwords*partitions_per_word*sizeof(*partword));
+      int beginoff=info->begin/ch;
+      
+      for(i=0;i<ch;i++)if(nonzero[i])break;
+      if(i==ch)return(0); /* no nonzero vectors */
+      
+      samples_per_partition/=ch;
+      
+      for(s=0;s<info->stages;s++){
 	for(i=0;i<partvals;){
+	  
 	  if(s==0){
-	    /* fetch the partition word for each channel */
-	    
-	    partword[0][i+partitions_per_word-1]=1;
+	    int temp;
+	    partword[i+partitions_per_word-1]=1;
 	    for(k=partitions_per_word-2;k>=0;k--)
-	      partword[0][i+k]=partword[0][i+k+1]*info->partitions;
+	      partword[i+k]=partword[i+k+1]*info->partitions;
 	    
-	    for(j=1;j<ch;j++)
-	      for(k=partitions_per_word-1;k>=0;k--)
-		partword[j][i+k]=partword[j-1][i+k];
+	    /* fetch the partition word */
+	    temp=vorbis_book_decode(phrasebook,&vd->opb);
+	    if(oggpack_eop(&vd->opb))goto eopbreak;
 	    
-	    for(j=0;j<ch;j++){
-	      int temp=vorbis_book_decode(phrasebook,&vd->opb);
-	      if(oggpack_eop(&vd->opb))goto eopbreak;
-	      
-	      /* this can be done quickly in assembly due to the quotient
-		 always being at most six bits */
-	      for(k=0;k<partitions_per_word;k++){
-		ogg_uint32_t div=partword[j][i+k];
-		partword[j][i+k]=temp/div;
-		temp-=partword[j][i+k]*div;
-	      }
-	      
+	    /* this can be done quickly in assembly due to the quotient
+	       always being at most six bits */
+	    for(k=0;k<partitions_per_word;k++){
+	      ogg_uint32_t div=partword[i+k];
+	      partword[i+k]=temp/div;
+	      temp-=partword[i+k]*div;
 	    }
 	  }
 	  
 	  /* now we decode residual values for the partitions */
 	  for(k=0;k<partitions_per_word && i<partvals;k++,i++)
-	    for(j=0;j<ch;j++){
-	      long offset=info->begin+i*samples_per_partition;
-	      if(info->stagemasks[partword[j][i]]&(1<<s)){
-		codebook *stagebook=ci->book_param+
-		  info->stagebooks[(partword[j][i]<<3)+s];
-		if(info->type){
-		  if(vorbis_book_decodev_add(stagebook,in[j]+offset,&vd->opb,
-					     samples_per_partition,-8)==-1)
-		    goto eopbreak;
-		}else{
-		  if(vorbis_book_decodevs_add(stagebook,in[j]+offset,&vd->opb,
-					      samples_per_partition,-8)==-1)
-		    goto eopbreak;
-		}
-	      }
+	    if(info->stagemasks[(int)partword[i]]&(1<<s)){
+	      codebook *stagebook=ci->book_param+
+		info->stagebooks[(partword[i]<<3)+s];
+	      if(vorbis_book_decodevv_add(stagebook,in,
+					  i*samples_per_partition+beginoff,ch,
+					  &vd->opb,
+					  samples_per_partition,-8)==-1)
+		goto eopbreak;
 	    }
 	}
       } 
     }
-  }else{
-    
-    char *partword=
-      (char *)alloca(partwords*partitions_per_word*sizeof(*partword));
-    int beginoff=info->begin/ch;
-    
-    for(i=0;i<ch;i++)if(nonzero[i])break;
-    if(i==ch)return(0); /* no nonzero vectors */
-    
-    samples_per_partition/=ch;
-    
-    for(s=0;s<info->stages;s++){
-      for(i=0;i<partvals;){
-	
-	if(s==0){
-	  int temp;
-	  partword[i+partitions_per_word-1]=1;
-	  for(k=partitions_per_word-2;k>=0;k--)
-	    partword[i+k]=partword[i+k+1]*info->partitions;
-	  
-	  /* fetch the partition word */
-	  temp=vorbis_book_decode(phrasebook,&vd->opb);
-	  if(oggpack_eop(&vd->opb))goto eopbreak;
-	  
-	  /* this can be done quickly in assembly due to the quotient
-	     always being at most six bits */
-	  for(k=0;k<partitions_per_word;k++){
-	    ogg_uint32_t div=partword[i+k];
-	    partword[i+k]=temp/div;
-	    temp-=partword[i+k]*div;
-	  }
-	}
-	
-	/* now we decode residual values for the partitions */
-	for(k=0;k<partitions_per_word && i<partvals;k++,i++)
-	  if(info->stagemasks[partword[i]]&(1<<s)){
-	    codebook *stagebook=ci->book_param+
-	      info->stagebooks[(partword[i]<<3)+s];
-	    if(vorbis_book_decodevv_add(stagebook,in,
-					i*samples_per_partition+beginoff,ch,
-					&vd->opb,
-					samples_per_partition,-8)==-1)
-	      goto eopbreak;
-	  }
-      }
-    } 
   }
-  
  errout:
  eopbreak: