Tremor mainline succeptible to bug #340; corrected.

git-svn-id: https://svn.xiph.org/trunk/Tremor@13173 0101bb08-14d6-0310-b084-bc0e0c8e3800

2 files changed, 176 insertions, 161 deletions

diff --git a/codebook.c b/codebook.c
index 4cff584..a5b7ffa 100644
--- a/codebook.c
+++ b/codebook.c
@@ -210,66 +210,72 @@ STIN long decode_packed_entry_number(codebook *book,
 
 /* returns the [original, not compacted] entry number or -1 on eof *********/
 long vorbis_book_decode(codebook *book, oggpack_buffer *b){
-  long packed_entry=decode_packed_entry_number(book,b);
-  if(packed_entry>=0)
-    return(book->dec_index[packed_entry]);
-  
+  if(book->used_entries>0){
+    long packed_entry=decode_packed_entry_number(book,b);
+    if(packed_entry>=0)
+      return(book->dec_index[packed_entry]);
+  }
+
   /* if there's no dec_index, the codebook unpacking isn't collapsed */
-  return(packed_entry);
+  return(-1);
 }
 
 /* returns 0 on OK or -1 on eof *************************************/
 long vorbis_book_decodevs_add(codebook *book,ogg_int32_t *a,
 			      oggpack_buffer *b,int n,int point){
-  int step=n/book->dim;
-  long *entry = (long *)alloca(sizeof(*entry)*step);
-  ogg_int32_t **t = (ogg_int32_t **)alloca(sizeof(*t)*step);
-  int i,j,o;
-  int shift=point-book->binarypoint;
-
-  if(shift>=0){
-    for (i = 0; i < step; i++) {
-      entry[i]=decode_packed_entry_number(book,b);
-      if(entry[i]==-1)return(-1);
-      t[i] = book->valuelist+entry[i]*book->dim;
-    }
-    for(i=0,o=0;i<book->dim;i++,o+=step)
-      for (j=0;j<step;j++)
-	a[o+j]+=t[j][i]>>shift;
-  }else{
-    for (i = 0; i < step; i++) {
-      entry[i]=decode_packed_entry_number(book,b);
-      if(entry[i]==-1)return(-1);
-      t[i] = book->valuelist+entry[i]*book->dim;
+  if(book->used_entries>0){  
+    int step=n/book->dim;
+    long *entry = (long *)alloca(sizeof(*entry)*step);
+    ogg_int32_t **t = (ogg_int32_t **)alloca(sizeof(*t)*step);
+    int i,j,o;
+    int shift=point-book->binarypoint;
+    
+    if(shift>=0){
+      for (i = 0; i < step; i++) {
+	entry[i]=decode_packed_entry_number(book,b);
+	if(entry[i]==-1)return(-1);
+	t[i] = book->valuelist+entry[i]*book->dim;
+      }
+      for(i=0,o=0;i<book->dim;i++,o+=step)
+	for (j=0;j<step;j++)
+	  a[o+j]+=t[j][i]>>shift;
+    }else{
+      for (i = 0; i < step; i++) {
+	entry[i]=decode_packed_entry_number(book,b);
+	if(entry[i]==-1)return(-1);
+	t[i] = book->valuelist+entry[i]*book->dim;
+      }
+      for(i=0,o=0;i<book->dim;i++,o+=step)
+	for (j=0;j<step;j++)
+	  a[o+j]+=t[j][i]<<-shift;
     }
-    for(i=0,o=0;i<book->dim;i++,o+=step)
-      for (j=0;j<step;j++)
-	a[o+j]+=t[j][i]<<-shift;
   }
   return(0);
 }
 
 long vorbis_book_decodev_add(codebook *book,ogg_int32_t *a,
 			     oggpack_buffer *b,int n,int point){
-  int i,j,entry;
-  ogg_int32_t *t;
-  int shift=point-book->binarypoint;
-  
-  if(shift>=0){
-    for(i=0;i<n;){
-      entry = decode_packed_entry_number(book,b);
-      if(entry==-1)return(-1);
-      t     = book->valuelist+entry*book->dim;
-      for (j=0;j<book->dim;)
-	a[i++]+=t[j++]>>shift;
-    }
-  }else{
-    for(i=0;i<n;){
-      entry = decode_packed_entry_number(book,b);
-      if(entry==-1)return(-1);
-      t     = book->valuelist+entry*book->dim;
-      for (j=0;j<book->dim;)
-	a[i++]+=t[j++]<<-shift;
+  if(book->used_entries>0){
+    int i,j,entry;
+    ogg_int32_t *t;
+    int shift=point-book->binarypoint;
+    
+    if(shift>=0){
+      for(i=0;i<n;){
+	entry = decode_packed_entry_number(book,b);
+	if(entry==-1)return(-1);
+	t     = book->valuelist+entry*book->dim;
+	for (j=0;j<book->dim;)
+	  a[i++]+=t[j++]>>shift;
+      }
+    }else{
+      for(i=0;i<n;){
+	entry = decode_packed_entry_number(book,b);
+	if(entry==-1)return(-1);
+	t     = book->valuelist+entry*book->dim;
+	for (j=0;j<book->dim;)
+	  a[i++]+=t[j++]<<-shift;
+      }
     }
   }
   return(0);
@@ -277,28 +283,38 @@ long vorbis_book_decodev_add(codebook *book,ogg_int32_t *a,
 
 long vorbis_book_decodev_set(codebook *book,ogg_int32_t *a,
 			     oggpack_buffer *b,int n,int point){
-  int i,j,entry;
-  ogg_int32_t *t;
-  int shift=point-book->binarypoint;
-  
-  if(shift>=0){
-
-    for(i=0;i<n;){
-      entry = decode_packed_entry_number(book,b);
-      if(entry==-1)return(-1);
-      t     = book->valuelist+entry*book->dim;
-      for (j=0;j<book->dim;){
-	a[i++]=t[j++]>>shift;
+  if(book->used_entries>0){
+    int i,j,entry;
+    ogg_int32_t *t;
+    int shift=point-book->binarypoint;
+    
+    if(shift>=0){
+      
+      for(i=0;i<n;){
+	entry = decode_packed_entry_number(book,b);
+	if(entry==-1)return(-1);
+	t     = book->valuelist+entry*book->dim;
+	for (j=0;j<book->dim;){
+	  a[i++]=t[j++]>>shift;
+	}
+      }
+    }else{
+      
+      for(i=0;i<n;){
+	entry = decode_packed_entry_number(book,b);
+	if(entry==-1)return(-1);
+	t     = book->valuelist+entry*book->dim;
+	for (j=0;j<book->dim;){
+	  a[i++]=t[j++]<<-shift;
+	}
       }
     }
   }else{
 
+    int i,j;
     for(i=0;i<n;){
-      entry = decode_packed_entry_number(book,b);
-      if(entry==-1)return(-1);
-      t     = book->valuelist+entry*book->dim;
       for (j=0;j<book->dim;){
-	a[i++]=t[j++]<<-shift;
+	a[i++]=0;
       }
     }
   }
@@ -308,38 +324,40 @@ long vorbis_book_decodev_set(codebook *book,ogg_int32_t *a,
 long vorbis_book_decodevv_add(codebook *book,ogg_int32_t **a,\
 			      long offset,int ch,
 			      oggpack_buffer *b,int n,int point){
-  long i,j,entry;
-  int chptr=0;
-  int shift=point-book->binarypoint;
-  
-  if(shift>=0){
+  if(book->used_entries>0){
+    long i,j,entry;
+    int chptr=0;
+    int shift=point-book->binarypoint;
     
-    for(i=offset;i<offset+n;){
-      entry = decode_packed_entry_number(book,b);
-      if(entry==-1)return(-1);
-      {
-	const ogg_int32_t *t = book->valuelist+entry*book->dim;
-	for (j=0;j<book->dim;j++){
-	  a[chptr++][i]+=t[j]>>shift;
-	  if(chptr==ch){
-	    chptr=0;
-	    i++;
+    if(shift>=0){
+      
+      for(i=offset;i<offset+n;){
+	entry = decode_packed_entry_number(book,b);
+	if(entry==-1)return(-1);
+	{
+	  const ogg_int32_t *t = book->valuelist+entry*book->dim;
+	  for (j=0;j<book->dim;j++){
+	    a[chptr++][i]+=t[j]>>shift;
+	    if(chptr==ch){
+	      chptr=0;
+	      i++;
+	    }
 	  }
 	}
       }
-    }
-  }else{
-
-    for(i=offset;i<offset+n;){
-      entry = decode_packed_entry_number(book,b);
-      if(entry==-1)return(-1);
-      {
-	const ogg_int32_t *t = book->valuelist+entry*book->dim;
-	for (j=0;j<book->dim;j++){
-	  a[chptr++][i]+=t[j]<<-shift;
-	  if(chptr==ch){
-	    chptr=0;
-	    i++;
+    }else{
+      
+      for(i=offset;i<offset+n;){
+	entry = decode_packed_entry_number(book,b);
+	if(entry==-1)return(-1);
+	{
+	  const ogg_int32_t *t = book->valuelist+entry*book->dim;
+	  for (j=0;j<book->dim;j++){
+	    a[chptr++][i]+=t[j]<<-shift;
+	    if(chptr==ch){
+	      chptr=0;
+	      i++;
+	    }
 	  }
 	}
       }
diff --git a/sharedbook.c b/sharedbook.c
index b982c7a..8e07492 100644
--- a/sharedbook.c
+++ b/sharedbook.c
@@ -334,20 +334,17 @@ int vorbis_book_init_decode(codebook *c,const static_codebook *s){
   c->used_entries=n;
   c->dim=s->dim;
 
-  c->q_min=s->q_min;
-  c->q_delta=s->q_delta;
-
-  /* two different remappings go on here.  
-
-     First, we collapse the likely sparse codebook down only to
-     actually represented values/words.  This collapsing needs to be
-     indexed as map-valueless books are used to encode original entry
-     positions as integers.
-
-     Second, we reorder all vectors, including the entry index above,
-     by sorted bitreversed codeword to allow treeless decode. */
-
-  {
+  if(n>0){
+    /* two different remappings go on here.  
+       
+       First, we collapse the likely sparse codebook down only to
+       actually represented values/words.  This collapsing needs to be
+       indexed as map-valueless books are used to encode original entry
+       positions as integers.
+       
+       Second, we reorder all vectors, including the entry index above,
+       by sorted bitreversed codeword to allow treeless decode. */
+    
     /* perform sort */
     ogg_uint32_t *codes=_make_words(s->lengthlist,s->entries,c->used_entries);
     ogg_uint32_t **codep=(ogg_uint32_t **)alloca(sizeof(*codep)*n);
@@ -372,67 +369,67 @@ int vorbis_book_init_decode(codebook *c,const static_codebook *s){
     for(i=0;i<n;i++)
       c->codelist[sortindex[i]]=codes[i];
     _ogg_free(codes);
-  }
-
-  
-  c->valuelist=_book_unquantize(s,n,sortindex,&c->binarypoint);
-  c->dec_index=(int *)_ogg_malloc(n*sizeof(*c->dec_index));
-  
-  for(n=0,i=0;i<s->entries;i++)
-    if(s->lengthlist[i]>0)
-      c->dec_index[sortindex[n++]]=i;
- 
-  c->dec_codelengths=(char *)_ogg_malloc(n*sizeof(*c->dec_codelengths));
-  for(n=0,i=0;i<s->entries;i++)
-    if(s->lengthlist[i]>0)
-      c->dec_codelengths[sortindex[n++]]=s->lengthlist[i];
-
-  c->dec_firsttablen=_ilog(c->used_entries)-4; /* this is magic */
-  if(c->dec_firsttablen<5)c->dec_firsttablen=5;
-  if(c->dec_firsttablen>8)c->dec_firsttablen=8;
-
-  tabn=1<<c->dec_firsttablen;
-  c->dec_firsttable=(ogg_uint32_t *)_ogg_calloc(tabn,sizeof(*c->dec_firsttable));
-  c->dec_maxlength=0;
-
-  for(i=0;i<n;i++){
-    if(c->dec_maxlength<c->dec_codelengths[i])
-      c->dec_maxlength=c->dec_codelengths[i];
-    if(c->dec_codelengths[i]<=c->dec_firsttablen){
-      ogg_uint32_t orig=bitreverse(c->codelist[i]);
-      for(j=0;j<(1<<(c->dec_firsttablen-c->dec_codelengths[i]));j++)
-	c->dec_firsttable[orig|(j<<c->dec_codelengths[i])]=i+1;
+    
+    
+    
+    c->valuelist=_book_unquantize(s,n,sortindex,&c->binarypoint);
+    c->dec_index=(int *)_ogg_malloc(n*sizeof(*c->dec_index));
+    
+    for(n=0,i=0;i<s->entries;i++)
+      if(s->lengthlist[i]>0)
+	c->dec_index[sortindex[n++]]=i;
+    
+    c->dec_codelengths=(char *)_ogg_malloc(n*sizeof(*c->dec_codelengths));
+    for(n=0,i=0;i<s->entries;i++)
+      if(s->lengthlist[i]>0)
+	c->dec_codelengths[sortindex[n++]]=s->lengthlist[i];
+    
+    c->dec_firsttablen=_ilog(c->used_entries)-4; /* this is magic */
+    if(c->dec_firsttablen<5)c->dec_firsttablen=5;
+    if(c->dec_firsttablen>8)c->dec_firsttablen=8;
+    
+    tabn=1<<c->dec_firsttablen;
+    c->dec_firsttable=(ogg_uint32_t *)_ogg_calloc(tabn,sizeof(*c->dec_firsttable));
+    c->dec_maxlength=0;
+    
+    for(i=0;i<n;i++){
+      if(c->dec_maxlength<c->dec_codelengths[i])
+	c->dec_maxlength=c->dec_codelengths[i];
+      if(c->dec_codelengths[i]<=c->dec_firsttablen){
+	ogg_uint32_t orig=bitreverse(c->codelist[i]);
+	for(j=0;j<(1<<(c->dec_firsttablen-c->dec_codelengths[i]));j++)
+	  c->dec_firsttable[orig|(j<<c->dec_codelengths[i])]=i+1;
+      }
     }
-  }
-
-  /* now fill in 'unused' entries in the firsttable with hi/lo search
-     hints for the non-direct-hits */
-  {
-    ogg_uint32_t mask=0xfffffffeUL<<(31-c->dec_firsttablen);
-    long lo=0,hi=0;
-
-    for(i=0;i<tabn;i++){
-      ogg_uint32_t word=i<<(32-c->dec_firsttablen);
-      if(c->dec_firsttable[bitreverse(word)]==0){
-	while((lo+1)<n && c->codelist[lo+1]<=word)lo++;
-	while(    hi<n && word>=(c->codelist[hi]&mask))hi++;
-	
-	/* we only actually have 15 bits per hint to play with here.
-           In order to overflow gracefully (nothing breaks, efficiency
-           just drops), encode as the difference from the extremes. */
-	{
-	  unsigned long loval=lo;
-	  unsigned long hival=n-hi;
-
-	  if(loval>0x7fff)loval=0x7fff;
-	  if(hival>0x7fff)hival=0x7fff;
-	  c->dec_firsttable[bitreverse(word)]=
-	    0x80000000UL | (loval<<15) | hival;
+    
+    /* now fill in 'unused' entries in the firsttable with hi/lo search
+       hints for the non-direct-hits */
+    {
+      ogg_uint32_t mask=0xfffffffeUL<<(31-c->dec_firsttablen);
+      long lo=0,hi=0;
+      
+      for(i=0;i<tabn;i++){
+	ogg_uint32_t word=i<<(32-c->dec_firsttablen);
+	if(c->dec_firsttable[bitreverse(word)]==0){
+	  while((lo+1)<n && c->codelist[lo+1]<=word)lo++;
+	  while(    hi<n && word>=(c->codelist[hi]&mask))hi++;
+	  
+	  /* we only actually have 15 bits per hint to play with here.
+	     In order to overflow gracefully (nothing breaks, efficiency
+	     just drops), encode as the difference from the extremes. */
+	  {
+	    unsigned long loval=lo;
+	    unsigned long hival=n-hi;
+	    
+	    if(loval>0x7fff)loval=0x7fff;
+	    if(hival>0x7fff)hival=0x7fff;
+	    c->dec_firsttable[bitreverse(word)]=
+	      0x80000000UL | (loval<<15) | hival;
+	  }
 	}
       }
     }
   }
-  
 
   return(0);
  err_out: