Bug#9487 - myisampack segmentation violation and bus error

Fixed some casts for 64-bit systems.
Fixed a possible buffer overflow.


myisam/myisampack.c:
  Bug#9487 - myisampack segmentation violation and bus error
  Added or changed proper casts for the argument of flush_buffer().
  Added a check for the counterstrike of the trick to have
  file_buffer.end some bytes below the real end of the buffer.

1 files changed, 16 insertions, 3 deletions

diff --git a/myisam/myisampack.c b/myisam/myisampack.c
index eae75b07760..263a468d96d 100644
--- a/myisam/myisampack.c
+++ b/myisam/myisampack.c
@@ -1709,7 +1709,7 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts)
     ulong tot_blob_length=0;
     if (! error)
     {
-      if (flush_buffer(max_calc_length+max_pack_length))
+      if (flush_buffer((ulong) max_calc_length + (ulong) max_pack_length))
 	break;
       record_pos=file_buffer.pos;
       file_buffer.pos+=max_pack_length;
@@ -1930,7 +1930,20 @@ static void init_file_buffer(File file, pbool read_buffer)
 static int flush_buffer(ulong neaded_length)
 {
   ulong length;
-  if ((ulong) (file_buffer.end - file_buffer.pos) > neaded_length)
+
+  /*
+    file_buffer.end is 8 bytes lower than the real end of the buffer.
+    This is done so that the end-of-buffer condition does not need to be
+    checked for every byte (see write_bits()). Consequently,
+    file_buffer.pos can become greater than file_buffer.end. The
+    algorithms in the other functions ensure that there will never be
+    more than 8 bytes written to the buffer without an end-of-buffer
+    check. So the buffer cannot be overrun. But we need to check for the
+    near-to-buffer-end condition to avoid a negative result, which is
+    casted to unsigned and thus becomes giant.
+  */
+  if ((file_buffer.pos < file_buffer.end) &&
+      ((ulong) (file_buffer.end - file_buffer.pos) > neaded_length))
     return 0;
   length=(ulong) (file_buffer.pos-file_buffer.buffer);
   file_buffer.pos=file_buffer.buffer;
@@ -2002,7 +2015,7 @@ static void write_bits (register ulong value, register uint bits)
     }
 #endif
     if (file_buffer.pos >= file_buffer.end)
-      VOID(flush_buffer((uint) ~0));
+      VOID(flush_buffer(~ (ulong) 0));
     file_buffer.bits=(int) (BITS_SAVED - bits);
     file_buffer.current_byte=(uint) (value << (BITS_SAVED - bits));
   }