Bug#24751 - Possible infinit loop in init_io_cache() when insufficient memory

- When cache memory can't be allocated size is recaclulated using 3/4 of
  the requested memory. This number is rounded up to the nearest 
  min_cache step. 
  However with the previous implementation the new cache size might
  become bigger than requested because of this rounding and thus we get
  an infinit loop.
- This patch fixes this problem by ensuring that the new cache size
  always will be smaller on the second and subsequent iterations until
  we reach min_cache.


mysys/mf_iocache.c:
  - Added mask to cachesize to ensure that algorithm always produce a 
    smaller cache size than current, until we reach 'min_cache' size.

1 files changed, 4 insertions, 3 deletions

diff --git a/mysys/mf_iocache.c b/mysys/mf_iocache.c
index b17df3da260..a91002d3b4c 100644
--- a/mysys/mf_iocache.c
+++ b/mysys/mf_iocache.c
@@ -199,11 +199,11 @@ int init_io_cache(IO_CACHE *info, File file, uint cachesize,
   if (type != READ_NET && type != WRITE_NET)
   {
     /* Retry allocating memory in smaller blocks until we get one */
+    cachesize=(uint) ((ulong) (cachesize + min_cache-1) &
+			(ulong) ~(min_cache-1));
     for (;;)
     {
       uint buffer_block;
-      cachesize=(uint) ((ulong) (cachesize + min_cache-1) &
-			(ulong) ~(min_cache-1));
       if (cachesize < min_cache)
 	cachesize = min_cache;
       buffer_block = cachesize;
@@ -222,7 +222,8 @@ int init_io_cache(IO_CACHE *info, File file, uint cachesize,
       }
       if (cachesize == min_cache)
 	DBUG_RETURN(2);				/* Can't alloc cache */
-      cachesize= (uint) ((long) cachesize*3/4); /* Try with less memory */
+      /* Try with less memory */
+      cachesize= (uint) ((ulong) cachesize*3/4 & (ulong)~(min_cache-1));
     }
   }