Fix for Bug#37408 - Compressed MyISAM files should not require/use mmap()

                  
When compressed myisam files are opened, they are always memory mapped
sometimes causing memory swapping problems.

When we mmap the myisam compressed tables of size greater than the memory 
available, the kswapd0 process utilization is very high consuming 30-40% of 
the cpu. This happens only with linux kernels older than 2.6.9

With newer linux kernels, we don't have this problem of high cpu consumption
and this option may not be required.
 
The option 'myisam_mmap_size' is added to limit the amount of memory used for
memory mapping of myisam files. This option is not dynamic.

The default value on 32 bit system is 4294967295 bytes and on 64 bit system it
is 18446744073709547520 bytes.

Note: Testcase only tests the option variable. The actual bug has be to 
tested manually.

include/my_global.h:
  Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
  
  define SIZE_T_MAX
include/myisam.h:
  Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
  
  declare 'myisam_mmap_size' and 'myisam_mmap_used' variables and the mutex
  THR_LOCK_myisam_mmap
myisam/mi_packrec.c:
  Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
  
  add 'myisam_mmap_size' option which limits the memory available to mmap of 
  myisam files
myisam/mi_static.c:
  Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
  
  declare 'myisam_mmap_size' and 'myisam_mmap_used' variables and the mutex
  THR_LOCK_myisam_mmap
myisam/myisamdef.h:
  Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
  
  move MEMMAP_EXTRA_MARGIN to myisam.h so that it can be used in mysqld.cc
mysql-test/r/variables.result:
  Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
  
  Testcase for BUG#37408 to test the myisam_mmap_size option
mysql-test/t/variables.test:
  Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
  
  Testcase for BUG#37408 to test the myisam_mmap_size option
mysys/my_thr_init.c:
  Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
  
  intialize the lock THR_LOCK_myisam_mmap
sql/mysqld.cc:
  Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
  
  add the 'myisam_mmap_size' option
sql/set_var.cc:
  Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
  
  add the 'myisam_mmap_size' to the SHOW VARIABLES list

3 files changed, 36 insertions, 3 deletions

diff --git a/myisam/mi_packrec.c b/myisam/mi_packrec.c
index 75cf45d5434..f04c65da505 100644
--- a/myisam/mi_packrec.c
+++ b/myisam/mi_packrec.c
@@ -1499,12 +1499,26 @@ my_bool _mi_memmap_file(MI_INFO *info)
 {
   byte *file_map;
   MYISAM_SHARE *share=info->s;
+  my_bool eom;
+
   DBUG_ENTER("mi_memmap_file");
 
   if (!share->file_map)
   {
     my_off_t data_file_length= share->state.state.data_file_length;
-    if (data_file_length > (my_off_t) (~((size_t) 0)) - MEMMAP_EXTRA_MARGIN)
+
+    if (myisam_mmap_size != SIZE_T_MAX)
+    {
+      pthread_mutex_lock(&THR_LOCK_myisam_mmap);
+      eom= data_file_length > myisam_mmap_size - myisam_mmap_used - MEMMAP_EXTRA_MARGIN;
+      if (!eom)
+        myisam_mmap_used+= data_file_length + MEMMAP_EXTRA_MARGIN;
+      pthread_mutex_unlock(&THR_LOCK_myisam_mmap);
+    }
+    else
+      eom= data_file_length > myisam_mmap_size - MEMMAP_EXTRA_MARGIN;
+
+    if (eom)
     {
       DBUG_PRINT("warning", ("File is too large for mmap"));
       DBUG_RETURN(0);
@@ -1513,6 +1527,12 @@ my_bool _mi_memmap_file(MI_INFO *info)
         data_file_length + MEMMAP_EXTRA_MARGIN)
     {
       DBUG_PRINT("warning",("File isn't extended for memmap"));
+      if (myisam_mmap_size != SIZE_T_MAX)
+      {
+        pthread_mutex_lock(&THR_LOCK_myisam_mmap);
+        myisam_mmap_used-= data_file_length + MEMMAP_EXTRA_MARGIN;
+        pthread_mutex_unlock(&THR_LOCK_myisam_mmap);
+      }
       DBUG_RETURN(0);
     }
     file_map=(byte*)
@@ -1522,6 +1542,12 @@ my_bool _mi_memmap_file(MI_INFO *info)
     {
       DBUG_PRINT("warning",("mmap failed: errno: %d",errno));
       my_errno=errno;
+      if (myisam_mmap_size != SIZE_T_MAX)
+      {
+        pthread_mutex_lock(&THR_LOCK_myisam_mmap);
+        myisam_mmap_used-= data_file_length + MEMMAP_EXTRA_MARGIN;
+        pthread_mutex_unlock(&THR_LOCK_myisam_mmap);
+      }
       DBUG_RETURN(0);
     }
     share->file_map= file_map;
@@ -1538,6 +1564,13 @@ void _mi_unmap_file(MI_INFO *info)
   VOID(my_munmap(info->s->file_map,
 	      (size_t) info->s->state.state.data_file_length+
 	      MEMMAP_EXTRA_MARGIN));
+
+  if (myisam_mmap_size != SIZE_T_MAX)
+  {
+    pthread_mutex_lock(&THR_LOCK_myisam_mmap);
+    myisam_mmap_used-= info->s->state.state.data_file_length + MEMMAP_EXTRA_MARGIN;
+    pthread_mutex_unlock(&THR_LOCK_myisam_mmap);
+  }
 }
 
 
diff --git a/myisam/mi_static.c b/myisam/mi_static.c
index ab7d3dc592b..b6464e452d7 100644
--- a/myisam/mi_static.c
+++ b/myisam/mi_static.c
@@ -40,7 +40,8 @@ ulong myisam_concurrent_insert= 0;
 my_off_t myisam_max_temp_length= MAX_FILE_SIZE;
 ulong    myisam_bulk_insert_tree_size=8192*1024;
 ulong    myisam_data_pointer_size=4;
-
+ulonglong    myisam_mmap_size= SIZE_T_MAX, myisam_mmap_used= 0;
+pthread_mutex_t  THR_LOCK_myisam_mmap;
 
 static int always_valid(const char *filename __attribute__((unused)))
 {
diff --git a/myisam/myisamdef.h b/myisam/myisamdef.h
index 4ebd5648d26..13ca945eae8 100644
--- a/myisam/myisamdef.h
+++ b/myisam/myisamdef.h
@@ -428,7 +428,6 @@ typedef struct st_mi_sort_param
 #define MI_MAX_BLOCK_LENGTH	((((ulong) 1 << 24)-1) & (~ (ulong) (MI_DYN_ALIGN_SIZE-1)))
 #define MI_REC_BUFF_OFFSET      ALIGN_SIZE(MI_DYN_DELETE_BLOCK_HEADER+sizeof(uint32))
 
-#define MEMMAP_EXTRA_MARGIN	7	/* Write this as a suffix for file */
 
 #define PACK_TYPE_SELECTED	1	/* Bits in field->pack_type */
 #define PACK_TYPE_SPACE_FIELDS	2