summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSatya B <satya.bn@sun.com>2009-12-17 16:55:50 +0530
committerSatya B <satya.bn@sun.com>2009-12-17 16:55:50 +0530
commitcf9966f86f3c6245f33342f2f5498a0c5efa96e4 (patch)
tree87609fa5cc213739e3ae187b8aab891ce002534d
parent06be03f77cc1dbd3f52232d98fbab2d220d93fc5 (diff)
downloadmariadb-git-cf9966f86f3c6245f33342f2f5498a0c5efa96e4.tar.gz
Fix for Bug#37408 - Compressed MyISAM files should not require/use mmap()
When compressed myisam files are opened, they are always memory mapped sometimes causing memory swapping problems. When we mmap the myisam compressed tables of size greater than the memory available, the kswapd0 process utilization is very high consuming 30-40% of the cpu. This happens only with linux kernels older than 2.6.9 With newer linux kernels, we don't have this problem of high cpu consumption and this option may not be required. The option 'myisam_mmap_size' is added to limit the amount of memory used for memory mapping of myisam files. This option is not dynamic. The default value on 32 bit system is 4294967295 bytes and on 64 bit system it is 18446744073709547520 bytes. Note: Testcase only tests the option variable. The actual bug has be to tested manually. include/my_global.h: Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap() define SIZE_T_MAX include/myisam.h: Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap() declare 'myisam_mmap_size' and 'myisam_mmap_used' variables and the mutex THR_LOCK_myisam_mmap myisam/mi_packrec.c: Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap() add 'myisam_mmap_size' option which limits the memory available to mmap of myisam files myisam/mi_static.c: Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap() declare 'myisam_mmap_size' and 'myisam_mmap_used' variables and the mutex THR_LOCK_myisam_mmap myisam/myisamdef.h: Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap() move MEMMAP_EXTRA_MARGIN to myisam.h so that it can be used in mysqld.cc mysql-test/r/variables.result: Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap() Testcase for BUG#37408 to test the myisam_mmap_size option mysql-test/t/variables.test: Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap() Testcase for BUG#37408 to test the myisam_mmap_size option mysys/my_thr_init.c: Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap() intialize the lock THR_LOCK_myisam_mmap sql/mysqld.cc: Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap() add the 'myisam_mmap_size' option sql/set_var.cc: Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap() add the 'myisam_mmap_size' to the SHOW VARIABLES list
-rw-r--r--include/my_global.h3
-rw-r--r--include/myisam.h4
-rw-r--r--myisam/mi_packrec.c35
-rw-r--r--myisam/mi_static.c3
-rw-r--r--myisam/myisamdef.h1
-rw-r--r--mysql-test/r/variables.result6
-rw-r--r--mysql-test/t/variables.test6
-rw-r--r--mysys/my_thr_init.c6
-rw-r--r--sql/mysqld.cc7
-rw-r--r--sql/set_var.cc12
10 files changed, 78 insertions, 5 deletions
diff --git a/include/my_global.h b/include/my_global.h
index 6910ae092e1..595c7cd793b 100644
--- a/include/my_global.h
+++ b/include/my_global.h
@@ -793,6 +793,9 @@ typedef SOCKET_SIZE_TYPE size_socket;
#define DBL_MAX 1.79769313486231470e+308
#define FLT_MAX ((float)3.40282346638528860e+38)
#endif
+#ifndef SIZE_T_MAX
+#define SIZE_T_MAX (~((size_t) 0))
+#endif
#ifndef HAVE_FINITE
#define finite(x) (1.0 / fabs(x) > 0.0)
diff --git a/include/myisam.h b/include/myisam.h
index ad585f79608..acc2d689b75 100644
--- a/include/myisam.h
+++ b/include/myisam.h
@@ -270,6 +270,8 @@ extern ulong myisam_bulk_insert_tree_size, myisam_data_pointer_size;
/* usually used to check if a symlink points into the mysql data home */
/* which is normally forbidden */
extern int (*myisam_test_invalid_symlink)(const char *filename);
+extern ulonglong myisam_mmap_size, myisam_mmap_used;
+extern pthread_mutex_t THR_LOCK_myisam_mmap;
/* Prototypes for myisam-functions */
@@ -315,6 +317,8 @@ extern int mi_delete_all_rows(struct st_myisam_info *info);
extern ulong _mi_calc_blob_length(uint length , const byte *pos);
extern uint mi_get_pointer_length(ulonglong file_length, uint def);
+#define MEMMAP_EXTRA_MARGIN 7 /* Write this as a suffix for mmap file */
+
/* this is used to pass to mysql_myisamchk_table -- by Sasha Pachev */
#define MYISAMCHK_REPAIR 1 /* equivalent to myisamchk -r */
diff --git a/myisam/mi_packrec.c b/myisam/mi_packrec.c
index 75cf45d5434..f04c65da505 100644
--- a/myisam/mi_packrec.c
+++ b/myisam/mi_packrec.c
@@ -1499,12 +1499,26 @@ my_bool _mi_memmap_file(MI_INFO *info)
{
byte *file_map;
MYISAM_SHARE *share=info->s;
+ my_bool eom;
+
DBUG_ENTER("mi_memmap_file");
if (!share->file_map)
{
my_off_t data_file_length= share->state.state.data_file_length;
- if (data_file_length > (my_off_t) (~((size_t) 0)) - MEMMAP_EXTRA_MARGIN)
+
+ if (myisam_mmap_size != SIZE_T_MAX)
+ {
+ pthread_mutex_lock(&THR_LOCK_myisam_mmap);
+ eom= data_file_length > myisam_mmap_size - myisam_mmap_used - MEMMAP_EXTRA_MARGIN;
+ if (!eom)
+ myisam_mmap_used+= data_file_length + MEMMAP_EXTRA_MARGIN;
+ pthread_mutex_unlock(&THR_LOCK_myisam_mmap);
+ }
+ else
+ eom= data_file_length > myisam_mmap_size - MEMMAP_EXTRA_MARGIN;
+
+ if (eom)
{
DBUG_PRINT("warning", ("File is too large for mmap"));
DBUG_RETURN(0);
@@ -1513,6 +1527,12 @@ my_bool _mi_memmap_file(MI_INFO *info)
data_file_length + MEMMAP_EXTRA_MARGIN)
{
DBUG_PRINT("warning",("File isn't extended for memmap"));
+ if (myisam_mmap_size != SIZE_T_MAX)
+ {
+ pthread_mutex_lock(&THR_LOCK_myisam_mmap);
+ myisam_mmap_used-= data_file_length + MEMMAP_EXTRA_MARGIN;
+ pthread_mutex_unlock(&THR_LOCK_myisam_mmap);
+ }
DBUG_RETURN(0);
}
file_map=(byte*)
@@ -1522,6 +1542,12 @@ my_bool _mi_memmap_file(MI_INFO *info)
{
DBUG_PRINT("warning",("mmap failed: errno: %d",errno));
my_errno=errno;
+ if (myisam_mmap_size != SIZE_T_MAX)
+ {
+ pthread_mutex_lock(&THR_LOCK_myisam_mmap);
+ myisam_mmap_used-= data_file_length + MEMMAP_EXTRA_MARGIN;
+ pthread_mutex_unlock(&THR_LOCK_myisam_mmap);
+ }
DBUG_RETURN(0);
}
share->file_map= file_map;
@@ -1538,6 +1564,13 @@ void _mi_unmap_file(MI_INFO *info)
VOID(my_munmap(info->s->file_map,
(size_t) info->s->state.state.data_file_length+
MEMMAP_EXTRA_MARGIN));
+
+ if (myisam_mmap_size != SIZE_T_MAX)
+ {
+ pthread_mutex_lock(&THR_LOCK_myisam_mmap);
+ myisam_mmap_used-= info->s->state.state.data_file_length + MEMMAP_EXTRA_MARGIN;
+ pthread_mutex_unlock(&THR_LOCK_myisam_mmap);
+ }
}
diff --git a/myisam/mi_static.c b/myisam/mi_static.c
index ab7d3dc592b..b6464e452d7 100644
--- a/myisam/mi_static.c
+++ b/myisam/mi_static.c
@@ -40,7 +40,8 @@ ulong myisam_concurrent_insert= 0;
my_off_t myisam_max_temp_length= MAX_FILE_SIZE;
ulong myisam_bulk_insert_tree_size=8192*1024;
ulong myisam_data_pointer_size=4;
-
+ulonglong myisam_mmap_size= SIZE_T_MAX, myisam_mmap_used= 0;
+pthread_mutex_t THR_LOCK_myisam_mmap;
static int always_valid(const char *filename __attribute__((unused)))
{
diff --git a/myisam/myisamdef.h b/myisam/myisamdef.h
index 4ebd5648d26..13ca945eae8 100644
--- a/myisam/myisamdef.h
+++ b/myisam/myisamdef.h
@@ -428,7 +428,6 @@ typedef struct st_mi_sort_param
#define MI_MAX_BLOCK_LENGTH ((((ulong) 1 << 24)-1) & (~ (ulong) (MI_DYN_ALIGN_SIZE-1)))
#define MI_REC_BUFF_OFFSET ALIGN_SIZE(MI_DYN_DELETE_BLOCK_HEADER+sizeof(uint32))
-#define MEMMAP_EXTRA_MARGIN 7 /* Write this as a suffix for file */
#define PACK_TYPE_SELECTED 1 /* Bits in field->pack_type */
#define PACK_TYPE_SPACE_FIELDS 2
diff --git a/mysql-test/r/variables.result b/mysql-test/r/variables.result
index 496f0b69fb8..e24117187d2 100644
--- a/mysql-test/r/variables.result
+++ b/mysql-test/r/variables.result
@@ -904,3 +904,9 @@ set global server_id =@my_server_id;
set global slow_launch_time =@my_slow_launch_time;
set global storage_engine =@my_storage_engine;
set global thread_cache_size =@my_thread_cache_size;
+#
+# BUG#37408 - Compressed MyISAM files should not require/use mmap()
+#
+# Test 'myisam_mmap_size' option is not dynamic
+SET @@myisam_mmap_size= 500M;
+ERROR HY000: Variable 'myisam_mmap_size' is a read only variable
diff --git a/mysql-test/t/variables.test b/mysql-test/t/variables.test
index 91f75cf6cd4..cff6a5a0767 100644
--- a/mysql-test/t/variables.test
+++ b/mysql-test/t/variables.test
@@ -765,3 +765,9 @@ set global slow_launch_time =@my_slow_launch_time;
set global storage_engine =@my_storage_engine;
set global thread_cache_size =@my_thread_cache_size;
+--echo #
+--echo # BUG#37408 - Compressed MyISAM files should not require/use mmap()
+--echo #
+--echo # Test 'myisam_mmap_size' option is not dynamic
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SET @@myisam_mmap_size= 500M;
diff --git a/mysys/my_thr_init.c b/mysys/my_thr_init.c
index 64fc99952e9..8dcffaebbbb 100644
--- a/mysys/my_thr_init.c
+++ b/mysys/my_thr_init.c
@@ -30,7 +30,9 @@ pthread_key(struct st_my_thread_var, THR_KEY_mysys);
#endif /* USE_TLS */
pthread_mutex_t THR_LOCK_malloc,THR_LOCK_open,
THR_LOCK_lock,THR_LOCK_isam,THR_LOCK_myisam,THR_LOCK_heap,
- THR_LOCK_net, THR_LOCK_charset, THR_LOCK_threads;
+ THR_LOCK_net, THR_LOCK_charset, THR_LOCK_threads,
+ THR_LOCK_myisam_mmap;
+
pthread_cond_t THR_COND_threads;
uint THR_thread_count= 0;
uint my_thread_end_wait_time= 5;
@@ -143,6 +145,7 @@ my_bool my_thread_global_init(void)
pthread_mutex_init(&THR_LOCK_lock,MY_MUTEX_INIT_FAST);
pthread_mutex_init(&THR_LOCK_isam,MY_MUTEX_INIT_SLOW);
pthread_mutex_init(&THR_LOCK_myisam,MY_MUTEX_INIT_SLOW);
+ pthread_mutex_init(&THR_LOCK_myisam_mmap,MY_MUTEX_INIT_FAST);
pthread_mutex_init(&THR_LOCK_heap,MY_MUTEX_INIT_FAST);
pthread_mutex_init(&THR_LOCK_net,MY_MUTEX_INIT_FAST);
pthread_mutex_init(&THR_LOCK_charset,MY_MUTEX_INIT_FAST);
@@ -208,6 +211,7 @@ void my_thread_global_end(void)
pthread_mutex_destroy(&THR_LOCK_lock);
pthread_mutex_destroy(&THR_LOCK_isam);
pthread_mutex_destroy(&THR_LOCK_myisam);
+ pthread_mutex_destroy(&THR_LOCK_myisam_mmap);
pthread_mutex_destroy(&THR_LOCK_heap);
pthread_mutex_destroy(&THR_LOCK_net);
pthread_mutex_destroy(&THR_LOCK_charset);
diff --git a/sql/mysqld.cc b/sql/mysqld.cc
index b4eb96ab65a..4b0739cc2d2 100644
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@ -4975,7 +4975,8 @@ enum options_mysqld
OPT_MAX_WRITE_LOCK_COUNT, OPT_BULK_INSERT_BUFFER_SIZE,
OPT_MAX_ERROR_COUNT, OPT_MULTI_RANGE_COUNT, OPT_MYISAM_DATA_POINTER_SIZE,
OPT_MYISAM_BLOCK_SIZE, OPT_MYISAM_MAX_EXTRA_SORT_FILE_SIZE,
- OPT_MYISAM_MAX_SORT_FILE_SIZE, OPT_MYISAM_SORT_BUFFER_SIZE,
+ OPT_MYISAM_MAX_SORT_FILE_SIZE, OPT_MYISAM_MMAP_SIZE,
+ OPT_MYISAM_SORT_BUFFER_SIZE,
OPT_MYISAM_STATS_METHOD,
OPT_NET_BUFFER_LENGTH, OPT_NET_RETRY_COUNT,
OPT_NET_READ_TIMEOUT, OPT_NET_WRITE_TIMEOUT,
@@ -6255,6 +6256,10 @@ The minimum value for this variable is 4096.",
(gptr*) &max_system_variables.myisam_max_sort_file_size, 0,
GET_ULL, REQUIRED_ARG, (longlong) LONG_MAX, 0, (ulonglong) MAX_FILE_SIZE,
0, 1024*1024, 0},
+ {"myisam_mmap_size", OPT_MYISAM_MMAP_SIZE,
+ "Can be used to restrict the total memory used for memory mmaping of myisam files",
+ (gptr*) &myisam_mmap_size, (gptr*) &myisam_mmap_size, 0,
+ GET_ULL, REQUIRED_ARG, SIZE_T_MAX, MEMMAP_EXTRA_MARGIN, SIZE_T_MAX, 0, 1, 0},
{"myisam_repair_threads", OPT_MYISAM_REPAIR_THREADS,
"Number of threads to use when repairing MyISAM tables. The value of 1 disables parallel repair.",
(gptr*) &global_system_variables.myisam_repair_threads,
diff --git a/sql/set_var.cc b/sql/set_var.cc
index c885f7160e9..4871afd2c56 100644
--- a/sql/set_var.cc
+++ b/sql/set_var.cc
@@ -122,6 +122,7 @@ static byte *get_error_count(THD *thd);
static byte *get_warning_count(THD *thd);
static byte *get_have_innodb(THD *thd);
static byte *get_tmpdir(THD *thd);
+static byte *get_myisam_mmap_size(THD *thd);
/*
Variable definition list
@@ -623,6 +624,10 @@ sys_var_thd_bool sys_keep_files_on_create("keep_files_on_create",
&SV::keep_files_on_create);
+static sys_var_readonly sys_myisam_mmap_size("myisam_mmap_size",
+ OPT_GLOBAL,
+ SHOW_LONGLONG,
+ get_myisam_mmap_size);
/*
@@ -723,6 +728,7 @@ sys_var *sys_variables[]=
&sys_multi_range_count,
&sys_myisam_data_pointer_size,
&sys_myisam_max_sort_file_size,
+ &sys_myisam_mmap_size,
&sys_myisam_repair_threads,
&sys_myisam_sort_buffer_size,
&sys_myisam_stats_method,
@@ -1026,6 +1032,7 @@ struct show_var_st init_vars[]= {
{sys_myisam_data_pointer_size.name, (char*) &sys_myisam_data_pointer_size, SHOW_SYS},
{sys_myisam_max_sort_file_size.name, (char*) &sys_myisam_max_sort_file_size,
SHOW_SYS},
+ {sys_myisam_mmap_size.name, (char*) &sys_myisam_mmap_size, SHOW_SYS},
{"myisam_recover_options", (char*) &myisam_recover_options_str, SHOW_CHAR_PTR},
{sys_myisam_repair_threads.name, (char*) &sys_myisam_repair_threads,
SHOW_SYS},
@@ -3181,6 +3188,11 @@ static byte *get_tmpdir(THD *thd)
return (byte*)mysql_tmpdir;
}
+static byte *get_myisam_mmap_size(THD *thd)
+{
+ return (byte *)&myisam_mmap_size;
+}
+
/****************************************************************************
Main handling of variables:
- Initialisation