WL#86: Partitioned key cache for MyISAM.

This is the base patch for the task.

16 files changed, 62 insertions, 33 deletions

diff --git a/storage/myisam/mi_check.c b/storage/myisam/mi_check.c
index 9fe954273fe..a13387b140d 100644
--- a/storage/myisam/mi_check.c
+++ b/storage/myisam/mi_check.c
@@ -332,7 +332,8 @@ int chk_size(HA_CHECK *param, register MI_INFO *info)
 
   /* The following is needed if called externally (not from myisamchk) */
   flush_key_blocks(info->s->key_cache,
-		   info->s->kfile, FLUSH_FORCE_WRITE);
+		   info->s->kfile, &info->s->dirty_part_map,
+                   FLUSH_FORCE_WRITE);
 
   size= my_seek(info->s->kfile, 0L, MY_SEEK_END, MYF(MY_THREADSAFE));
   if ((skr=(my_off_t) info->state->key_file_length) != size)
@@ -1474,6 +1475,7 @@ static int mi_drop_all_indexes(HA_CHECK *param, MI_INFO *info, my_bool force)
       */
       DBUG_PRINT("repair", ("all disabled are empty: create missing"));
       error= flush_key_blocks(share->key_cache, share->kfile,
+                              &share->dirty_part_map,
                               FLUSH_FORCE_WRITE);
       goto end;
     }
@@ -1488,6 +1490,7 @@ static int mi_drop_all_indexes(HA_CHECK *param, MI_INFO *info, my_bool force)
 
   /* Remove all key blocks of this index file from key cache. */
   if ((error= flush_key_blocks(share->key_cache, share->kfile,
+                               &share->dirty_part_map,
                                FLUSH_IGNORE_CHANGED)))
     goto end; /* purecov: inspected */
 
@@ -1549,7 +1552,7 @@ int mi_repair(HA_CHECK *param, register MI_INFO *info,
 
   if (!param->using_global_keycache)
     VOID(init_key_cache(dflt_key_cache, param->key_cache_block_size,
-                        (size_t) param->use_buffers, 0, 0));
+                        (size_t) param->use_buffers, 0, 0, 0));
 
   if (init_io_cache(&param->read_cache,info->dfile,
 		    (uint) param->read_buffer_length,
@@ -1762,7 +1765,8 @@ err:
   VOID(end_io_cache(&param->read_cache));
   info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
   VOID(end_io_cache(&info->rec_cache));
-  got_error|=flush_blocks(param, share->key_cache, share->kfile);
+  got_error|=flush_blocks(param, share->key_cache, share->kfile,
+                          &share->dirty_part_map);
   if (!got_error && param->testflag & T_UNPACK)
   {
     share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
@@ -1908,9 +1912,10 @@ void lock_memory(HA_CHECK *param __attribute__((unused)))
 
 	/* Flush all changed blocks to disk */
 
-int flush_blocks(HA_CHECK *param, KEY_CACHE *key_cache, File file)
+int flush_blocks(HA_CHECK *param, KEY_CACHE *key_cache, File file,
+                 ulonglong *dirty_part_map)
 {
-  if (flush_key_blocks(key_cache, file, FLUSH_RELEASE))
+  if (flush_key_blocks(key_cache, file, dirty_part_map, FLUSH_RELEASE))
   {
     mi_check_print_error(param,"%d when trying to write bufferts",my_errno);
     return(1);
@@ -1977,7 +1982,8 @@ int mi_sort_index(HA_CHECK *param, register MI_INFO *info, char * name)
   }
 
   /* Flush key cache for this file if we are calling this outside myisamchk */
-  flush_key_blocks(share->key_cache,share->kfile, FLUSH_IGNORE_CHANGED);
+  flush_key_blocks(share->key_cache, share->kfile, &share->dirty_part_map,
+                   FLUSH_IGNORE_CHANGED);
 
   share->state.version=(ulong) time((time_t*) 0);
   old_state= share->state;			/* save state if not stored */
@@ -2535,7 +2541,8 @@ int mi_repair_by_sort(HA_CHECK *param, register MI_INFO *info,
     memcpy( &share->state.state, info->state, sizeof(*info->state));
 
 err:
-  got_error|= flush_blocks(param, share->key_cache, share->kfile);
+  got_error|= flush_blocks(param, share->key_cache, share->kfile,
+                           &share->dirty_part_map);
   VOID(end_io_cache(&info->rec_cache));
   if (!got_error)
   {
@@ -3059,7 +3066,8 @@ int mi_repair_parallel(HA_CHECK *param, register MI_INFO *info,
     memcpy(&share->state.state, info->state, sizeof(*info->state));
 
 err:
-  got_error|= flush_blocks(param, share->key_cache, share->kfile);
+  got_error|= flush_blocks(param, share->key_cache, share->kfile,
+                           &share->dirty_part_map);
   /*
     Destroy the write cache. The master thread did already detach from
     the share by remove_io_thread() or it was not yet started (if the
diff --git a/storage/myisam/mi_close.c b/storage/myisam/mi_close.c
index fbd24237286..a8d678494f5 100644
--- a/storage/myisam/mi_close.c
+++ b/storage/myisam/mi_close.c
@@ -64,6 +64,7 @@ int mi_close(register MI_INFO *info)
                     if (share->kfile >= 0) abort(););
     if (share->kfile >= 0 &&
 	flush_key_blocks(share->key_cache, share->kfile,
+                         &share->dirty_part_map,
                          ((share->temporary || share->deleting) ?
                           FLUSH_IGNORE_CHANGED :
                           FLUSH_RELEASE)))
diff --git a/storage/myisam/mi_delete_all.c b/storage/myisam/mi_delete_all.c
index e2bbb04ab3c..18fcdee77e2 100644
--- a/storage/myisam/mi_delete_all.c
+++ b/storage/myisam/mi_delete_all.c
@@ -52,7 +52,8 @@ int mi_delete_all_rows(MI_INFO *info)
     If we are using delayed keys or if the user has done changes to the tables
     since it was locked then there may be key blocks in the key cache
   */
-  flush_key_blocks(share->key_cache, share->kfile, FLUSH_IGNORE_CHANGED);
+  flush_key_blocks(share->key_cache, share->kfile, &share->dirty_part_map,
+                   FLUSH_IGNORE_CHANGED);
 #ifdef HAVE_MMAP
   if (share->file_map)
     _mi_unmap_file(info);
diff --git a/storage/myisam/mi_extra.c b/storage/myisam/mi_extra.c
index 64e7a55231d..32516d0115e 100644
--- a/storage/myisam/mi_extra.c
+++ b/storage/myisam/mi_extra.c
@@ -268,6 +268,7 @@ int mi_extra(MI_INFO *info, enum ha_extra_function function, void *extra_arg)
     pthread_mutex_lock(&share->intern_lock);
     /* Flush pages that we don't need anymore */
     if (flush_key_blocks(share->key_cache, share->kfile,
+                         &share->dirty_part_map,
 			 (function == HA_EXTRA_PREPARE_FOR_DROP ?
                           FLUSH_IGNORE_CHANGED : FLUSH_RELEASE)))
     {
@@ -326,7 +327,8 @@ int mi_extra(MI_INFO *info, enum ha_extra_function function, void *extra_arg)
     break;
   case HA_EXTRA_FLUSH:
     if (!share->temporary)
-      flush_key_blocks(share->key_cache, share->kfile, FLUSH_KEEP);
+      flush_key_blocks(share->key_cache, share->kfile, &share->dirty_part_map,
+                       FLUSH_KEEP);
 #ifdef HAVE_PWRITE
     _mi_decrement_open_count(info);
 #endif
diff --git a/storage/myisam/mi_keycache.c b/storage/myisam/mi_keycache.c
index 5cf3fede1ae..be0cdc470f1 100644
--- a/storage/myisam/mi_keycache.c
+++ b/storage/myisam/mi_keycache.c
@@ -75,7 +75,8 @@ int mi_assign_to_key_cache(MI_INFO *info,
     in the old key cache.
   */
 
-  if (flush_key_blocks(share->key_cache, share->kfile, FLUSH_RELEASE))
+  if (flush_key_blocks(share->key_cache, share->kfile, &share->dirty_part_map,
+                       FLUSH_RELEASE))
   {
     error= my_errno;
     mi_print_error(info->s, HA_ERR_CRASHED);
@@ -90,7 +91,8 @@ int mi_assign_to_key_cache(MI_INFO *info,
     (This can never fail as there is never any not written data in the
     new key cache)
   */
-  (void) flush_key_blocks(key_cache, share->kfile, FLUSH_RELEASE);
+  (void) flush_key_blocks(key_cache, share->kfile, &share->dirty_part_map,
+                          FLUSH_RELEASE);
 
   /*
     ensure that setting the key cache and changing the multi_key_cache
@@ -102,6 +104,7 @@ int mi_assign_to_key_cache(MI_INFO *info,
     This should be seen at the lastes for the next call to an myisam function.
   */
   share->key_cache= key_cache;
+  share->dirty_part_map= 0;
 
   /* store the key cache in the global hash structure for future opens */
   if (multi_key_cache_set((uchar*) share->unique_file_name,
diff --git a/storage/myisam/mi_locking.c b/storage/myisam/mi_locking.c
index b935d517c81..a76790c7d7d 100644
--- a/storage/myisam/mi_locking.c
+++ b/storage/myisam/mi_locking.c
@@ -68,7 +68,9 @@ int mi_lock_database(MI_INFO *info, int lock_type)
       --share->tot_locks;
       if (info->lock_type == F_WRLCK && !share->w_locks &&
 	  !share->delay_key_write && flush_key_blocks(share->key_cache,
-						      share->kfile,FLUSH_KEEP))
+						      share->kfile,
+                                                      &share->dirty_part_map,
+                                                      FLUSH_KEEP))
       {
 	error=my_errno;
         mi_print_error(info->s, HA_ERR_CRASHED);
@@ -513,7 +515,8 @@ int _mi_test_if_changed(register MI_INFO *info)
   {						/* Keyfile has changed */
     DBUG_PRINT("info",("index file changed"));
     if (share->state.process != share->this_process)
-      VOID(flush_key_blocks(share->key_cache, share->kfile, FLUSH_RELEASE));
+      VOID(flush_key_blocks(share->key_cache, share->kfile,
+                            &share->dirty_part_map, FLUSH_RELEASE));
     share->last_process=share->state.process;
     info->last_unique=	share->state.unique;
     info->last_loop=	share->state.update_count;
diff --git a/storage/myisam/mi_page.c b/storage/myisam/mi_page.c
index a05a96e514b..70a26c5ba25 100644
--- a/storage/myisam/mi_page.c
+++ b/storage/myisam/mi_page.c
@@ -94,10 +94,11 @@ int _mi_write_keypage(register MI_INFO *info, register MI_KEYDEF *keyinfo,
   }
 #endif
   DBUG_RETURN((key_cache_write(info->s->key_cache,
-                         info->s->kfile,page, level, (uchar*) buff,length,
-			 (uint) keyinfo->block_length,
-			 (int) ((info->lock_type != F_UNLCK) ||
-				info->s->delay_key_write))));
+			       info->s->kfile, &info->s->dirty_part_map,
+                               page, level, (uchar*) buff, length,
+			       (uint) keyinfo->block_length,
+			       (int) ((info->lock_type != F_UNLCK) ||
+				     info->s->delay_key_write))));
 } /* mi_write_keypage */
 
 
@@ -116,7 +117,8 @@ int _mi_dispose(register MI_INFO *info, MI_KEYDEF *keyinfo, my_off_t pos,
   mi_sizestore(buff,old_link);
   info->s->state.changed|= STATE_NOT_SORTED_PAGES;
   DBUG_RETURN(key_cache_write(info->s->key_cache,
-                              info->s->kfile, pos , level, buff,
+                              info->s->kfile, &info->s->dirty_part_map,
+                              pos , level, buff,
 			      sizeof(buff),
 			      (uint) keyinfo->block_length,
 			      (int) (info->lock_type != F_UNLCK)));
diff --git a/storage/myisam/mi_panic.c b/storage/myisam/mi_panic.c
index 74c93761b61..2d06b66f834 100644
--- a/storage/myisam/mi_panic.c
+++ b/storage/myisam/mi_panic.c
@@ -47,7 +47,8 @@ int mi_panic(enum ha_panic_function flag)
       if (info->s->options & HA_OPTION_READ_ONLY_DATA)
 	break;
 #endif
-      if (flush_key_blocks(info->s->key_cache, info->s->kfile, FLUSH_RELEASE))
+      if (flush_key_blocks(info->s->key_cache, info->s->kfile,
+                           &info->s->dirty_part_map, FLUSH_RELEASE))
 	error=my_errno;
       if (info->opt_flag & WRITE_CACHE_USED)
 	if (flush_io_cache(&info->rec_cache))
diff --git a/storage/myisam/mi_preload.c b/storage/myisam/mi_preload.c
index 60ab55106cb..115f0f0c70c 100644
--- a/storage/myisam/mi_preload.c
+++ b/storage/myisam/mi_preload.c
@@ -65,7 +65,7 @@ int mi_preload(MI_INFO *info, ulonglong key_map, my_bool ignore_leaves)
     }
   }
   else
-    block_length= share->key_cache->key_cache_block_size;
+    block_length= share->key_cache->param_block_size;
 
   length= info->preload_buff_size/block_length * block_length;
   set_if_bigger(length, block_length);
@@ -73,7 +73,8 @@ int mi_preload(MI_INFO *info, ulonglong key_map, my_bool ignore_leaves)
   if (!(buff= (uchar *) my_malloc(length, MYF(MY_WME))))
     DBUG_RETURN(my_errno= HA_ERR_OUT_OF_MEM);
 
-  if (flush_key_blocks(share->key_cache,share->kfile, FLUSH_RELEASE))
+  if (flush_key_blocks(share->key_cache, share->kfile, &share->dirty_part_map,
+                       FLUSH_RELEASE))
     goto err;
 
   do
diff --git a/storage/myisam/mi_test1.c b/storage/myisam/mi_test1.c
index 8e491823939..aea8d86e380 100644
--- a/storage/myisam/mi_test1.c
+++ b/storage/myisam/mi_test1.c
@@ -49,7 +49,8 @@ int main(int argc,char *argv[])
   MY_INIT(argv[0]);
   my_init();
   if (key_cacheing)
-    init_key_cache(dflt_key_cache,KEY_CACHE_BLOCK_SIZE,IO_SIZE*16,0,0);
+    init_key_cache(dflt_key_cache,KEY_CACHE_BLOCK_SIZE,IO_SIZE*16,0,0,
+                   DEFAULT_KEY_CACHE_PARTITIONS);
   get_options(argc,argv);
 
   exit(run_test("test1"));
diff --git a/storage/myisam/mi_test2.c b/storage/myisam/mi_test2.c
index 9844126e7dd..67da89e55ef 100644
--- a/storage/myisam/mi_test2.c
+++ b/storage/myisam/mi_test2.c
@@ -215,7 +215,8 @@ int main(int argc, char *argv[])
   if (!silent)
     printf("- Writing key:s\n");
   if (key_cacheing)
-    init_key_cache(dflt_key_cache,key_cache_block_size,key_cache_size,0,0);
+    init_key_cache(dflt_key_cache,key_cache_block_size,key_cache_size,0,0,
+                   DEFAULT_KEY_CACHE_PARTITIONS);
   if (do_locking)
     mi_lock_database(file,F_WRLCK);
   if (write_cacheing)
diff --git a/storage/myisam/mi_test3.c b/storage/myisam/mi_test3.c
index e792612a313..e4527bf441d 100644
--- a/storage/myisam/mi_test3.c
+++ b/storage/myisam/mi_test3.c
@@ -177,7 +177,8 @@ void start_test(int id)
     exit(1);
   }
   if (key_cacheing && rnd(2) == 0)
-    init_key_cache(dflt_key_cache, KEY_CACHE_BLOCK_SIZE, 65536L, 0, 0);
+    init_key_cache(dflt_key_cache, KEY_CACHE_BLOCK_SIZE, 65536L, 0, 0,
+                   DEFAULT_KEY_CACHE_PARTITIONS);
   printf("Process %d, pid: %ld\n", id, (long) getpid());
   fflush(stdout);
 
diff --git a/storage/myisam/myisam_ftdump.c b/storage/myisam/myisam_ftdump.c
index 3eecb08bd99..4436d06e36b 100644
--- a/storage/myisam/myisam_ftdump.c
+++ b/storage/myisam/myisam_ftdump.c
@@ -83,7 +83,7 @@ int main(int argc,char *argv[])
       usage();
   }
 
-  init_key_cache(dflt_key_cache,MI_KEY_BLOCK_LENGTH,USE_BUFFER_INIT, 0, 0);
+  init_key_cache(dflt_key_cache,MI_KEY_BLOCK_LENGTH,USE_BUFFER_INIT, 0, 0, 0);
 
   if (!(info=mi_open(argv[0], O_RDONLY,
                      HA_OPEN_ABORT_IF_LOCKED|HA_OPEN_FROM_SQL_LAYER)))
diff --git a/storage/myisam/myisamchk.c b/storage/myisam/myisamchk.c
index f8dd3b9c51f..d5d1c548717 100644
--- a/storage/myisam/myisamchk.c
+++ b/storage/myisam/myisamchk.c
@@ -1102,7 +1102,7 @@ static int myisamchk(HA_CHECK *param, char * filename)
       {
 	if (param->testflag & (T_EXTEND | T_MEDIUM))
 	  VOID(init_key_cache(dflt_key_cache,opt_key_cache_block_size,
-                              (size_t) param->use_buffers, 0, 0));
+                              (size_t) param->use_buffers, 0, 0, 0));
 	VOID(init_io_cache(&param->read_cache,datafile,
 			   (uint) param->read_buffer_length,
 			   READ_CACHE,
@@ -1116,7 +1116,8 @@ static int myisamchk(HA_CHECK *param, char * filename)
 				 HA_OPTION_COMPRESS_RECORD)) ||
 	    (param->testflag & (T_EXTEND | T_MEDIUM)))
 	  error|=chk_data_link(param, info, test(param->testflag & T_EXTEND));
-	error|=flush_blocks(param, share->key_cache, share->kfile);
+	error|=flush_blocks(param, share->key_cache, share->kfile,
+                            &share->dirty_part_map);
 	VOID(end_io_cache(&param->read_cache));
       }
       if (!error)
@@ -1526,7 +1527,7 @@ static int mi_sort_records(HA_CHECK *param,
     DBUG_RETURN(0);				/* Nothing to do */
 
   init_key_cache(dflt_key_cache, opt_key_cache_block_size,
-                 (size_t) param->use_buffers, 0, 0);
+                 (size_t) param->use_buffers, 0, 0, 0);
   if (init_io_cache(&info->rec_cache,-1,(uint) param->write_buffer_length,
 		   WRITE_CACHE,share->pack.header_length,1,
 		   MYF(MY_WME | MY_WAIT_IF_FULL)))
@@ -1641,8 +1642,8 @@ err:
   my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR));
   sort_info.buff=0;
   share->state.sortkey=sort_key;
-  DBUG_RETURN(flush_blocks(param, share->key_cache, share->kfile) |
-	      got_error);
+  DBUG_RETURN(flush_blocks(param, share->key_cache, share->kfile,
+                           &share->dirty_part_map) | got_error);
 } /* sort_records */
 
 
diff --git a/storage/myisam/myisamdef.h b/storage/myisam/myisamdef.h
index 48e5d92a9a9..20e5afbe1e2 100644
--- a/storage/myisam/myisamdef.h
+++ b/storage/myisam/myisamdef.h
@@ -174,6 +174,8 @@ typedef struct st_mi_isam_share
    *index_file_name;
   uchar *file_map;                       /* mem-map of file if possible */
   KEY_CACHE *key_cache;                 /* ref to the current key cache */
+  /* To mark the key cache partitions containing dirty pages for this file */ 
+  ulonglong dirty_part_map;   
   MI_DECODE_TREE *decode_trees;
   uint16 *decode_tables;
   /* Function to use for a row checksum. */
@@ -733,7 +735,8 @@ void mi_check_print_info _VARARGS((HA_CHECK *param, const char *fmt, ...));
 #ifdef THREAD
 pthread_handler_t thr_find_all_keys(void *arg);
 #endif
-int flush_blocks(HA_CHECK *param, KEY_CACHE *key_cache, File file);
+int flush_blocks(HA_CHECK *param, KEY_CACHE *key_cache, File file,
+                 ulonglong *dirty_part_map);
 #ifdef __cplusplus
 }
 #endif
diff --git a/storage/myisam/myisamlog.c b/storage/myisam/myisamlog.c
index 679dc0b731b..8b4ccfe3256 100644
--- a/storage/myisam/myisamlog.c
+++ b/storage/myisam/myisamlog.c
@@ -333,7 +333,7 @@ static int examine_log(char * file_name, char **table_names)
   init_tree(&tree,0,0,sizeof(file_info),(qsort_cmp2) file_info_compare,1,
 	    (tree_element_free) file_info_free, NULL);
   VOID(init_key_cache(dflt_key_cache,KEY_CACHE_BLOCK_SIZE,KEY_CACHE_SIZE,
-                      0, 0));
+                      0, 0, 0));
 
   files_open=0; access_time=0;
   while (access_time++ != number_of_commands &&