Implementation of rows-in-block

- Fixes some things missed in myisam->maria port
- Moved variables that holds the state for the current row to 'cur_row'
- Changed most uchar * to byte * to be able to remove a lot of casts
- Removed RAID support
- Added CHECK for rows-in-block
- Added allocate_dynamic() for easier usage of dynamic rows when we know how many entries we will need
- Reorder columns after CREATE for more optimal row storage (for rows-in-block)
- Removed flag 'RRND_PRESERVER_LASTINX' (not needed)
- Extended ma_test_all.sh to test more completely all row formats
- New structs and variables to hold rows-in-block and bitmap information
- Added org_data_file_type in header to allow easy restore of old record format when doing maria_pack / maria_chk -u
- More virtual functions to handle different row types
- Pointer to row is now MARIA_RECORD_POS instead of my_off_t
- New header signature for MARIA index files
- Fixed bugs in ma_test1.c and ma_test2.c
- All key and row blocks are now of same size
- We now only have one link chain for deleted key blocks


include/m_string.h:
  Define bzero_if_purify
include/maria.h:
  Implementation of rows-in-block
include/my_base.h:
  Implementation of rows-in-block
include/my_handler.h:
  Cleanup macros
  Added size_to_store_key_length()
include/my_sys.h:
  Added 'allocate_dynamic()'
include/myisamchk.h:
  Implementation of rows-in-block
mysys/array.c:
  Added allocate_dynamic()
mysys/mf_keycache.c:
  Moved DBUG_ENTER to it's right position
mysys/my_pread.c:
  Ensure my_errno is always set
sql/filesort.cc:
  Fixed some compiler warnings
sql/gen_lex_hash.cc:
  Removed not needed 'inline'
sql/ha_maria.cc:
  Implementation of rows-in-block
  Fixed compiler warnings
sql/mysqld.cc:
  Fixed setting of wrong variable
sql/uniques.cc:
  Fixed compiler warnings
storage/maria/Makefile.am:
  Implementation of rows-in-block
storage/maria/ma_check.c:
  Removed RAID functions
  Added support for CHECK of rows-in-blocks rows
storage/maria/ma_checksum.c:
  Implementation of rows-in-block
storage/maria/ma_close.c:
  Implementation of rows-in-block
storage/maria/ma_create.c:
  Implementation of rows-in-block:
  - Reorder columns
  - All key blocks are now of same size
  - Removed old RAID support
storage/maria/ma_dbug.c:
  Implementation of rows-in-block
storage/maria/ma_delete.c:
  Implementation of rows-in-block
storage/maria/ma_delete_all.c:
  Implementation of rows-in-block
storage/maria/ma_dynrec.c:
  info->rec_buff is now allocated through _ma_alloc_buffer()
  Use new info->cur_row structure
storage/maria/ma_extra.c:
  Implementation of rows-in-block
storage/maria/ma_ft_boolean_search.c:
  Removed compiler warnings
  Indentation fixes
storage/maria/ma_ft_nlq_search.c:
  Removed compiler warnings
  Indentation fixes
storage/maria/ma_ft_update.c:
  Removed some casts
storage/maria/ma_fulltext.h:
  Changed pointer type
storage/maria/ma_info.c:
  Implementation of rows-in-block
  More general _ma_report_error()
storage/maria/ma_init.c:
  Implementation of rows-in-block
storage/maria/ma_key.c:
  Implementation of rows-in-block
  Removed some casts
storage/maria/ma_keycache.c:
  Fixed DBUG entry
storage/maria/ma_locking.c:
  Implementation of rows-in-block
storage/maria/ma_open.c:
  Implementation of rows-in-block
storage/maria/ma_packrec.c:
  Indentation fixes
  Changed uchar * to byte * to make it possible to remove some casts
storage/maria/ma_page.c:
  Implementation of rows-in-block
storage/maria/ma_range.c:
  Implementation of rows-in-block
storage/maria/ma_rfirst.c:
  Implementation of rows-in-block
storage/maria/ma_rkey.c:
  Implementation of rows-in-block
  Indentation fixes
storage/maria/ma_rlast.c:
  Implementation of rows-in-block
storage/maria/ma_rnext.c:
  Implementation of rows-in-block
storage/maria/ma_rnext_same.c:
  Implementation of rows-in-block
storage/maria/ma_rprev.c:
  Implementation of rows-in-block
storage/maria/ma_rrnd.c:
  Implementation of rows-in-block
  Removed flag 'RRND_PRESERVER_LASTINX', by not resetting lastinx (This is reset by maria_scan_init())
storage/maria/ma_rsame.c:
  Implementation of rows-in-block
storage/maria/ma_rsamepos.c:
  Implementation of rows-in-block
storage/maria/ma_rt_index.c:
  Implementation of rows-in-block
storage/maria/ma_rt_index.h:
  Implementation of rows-in-block
storage/maria/ma_rt_key.c:
  Implementation of rows-in-block
storage/maria/ma_rt_key.h:
  Implementation of rows-in-block
storage/maria/ma_rt_mbr.c:
  Implementation of rows-in-block
storage/maria/ma_rt_mbr.h:
  Implementation of rows-in-block
storage/maria/ma_rt_split.c:
  Implementation of rows-in-block
storage/maria/ma_rt_test.c:
  Indentation fix
storage/maria/ma_scan.c:
  Implementation of rows-in-block
  Added 'maria_scan_end()'
storage/maria/ma_search.c:
  Implementation of rows-in-block
storage/maria/ma_sort.c:
  Indentation fixes
  uchar -> byte to be able to remove some casts
storage/maria/ma_sp_defs.h:
  uchar * -> byte *
storage/maria/ma_sp_key.c:
  uchar * -> byte *
storage/maria/ma_sp_test.c:
  Indentation fixes
storage/maria/ma_static.c:
  New header signature for MARIA
storage/maria/ma_statrec.c:
  int -> my_bool functions
  my_off_t -> MARIA_RECORD_POS
  Fixed argument order for _ma_read_static_record()
storage/maria/ma_test1.c:
  Implementation of rows-in-block
  Fixed some bugs in VARCHAR and BLOB testing
storage/maria/ma_test2.c:
  Implementation of rows-in-block
  Fixed bug in BLOB testing
storage/maria/ma_test3.c:
  Implementation of rows-in-block
storage/maria/ma_test_all.sh:
  Run all tests with dynamic, static and block row formats
  (For the moment we skip REPAIR test of rows-in-block as this is not yet implemented)
storage/maria/ma_unique.c:
  Implementation of rows-in-block
storage/maria/ma_update.c:
  Implementation of rows-in-block
storage/maria/ma_write.c:
  Implementation of rows-in-block
  Write of row is split into two parts, as rows-in-block format require us to do write of row before keys (to get row position) in contrast to all other row formats
storage/maria/maria_chk.c:
  Implementation of rows-in-block
storage/maria/maria_def.h:
  Implementation of rows-in-block
  - New structs and variables to hold rows-in-block and bitmap information
  - Added org_data_file_type in header to allow easy restore of old record format when doing maria_pack / maria_chk -u
  - More virtual functions to handle different row types
  - Pointer to row is now MARIA_RECORD_POS instead of my_off_t
  - uchar -> byte for many parameters to avoid casts
storage/maria/maria_ftdump.c:
  Implementation of rows-in-block
storage/maria/maria_pack.c:
  Implementation of rows-in-block
storage/myisam/mi_check.c:
  Added new row types into switch to avoid compiler warnings
  Added some casts to avoid warnings after changing type of lastkey and buff
storage/myisam/mi_create.c:
  Fix that 'pack_fields' is calculated correctly
storage/myisam/mi_rsamepos.c:
  Implementation of rows-in-block
storage/myisam/mi_test2.c:
  Fixed wrong printf
storage/myisam/sort.c:
  uchar * -> byte *
support-files/magic:
  Added support for Maria files
  Fided wrong entry's for MyISAM files
storage/maria/ma_bitmap.c:
  New BitKeeper file ``storage/maria/ma_bitmap.c''
storage/maria/ma_blockrec.c:
  New BitKeeper file ``storage/maria/ma_blockrec.c''
storage/maria/ma_blockrec.h:
  New BitKeeper file ``storage/maria/ma_blockrec.h''

1 files changed, 54 insertions, 42 deletions

diff --git a/include/maria.h b/include/maria.h
index 705ba7c91c7..b917efe1f6c 100644
--- a/include/maria.h
+++ b/include/maria.h
@@ -55,6 +55,7 @@ extern "C" {
 #define MARIA_MIN_KEY_BLOCK_LENGTH	1024	/* Min key block length */
 #define MARIA_MAX_KEY_BLOCK_LENGTH	32768
 #define maria_portable_sizeof_char_ptr 8
+#define MARIA_MAX_KEY_LENGTH    1000        	/* Max length in bytes */
 
 /*
   In the following macros '_keyno_' is 0 .. keys-1.
@@ -109,13 +110,15 @@ extern "C" {
 
 	/* Param to/from maria_info */
 
+typedef ulonglong MARIA_RECORD_POS;
+
 typedef struct st_maria_isaminfo	/* Struct from h_info */
 {
   ha_rows records;			/* Records in database */
   ha_rows deleted;			/* Deleted records in database */
-  my_off_t recpos;			/* Pos for last used record */
-  my_off_t newrecpos;			/* Pos if we write new record */
-  my_off_t dupp_key_pos;		/* Position to record with dup key */
+  MARIA_RECORD_POS recpos;		/* Pos for last used record */
+  MARIA_RECORD_POS newrecpos;		/* Pos if we write new record */
+  MARIA_RECORD_POS dup_key_pos;		/* Position to record with dup key */
   my_off_t data_file_length;            /* Length of data file */
   my_off_t max_data_file_length, index_file_length;
   my_off_t max_index_file_length, delete_length;
@@ -146,9 +149,11 @@ typedef struct st_maria_create_info
   ulonglong auto_increment;
   ulonglong data_file_length;
   ulonglong key_file_length;
+  uint null_bytes;
   uint old_options;
+  enum data_file_type org_data_file_type;
   uint8 language;
-  my_bool with_auto_increment;
+  my_bool with_auto_increment, transactional;
 } MARIA_CREATE_INFO;
 
 struct st_maria_info;				/* For referense */
@@ -168,25 +173,24 @@ typedef struct st_maria_keydef          /* Key definition with open & info */
   uint16 keylength;                     /* Tot length of keyparts (auto) */
   uint16 minlength;                     /* min length of (packed) key (auto) */
   uint16 maxlength;                     /* max length of (packed) key (auto) */
-  uint16 block_size_index;              /* block_size (auto) */
   uint32 version;                       /* For concurrent read/write */
   uint32 ftparser_nr;                   /* distinct ftparser number */
 
   HA_KEYSEG *seg, *end;
   struct st_mysql_ftparser *parser;     /* Fulltext [pre]parser */
-  int(*bin_search) (struct st_maria_info *info,
-		    struct st_maria_keydef *keyinfo, uchar *page, uchar *key,
-		    uint key_len, uint comp_flag, uchar **ret_pos,
-		    uchar *buff, my_bool *was_last_key);
-    uint(*get_key) (struct st_maria_keydef *keyinfo, uint nod_flag,
-		    uchar **page, uchar *key);
-  int(*pack_key) (struct st_maria_keydef *keyinfo, uint nod_flag,
-		  uchar *next_key, uchar *org_key, uchar *prev_key,
-		  uchar *key, struct st_maria_s_param *s_temp);
-  void(*store_key) (struct st_maria_keydef *keyinfo, uchar *key_pos,
+  int (*bin_search)(struct st_maria_info *info,
+		    struct st_maria_keydef *keyinfo, byte *page, byte *key,
+		    uint key_len, uint comp_flag, byte **ret_pos,
+		    byte *buff, my_bool *was_last_key);
+    uint(*get_key)(struct st_maria_keydef *keyinfo, uint nod_flag,
+                   byte **page, byte *key);
+  int (*pack_key)(struct st_maria_keydef *keyinfo, uint nod_flag,
+		  byte *next_key, byte *org_key, byte *prev_key,
+		  const byte *key, struct st_maria_s_param *s_temp);
+  void (*store_key)(struct st_maria_keydef *keyinfo, byte *key_pos,
 		    struct st_maria_s_param *s_temp);
-  int(*ck_insert) (struct st_maria_info *inf, uint k_nr, uchar *k, uint klen);
-  int(*ck_delete) (struct st_maria_info *inf, uint k_nr, uchar *k, uint klen);
+  int (*ck_insert)(struct st_maria_info *inf, uint k_nr, byte *k, uint klen);
+  int (*ck_delete)(struct st_maria_info *inf, uint k_nr, byte *k, uint klen);
 } MARIA_KEYDEF;
 
 
@@ -195,7 +199,7 @@ typedef struct st_maria_keydef          /* Key definition with open & info */
 typedef struct st_maria_unique_def	/* Segment definition of unique */
 {
   uint16 keysegs;                       /* Number of key-segment */
-  uchar key;                            /* Mapped to which key */
+  uint8 key;                            /* Mapped to which key */
   uint8 null_are_equal;
   HA_KEYSEG *seg, *end;
 } MARIA_UNIQUEDEF;
@@ -218,16 +222,19 @@ struct st_maria_bit_buff;
 
 typedef struct st_maria_columndef		/* column information */
 {
-  int16 type;					/* en_fieldtype */
+  uint64 offset;				/* Offset to position in row */
+  enum en_fieldtype type;
   uint16 length;				/* length of field */
-  uint32 offset;				/* Offset to position in row */
-  uint8 null_bit;				/* If column may be 0 */
-  uint16 null_pos;				/* position for null marker */
+  uint16 fill_length;
+  uint16 null_pos;				/* Position for null marker */
+  uint16 empty_pos;                             /* Position for empty marker */
+  uint8 null_bit;				/* If column may be NULL */
+  uint8 empty_bit;				/* If column may be empty */
 
 #ifndef NOT_PACKED_DATABASES
-  void(*unpack) (struct st_maria_columndef *rec,
-                 struct st_maria_bit_buff *buff,
-		 uchar *start, uchar *end);
+  void(*unpack)(struct st_maria_columndef *rec,
+                struct st_maria_bit_buff *buff,
+                byte *start, byte *end);
   enum en_fieldtype base_type;
   uint space_length_bits, pack_type;
   MARIA_DECODE_TREE *huff_tree;
@@ -237,7 +244,8 @@ typedef struct st_maria_columndef		/* column information */
 
 extern ulong maria_block_size;
 extern ulong maria_concurrent_insert;
-extern my_bool maria_flush, maria_delay_key_write, maria_single_user;
+extern my_bool maria_flush, maria_single_user;
+extern my_bool maria_delay_key_write, maria_delay_rec_write;
 extern my_off_t maria_max_temp_length;
 extern ulong maria_bulk_insert_tree_size, maria_data_pointer_size;
 extern KEY_CACHE maria_key_cache_var, *maria_key_cache;
@@ -260,19 +268,22 @@ extern int maria_rlast(struct st_maria_info *file, byte *buf, int inx);
 extern int maria_rnext(struct st_maria_info *file, byte *buf, int inx);
 extern int maria_rnext_same(struct st_maria_info *info, byte *buf);
 extern int maria_rprev(struct st_maria_info *file, byte *buf, int inx);
-extern int maria_rrnd(struct st_maria_info *file, byte *buf, my_off_t pos);
+extern int maria_rrnd(struct st_maria_info *file, byte *buf,
+                      MARIA_RECORD_POS pos);
 extern int maria_scan_init(struct st_maria_info *file);
 extern int maria_scan(struct st_maria_info *file, byte *buf);
+extern void maria_scan_end(struct st_maria_info *file);
 extern int maria_rsame(struct st_maria_info *file, byte *record, int inx);
 extern int maria_rsame_with_pos(struct st_maria_info *file, byte *record,
-				int inx, my_off_t pos);
+				int inx, MARIA_RECORD_POS pos);
 extern int maria_update(struct st_maria_info *file, const byte *old,
 			byte *new_record);
 extern int maria_write(struct st_maria_info *file, byte *buff);
-extern my_off_t maria_position(struct st_maria_info *file);
+extern MARIA_RECORD_POS maria_position(struct st_maria_info *file);
 extern int maria_status(struct st_maria_info *info, MARIA_INFO *x, uint flag);
 extern int maria_lock_database(struct st_maria_info *file, int lock_type);
-extern int maria_create(const char *name, uint keys, MARIA_KEYDEF *keydef,
+extern int maria_create(const char *name, enum data_file_type record_type,
+                        uint keys, MARIA_KEYDEF *keydef,
 			uint columns, MARIA_COLUMNDEF *columndef,
 			uint uniques, MARIA_UNIQUEDEF *uniquedef,
 			MARIA_CREATE_INFO *create_info, uint flags);
@@ -324,7 +335,7 @@ typedef struct st_maria_sort_param
   MARIA_KEYDEF *keyinfo;
   MARIA_SORT_INFO *sort_info;
   HA_KEYSEG *seg;
-  uchar **sort_keys;
+  byte **sort_keys;
   byte *rec_buff;
   void *wordlist, *wordptr;
   MEM_ROOT wordroot;
@@ -338,16 +349,17 @@ typedef struct st_maria_sort_param
   ulonglong unique[HA_MAX_KEY_SEG+1];
   ulonglong notnull[HA_MAX_KEY_SEG+1];
 
-  my_off_t pos,max_pos,filepos,start_recpos;
+  MARIA_RECORD_POS pos,max_pos,filepos,start_recpos;
   uint key, key_length,real_key_length,sortbuff_size;
   uint maxbuffers, keys, find_length, sort_keys_length;
   my_bool fix_datafile, master;
+  my_size_t rec_buff_size;
 
   int (*key_cmp)(struct st_maria_sort_param *, const void *, const void *);
-  int (*key_read)(struct st_maria_sort_param *,void *);
-  int (*key_write)(struct st_maria_sort_param *, const void *);
+  int (*key_read)(struct st_maria_sort_param *, byte *);
+  int (*key_write)(struct st_maria_sort_param *, const byte *);
   void (*lock_in_memory)(HA_CHECK *);
-  NEAR int (*write_keys)(struct st_maria_sort_param *, register uchar **,
+  NEAR int (*write_keys)(struct st_maria_sort_param *, register byte **,
                          uint , struct st_buffpek *, IO_CACHE *);
   NEAR uint (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint);
   NEAR int (*write_key)(struct st_maria_sort_param *, IO_CACHE *,char *,
@@ -356,7 +368,7 @@ typedef struct st_maria_sort_param
 
 
 /* functions in maria_check */
-void mariachk_init(HA_CHECK *param);
+void maria_chk_init(HA_CHECK *param);
 int maria_chk_status(HA_CHECK *param, MARIA_HA *info);
 int maria_chk_del(HA_CHECK *param, register MARIA_HA *info, uint test_flag);
 int maria_chk_size(HA_CHECK *param, MARIA_HA *info);
@@ -371,19 +383,19 @@ int maria_repair_by_sort(HA_CHECK *param, register MARIA_HA *info,
 int maria_repair_parallel(HA_CHECK *param, register MARIA_HA *info,
 			  const char *name, int rep_quick);
 int maria_change_to_newfile(const char *filename, const char *old_ext,
-		      const char *new_ext, uint raid_chunks, myf myflags);
+                            const char *new_ext, myf myflags);
 void maria_lock_memory(HA_CHECK *param);
 int maria_update_state_info(HA_CHECK *param, MARIA_HA *info, uint update);
 void maria_update_key_parts(MARIA_KEYDEF *keyinfo, ulong *rec_per_key_part,
-		      ulonglong *unique, ulonglong *notnull,
-		      ulonglong records);
+                            ulonglong *unique, ulonglong *notnull,
+                            ulonglong records);
 int maria_filecopy(HA_CHECK *param, File to, File from, my_off_t start,
-	     my_off_t length, const char *type);
+                   my_off_t length, const char *type);
 int maria_movepoint(MARIA_HA *info, byte *record, my_off_t oldpos,
-	      my_off_t newpos, uint prot_key);
+                    my_off_t newpos, uint prot_key);
 int maria_write_data_suffix(MARIA_SORT_INFO *sort_info, my_bool fix_datafile);
 int maria_test_if_almost_full(MARIA_HA *info);
-int maria_recreate_table(HA_CHECK *param, MARIA_HA ** org_info, char *filename);
+int maria_recreate_table(HA_CHECK *param, MARIA_HA **org_info, char *filename);
 int maria_disable_indexes(MARIA_HA *info);
 int maria_enable_indexes(MARIA_HA *info);
 int maria_indexes_are_disabled(MARIA_HA *info);