From 571843804c1b22e7ff641f5dfe223958c4af70fc Mon Sep 17 00:00:00 2001 From: Mats Kindahl Date: Mon, 14 Dec 2009 12:04:55 +0100 Subject: WL#5151: Conversion between different types when replicating Row-based replication requires the types of columns on the master and slave to be approximately the same (some safe conversions between strings are allowed), but does not allow safe conversions between fields of similar types such as TINYINT and INT. This patch implement type conversions between similar fields on the master and slave. The conversions are controlled using a new variable SLAVE_TYPE_CONVERSIONS of type SET('ALL_LOSSY','ALL_NON_LOSSY'). Non-lossy conversions are any conversions that do not run the risk of losing any information, while lossy conversions can potentially truncate the value. The column definitions are checked to decide if the conversion is acceptable. If neither conversion is enabled, it is required that the definitions of the columns are identical on master and slave. Conversion is done by creating an internal conversion table, unpacking the master data into it, and then copy the data to the real table on the slave. .bzrignore: New files added client/Makefile.am: New files added client/mysqlbinlog.cc: Functions in rpl_utility.cc is now needed by mysqlbinlog.cc. libmysqld/Makefile.am: New files added mysql-test/extra/rpl_tests/check_type.inc: Test include file to check a single type conversion. mysql-test/extra/rpl_tests/rpl_extraSlave_Col.test: Switching to use INT instead of TEXT for column that should not have matching types. mysql-test/extra/rpl_tests/rpl_row_basic.test: Adding code to enable type conversions for BIT tests since InnoDB cannot handle them properly due to incorrect information stored as metadata. mysql-test/extra/rpl_tests/type_conversions.test: Test file to check a set of type conversions with current settings of slave_type_conversions. mysql-test/suite/rpl/t/rpl_typeconv.test: Test file to test conversions from master to slave with all possible values for slave_type_conversions. The test also checks that the slave_type_conversions variable works as expected. sql/field.cc: Changing definition of compatible_field_size to both check if two field with identical base types are compatible and give an order between them if they are compatible. This only implement checking on the slave, so it will not affect replication from an old master to a new slave. sql/field.h: Changing prototypes for functions: - compatible_field_size() - init_for_tmp_table() - row_pack_length() sql/log_event.cc: Changing compability checks to build a conversion table if the fields are compatible, but does not have the same base type. sql/log_event_old.cc: Changing compability checks to build a conversion table if the fields are compatible, but does not have the same base type. sql/mysql_priv.h: Adding global option variable for SLAVE_TYPE_CONVERSIONS sql/mysqld.cc: Adding SLAVE_TYPE_CONVERSIONS global server variable. sql/rpl_record.cc: Changing unpack_row to use the conversion table if present. sql/rpl_rli.h: Removing function get_tabledef and replacing it with get_table_data(). This function retrieve data for table opened for replication, not just table definition. sql/rpl_utility.cc: Function table_def::compatible_with is changed to compare table on master and slave for compatibility and generate a conversions table if they are compatible. Computing real type of fields from metadata for ENUM and SET types. Computing pack_length correctly for ENUM, SET, and BLOB types. Adding optimization to not check compatibility if no slave type conversions are enabled. sql/rpl_utility.h: Changing prototypes since implementation has changed. Modifying table_def::type() to return real type instead of stored type. sql/set_var.cc: Adding SLAVE_TYPE_CONVERSIONS variable. sql/set_var.h: Adding SLAVE_TYPE_CONVERSIONS variable. sql/share/errmsg.txt: Adding error messages for slave type conversions. sql/sql_class.h: Adding SLAVE_TYPE_CONVERSIONS variable. sql/sql_select.cc: Correcting create_virtual_tmp_table() to compute null bit positions correctly in the presence of bit fields. --- sql/field.cc | 267 +++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 195 insertions(+), 72 deletions(-) (limited to 'sql/field.cc') diff --git a/sql/field.cc b/sql/field.cc index 0df9b0fc2e4..b6a795d34aa 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -59,6 +59,8 @@ const char field_separator=','; #define ASSERT_COLUMN_MARKED_FOR_READ DBUG_ASSERT(!table || (!table->read_set || bitmap_is_set(table->read_set, field_index))) #define ASSERT_COLUMN_MARKED_FOR_WRITE DBUG_ASSERT(!table || (!table->write_set || bitmap_is_set(table->write_set, field_index))) +#define FLAGSTR(S,F) ((S) & (F) ? #F " " : "") + /* Rules for merging different types of fields in UNION @@ -996,6 +998,21 @@ test_if_important_data(CHARSET_INFO *cs, const char *str, const char *strend) } +/** + Template function to compare two objects. + */ +namespace { + template + int compare(A_type a, B_type b) + { + if (a < b) + return -1; + if (b < a) + return 1; + return 0; + } +} + /** Detect Item_result by given field type of UNION merge result. @@ -1367,22 +1384,46 @@ bool Field::send_binary(Protocol *protocol) /** Check to see if field size is compatible with destination. - This method is used in row-based replication to verify that the slave's - field size is less than or equal to the master's field size. The - encoded field metadata (from the master or source) is decoded and compared - to the size of this field (the slave or destination). + This method is used in row-based replication to verify that the + slave's field size is less than or equal to the master's field + size. The encoded field metadata (from the master or source) is + decoded and compared to the size of this field (the slave or + destination). + + @note + + The comparison is made so that if the source data (from the master) + is less than the target data (on the slave), -1 is returned in @c + *order_var. This implies that a conversion is + necessary, but that it is lossy and can result in truncation of the + value. + + If the source data is strictly greater than the target data, 1 is + returned in *order_var. This implies that the source + type can is contained in the target type and that a conversion is + necessary but is non-lossy. + + If no conversion is required to fit the source type in the target + type, 0 is returned in *order_var. @param field_metadata Encoded size in field metadata + @param order_var Pointer to variable where the order + between the source field and this field + will be returned. - @retval 0 if this field's size is < the source field's size - @retval 1 if this field's size is >= the source field's size + @return @c true if this field's size is compatible with the + master's field size, @c false otherwise. */ -int Field::compatible_field_size(uint field_metadata, - const Relay_log_info *rli_arg __attribute__((unused))) +bool Field::compatible_field_size(uint field_metadata, + Relay_log_info *rli_arg __attribute__((unused)), + int *order_var) { uint const source_size= pack_length_from_metadata(field_metadata); uint const destination_size= row_pack_length(); - return (source_size <= destination_size); + DBUG_PRINT("debug", ("real_type: %d, source_size: %u, destination_size: %u", + real_type(), source_size, destination_size)); + *order_var = compare(source_size, destination_size); + return true; } @@ -2907,33 +2948,15 @@ uint Field_new_decimal::pack_length_from_metadata(uint field_metadata) } -/** - Check to see if field size is compatible with destination. - - This method is used in row-based replication to verify that the slave's - field size is less than or equal to the master's field size. The - encoded field metadata (from the master or source) is decoded and compared - to the size of this field (the slave or destination). - - @param field_metadata Encoded size in field metadata - - @retval 0 if this field's size is < the source field's size - @retval 1 if this field's size is >= the source field's size -*/ -int Field_new_decimal::compatible_field_size(uint field_metadata, - const Relay_log_info * __attribute__((unused))) +bool Field_new_decimal::compatible_field_size(uint field_metadata, + Relay_log_info * __attribute__((unused)), + int *order_var) { - int compatible= 0; uint const source_precision= (field_metadata >> 8U) & 0x00ff; uint const source_decimal= field_metadata & 0x00ff; - uint const source_size= my_decimal_get_binary_size(source_precision, - source_decimal); - uint const destination_size= row_pack_length(); - compatible= (source_size <= destination_size); - if (compatible) - compatible= (source_precision <= precision) && - (source_decimal <= decimals()); - return (compatible); + int order= compare(source_precision, precision); + *order_var= order != 0 ? order : compare(source_decimal, dec); + return true; } @@ -6707,8 +6730,10 @@ check_field_for_37426(const void *param_arg) } #endif -int Field_string::compatible_field_size(uint field_metadata, - const Relay_log_info *rli_arg) +bool +Field_string::compatible_field_size(uint field_metadata, + Relay_log_info *rli_arg, + int *order_var) { #ifdef HAVE_REPLICATION const Check_field_param check_param = { this }; @@ -6716,7 +6741,7 @@ int Field_string::compatible_field_size(uint field_metadata, check_field_for_37426, &check_param)) return FALSE; // Not compatible field sizes #endif - return Field::compatible_field_size(field_metadata, rli_arg); + return Field::compatible_field_size(field_metadata, rli_arg, order_var); } @@ -6778,6 +6803,8 @@ uchar *Field_string::pack(uchar *to, const uchar *from, { uint length= min(field_length,max_length); uint local_char_length= max_length/field_charset->mbmaxlen; + DBUG_PRINT("debug", ("Packing field '%s' - length: %u ", field_name, length)); + if (length > local_char_length) local_char_length= my_charpos(field_charset, from, from+length, local_char_length); @@ -7625,6 +7652,7 @@ Field_blob::Field_blob(uchar *ptr_arg, uchar *null_ptr_arg, uchar null_bit_arg, cs), packlength(blob_pack_length) { + DBUG_ASSERT(blob_pack_length <= 4); // Only pack lengths 1-4 supported currently flags|= BLOB_FLAG; share->blob_fields++; /* TODO: why do not fill table->s->blob_field array here? */ @@ -8035,8 +8063,10 @@ int Field_blob::key_cmp(const uchar *a,const uchar *b) */ int Field_blob::do_save_field_metadata(uchar *metadata_ptr) { + DBUG_ENTER("Field_blob::do_save_field_metadata"); *metadata_ptr= pack_length_no_ptr(); - return 1; + DBUG_PRINT("debug", ("metadata: %u (pack_length_no_ptr)", *metadata_ptr)); + DBUG_RETURN(1); } @@ -8977,6 +9007,9 @@ Field_bit::Field_bit(uchar *ptr_arg, uint32 len_arg, uchar *null_ptr_arg, bit_ptr(bit_ptr_arg), bit_ofs(bit_ofs_arg), bit_len(len_arg & 7), bytes_in_rec(len_arg / 8) { + DBUG_ENTER("Field_bit::Field_bit"); + DBUG_PRINT("enter", ("ptr_arg: %p, null_ptr_arg: %p, len_arg: %u, bit_len: %u, bytes_in_rec: %u", + ptr_arg, null_ptr_arg, len_arg, bit_len, bytes_in_rec)); flags|= UNSIGNED_FLAG; /* Ensure that Field::eq() can distinguish between two different bit fields. @@ -8984,6 +9017,7 @@ Field_bit::Field_bit(uchar *ptr_arg, uint32 len_arg, uchar *null_ptr_arg, */ if (!null_ptr_arg) null_bit= bit_ofs_arg; + DBUG_VOID_RETURN; } @@ -9268,9 +9302,12 @@ uint Field_bit::get_key_image(uchar *buff, uint length, imagetype type_arg) */ int Field_bit::do_save_field_metadata(uchar *metadata_ptr) { + DBUG_ENTER("Field_bit::do_save_field_metadata"); + DBUG_PRINT("debug", ("bit_len: %d, bytes_in_rec: %d", + bit_len, bytes_in_rec)); *metadata_ptr= bit_len; *(metadata_ptr + 1)= bytes_in_rec; - return 2; + DBUG_RETURN(2); } @@ -9295,34 +9332,20 @@ uint Field_bit::pack_length_from_metadata(uint field_metadata) } -/** - Check to see if field size is compatible with destination. - - This method is used in row-based replication to verify that the slave's - field size is less than or equal to the master's field size. The - encoded field metadata (from the master or source) is decoded and compared - to the size of this field (the slave or destination). - - @param field_metadata Encoded size in field metadata - - @retval 0 if this field's size is < the source field's size - @retval 1 if this field's size is >= the source field's size -*/ -int Field_bit::compatible_field_size(uint field_metadata, - const Relay_log_info * __attribute__((unused))) +bool +Field_bit::compatible_field_size(uint field_metadata, + Relay_log_info * __attribute__((unused)), + int *order_var) { - int compatible= 0; - uint const source_size= pack_length_from_metadata(field_metadata); - uint const destination_size= row_pack_length(); - uint const from_bit_len= field_metadata & 0x00ff; - uint const from_len= (field_metadata >> 8U) & 0x00ff; - if ((bit_len == 0) || (from_bit_len == 0)) - compatible= (source_size <= destination_size); - else if (from_bit_len > bit_len) - compatible= (from_len < bytes_in_rec); - else - compatible= ((from_bit_len <= bit_len) && (from_len <= bytes_in_rec)); - return (compatible); + DBUG_ENTER("Field_bit::compatible_field_size"); + DBUG_ASSERT((field_metadata >> 16) == 0); + uint const from_bit_len= + 8 * (field_metadata >> 8) + (field_metadata & 0xff); + uint const to_bit_len= max_display_length(); + DBUG_PRINT("debug", ("from_bit_len: %u, to_bit_len: %u", + from_bit_len, to_bit_len)); + *order_var= compare(from_bit_len, to_bit_len); + DBUG_RETURN(TRUE); } @@ -9388,8 +9411,15 @@ const uchar * Field_bit::unpack(uchar *to, const uchar *from, uint param_data, bool low_byte_first __attribute__((unused))) { + DBUG_ENTER("Field_bit::unpack"); + DBUG_PRINT("enter", ("to: %p, from: %p, param_data: 0x%x", + to, from, param_data)); + DBUG_PRINT("debug", ("bit_ptr: %p, bit_len: %u, bit_ofs: %u", + bit_ptr, bit_len, bit_ofs)); uint const from_len= (param_data >> 8U) & 0x00ff; uint const from_bit_len= param_data & 0x00ff; + DBUG_PRINT("debug", ("from_len: %u, from_bit_len: %u", + from_len, from_bit_len)); /* If the parameter data is zero (i.e., undefined), or if the master and slave have the same sizes, then use the old unpack() method. @@ -9410,7 +9440,7 @@ Field_bit::unpack(uchar *to, const uchar *from, uint param_data, from++; } memcpy(to, from, bytes_in_rec); - return from + bytes_in_rec; + DBUG_RETURN(from + bytes_in_rec); } /* @@ -9436,7 +9466,7 @@ Field_bit::unpack(uchar *to, const uchar *from, uint param_data, bitmap_set_bit(table->write_set,field_index); store(value, new_len, system_charset_info); my_afree(value); - return from + len; + DBUG_RETURN(from + len); } @@ -9564,8 +9594,11 @@ void Create_field::create_length_to_internal_length(void) */ void Create_field::init_for_tmp_table(enum_field_types sql_type_arg, uint32 length_arg, uint32 decimals_arg, - bool maybe_null, bool is_unsigned) + bool maybe_null, bool is_unsigned, + uint pack_length) { + DBUG_ENTER("Create_field::init_for_tmp_table"); + field_name= ""; sql_type= sql_type_arg; char_length= length= length_arg;; @@ -9573,10 +9606,92 @@ void Create_field::init_for_tmp_table(enum_field_types sql_type_arg, interval= 0; charset= &my_charset_bin; geom_type= Field::GEOM_GEOMETRY; - pack_flag= (FIELDFLAG_NUMBER | - ((decimals_arg & FIELDFLAG_MAX_DEC) << FIELDFLAG_DEC_SHIFT) | - (maybe_null ? FIELDFLAG_MAYBE_NULL : 0) | - (is_unsigned ? 0 : FIELDFLAG_DECIMAL)); + + DBUG_PRINT("enter", ("sql_type: %d, length: %u, pack_length: %u", + sql_type_arg, length_arg, pack_length)); + + /* + These pack flags are crafted to get it correctly through the + branches of make_field(). + */ + switch (sql_type_arg) + { + case MYSQL_TYPE_VARCHAR: + case MYSQL_TYPE_VAR_STRING: + case MYSQL_TYPE_STRING: + case MYSQL_TYPE_SET: + pack_flag= 0; + break; + + case MYSQL_TYPE_GEOMETRY: + pack_flag= FIELDFLAG_GEOM; + break; + + case MYSQL_TYPE_ENUM: + pack_flag= FIELDFLAG_INTERVAL; + break; + + case MYSQL_TYPE_DECIMAL: + case MYSQL_TYPE_NEWDECIMAL: + case MYSQL_TYPE_FLOAT: + case MYSQL_TYPE_DOUBLE: + pack_flag= FIELDFLAG_DECIMAL | FIELDFLAG_NUMBER | + (decimals_arg & FIELDFLAG_MAX_DEC) << FIELDFLAG_DEC_SHIFT; + break; + + case MYSQL_TYPE_TINY_BLOB: + case MYSQL_TYPE_MEDIUM_BLOB: + case MYSQL_TYPE_LONG_BLOB: + case MYSQL_TYPE_BLOB: + pack_flag= FIELDFLAG_BLOB; + break; + + case MYSQL_TYPE_BIT: + pack_flag= FIELDFLAG_NUMBER | FIELDFLAG_TREAT_BIT_AS_CHAR; + break; + + default: + pack_flag= FIELDFLAG_NUMBER; + break; + } + + /* + Set the pack flag correctly for the blob-like types. This sets the + packtype to something that make_field can use. If the pack type is + not set correctly, the packlength will be reeeeally wierd (like + 129 or so). + */ + switch (sql_type_arg) + { + case MYSQL_TYPE_ENUM: + case MYSQL_TYPE_SET: + case MYSQL_TYPE_TINY_BLOB: + case MYSQL_TYPE_MEDIUM_BLOB: + case MYSQL_TYPE_LONG_BLOB: + case MYSQL_TYPE_BLOB: + case MYSQL_TYPE_GEOMETRY: + // If you are going to use the above types, you have to pass a + // pack_length as parameter. Assert that is really done. + DBUG_ASSERT(pack_length != ~0U); + pack_flag|= pack_length_to_packflag(pack_length); + break; + default: + /* Nothing */ + break; + } + + pack_flag|= + (maybe_null ? FIELDFLAG_MAYBE_NULL : 0) | + (is_unsigned ? 0 : FIELDFLAG_DECIMAL); + + DBUG_PRINT("debug", ("pack_flag: %s%s%s%s%s, pack_type: %d", + FLAGSTR(pack_flag, FIELDFLAG_BINARY), + FLAGSTR(pack_flag, FIELDFLAG_NUMBER), + FLAGSTR(pack_flag, FIELDFLAG_INTERVAL), + FLAGSTR(pack_flag, FIELDFLAG_GEOM), + FLAGSTR(pack_flag, FIELDFLAG_BLOB), + f_packtype(pack_flag))); + DBUG_VOID_RETURN; } @@ -10073,6 +10188,14 @@ Field *make_field(TABLE_SHARE *share, uchar *ptr, uint32 field_length, default: break; } + DBUG_PRINT("debug", ("field_type: %d, field_length: %u, interval: %p, pack_flag: %s%s%s%s%s", + field_type, field_length, interval, + FLAGSTR(pack_flag, FIELDFLAG_BINARY), + FLAGSTR(pack_flag, FIELDFLAG_INTERVAL), + FLAGSTR(pack_flag, FIELDFLAG_NUMBER), + FLAGSTR(pack_flag, FIELDFLAG_PACK), + FLAGSTR(pack_flag, FIELDFLAG_BLOB))); + if (f_is_alpha(pack_flag)) { if (!f_is_packed(pack_flag)) -- cgit v1.2.1 From 9e980bf79ef0c727c630e79c1bc043c48bc947ee Mon Sep 17 00:00:00 2001 From: Mats Kindahl Date: Tue, 15 Dec 2009 16:11:44 +0100 Subject: BUG#49618: Field length stored incorrectly in binary log for InnoDB The class Field_bit_as_char stores the metadata for the field incorrecly because bytes_in_rec and bit_len are set to (field_length + 7 ) / 8 and 0 respectively, while Field_bit has the correct values field_length / 8 and field_length % 8. Solved the problem by re-computing the values for the metadata based on the field_length instead of using the bytes_in_rec and bit_len variables. To handle compatibility with old server, a table map flag was added to indicate that the bit computation is exact. If the flag is clear, the slave computes the number of bytes required to store the bit field and compares that instead, effectively allowing replication *without conversion* from any field length that require the same number of bytes to store. mysql-test/suite/rpl/t/rpl_typeconv_innodb.test: Adding test to check compatibility for bit field replication when using InnoDB. sql/field.cc: Extending compatible_field_size() with flags from table map to allow fields to check master info. sql/field.h: Extending compatible_field_size() with flags from table map to allow fields to check master info. sql/log.cc: Removing table map flags since they are not used outside table map class. sql/log_event.cc: Removing flags parameter from table map constructor since it is not used and does not have to be exposed. sql/log_event.h: Adding flag to denote that bit length for bit field type is exact and not potentially rounded to even bytes. sql/rpl_utility.cc: Adding fields to table_def to store table map flags. sql/rpl_utility.h: Removing obsolete comment and adding flags to store table map flags from master. --- sql/field.cc | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) (limited to 'sql/field.cc') diff --git a/sql/field.cc b/sql/field.cc index b6a795d34aa..477cdc0a993 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -1407,6 +1407,7 @@ bool Field::send_binary(Protocol *protocol) type, 0 is returned in *order_var. @param field_metadata Encoded size in field metadata + @param mflags Flags from the table map event for the table. @param order_var Pointer to variable where the order between the source field and this field will be returned. @@ -1416,6 +1417,7 @@ bool Field::send_binary(Protocol *protocol) */ bool Field::compatible_field_size(uint field_metadata, Relay_log_info *rli_arg __attribute__((unused)), + uint16 mflags __attribute__((unused)), int *order_var) { uint const source_size= pack_length_from_metadata(field_metadata); @@ -2950,6 +2952,7 @@ uint Field_new_decimal::pack_length_from_metadata(uint field_metadata) bool Field_new_decimal::compatible_field_size(uint field_metadata, Relay_log_info * __attribute__((unused)), + uint16 mflags __attribute__((unused)), int *order_var) { uint const source_precision= (field_metadata >> 8U) & 0x00ff; @@ -6733,6 +6736,7 @@ check_field_for_37426(const void *param_arg) bool Field_string::compatible_field_size(uint field_metadata, Relay_log_info *rli_arg, + uint16 mflags __attribute__((unused)), int *order_var) { #ifdef HAVE_REPLICATION @@ -6741,7 +6745,7 @@ Field_string::compatible_field_size(uint field_metadata, check_field_for_37426, &check_param)) return FALSE; // Not compatible field sizes #endif - return Field::compatible_field_size(field_metadata, rli_arg, order_var); + return Field::compatible_field_size(field_metadata, rli_arg, mflags, order_var); } @@ -9305,8 +9309,13 @@ int Field_bit::do_save_field_metadata(uchar *metadata_ptr) DBUG_ENTER("Field_bit::do_save_field_metadata"); DBUG_PRINT("debug", ("bit_len: %d, bytes_in_rec: %d", bit_len, bytes_in_rec)); - *metadata_ptr= bit_len; - *(metadata_ptr + 1)= bytes_in_rec; + /* + Since this class and Field_bit_as_char have different ideas of + what should be stored here, we compute the values of the metadata + explicitly using the field_length. + */ + metadata_ptr[0]= field_length % 8; + metadata_ptr[1]= field_length / 8; DBUG_RETURN(2); } @@ -9335,15 +9344,29 @@ uint Field_bit::pack_length_from_metadata(uint field_metadata) bool Field_bit::compatible_field_size(uint field_metadata, Relay_log_info * __attribute__((unused)), + uint16 mflags, int *order_var) { DBUG_ENTER("Field_bit::compatible_field_size"); DBUG_ASSERT((field_metadata >> 16) == 0); - uint const from_bit_len= + uint from_bit_len= 8 * (field_metadata >> 8) + (field_metadata & 0xff); - uint const to_bit_len= max_display_length(); + uint to_bit_len= max_display_length(); DBUG_PRINT("debug", ("from_bit_len: %u, to_bit_len: %u", from_bit_len, to_bit_len)); + /* + If the bit length exact flag is clear, we are dealing with an old + master, so we allow some less strict behaviour if replicating by + moving both bit lengths to an even multiple of 8. + + We do this by computing the number of bytes to store the field + instead, and then compare the result. + */ + if (!(mflags & Table_map_log_event::TM_BIT_LEN_EXACT_F)) { + from_bit_len= (from_bit_len + 7) / 8; + to_bit_len= (to_bit_len + 7) / 8; + } + *order_var= compare(from_bit_len, to_bit_len); DBUG_RETURN(TRUE); } -- cgit v1.2.1 From 74577209ac292807550ee0c57f94a9db320941dc Mon Sep 17 00:00:00 2001 From: Mats Kindahl Date: Wed, 16 Dec 2009 09:32:58 +0100 Subject: WL#5151: Conversion between different types when replicating Fixes to get it to compile on MacOSX. --- sql/field.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'sql/field.cc') diff --git a/sql/field.cc b/sql/field.cc index 477cdc0a993..1fa4b698e72 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -999,18 +999,19 @@ test_if_important_data(CHARSET_INFO *cs, const char *str, const char *strend) /** - Template function to compare two objects. + Function to compare two unsigned integers for their relative order. + Used below. In an anonymous namespace to not clash with definitions + in other files. */ namespace { - template - int compare(A_type a, B_type b) + int compare(unsigned int a, unsigned int b) { if (a < b) return -1; if (b < a) return 1; return 0; - } +} } /** -- cgit v1.2.1 From 12f364ece7663663cabdc29f106ca69af63fe4e7 Mon Sep 17 00:00:00 2001 From: Alexey Kopytov Date: Tue, 22 Dec 2009 19:23:13 +0300 Subject: Backport of WL #2934: Make/find library for doing float/double to string conversions and vice versa" Initial import of the dtoa.c code and custom wrappers around it to allow its usage from the server code. Conversion of FLOAT/DOUBLE values to DECIMAL ones or strings and vice versa has been significantly reworked. As the new algoritms are more precise than the older ones, results of such conversions may not always match those obtained from older server versions. This in turn may break compatibility for some applications. This patch also fixes the following bugs: - bug #12860 "Difference in zero padding of exponent between Unix and Windows" - bug #21497 "DOUBLE truncated to unusable value" - bug #26788 "mysqld (debug) aborts when inserting specific numbers into char fields" - bug #24541 "Data truncated..." on decimal type columns without any good reason" --- sql/field.cc | 243 ++++++++--------------------------------------------------- 1 file changed, 30 insertions(+), 213 deletions(-) (limited to 'sql/field.cc') diff --git a/sql/field.cc b/sql/field.cc index 0934bb04ccd..56da32959f9 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -31,9 +31,6 @@ #include "slave.h" // Pull in rpl_master_has_bug() #include #include -#ifdef HAVE_FCONVERT -#include -#endif // Maximum allowed exponent value for converting string to decimal #define MAX_EXPONENT 1024 @@ -50,7 +47,7 @@ template class List_iterator; uchar Field_null::null[1]={1}; const char field_separator=','; -#define DOUBLE_TO_STRING_CONVERSION_BUFFER_SIZE 320 +#define DOUBLE_TO_STRING_CONVERSION_BUFFER_SIZE FLOATING_POINT_BUFFER #define LONGLONG_TO_STRING_CONVERSION_BUFFER_SIZE 128 #define DECIMAL_TO_STRING_CONVERSION_BUFFER_SIZE 128 #define BLOB_PACK_LENGTH_TO_MAX_LENGH(arg) \ @@ -2304,13 +2301,7 @@ int Field_decimal::store(double nr) char buff[DOUBLE_TO_STRING_CONVERSION_BUFFER_SIZE]; fyllchar = zerofill ? (char) '0' : (char) ' '; -#ifdef HAVE_SNPRINTF - buff[sizeof(buff)-1]=0; // Safety - snprintf(buff,sizeof(buff)-1, "%.*f",(int) dec,nr); - length= strlen(buff); -#else - length= my_sprintf(buff,(buff,"%.*f",dec,nr)); -#endif + length= my_fcvt(nr, dec, buff, NULL); if (length > field_length) { @@ -2723,17 +2714,6 @@ int Field_new_decimal::store(double nr) err= double2my_decimal(E_DEC_FATAL_ERROR & ~E_DEC_OVERFLOW, nr, &decimal_value); - /* - TODO: fix following when double2my_decimal when double2decimal - will return E_DEC_TRUNCATED always correctly - */ - if (!err) - { - double nr2; - my_decimal2double(E_DEC_FATAL_ERROR, &decimal_value, &nr2); - if (nr2 != nr) - err= E_DEC_TRUNCATED; - } if (err) { if (check_overflow(err)) @@ -4231,67 +4211,20 @@ String *Field_float::val_str(String *val_buffer, uint to_length=max(field_length,70); val_buffer->alloc(to_length); char *to=(char*) val_buffer->ptr(); + size_t len; if (dec >= NOT_FIXED_DEC) - { - sprintf(to,"%-*.*g",(int) field_length,FLT_DIG,nr); - to=strcend(to,' '); - *to=0; - } + len= my_gcvt(nr, MY_GCVT_ARG_FLOAT, to_length - 1, to, NULL); else { -#ifdef HAVE_FCONVERT - char buff[70],*pos=buff; - int decpt,sign,tmp_dec=dec; - - (void) sfconvert(&nr,tmp_dec,&decpt,&sign,buff); - if (sign) - { - *to++='-'; - } - if (decpt < 0) - { /* val_buffer is < 0 */ - *to++='0'; - if (!tmp_dec) - goto end; - *to++='.'; - if (-decpt > tmp_dec) - decpt= - (int) tmp_dec; - tmp_dec=(uint) ((int) tmp_dec+decpt); - while (decpt++ < 0) - *to++='0'; - } - else if (decpt == 0) - { - *to++= '0'; - if (!tmp_dec) - goto end; - *to++='.'; - } - else - { - while (decpt-- > 0) - *to++= *pos++; - if (!tmp_dec) - goto end; - *to++='.'; - } - while (tmp_dec--) - *to++= *pos++; -#else -#ifdef HAVE_SNPRINTF - to[to_length-1]=0; // Safety - snprintf(to,to_length-1,"%.*f",dec,nr); - to=strend(to); -#else - to+= my_sprintf(to,(to,"%.*f",dec,nr)); -#endif -#endif + /* + We are safe here because the buffer length is >= 70, and + fabs(float) < 10^39, dec < NOT_FIXED_DEC. So the resulting string + will be not longer than 69 chars + terminating '\0'. + */ + len= my_fcvt(nr, dec, to, NULL); } -#ifdef HAVE_FCONVERT - end: -#endif - val_buffer->length((uint) (to-val_buffer->ptr())); + val_buffer->length((uint) len); if (zerofill) prepend_zeros(val_buffer); return val_buffer; @@ -4479,8 +4412,12 @@ int Field_real::truncate(double *nr, double max_value) max_value*= log_10[order]; max_value-= 1.0 / log_10[dec]; - double tmp= rint((res - floor(res)) * log_10[dec]) / log_10[dec]; - res= floor(res) + tmp; + /* Check for infinity so we don't get NaN in calculations */ + if (!my_isinf(res)) + { + double tmp= rint((res - floor(res)) * log_10[dec]) / log_10[dec]; + res= floor(res) + tmp; + } } if (res < -max_value) @@ -4590,68 +4527,14 @@ String *Field_double::val_str(String *val_buffer, uint to_length=max(field_length, DOUBLE_TO_STRING_CONVERSION_BUFFER_SIZE); val_buffer->alloc(to_length); char *to=(char*) val_buffer->ptr(); + size_t len; if (dec >= NOT_FIXED_DEC) - { - sprintf(to,"%-*.*g",(int) field_length,DBL_DIG,nr); - to=strcend(to,' '); - } + len= my_gcvt(nr, MY_GCVT_ARG_DOUBLE, to_length - 1, to, NULL); else - { -#ifdef HAVE_FCONVERT - char buff[DOUBLE_TO_STRING_CONVERSION_BUFFER_SIZE]; - char *pos= buff; - int decpt,sign,tmp_dec=dec; - - (void) fconvert(nr,tmp_dec,&decpt,&sign,buff); - if (sign) - { - *to++='-'; - } - if (decpt < 0) - { /* val_buffer is < 0 */ - *to++='0'; - if (!tmp_dec) - goto end; - *to++='.'; - if (-decpt > tmp_dec) - decpt= - (int) tmp_dec; - tmp_dec=(uint) ((int) tmp_dec+decpt); - while (decpt++ < 0) - *to++='0'; - } - else if (decpt == 0) - { - *to++= '0'; - if (!tmp_dec) - goto end; - *to++='.'; - } - else - { - while (decpt-- > 0) - *to++= *pos++; - if (!tmp_dec) - goto end; - *to++='.'; - } - while (tmp_dec--) - *to++= *pos++; -#else -#ifdef HAVE_SNPRINTF - to[to_length-1]=0; // Safety - snprintf(to,to_length-1,"%.*f",dec,nr); - to=strend(to); -#else - to+= my_sprintf(to,(to,"%.*f",dec,nr)); -#endif -#endif - } -#ifdef HAVE_FCONVERT - end: -#endif + len= my_fcvt(nr, dec, to, NULL); - val_buffer->length((uint) (to-val_buffer->ptr())); + val_buffer->length((uint) len); if (zerofill) prepend_zeros(val_buffer); return val_buffer; @@ -6448,84 +6331,18 @@ int Field_str::store(double nr) { ASSERT_COLUMN_MARKED_FOR_WRITE; char buff[DOUBLE_TO_STRING_CONVERSION_BUFFER_SIZE]; - uint length; uint local_char_length= field_length / charset()->mbmaxlen; - double anr= fabs(nr); - bool fractional= (anr != floor(anr)); - int neg= (nr < 0.0) ? 1 : 0; - uint max_length; - int exp; - uint digits; - uint i; - - /* Calculate the exponent from the 'e'-format conversion */ - if (anr < 1.0 && anr > 0) - { - for (exp= 0; anr < 1e-100; exp-= 100, anr*= 1e100) ; - for (; anr < 1e-10; exp-= 10, anr*= 1e10) ; - for (i= 1; anr < 1 / log_10[i]; exp--, i++) ; - exp--; - } - else - { - for (exp= 0; anr > 1e100; exp+= 100, anr/= 1e100) ; - for (; anr > 1e10; exp+= 10, anr/= 1e10) ; - for (i= 1; anr > log_10[i]; exp++, i++) ; - } - - max_length= local_char_length - neg; - - /* - Since in sprintf("%g") precision means the number of significant digits, - calculate the maximum number of significant digits if the 'f'-format - would be used (+1 for decimal point if the number has a fractional part). - */ - digits= max(1, (int) max_length - fractional); - /* - If the exponent is negative, decrease digits by the number of leading zeros - after the decimal point that do not count as significant digits. - */ - if (exp < 0) - digits= max(1, (int) digits + exp); - /* - 'e'-format is used only if the exponent is less than -4 or greater than or - equal to the precision. In this case we need to adjust the number of - significant digits to take "e+NN" + decimal point into account (hence -5). - We also have to reserve one additional character if abs(exp) >= 100. - */ - if (exp >= (int) digits || exp < -4) - digits= max(1, (int) (max_length - 5 - (exp >= 100 || exp <= -100))); - - /* Limit precision to DBL_DIG to avoid garbage past significant digits */ - set_if_smaller(digits, DBL_DIG); - - length= (uint) my_sprintf(buff, (buff, "%-.*g", digits, nr)); + size_t length; + my_bool error; -#ifdef __WIN__ - /* - Windows always zero-pads the exponent to 3 digits, we want to remove the - leading 0 to match the sprintf() output on other platforms. - */ - if ((exp >= (int) digits || exp < -4) && exp > -100 && exp < 100) + length= my_gcvt(nr, MY_GCVT_ARG_DOUBLE, local_char_length, buff, &error); + if (error) { - DBUG_ASSERT(length >= 6); /* 1e+NNN */ - uint tmp= length - 3; - buff[tmp]= buff[tmp + 1]; - tmp++; - buff[tmp]= buff[tmp + 1]; - length--; + if (table->in_use->abort_on_warning) + set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_DATA_TOO_LONG, 1); + else + set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, WARN_DATA_TRUNCATED, 1); } -#endif - - /* - +1 below is because "precision" in %g above means the - max. number of significant digits, not the output width. - Thus the width can be larger than number of significant digits by 1 - (for decimal point) - the test for local_char_length < 5 is for extreme cases, - like inserting 500.0 in char(1) - */ - DBUG_ASSERT(local_char_length < 5 || length <= local_char_length+1); return store(buff, length, charset()); } @@ -7601,7 +7418,7 @@ oom_error: int Field_blob::store(double nr) { CHARSET_INFO *cs=charset(); - value.set_real(nr, 2, cs); + value.set_real(nr, NOT_FIXED_DEC, cs); return Field_blob::store(value.ptr(),(uint) value.length(), cs); } -- cgit v1.2.1