/***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 of the License. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA *****************************************************************************/ /********************************************************************//** @file data/data0data.cc SQL data field and tuple Created 5/30/1994 Heikki Tuuri *************************************************************************/ #include "ha_prototypes.h" #include "data0data.h" #ifdef UNIV_NONINL #include "data0data.ic" #endif #include "rem0rec.h" #include "rem0cmp.h" #include "page0page.h" #include "page0zip.h" #include "dict0dict.h" #include "btr0cur.h" #include "row0upd.h" #ifdef UNIV_DEBUG /** Dummy variable to catch access to uninitialized fields. In the debug version, dtuple_create() will make all fields of dtuple_t point to data_error. */ byte data_error; # ifndef UNIV_DEBUG_VALGRIND /** this is used to fool the compiler in dtuple_validate */ ulint data_dummy; # endif /* !UNIV_DEBUG_VALGRIND */ #endif /* UNIV_DEBUG */ /** Compare two data tuples. @param[in] tuple1 first data tuple @param[in] tuple2 second data tuple @return positive, 0, negative if tuple1 is greater, equal, less, than tuple2, respectively */ int dtuple_coll_cmp( const dtuple_t* tuple1, const dtuple_t* tuple2) { ulint n_fields; ulint i; int cmp; ut_ad(tuple1 != NULL); ut_ad(tuple2 != NULL); ut_ad(tuple1->magic_n == DATA_TUPLE_MAGIC_N); ut_ad(tuple2->magic_n == DATA_TUPLE_MAGIC_N); ut_ad(dtuple_check_typed(tuple1)); ut_ad(dtuple_check_typed(tuple2)); n_fields = dtuple_get_n_fields(tuple1); cmp = (int) n_fields - (int) dtuple_get_n_fields(tuple2); for (i = 0; cmp == 0 && i < n_fields; i++) { const dfield_t* field1 = dtuple_get_nth_field(tuple1, i); const dfield_t* field2 = dtuple_get_nth_field(tuple2, i); cmp = cmp_dfield_dfield(field1, field2); } return(cmp); } /*********************************************************************//** Sets number of fields used in a tuple. Normally this is set in dtuple_create, but if you want later to set it smaller, you can use this. */ void dtuple_set_n_fields( /*================*/ dtuple_t* tuple, /*!< in: tuple */ ulint n_fields) /*!< in: number of fields */ { ut_ad(tuple); tuple->n_fields = n_fields; tuple->n_fields_cmp = n_fields; } /**********************************************************//** Checks that a data field is typed. @return TRUE if ok */ static ibool dfield_check_typed_no_assert( /*=========================*/ const dfield_t* field) /*!< in: data field */ { if (dfield_get_type(field)->mtype > DATA_MTYPE_CURRENT_MAX || dfield_get_type(field)->mtype < DATA_MTYPE_CURRENT_MIN) { ib::error() << "Data field type " << dfield_get_type(field)->mtype << ", len " << dfield_get_len(field); return(FALSE); } return(TRUE); } /**********************************************************//** Checks that a data tuple is typed. @return TRUE if ok */ ibool dtuple_check_typed_no_assert( /*=========================*/ const dtuple_t* tuple) /*!< in: tuple */ { const dfield_t* field; ulint i; if (dtuple_get_n_fields(tuple) > REC_MAX_N_FIELDS) { ib::error() << "Index entry has " << dtuple_get_n_fields(tuple) << " fields"; dump: fputs("InnoDB: Tuple contents: ", stderr); dtuple_print(stderr, tuple); putc('\n', stderr); return(FALSE); } for (i = 0; i < dtuple_get_n_fields(tuple); i++) { field = dtuple_get_nth_field(tuple, i); if (!dfield_check_typed_no_assert(field)) { goto dump; } } return(TRUE); } #ifdef UNIV_DEBUG /**********************************************************//** Checks that a data field is typed. Asserts an error if not. @return TRUE if ok */ ibool dfield_check_typed( /*===============*/ const dfield_t* field) /*!< in: data field */ { if (dfield_get_type(field)->mtype > DATA_MTYPE_CURRENT_MAX || dfield_get_type(field)->mtype < DATA_MTYPE_CURRENT_MIN) { ib::fatal() << "Data field type " << dfield_get_type(field)->mtype << ", len " << dfield_get_len(field); } return(TRUE); } /**********************************************************//** Checks that a data tuple is typed. Asserts an error if not. @return TRUE if ok */ ibool dtuple_check_typed( /*===============*/ const dtuple_t* tuple) /*!< in: tuple */ { const dfield_t* field; ulint i; for (i = 0; i < dtuple_get_n_fields(tuple); i++) { field = dtuple_get_nth_field(tuple, i); ut_a(dfield_check_typed(field)); } return(TRUE); } /**********************************************************//** Validates the consistency of a tuple which must be complete, i.e, all fields must have been set. @return TRUE if ok */ ibool dtuple_validate( /*============*/ const dtuple_t* tuple) /*!< in: tuple */ { const dfield_t* field; ulint n_fields; ulint len; ulint i; ut_ad(tuple->magic_n == DATA_TUPLE_MAGIC_N); n_fields = dtuple_get_n_fields(tuple); /* We dereference all the data of each field to test for memory traps */ for (i = 0; i < n_fields; i++) { field = dtuple_get_nth_field(tuple, i); len = dfield_get_len(field); if (!dfield_is_null(field)) { const byte* data; data = static_cast(dfield_get_data(field)); #ifndef UNIV_DEBUG_VALGRIND ulint j; for (j = 0; j < len; j++) { data_dummy += *data; /* fool the compiler not to optimize out this code */ data++; } #endif /* !UNIV_DEBUG_VALGRIND */ UNIV_MEM_ASSERT_RW(data, len); } } ut_a(dtuple_check_typed(tuple)); return(TRUE); } #endif /* UNIV_DEBUG */ /*************************************************************//** Pretty prints a dfield value according to its data type. */ void dfield_print( /*=========*/ const dfield_t* dfield) /*!< in: dfield */ { const byte* data; ulint len; ulint i; len = dfield_get_len(dfield); data = static_cast(dfield_get_data(dfield)); if (dfield_is_null(dfield)) { fputs("NULL", stderr); return; } switch (dtype_get_mtype(dfield_get_type(dfield))) { case DATA_CHAR: case DATA_VARCHAR: for (i = 0; i < len; i++) { int c = *data++; putc(isprint(c) ? c : ' ', stderr); } if (dfield_is_ext(dfield)) { fputs("(external)", stderr); } break; case DATA_INT: ut_a(len == 4); /* only works for 32-bit integers */ fprintf(stderr, "%d", (int) mach_read_from_4(data)); break; default: ut_error; } } /*************************************************************//** Pretty prints a dfield value according to its data type. Also the hex string is printed if a string contains non-printable characters. */ void dfield_print_also_hex( /*==================*/ const dfield_t* dfield) /*!< in: dfield */ { const byte* data; ulint len; ulint prtype; ulint i; ibool print_also_hex; len = dfield_get_len(dfield); data = static_cast(dfield_get_data(dfield)); if (dfield_is_null(dfield)) { fputs("NULL", stderr); return; } prtype = dtype_get_prtype(dfield_get_type(dfield)); switch (dtype_get_mtype(dfield_get_type(dfield))) { ib_id_t id; case DATA_INT: switch (len) { ulint val; case 1: val = mach_read_from_1(data); if (!(prtype & DATA_UNSIGNED)) { val &= ~0x80; fprintf(stderr, "%ld", (long) val); } else { fprintf(stderr, "%lu", (ulong) val); } break; case 2: val = mach_read_from_2(data); if (!(prtype & DATA_UNSIGNED)) { val &= ~0x8000; fprintf(stderr, "%ld", (long) val); } else { fprintf(stderr, "%lu", (ulong) val); } break; case 3: val = mach_read_from_3(data); if (!(prtype & DATA_UNSIGNED)) { val &= ~0x800000; fprintf(stderr, "%ld", (long) val); } else { fprintf(stderr, "%lu", (ulong) val); } break; case 4: val = mach_read_from_4(data); if (!(prtype & DATA_UNSIGNED)) { val &= ~0x80000000; fprintf(stderr, "%ld", (long) val); } else { fprintf(stderr, "%lu", (ulong) val); } break; case 6: id = mach_read_from_6(data); fprintf(stderr, IB_ID_FMT, id); break; case 7: id = mach_read_from_7(data); fprintf(stderr, IB_ID_FMT, id); break; case 8: id = mach_read_from_8(data); fprintf(stderr, IB_ID_FMT, id); break; default: goto print_hex; } break; case DATA_SYS: switch (prtype & DATA_SYS_PRTYPE_MASK) { case DATA_TRX_ID: id = mach_read_from_6(data); fprintf(stderr, "trx_id " TRX_ID_FMT, id); break; case DATA_ROLL_PTR: id = mach_read_from_7(data); fprintf(stderr, "roll_ptr " TRX_ID_FMT, id); break; case DATA_ROW_ID: id = mach_read_from_6(data); fprintf(stderr, "row_id " TRX_ID_FMT, id); break; default: goto print_hex; } break; case DATA_CHAR: case DATA_VARCHAR: print_also_hex = FALSE; for (i = 0; i < len; i++) { int c = *data++; if (!isprint(c)) { print_also_hex = TRUE; fprintf(stderr, "\\x%02x", (unsigned char) c); } else { putc(c, stderr); } } if (dfield_is_ext(dfield)) { fputs("(external)", stderr); } if (!print_also_hex) { break; } data = static_cast(dfield_get_data(dfield)); /* fall through */ case DATA_BINARY: default: print_hex: fputs(" Hex: ",stderr); for (i = 0; i < len; i++) { fprintf(stderr, "%02lx", static_cast(*data++)); } if (dfield_is_ext(dfield)) { fputs("(external)", stderr); } } } /*************************************************************//** Print a dfield value using ut_print_buf. */ static void dfield_print_raw( /*=============*/ FILE* f, /*!< in: output stream */ const dfield_t* dfield) /*!< in: dfield */ { ulint len = dfield_get_len(dfield); if (!dfield_is_null(dfield)) { ulint print_len = ut_min(len, static_cast(1000)); ut_print_buf(f, dfield_get_data(dfield), print_len); if (len != print_len) { fprintf(f, "(total %lu bytes%s)", (ulong) len, dfield_is_ext(dfield) ? ", external" : ""); } } else { fputs(" SQL NULL", f); } } /**********************************************************//** The following function prints the contents of a tuple. */ void dtuple_print( /*=========*/ FILE* f, /*!< in: output stream */ const dtuple_t* tuple) /*!< in: tuple */ { ulint n_fields; ulint i; n_fields = dtuple_get_n_fields(tuple); fprintf(f, "DATA TUPLE: %lu fields;\n", (ulong) n_fields); for (i = 0; i < n_fields; i++) { fprintf(f, " %lu:", (ulong) i); dfield_print_raw(f, dtuple_get_nth_field(tuple, i)); putc(';', f); putc('\n', f); } ut_ad(dtuple_validate(tuple)); } /** Print the contents of a tuple. @param[out] o output stream @param[in] field array of data fields @param[in] n number of data fields */ void dfield_print( std::ostream& o, const dfield_t* field, ulint n) { for (ulint i = 0; i < n; i++, field++) { const void* data = dfield_get_data(field); const ulint len = dfield_get_len(field); if (i) { o << ','; } if (dfield_is_null(field)) { o << "NULL"; } else if (dfield_is_ext(field)) { ulint local_len = len - BTR_EXTERN_FIELD_REF_SIZE; ut_ad(len >= BTR_EXTERN_FIELD_REF_SIZE); o << '[' << local_len << '+' << BTR_EXTERN_FIELD_REF_SIZE << ']'; ut_print_buf(o, data, local_len); ut_print_buf_hex(o, static_cast(data) + local_len, BTR_EXTERN_FIELD_REF_SIZE); } else { o << '[' << len << ']'; ut_print_buf(o, data, len); } } } /** Print the contents of a tuple. @param[out] o output stream @param[in] tuple data tuple */ void dtuple_print( std::ostream& o, const dtuple_t* tuple) { const ulint n = dtuple_get_n_fields(tuple); o << "TUPLE (info_bits=" << dtuple_get_info_bits(tuple) << ", " << n << " fields): {"; dfield_print(o, tuple->fields, n); o << "}"; } /**************************************************************//** Moves parts of long fields in entry to the big record vector so that the size of tuple drops below the maximum record size allowed in the database. Moves data only from those fields which are not necessary to determine uniquely the insertion place of the tuple in the index. @return own: created big record vector, NULL if we are not able to shorten the entry enough, i.e., if there are too many fixed-length or short fields in entry or the index is clustered */ big_rec_t* dtuple_convert_big_rec( /*===================*/ dict_index_t* index, /*!< in: index */ upd_t* upd, /*!< in/out: update vector */ dtuple_t* entry, /*!< in/out: index entry */ ulint* n_ext) /*!< in/out: number of externally stored columns */ { mem_heap_t* heap; big_rec_t* vector; dfield_t* dfield; dict_field_t* ifield; ulint size; ulint n_fields; ulint local_len; ulint local_prefix_len; if (!dict_index_is_clust(index)) { return(NULL); } if (dict_table_get_format(index->table) < UNIV_FORMAT_B) { /* up to MySQL 5.1: store a 768-byte prefix locally */ local_len = BTR_EXTERN_FIELD_REF_SIZE + DICT_ANTELOPE_MAX_INDEX_COL_LEN; } else { /* new-format table: do not store any BLOB prefix locally */ local_len = BTR_EXTERN_FIELD_REF_SIZE; } ut_a(dtuple_check_typed_no_assert(entry)); size = rec_get_converted_size(index, entry, *n_ext); if (UNIV_UNLIKELY(size > 1000000000)) { ib::warn() << "Tuple size is very big: " << size; fputs("InnoDB: Tuple contents: ", stderr); dtuple_print(stderr, entry); putc('\n', stderr); } heap = mem_heap_create(size + dtuple_get_n_fields(entry) * sizeof(big_rec_field_t) + 1000); vector = big_rec_t::alloc(heap, dtuple_get_n_fields(entry)); /* Decide which fields to shorten: the algorithm is to look for a variable-length field that yields the biggest savings when stored externally */ n_fields = 0; while (page_zip_rec_needs_ext(rec_get_converted_size(index, entry, *n_ext), dict_table_is_comp(index->table), dict_index_get_n_fields(index), dict_table_page_size(index->table))) { ulint i; ulint longest = 0; ulint longest_i = ULINT_MAX; byte* data; for (i = dict_index_get_n_unique_in_tree(index); i < dtuple_get_n_fields(entry); i++) { ulint savings; dfield = dtuple_get_nth_field(entry, i); ifield = dict_index_get_nth_field(index, i); /* Skip fixed-length, NULL, externally stored, or short columns */ if (ifield->fixed_len || dfield_is_null(dfield) || dfield_is_ext(dfield) || dfield_get_len(dfield) <= local_len || dfield_get_len(dfield) <= BTR_EXTERN_LOCAL_STORED_MAX_SIZE) { goto skip_field; } savings = dfield_get_len(dfield) - local_len; /* Check that there would be savings */ if (longest >= savings) { goto skip_field; } /* In DYNAMIC and COMPRESSED format, store locally any non-BLOB columns whose maximum length does not exceed 256 bytes. This is because there is no room for the "external storage" flag when the maximum length is 255 bytes or less. This restriction trivially holds in REDUNDANT and COMPACT format, because there we always store locally columns whose length is up to local_len == 788 bytes. @see rec_init_offsets_comp_ordinary */ if (!DATA_BIG_COL(ifield->col)) { goto skip_field; } longest_i = i; longest = savings; skip_field: continue; } if (!longest) { /* Cannot shorten more */ mem_heap_free(heap); return(NULL); } /* Move data from field longest_i to big rec vector. We store the first bytes locally to the record. Then we can calculate all ordering fields in all indexes from locally stored data. */ dfield = dtuple_get_nth_field(entry, longest_i); ifield = dict_index_get_nth_field(index, longest_i); local_prefix_len = local_len - BTR_EXTERN_FIELD_REF_SIZE; vector->append( big_rec_field_t( longest_i, dfield_get_len(dfield) - local_prefix_len, static_cast(dfield_get_data(dfield)) + local_prefix_len)); /* Allocate the locally stored part of the column. */ data = static_cast(mem_heap_alloc(heap, local_len)); /* Copy the local prefix. */ memcpy(data, dfield_get_data(dfield), local_prefix_len); /* Clear the extern field reference (BLOB pointer). */ memset(data + local_prefix_len, 0, BTR_EXTERN_FIELD_REF_SIZE); #if 0 /* The following would fail the Valgrind checks in page_cur_insert_rec_low() and page_cur_insert_rec_zip(). The BLOB pointers in the record will be initialized after the record and the BLOBs have been written. */ UNIV_MEM_ALLOC(data + local_prefix_len, BTR_EXTERN_FIELD_REF_SIZE); #endif dfield_set_data(dfield, data, local_len); dfield_set_ext(dfield); n_fields++; (*n_ext)++; ut_ad(n_fields < dtuple_get_n_fields(entry)); if (upd && !upd->is_modified(longest_i)) { DEBUG_SYNC_C("ib_mv_nonupdated_column_offpage"); upd_field_t upd_field; upd_field.field_no = unsigned(longest_i); upd_field.orig_len = 0; upd_field.exp = NULL; upd_field.old_v_val = NULL; dfield_copy(&upd_field.new_val, dfield->clone(upd->heap)); upd->append(upd_field); ut_ad(upd->is_modified(longest_i)); ut_ad(upd_field.new_val.len >= BTR_EXTERN_FIELD_REF_SIZE); ut_ad(upd_field.new_val.len == local_len); ut_ad(upd_field.new_val.len == dfield_get_len(dfield)); } } ut_ad(n_fields == vector->n_fields); return(vector); } /**************************************************************//** Puts back to entry the data stored in vector. Note that to ensure the fields in entry can accommodate the data, vector must have been created from entry with dtuple_convert_big_rec. */ void dtuple_convert_back_big_rec( /*========================*/ dict_index_t* index MY_ATTRIBUTE((unused)), /*!< in: index */ dtuple_t* entry, /*!< in: entry whose data was put to vector */ big_rec_t* vector) /*!< in, own: big rec vector; it is freed in this function */ { big_rec_field_t* b = vector->fields; const big_rec_field_t* const end = b + vector->n_fields; for (; b < end; b++) { dfield_t* dfield; ulint local_len; dfield = dtuple_get_nth_field(entry, b->field_no); local_len = dfield_get_len(dfield); ut_ad(dfield_is_ext(dfield)); ut_ad(local_len >= BTR_EXTERN_FIELD_REF_SIZE); local_len -= BTR_EXTERN_FIELD_REF_SIZE; /* Only in REDUNDANT and COMPACT format, we store up to DICT_ANTELOPE_MAX_INDEX_COL_LEN (768) bytes locally */ ut_ad(local_len <= DICT_ANTELOPE_MAX_INDEX_COL_LEN); dfield_set_data(dfield, (char*) b->data - local_len, b->len + local_len); } mem_heap_free(vector->heap); } /** Allocate a big_rec_t object in the given memory heap, and for storing n_fld number of fields. @param[in] heap memory heap in which this object is allocated @param[in] n_fld maximum number of fields that can be stored in this object @return the allocated object */ big_rec_t* big_rec_t::alloc( mem_heap_t* heap, ulint n_fld) { big_rec_t* rec = static_cast( mem_heap_alloc(heap, sizeof(big_rec_t))); new(rec) big_rec_t(n_fld); rec->heap = heap; rec->fields = static_cast( mem_heap_alloc(heap, n_fld * sizeof(big_rec_field_t))); rec->n_fields = 0; return(rec); } /** Create a deep copy of this object @param[in] heap the memory heap in which the clone will be created. @return the cloned object. */ dfield_t* dfield_t::clone( mem_heap_t* heap) { const ulint size = len == UNIV_SQL_NULL ? 0 : len; dfield_t* obj = static_cast( mem_heap_alloc(heap, sizeof(dfield_t) + size)); obj->ext = ext; obj->len = len; obj->type = type; obj->spatial_status = spatial_status; if (len != UNIV_SQL_NULL) { obj->data = obj + 1; memcpy(obj->data, data, len); } else { obj->data = 0; } return(obj); }