From 5d48ea7d07b481ae3930486b4b039e1454273190 Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Fri, 27 Jul 2018 22:55:18 +0300 Subject: MDEV-10963 Fragmented BINLOG query The problem was originally stated in http://bugs.mysql.com/bug.php?id=82212 The size of an base64-encoded Rows_log_event exceeds its vanilla byte representation in 4/3 times. When a binlogged event size is about 1GB mysqlbinlog generates a BINLOG query that can't be send out due to its size. It is fixed with fragmenting the BINLOG argument C-string into (approximate) halves when the base64 encoded event is over 1GB size. The mysqlbinlog in such case puts out SET @binlog_fragment_0='base64-encoded-fragment_0'; SET @binlog_fragment_1='base64-encoded-fragment_1'; BINLOG @binlog_fragment_0, @binlog_fragment_1; to represent a big BINLOG. For prompt memory release BINLOG handler is made to reset the BINLOG argument user variables in the middle of processing, as if @binlog_fragment_{0,1} = NULL is assigned. Notice the 2 fragments are enough, though the client and server still may need to tweak their @@max_allowed_packet to satisfy to the fragment size (which they would have to do anyway with greater number of fragments, should that be desired). On the lower level the following changes are made: Log_event::print_base64() remains to call encoder and store the encoded data into a cache but now *without* doing any formatting. The latter is left for time when the cache is copied to an output file (e.g mysqlbinlog output). No formatting behavior is also reflected by the change in the meaning of the last argument which specifies whether to cache the encoded data. Rows_log_event::print_helper() is made to invoke a specialized fragmented cache-to-file copying function which is copy_cache_to_file_wrapped() that takes care of fragmenting also optionally wraps encoded strings (fragments) into SQL stanzas. my_b_copy_to_file() is refactored to into my_b_copy_all_to_file(). The former function is generalized to accepts more a limit argument to constraint the copying and does not reinitialize anymore the cache into reading mode. The limit does not do any effect on the fully read cache. --- mysys/mf_iocache2.c | 61 +++++++++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 28 deletions(-) (limited to 'mysys') diff --git a/mysys/mf_iocache2.c b/mysys/mf_iocache2.c index fa3b6e672d7..f3877331664 100644 --- a/mysys/mf_iocache2.c +++ b/mysys/mf_iocache2.c @@ -23,51 +23,56 @@ #include #include -/* - Copy contents of an IO_CACHE to a file. - - SYNOPSIS - my_b_copy_to_file() - cache IO_CACHE to copy from - file File to copy to - - DESCRIPTION - Copy the contents of the cache to the file. The cache will be - re-inited to a read cache and will read from the beginning of the - cache. - - If a failure to write fully occurs, the cache is only copied - partially. +/** + Copy the cache to the file. Copying can be constrained to @c count + number of bytes when the parameter is less than SIZE_T_MAX. The + cache will be optionally re-inited to a read cache and will read + from the beginning of the cache. If a failure to write fully + occurs, the cache is only copied partially. TODO - Make this function solid by handling partial reads from the cache - in a correct manner: it should be atomic. - - RETURN VALUE - 0 All OK - 1 An error occurred + Make this function solid by handling partial reads from the cache + in a correct manner: it should be atomic. + + @param cache IO_CACHE to copy from + @param file File to copy to + @param count the copied size or the max of the type + when the whole cache is to be copied. + @return + 0 All OK + 1 An error occurred */ int -my_b_copy_to_file(IO_CACHE *cache, FILE *file) +my_b_copy_to_file(IO_CACHE *cache, FILE *file, + size_t count) { - size_t bytes_in_cache; + size_t curr_write, bytes_in_cache; DBUG_ENTER("my_b_copy_to_file"); - /* Reinit the cache to read from the beginning of the cache */ - if (reinit_io_cache(cache, READ_CACHE, 0L, FALSE, FALSE)) - DBUG_RETURN(1); bytes_in_cache= my_b_bytes_in_cache(cache); do { - if (my_fwrite(file, cache->read_pos, bytes_in_cache, + curr_write= MY_MIN(bytes_in_cache, count); + if (my_fwrite(file, cache->read_pos, curr_write, MYF(MY_WME | MY_NABP)) == (size_t) -1) DBUG_RETURN(1); - } while ((bytes_in_cache= my_b_fill(cache))); + + cache->read_pos += curr_write; + count -= curr_write; + } while (count && (bytes_in_cache= my_b_fill(cache))); if(cache->error == -1) DBUG_RETURN(1); DBUG_RETURN(0); } +int my_b_copy_all_to_file(IO_CACHE *cache, FILE *file) +{ + DBUG_ENTER("my_b_copy_all_to_file"); + /* Reinit the cache to read from the beginning of the cache */ + if (reinit_io_cache(cache, READ_CACHE, 0L, FALSE, FALSE)) + DBUG_RETURN(1); + DBUG_RETURN(my_b_copy_to_file(cache, file, SIZE_T_MAX)); +} my_off_t my_b_append_tell(IO_CACHE* info) { -- cgit v1.2.1 From 94e6a226a3145c3cc0871ea5cdc2f8c55ff313ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 1 Feb 2019 09:14:10 +0200 Subject: my_malloc(): Invoke TRASH_ALLOC even WITH_SAFEMALLOC=OFF --- mysys/my_malloc.c | 1 + 1 file changed, 1 insertion(+) (limited to 'mysys') diff --git a/mysys/my_malloc.c b/mysys/my_malloc.c index 719c13a040e..99a15e6f432 100644 --- a/mysys/my_malloc.c +++ b/mysys/my_malloc.c @@ -117,6 +117,7 @@ void *my_malloc(size_t size, myf my_flags) MY_TEST(my_flags & MY_THREAD_SPECIFIC)); update_malloc_size(size + MALLOC_PREFIX_SIZE, MY_TEST(my_flags & MY_THREAD_SPECIFIC)); + TRASH_ALLOC(point, size); DBUG_EXECUTE_IF("simulate_out_of_memory", { /* my_free() handles memory accounting */ -- cgit v1.2.1