From 69dd773b67a9892561f148964e51d3be606f7c77 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Mon, 13 Sep 2010 15:22:11 -0700 Subject: An implementation of index intersect via a modified Unique class. This code is planned to be used for mwl#21. --- mysys/tree.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'mysys') diff --git a/mysys/tree.c b/mysys/tree.c index e4854581204..70f2972b03f 100644 --- a/mysys/tree.c +++ b/mysys/tree.c @@ -221,6 +221,8 @@ TREE_ELEMENT *tree_insert(TREE *tree, void *key, uint key_size, } if (element == &tree->null_element) { + if (tree->flag & TREE_ONLY_DUPS) + return((TREE_ELEMENT *) 1); uint alloc_size=sizeof(TREE_ELEMENT)+key_size+tree->size_of_element; tree->allocated+=alloc_size; -- cgit v1.2.1 From 9d68ccdeb8692b05d9cc0479d8d3757edbc49368 Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Sun, 7 Nov 2010 14:25:29 +0200 Subject: Added option BACKUP_ALL to mysqld --myisam-recover to also get a backup of the index file before it's repaired. Removed wrong call to translog_buffer_unlock() that caused 'unlocking not locked mutex' failure in Aria log handler. extra/replace.c: Updated call to my_redel() include/maria.h: Updated prototype for maria_change_to_newfile include/my_sys.h: Updated prototypes Added my_create_backup_name include/myisam.h: Updated prototypes include/myisamchk.h: Added 'backup_time' to st_handler_check_param to be able to generate same name for backuped data and index file mysys/my_redel.c: Added time_t option to my_redel() to be able to generate same backup extensions for many files sql/handler.cc: Added start_time to st_ha_check_opt sql/handler.h: Added start_time to HA_CHECK_OPT sql/mysqld.cc: Added option BACKUP_ALL to --myisam-recover storage/maria/ha_maria.cc: Remember start time for repair storage/maria/ma_check.c: Use remembered start time for backup file names Removed some dead code storage/maria/ma_loghandler.c: Removed wrong call to translog_buffer_unlock() that caused 'unlocking not locked mutex' failure in log handler. storage/maria/maria_chk.c: Removed dead code (O_NEW_INDEX was never set) Report if table was 'crashed on repair' storage/maria/maria_pack.c: Updated parameters to my_redel() storage/myisam/ha_myisam.cc: Added recover option BACKUP_ALL to get a backup of the index file before it's repaired. Print information to log if we make a backup of data as part of repair. storage/myisam/ha_myisam.h: Added HA_RECOVER_FULL_BACKUP storage/myisam/mi_check.c: Use remembered start time for backup file names Added mi_make_backup_of_index() storage/myisam/myisamchk.c: Removed dead code (O_NEW_INDEX was never set) Report if table was 'crashed on repair' storage/myisam/myisampack.c: Updated call to my_redel() --- mysys/my_redel.c | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) (limited to 'mysys') diff --git a/mysys/my_redel.c b/mysys/my_redel.c index cf0986a7821..2de989a2854 100644 --- a/mysys/my_redel.c +++ b/mysys/my_redel.c @@ -40,7 +40,8 @@ struct utimbuf { #define REDEL_EXT ".BAK" -int my_redel(const char *org_name, const char *tmp_name, myf MyFlags) +int my_redel(const char *org_name, const char *tmp_name, + time_t backup_time_stamp, myf MyFlags) { int error=1; DBUG_ENTER("my_redel"); @@ -51,13 +52,9 @@ int my_redel(const char *org_name, const char *tmp_name, myf MyFlags) goto end; if (MyFlags & MY_REDEL_MAKE_BACKUP) { - char name_buff[FN_REFLEN+20]; - char ext[20]; - ext[0]='-'; - get_date(ext+1,2+4,(time_t) 0); - strmov(strend(ext),REDEL_EXT); - if (my_rename(org_name, fn_format(name_buff, org_name, "", ext, 2), - MyFlags)) + char name_buff[FN_REFLEN + MY_BACKUP_NAME_EXTRA_LENGTH]; + my_create_backup_name(name_buff, org_name, backup_time_stamp); + if (my_rename(org_name, name_buff, MyFlags)) goto end; } else if (my_delete_allow_opened(org_name, MyFlags)) @@ -149,3 +146,23 @@ int my_copystat(const char *from, const char *to, int MyFlags) #endif return 0; } /* my_copystat */ + + +/** + Create a backup file name. + @fn my_create_backup_name() + @param to Store new file name here + @param from Original name + + @info + The backup name is made by adding -YYMMDDHHMMSS.BAK to the file name +*/ + +void my_create_backup_name(char *to, const char *from, time_t backup_start) +{ + char ext[MY_BACKUP_NAME_EXTRA_LENGTH+1]; + ext[0]='-'; + get_date(ext+1, GETDATE_SHORT_DATE | GETDATE_HHMMSSTIME, backup_start); + strmov(strend(ext),REDEL_EXT); + strmov(strmov(to, from), ext); +} -- cgit v1.2.1 From 7b047a31a09a7999353232cb6407b4e6338d5b70 Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Mon, 8 Nov 2010 13:43:54 +0200 Subject: Make SQLString reallocation addaptive Avoid doing reallocs Prealloc some strings / provide extension allocation size to some strings This gave a 25 % speedup in some mysql-test-run tests. mysys/safemalloc.c: More DBUG_PRINT sql/net_serv.cc: Make all mallocs() look the similar. (just-for-safety fix) sql/protocol.cc: Ensure that communication packet buffer is allocated. (It's freed by stored precedures and some DLL statements) sql/sp.cc: Fixed valgrind warning sql/sql_select.cc: Set extent allocation for buffer that has a lot of append() calls. sql/sql_show.cc: Fixed wrong usage of string buffer. Old code worked in test suite 'just-by-chance' sql/sql_string.cc: Call realloc_with_extra_if_needed() in append() functions. sql/sql_string.h: Added 'extra_alloc' member, to specify chunck size for realloc(). extra_alloc is addaptive to catch cases where preallocation of buffers is not done properly. Simplified free() to allow compiler to optimize things better (and to keep things consistent). Fixed shrink() to take into account the extra memory added to the Alloced_length in realloc(). This saves us a realloc() per query. sql/sql_test.cc: Set extent allocation for buffer that has a lot of append() calls. sql/table.cc: Set extent allocation for buffer that has a lot of append() calls. --- mysys/safemalloc.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'mysys') diff --git a/mysys/safemalloc.c b/mysys/safemalloc.c index 0d0ae2dabe5..0e489263c69 100644 --- a/mysys/safemalloc.c +++ b/mysys/safemalloc.c @@ -224,6 +224,8 @@ void *_myrealloc(register void *ptr, register size_t size, struct st_irem *irem; char *data; DBUG_ENTER("_myrealloc"); + DBUG_PRINT("my",("ptr: 0x%lx size: %lu my_flags: %d", (long) ptr, + (ulong) size, MyFlags)); if (!ptr && (MyFlags & MY_ALLOW_ZERO_PTR)) DBUG_RETURN(_mymalloc(size, filename, lineno, MyFlags)); @@ -245,6 +247,8 @@ void *_myrealloc(register void *ptr, register size_t size, (void) fflush(stderr); DBUG_RETURN((uchar*) NULL); } + DBUG_PRINT("my", ("old_size: %lu -> new_size: %lu", + (ulong) irem->datasize, (ulong) size)); if ((data= _mymalloc(size,filename,lineno,MyFlags))) /* Allocate new area */ { -- cgit v1.2.1 From 0de9a4ababac1e7f510558dfd0bece894cf7e69c Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 17 Nov 2010 13:24:20 +0100 Subject: MWL#74: Shared libmysqld.so library. Switch makefiles to use libtool to build libmysqld.so, as well as all its dependencies. The previous MYSQL_PLUGIN_DEPENDS_ON_MYSQL_INTERNALS() declaration is removed, as it does not work well with a libtool build. Instead, plugins that need it can specify an alternate object in MYSQL_PLUGIN_STATIC() that will be used for embedded library. The plugin must then take care itself of compiling the special object for embedded, rebuilding the source files previously listed in MYSQL_PLUGIN_DEPENDS_ON_MYSQL_INTERNALS() with @plugin_embedded_defs@ in CFLAGS/CXXFLAGS. The extra target @XXX_embedded_static_target@ is available for the special object, this will be empty when --without-embedded-server. All in-tree plugins are changed to build their static targets with libtool. Additional plugins that want to work with libmysqld.so will need to be similarly modified to build with libtool (or otherwise provide an -fPIC object). Dynamically loaded plugins are not affected. The old libraries like libmysys.a, libmyisam.a and similar libraries, which were installed by `make install` though this is of little use, are still built and installed to not break package scripts etc. that expect them. These libraries are kept static to avoid introducing new .so dependencies. The patch also fixes a handfull of duplicate symbol linker errors, where we included some object twice during linking; these for one reason or another did not produce errors before but caused problems on some platforms with this patch (eg. Mac OS X linker is more strict for shared objects). This patch only does what is necessary to build libmysqld.so. There are some more cleanups that are possible now that we are using libtool more fully, which could done in subsequent patches (though we may not bother as we are switching from autotools to CMake anyway): - In libmysql_r/, we should be able to just link libmysys.la etc, instead of symlinking and re-compiling sources into the directory. - In libmysql/, we can similarly avoid symlinking and recompiling sources if we instead build a libmysys_nothread.la library with appropriate CFLAGS and link that. - In sql/, we can build a separate target libmysql_int.la with appropriate CFLAGS for embedded and use that in libmysqld/ instead of symlinking sources. - libmysys.a, libmyisam.a and similar libraries could be installed as .so also to save on code size; or alternatively could be not installed at all. client/Makefile.am: Updated for using libtool config/ac-macros/plugins.m4: Replace MUSQL_PLUGIN_DEPENDS_ON_MYSQL_INTERNALS with mechanism for plugins to specify alternate object for embedded. configure.in: Fix linking duplicate objects related to THREAD_LOBJECTS. dbug/Makefile.am: Updated for using libtool extra/Makefile.am: Fix relative paths. libmysqld/Makefile.am: Build libmysqld.la using libtool libmysqld/examples/Makefile.am: Updated to use libtool mysys/Makefile.am: Updated to use libtool. Fix linking duplicate objects related to THREAD_LOBJECTS. mysys/my_uuid.c: Fix conflicting global mutex name by making it static. regex/Makefile.am: Updated to use libtool sql/Makefile.am: Updated to use libtool sql/item_func.cc: Fix conflicting mutex name. sql/mysql_priv.h: Fix conflicting mutex name sql/mysqld.cc: Fix conflicting mutex name. Add missing call of my_uuid_end(). storage/archive/Makefile.am: Updated to use libtool storage/archive/plug.in: Updated to use libtool storage/blackhole/Makefile.am: Updated to use libtool storage/blackhole/plug.in: Updated to use libtool storage/csv/Makefile.am: Updated to use libtool storage/csv/plug.in: Updated to use libtool storage/example/Makefile.am: Updated to use libtool storage/federated/Makefile.am: Updated to use libtool storage/federated/plug.in: Updated to use libtool storage/federatedx/Makefile.am: Updated to use libtool storage/federatedx/plug.in: Updated to use libtool storage/heap/Makefile.am: Updated to use libtool storage/heap/plug.in: Updated to use libtool storage/innobase/Makefile.am: Updated to use libtool storage/innobase/plug.in.disabled: Updated to use libtool storage/innodb_plugin/Makefile.am: Updated to use libtool storage/maria/CMakeLists.txt: Fix linking duplicate object in maria_dump_log, causes failure on Mac OS X storage/maria/Makefile.am: Updated to use libtool Fix linking duplicate object in maria_dump_log, causes link failure on Mac OS X storage/maria/ma_loghandler.c: Move maria_dump_log code to separate file to fix duplicate object link failures. storage/maria/ma_loghandler.h: Move maria_dump_log code to separate file to fix duplicate object link failures. storage/maria/maria_dump_log.c: Move maria_dump_log code to separate file to fix duplicate object link failures. storage/maria/plug.in: Updated to use libtool storage/myisam/Makefile.am: Updated to use libtool storage/myisam/plug.in: Updated to use libtool storage/myisammrg/Makefile.am: Updated to use libtool storage/myisammrg/plug.in: Updated to use libtool storage/pbxt/plug.in: Updated to use libtool storage/pbxt/src/Makefile.am: Updated to use libtool storage/xtradb/Makefile.am: Updated to use libtool storage/xtradb/plug.in: Updated to use libtool strings/Makefile.am: Updated to use libtool unittest/unit.pl: Don't attempt to run libtool internal files as unit tests. vio/Makefile.am: Updated to use libtool --- mysys/Makefile.am | 16 +++++++++++----- mysys/my_uuid.c | 2 +- 2 files changed, 12 insertions(+), 6 deletions(-) (limited to 'mysys') diff --git a/mysys/Makefile.am b/mysys/Makefile.am index dc0b2bfb84b..7d1c1ff062c 100644 --- a/mysys/Makefile.am +++ b/mysys/Makefile.am @@ -19,10 +19,11 @@ MYSQLBASEdir= $(prefix) INCLUDES = @ZLIB_INCLUDES@ -I$(top_builddir)/include \ -I$(top_srcdir)/include -I$(srcdir) pkglib_LIBRARIES = libmysys.a +noinst_LTLIBRARIES = libmysys.la LDADD = libmysys.a $(top_builddir)/strings/libmystrings.a $(top_builddir)/dbug/libdbug.a noinst_HEADERS = mysys_priv.h my_static.h my_handler_errors.h \ my_safehash.h -libmysys_a_SOURCES = my_init.c my_getwd.c mf_getdate.c my_mmap.c \ +libmysys_la_SOURCES = my_init.c my_getwd.c mf_getdate.c my_mmap.c \ mf_path.c mf_loadpath.c my_file.c \ my_open.c my_create.c my_dup.c my_seek.c my_read.c \ my_pread.c my_write.c my_getpagesize.c \ @@ -57,19 +58,22 @@ libmysys_a_SOURCES = my_init.c my_getwd.c mf_getdate.c my_mmap.c \ my_memmem.c stacktrace.c \ my_windac.c my_access.c base64.c my_libwrap.c \ wqueue.c +libmysys_la_LDFLAGS = $(AM_LDFLAGS) @WRAPLIBS@ +libmysys_la_LIBADD = $(ZLIB_LIBS) if NEED_THREAD # mf_keycache is used only in the server, so it is safe to leave the file # out of the non-threaded library. # In fact, it will currently not compile without thread support. -libmysys_a_SOURCES += mf_keycache.c mf_keycaches.c +libmysys_la_SOURCES += thr_alarm.c thr_lock.c thr_mutex.c thr_rwlock.c \ + my_pthread.c my_thr_init.c waiting_threads.c \ + mf_keycache.c mf_keycaches.c endif EXTRA_DIST = thr_alarm.c thr_lock.c my_pthread.c my_thr_init.c \ thr_mutex.c thr_rwlock.c waiting_threads.c \ CMakeLists.txt mf_soundex.c \ my_conio.c my_wincond.c my_winthread.c -libmysys_a_LIBADD = @THREAD_LOBJECTS@ # test_dir_DEPENDENCIES= $(LIBRARIES) # testhash_DEPENDENCIES= $(LIBRARIES) # test_charset_DEPENDENCIES= $(LIBRARIES) @@ -83,8 +87,6 @@ DEFS = -DDEFAULT_BASEDIR=\"$(prefix)\" \ -DDEFAULT_SYSCONFDIR="\"$(sysconfdir)\"" \ @DEFS@ -libmysys_a_DEPENDENCIES= @THREAD_LOBJECTS@ - # I hope this always does the right thing. Otherwise this is only test programs FLAGS=$(DEFS) $(INCLUDES) $(CPPFLAGS) $(CFLAGS) @NOINST_LDFLAGS@ @@ -95,6 +97,10 @@ CLEANFILES = test_bitmap$(EXEEXT) test_priority_queue$(EXEEXT) \ testhash$(EXEEXT) test_gethwaddr$(EXEEXT) \ test_base64$(EXEEXT) test_thr_mutex$(EXEEXT) +libmysys_a_SOURCES= +libmysys.a: libmysys.la + $(CP) .libs/libmysys.a $@ + # # The CP .. RM stuff is to avoid problems with some compilers (like alpha ccc) # which automaticly removes the object files you use to compile a final program diff --git a/mysys/my_uuid.c b/mysys/my_uuid.c index f115806b4e9..e2372391b06 100644 --- a/mysys/my_uuid.c +++ b/mysys/my_uuid.c @@ -50,7 +50,7 @@ static ulonglong uuid_time= 0; static uchar uuid_suffix[2+6]; /* clock_seq and node */ #ifdef THREAD -pthread_mutex_t LOCK_uuid_generator; +static pthread_mutex_t LOCK_uuid_generator; #endif /* -- cgit v1.2.1 From d54f869f8c34e5bb93ab187cfc6495f9e192c259 Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Tue, 30 Nov 2010 01:14:34 +0200 Subject: Fixed some compiler warnings configure.in: Added comment mysql-test/suite/innodb_plugin/t/innodb_bug56680.test: Disable test when run with valgrind as we get errors from buf_buddy_relocate() on work for this test. (Should probably be investigated as this may be an issue in xtradb, but probably harmless) Work is an amd-64 running openSUSE 1.11 and valgrind 3.4.1 mysys/charset.c: Remove static function if not used (to remove compiler warning) storage/xtradb/srv/srv0srv.c: Added casts to get rid of compiler warnings --- mysys/charset.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'mysys') diff --git a/mysys/charset.c b/mysys/charset.c index 8d2f68b7bbf..f751e59de82 100644 --- a/mysys/charset.c +++ b/mysys/charset.c @@ -202,6 +202,7 @@ static my_bool simple_cs_is_full(CHARSET_INFO *cs) } +#if defined(HAVE_UCA_COLLATIONS) && (defined(HAVE_CHARSET_ucs2) || defined(HAVE_CHARSET_utf8)) static void copy_uca_collation(CHARSET_INFO *to, CHARSET_INFO *from) { @@ -215,6 +216,7 @@ copy_uca_collation(CHARSET_INFO *to, CHARSET_INFO *from) to->state|= MY_CS_AVAILABLE | MY_CS_LOADED | MY_CS_STRNXFRM | MY_CS_UNICODE; } +#endif static int add_collation(CHARSET_INFO *cs) -- cgit v1.2.1 From 0eae06c43007d7e59275af2d437b3c19537bab46 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 6 Dec 2010 12:52:06 +0100 Subject: Bug#473914: mysql_client_test fail with in debug compilaton on windows x64 Reason: inconsistent compilation, federatedx is compiled without SAFEMALLOC flag, while anything else is compiled with SAFEMALLOC. As a consequence, my_hash_init used inside federatedx initialization does not provide correct caller info parameters (file, line) , so they are initialized with whatever is on stack. When info about allocated memory is output in COM_DEBUG command, the server crashes trying to output string starting at 0xcccccccccccccccc. The fix is to remove SAFEMALLOC preprocessor flags from every CMakeLists.txt, except the top-level one. Also, SAFEMALLOC is not defined by default now, instead there is WITH_DEBUG_FULL CMake option which adds -DSAFEMALLOC to C and C++ flags in debug compilation. This option is off by default, because 1) Debug C runtime already has heap debugging builtin with overwrite and leak detection 2)safemalloc considerably slows down the tests. Note also that - SAFEMALLOC is gone in MySQL5.5 - On Windows, heap related overflows can also be found using free pageheap utility (that is also part of application verifier). This is even more efficient if there are no other layers on top of Windows heap allocator, e.g it is most efficient with release version. --- mysys/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) (limited to 'mysys') diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index bf0c6551a11..7cbd34ef1af 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -13,7 +13,6 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DSAFEMALLOC -DSAFE_MUTEX") # Only the server link with this library, the client libraries and the client # executables all link with recompiles of source found in the "mysys" directory. -- cgit v1.2.1 From 84edaac4f14b2fbaf57fef8fbaeb45966b1feda3 Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Fri, 10 Dec 2010 17:04:09 +0200 Subject: Better warning message if lock test fails Made archive.test a bit more safe mysql-test/r/archive.result: Added removal of files to make rerun of failed test work mysql-test/t/archive.test: Added removal of files to make rerun of failed test work mysys/thr_lock.c: Better warning message if lock test fails --- mysys/thr_lock.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'mysys') diff --git a/mysys/thr_lock.c b/mysys/thr_lock.c index 341b2f0058e..60214c9af08 100644 --- a/mysys/thr_lock.c +++ b/mysys/thr_lock.c @@ -158,15 +158,13 @@ static int check_lock(struct st_lock_list *list, const char* lock_type, { THR_LOCK_DATA *data,**prev; uint count=0; - THR_LOCK_OWNER *UNINIT_VAR(first_owner); prev= &list->data; if (list->data) { - enum thr_lock_type last_lock_type=list->data->type; + enum thr_lock_type last_lock_type= list->data->type; + THR_LOCK_OWNER first_owner= list->data->owner; - if (same_owner && list->data) - first_owner= list->data->owner; for (data=list->data; data && count++ < MAX_LOCKS ; data=data->next) { if (data->type != last_lock_type) @@ -184,8 +182,8 @@ static int check_lock(struct st_lock_list *list, const char* lock_type, last_lock_type != TL_WRITE_CONCURRENT_INSERT) { fprintf(stderr, - "Warning: Found locks from different threads in %s: %s\n", - lock_type,where); + "Warning: Found locks from different threads in %s at '%s'. org_lock_type: %d last_lock_type: %d new_lock_type: %d\n", + lock_type, where, list->data->type, last_lock_type, data->type); return 1; } if (no_cond && data->cond) -- cgit v1.2.1 From 0b20943e9d7e22332ccc14b30b49a78a780cc23d Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Mon, 13 Dec 2010 15:27:13 +0200 Subject: Fixed typo that caused compile failure in thr_lock.c mysys/thr_lock.c: Fixed typo that caused compile failure --- mysys/thr_lock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mysys') diff --git a/mysys/thr_lock.c b/mysys/thr_lock.c index 60214c9af08..68a3caf6da7 100644 --- a/mysys/thr_lock.c +++ b/mysys/thr_lock.c @@ -163,7 +163,7 @@ static int check_lock(struct st_lock_list *list, const char* lock_type, if (list->data) { enum thr_lock_type last_lock_type= list->data->type; - THR_LOCK_OWNER first_owner= list->data->owner; + THR_LOCK_OWNER *first_owner= list->data->owner; for (data=list->data; data && count++ < MAX_LOCKS ; data=data->next) { -- cgit v1.2.1 From e63b5546c597f65696868eaf69159107bc4a8e44 Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Tue, 4 Jan 2011 00:55:41 +0200 Subject: Implementation of MWL#172: Add support for prepared statements to HANDLER READ It includes speed optimizations for HANDLER READ by caching as much as possible in HANDLER OPEN Other things: - Added mysqld option --disable-thr-alarm to be able to benchmark things without thr_alarm - Changed 'Locked' state to 'System lock' and 'Table lock' (these where used in the code but never shown to end user) - Better error message if mysql_install_db.sh fails - Moved handler function prototypes to sql_handler.h - Remove not anymore used 'thd->locked' member include/thr_alarm.h: Added my_disable_thr_alarm include/thr_lock.h: Add new member to THR_LOCK_DATA to remember original lock type state. This is needed as thr_unlock() resets type to TL_UNLOCK. mysql-test/include/check_no_concurrent_insert.inc: Locked -> Table lock mysql-test/include/handler.inc: Locked -> Table lock mysql-test/r/handler_innodb.result: Updated results for new tests mysql-test/r/handler_myisam.result: Updated results for new tests mysql-test/r/sp-threads.result: Locked -> Table lock mysql-test/suite/binlog/t/binlog_stm_row.test: Locked -> Table lock mysql-test/suite/funcs_1/datadict/processlist_val.inc: Locked -> Table lock mysql-test/suite/pbxt/t/lock_multi.test: Locked -> Table lock mysql-test/suite/sys_vars/r/concurrent_insert_func.result: Locked -> Table lock mysql-test/suite/sys_vars/t/concurrent_insert_func.test: Locked -> Table lock mysql-test/suite/sys_vars/t/delayed_insert_limit_func.test: Locked -> Table lock mysql-test/suite/sys_vars/t/query_cache_wlock_invalidate_func.test: Locked -> Table lock mysql-test/suite/sys_vars/t/sql_low_priority_updates_func.test: Locked -> Table lock mysql-test/t/insert_notembedded.test: Locked -> Table lock mysql-test/t/lock_multi.test: Locked -> Table lock mysql-test/t/merge-big.test: Locked -> Table lock mysql-test/t/multi_update.test: Locked -> Table lock mysql-test/t/query_cache_28249.test: Locked -> Table lock mysql-test/t/sp_notembedded.test: Locked -> Table lock mysql-test/t/sp_sync.test: Locked -> Table lock mysql-test/t/status.test: Locked -> Table lock mysql-test/t/trigger_notembedded.test: Locked -> Table lock mysys/thr_alarm.c: Added option to disable thr_alarm mysys/thr_lock.c: Detect loops scripts/mysql_install_db.sh: Give better error message if something goes wrong sql/Makefile.am: Added sql_handler.h sql/lock.cc: Split functions to allow one to cache value if store_lock() (for HANDLER functions). - Split mysql_lock_tables() into two functions, where first one allocates MYSQL_LOCK and other other one uses it. - Made get_lock_data() an external function. - Added argument to mysql_unlock_tables() to not free sql_lock. - Added argument to reset_lock_data() to reset lock structure to initial state (as after get_lock_data()) sql/mysql_priv.h: Moved handler function prototypes to sql_handler.h Added new lock functions. sql/mysqld.cc: Added --thread-alarm startup option sql/net_serv.cc: Don't call vio_blocking() if not needed sql/sql_base.cc: include sql_handler.h sql/sql_class.cc: include sql_handler.h Remove not anymore used 'thd->locked' member sql/sql_class.h: Remove not anymore used 'thd->locked' member sql/sql_db.cc: include sql_handler.h sql/sql_delete.cc: include sql_handler.h sql/sql_handler.cc: Rewrote all code to use SQL_HANDLER instead of TABLE_LIST (original interface) Rewrote mysql_ha_open() to cache all things from TABLE_LIST and items for field list, where etc. In mysql_ha_open() also cache MYSQL_LOCK structure from get_lock_data(). Split functions into smaller sub functions (needed to be able to implement mysql_ha_read_prepare()) Added mysql_ha_read_prepare() to allow one to prepare HANDLER READ. sql/sql_handler.h: Interface to sql_handler.cc sql/sql_parse.cc: include sql_handler.h sql/sql_prepare.cc: Added mysql_test_handler_read(), prepare for HANDLER READ sql/sql_rename.cc: include sql_handler.h sql/sql_show.cc: Removed usage of thd->locked sql/sql_table.cc: include sql_handler.h sql/sql_trigger.cc: include sql_handler.h --- mysys/thr_alarm.c | 26 ++++++++++++++++++-------- mysys/thr_lock.c | 11 ++++++++--- 2 files changed, 26 insertions(+), 11 deletions(-) (limited to 'mysys') diff --git a/mysys/thr_alarm.c b/mysys/thr_alarm.c index 15c24facac0..98c746af8ca 100644 --- a/mysys/thr_alarm.c +++ b/mysys/thr_alarm.c @@ -36,7 +36,7 @@ uint thr_client_alarm; static int alarm_aborted=1; /* No alarm thread */ -my_bool thr_alarm_inited= 0; +my_bool thr_alarm_inited= 0, my_disable_thr_alarm= 0; volatile my_bool alarm_thread_running= 0; time_t next_alarm_expire_time= ~ (time_t) 0; static sig_handler process_alarm_part2(int sig); @@ -173,6 +173,21 @@ my_bool thr_alarm(thr_alarm_t *alrm, uint sec, ALARM *alarm_data) DBUG_ENTER("thr_alarm"); DBUG_PRINT("enter",("thread: %s sec: %d",my_thread_name(),sec)); + if (my_disable_thr_alarm) + { + (*alrm)= &alarm_data->alarmed; + alarm_data->alarmed= 1; /* Abort if interrupted */ + DBUG_RETURN(0); + } + + if (unlikely(alarm_aborted)) + { /* No signal thread */ + DBUG_PRINT("info", ("alarm aborted")); + if (alarm_aborted > 0) + goto abort_no_unlock; + sec= 1; /* Abort mode */ + } + now= my_time(0); if (!alarm_data) { @@ -190,13 +205,6 @@ my_bool thr_alarm(thr_alarm_t *alrm, uint sec, ALARM *alarm_data) one_signal_hand_sigmask(SIG_BLOCK,&full_signal_set,&old_mask); pthread_mutex_lock(&LOCK_alarm); /* Lock from threads & alarms */ - if (unlikely(alarm_aborted)) - { /* No signal thread */ - DBUG_PRINT("info", ("alarm aborted")); - if (alarm_aborted > 0) - goto abort; - sec= 1; /* Abort mode */ - } if (alarm_queue.elements >= max_used_alarms) { if (alarm_queue.elements == alarm_queue.max_elements) @@ -251,6 +259,8 @@ void thr_end_alarm(thr_alarm_t *alarmed) #endif DBUG_ENTER("thr_end_alarm"); + if (my_disable_thr_alarm) + DBUG_VOID_RETURN; one_signal_hand_sigmask(SIG_BLOCK,&full_signal_set,&old_mask); alarm_data= (ALARM*) ((uchar*) *alarmed - offsetof(ALARM,alarmed)); pthread_mutex_lock(&LOCK_alarm); diff --git a/mysys/thr_lock.c b/mysys/thr_lock.c index 341b2f0058e..9d1b6f04b7a 100644 --- a/mysys/thr_lock.c +++ b/mysys/thr_lock.c @@ -230,14 +230,16 @@ static void check_locks(THR_LOCK *lock, const char *where, if (found_errors < MAX_FOUND_ERRORS) { - uint count=0; + uint count=0, count2= 0; THR_LOCK_DATA *data; for (data=lock->read.data ; data ; data=data->next) { + count2++; if (data->type == TL_READ_NO_INSERT) count++; /* Protect against infinite loop. */ - DBUG_ASSERT(count <= lock->read_no_write_count); + DBUG_ASSERT(count <= lock->read_no_write_count && + count2 <= MAX_LOCKS); } if (count != lock->read_no_write_count) { @@ -288,7 +290,10 @@ static void check_locks(THR_LOCK *lock, const char *where, if (lock->write.data->type == TL_WRITE_CONCURRENT_INSERT) { THR_LOCK_DATA *data; - for (data=lock->write.data->next ; data ; data=data->next) + uint count= 0; + for (data=lock->write.data->next; + data && count < MAX_LOCKS; + data=data->next) { if (data->type != TL_WRITE_CONCURRENT_INSERT) { -- cgit v1.2.1 From 505c663a1e19af4c8ee726b305691414ab5fc999 Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Mon, 10 Jan 2011 23:22:40 +0200 Subject: - Fixed that Aria works with HANDLER commands - Added test case for Aria - Tested HANDLER with HEAP (changes to HEAP code will be pushed in 5.3) - Moved all HANDLER test to suite/handler. mysql-test/Makefile.am: Added suite/handler mysql-test/mysql-test-run.pl: Added suite/handler mysql-test/r/lock_multi.result: Remove test that is already in handler test suite mysql-test/suite/handler/aria.result: Test for HANDLER with Aria storage engine mysql-test/suite/handler/aria.test: Test for HANDLER with Aria storage engine mysql-test/suite/handler/handler.inc: Extended the general handler test Moved interface testing to 'interface.test' mysql-test/suite/handler/init.inc: Common init for handler tests. mysql-test/suite/handler/innodb.result: New results mysql-test/suite/handler/innodb.test: Update to use new include files mysql-test/suite/handler/interface.result: Test of HANDLER interface (not storage engine dependent parts) mysql-test/suite/handler/interface.test: Test of HANDLER interface (not storage engine dependent parts) mysql-test/suite/handler/myisam.result: New results mysql-test/suite/handler/myisam.test: Update to use new include files mysql-test/t/lock_multi.test: Remove test that is already in handler test suite mysys/tree.c: Added missing handling of read previous (showed up in HEAP testing) sql/handler.cc: Don't marka 'HA_ERR_RECORD_CHANGED' as fatal (can be used with HANDLER READ, especially with MEMORY ENGINE) sql/handler.h: Added prototype for can_continue_handler_scan() sql/sql_handler.cc: Re-initialize search if we switch from key to table search. Check if handler can continue searching between calls (via can_continue_handler_scan()) Don't write common not fatal errors to log storage/maria/ma_extra.c: Don't set index 0 as default. This forces call to ma_check_index() to set up index variables. storage/maria/ma_ft_boolean_search.c: Ensure that info->last_key.keyinfo is set storage/maria/ma_open.c: Don't set index 0 as default. This forces call to ma_check_index() to set up index variables. storage/maria/ma_rkey.c: Trivial optimization storage/maria/ma_rnext.c: Added missing code from mi_rnext.c to ensure that handler next/prev works. storage/maria/ma_rsame.c: Simple optimizations storage/maria/ma_search.c: Initialize info->last_key once and for all when we change keys. storage/maria/ma_unique.c: Ensure that info->last_key.keyinfo is up to date. --- mysys/tree.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'mysys') diff --git a/mysys/tree.c b/mysys/tree.c index e4854581204..68eaecbb27b 100644 --- a/mysys/tree.c +++ b/mysys/tree.c @@ -375,6 +375,7 @@ void *tree_search_key(TREE *tree, const void *key, case HA_READ_KEY_EXACT: case HA_READ_KEY_OR_NEXT: case HA_READ_BEFORE_KEY: + case HA_READ_KEY_OR_PREV: last_equal_element= parents; cmp= 1; break; @@ -418,6 +419,9 @@ void *tree_search_key(TREE *tree, const void *key, case HA_READ_BEFORE_KEY: *last_pos= last_right_step_parent; break; + case HA_READ_KEY_OR_PREV: + *last_pos= last_equal_element ? last_equal_element : last_right_step_parent; + break; default: return NULL; } -- cgit v1.2.1 From 563da78d579f534e139f2e2b5e9c53d65d2d42af Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 14 Jan 2011 11:31:09 +0100 Subject: Removed some old comments. mysys/my_handler.c: Fixed typo --- mysys/my_handler.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mysys') diff --git a/mysys/my_handler.c b/mysys/my_handler.c index efee04194f7..79b0f020a85 100644 --- a/mysys/my_handler.c +++ b/mysys/my_handler.c @@ -659,7 +659,7 @@ void my_handler_error_register(void) /* If you got compilation error here about compile_time_assert array, check that every HA_ERR_xxx constant has a corresponding error message in - handler_error_messages[] list (check mysys/ma_handler_errors.h and + handler_error_messages[] list (check mysys/my_handler_errors.h and include/my_base.h). */ compile_time_assert(HA_ERR_FIRST + array_elements(handler_error_messages) == -- cgit v1.2.1 From 69fe020f01a7eefa9f737291f0da2be56f42a6a0 Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Wed, 26 Jan 2011 15:17:46 +0200 Subject: Fixed bugs found by buildbot: - Use -Wno-uninitialized if -DFORCE_INIT_OF_VARS is not used, to avoid warnings about not initialized variables. - Fixed compiler warnings - Added a name for each thr_lock to get better error messages (This is needed to find out why 'archive.test' sometimes fails) BUILD/SETUP.sh: Use -Wno-uninitialized if -DFORCE_INIT_OF_VARS is not used, to avoid warnings about not initialized variables. BUILD/build_mccge.sh: Use -Wno-uninitialized if -DFORCE_INIT_OF_VARS is not used, to avoid warnings about not initialized variables. client/mysqltest.cc: Fixed bug in remove_files_wildcards (the orignal code never removed anything) extra/libevent/devpoll.c: Fixed compiler warning include/thr_lock.h: Added a name for each thr_lock to get better error messages. mysql-test/suite/maria/t/maria3.test: Speed up test. mysys/thr_lock.c: Added a name for each thr_lock to get better error messages. Added a second 'check_locks' to find if something goes wrong in 'wake_up_waiters'. sql/lock.cc: Added a name for each thr_lock to get better error messages. storage/xtradb/fil/fil0fil.c: Fixed compiler warning --- mysys/thr_lock.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'mysys') diff --git a/mysys/thr_lock.c b/mysys/thr_lock.c index 68a3caf6da7..f7908ab57bc 100644 --- a/mysys/thr_lock.c +++ b/mysys/thr_lock.c @@ -182,8 +182,10 @@ static int check_lock(struct st_lock_list *list, const char* lock_type, last_lock_type != TL_WRITE_CONCURRENT_INSERT) { fprintf(stderr, - "Warning: Found locks from different threads in %s at '%s'. org_lock_type: %d last_lock_type: %d new_lock_type: %d\n", - lock_type, where, list->data->type, last_lock_type, data->type); + "Warning: Found locks from different threads for lock '%s' in '%s' at '%s'. org_lock_type: %d last_lock_type: %d new_lock_type: %d\n", + data->lock->name ? data->lock->name : "", + lock_type, where, list->data->type, last_lock_type, + data->type); return 1; } if (no_cond && data->cond) @@ -405,6 +407,7 @@ void thr_lock_data_init(THR_LOCK *lock,THR_LOCK_DATA *data, void *param) data->status_param=param; data->cond=0; data->priority= 0; + data->debug_print_param= 0; } @@ -879,6 +882,7 @@ void thr_unlock(THR_LOCK_DATA *data, uint unlock_flags) data->type=TL_UNLOCK; /* Mark unlocked */ check_locks(lock,"after releasing lock",1); wake_up_waiters(lock); + check_locks(lock,"end of thr_unlock",1); pthread_mutex_unlock(&lock->mutex); DBUG_VOID_RETURN; } -- cgit v1.2.1 From d3135162946d95e6279bd70a0e16aec19f569115 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 26 Jan 2011 14:16:13 +0100 Subject: fix "./configure --with-debug" builds (without CFLAGS=-DSAFEMALLOC). --- mysys/safemalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mysys') diff --git a/mysys/safemalloc.c b/mysys/safemalloc.c index 0d0ae2dabe5..16489ccc022 100644 --- a/mysys/safemalloc.c +++ b/mysys/safemalloc.c @@ -60,7 +60,7 @@ */ #ifndef SAFEMALLOC -#define SAFEMALLOC /* Get protos from my_sys */ +#define SAFEMALLOC 1 /* Get protos from my_sys */ #endif #include "mysys_priv.h" -- cgit v1.2.1 From b19e99865ccf3e946ebe59bf09add8e0452904ac Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 29 Jan 2011 18:51:12 +0100 Subject: MWL#55 : cherrypick MySQL 5.5 CMake/build improvements in order to be able to build MSI based installer --- mysys/CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) (limited to 'mysys') diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index 7cbd34ef1af..8068318e4c2 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -48,6 +48,4 @@ SET(MYSYS_SOURCES array.c charset-def.c charset.c checksum.c default.c default_ IF(NOT SOURCE_SUBLIBS) ADD_LIBRARY(mysys ${MYSYS_SOURCES}) - - INSTALL(TARGETS mysys DESTINATION lib/opt COMPONENT runtime) # TODO: Component? ENDIF(NOT SOURCE_SUBLIBS) -- cgit v1.2.1 From 7e206872128d891fffa5238c093c8d6506da8dbb Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 1 Feb 2011 14:19:58 +0100 Subject: Fix compile errors: * declaration in the middle of the block in C file. * round() is only available in C99. --- mysys/tree.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'mysys') diff --git a/mysys/tree.c b/mysys/tree.c index 70f2972b03f..c8571fd0337 100644 --- a/mysys/tree.c +++ b/mysys/tree.c @@ -221,9 +221,10 @@ TREE_ELEMENT *tree_insert(TREE *tree, void *key, uint key_size, } if (element == &tree->null_element) { + uint alloc_size; if (tree->flag & TREE_ONLY_DUPS) return((TREE_ELEMENT *) 1); - uint alloc_size=sizeof(TREE_ELEMENT)+key_size+tree->size_of_element; + alloc_size=sizeof(TREE_ELEMENT)+key_size+tree->size_of_element; tree->allocated+=alloc_size; if (tree->memory_limit && tree->elements_in_tree -- cgit v1.2.1 From b12e3796dc70466eca5ef3f25d51234b32af5113 Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Fri, 18 Feb 2011 17:43:59 +0200 Subject: Fix for lp:711565 "Index Condition Pushdown can make a thread hold MyISAM locks as well as be unKILLable for long time" - In Maria/MyISAM: Release/re-acquire locks to give queries that wait on them a chance to make progress - In Maria/MyISAM: Change from numeric constants to ICP_RES values. - In Maria: Do check index condition in maria_rprev() (was lost in the merge/backport?) - In Maria/MyISAM/XtraDB: Check if the query was killed, and return immediately if it was. Added new storage engine error: HA_ERR_ABORTED_BY_USER, for handler to signal that it detected a kill of the query and aborted Authors: Sergey Petrunia & Monty include/my_base.h: Added HA_ERR_ABORTED_BY_USER, for handler to signal that it detected a kill of the query and aborted include/my_handler.h: Added comment mysql-test/r/myisam_icp.result: Updated test mysql-test/t/myisam_icp.test: Drop used tables at start of test Added test case that can help with manual testing of killing index condition pushdown query. mysys/my_handler_errors.h: Text for new storage engine error sql/handler.cc: If engine got HA_ERR_ABORTED_BY_USER, send kill message. sql/multi_range_read.cc: Return error code storage/maria/ha_maria.cc: Added ma_killed_in_mariadb() to detect kill. Ensure that file->external_ref points to TABLE object. storage/maria/ma_extra.c: Dummy test-if-killed for standalone storage/maria/ma_key.c: If ma_check_index_cond() fails, set my_errno and info->cur_row.lastpos storage/maria/ma_rkey.c: Release/re-acquire locks to give queries that wait on them a chance to make progress Check if the query was killed, and return immediately if it was storage/maria/ma_rnext.c: Check if the query was killed, and return immediately if it was Added missing fast_ma_writeinfo(info) storage/maria/ma_rnext_same.c: Check if the query was killed, and return immediately if it was Added missing fast_ma_writeinfo(info) storage/maria/ma_rprev.c: Check if the query was killed, and return immediately if it was Added missing fast_ma_writeinfo(info) and ma_check_index_cond() storage/maria/ma_search.c: Give error message if we find a wrong key storage/maria/ma_static.c: Added pointer to test-if-killed function storage/maria/maria_def.h: New prototypes storage/myisam/ha_myisam.cc: Added mi_killed_in_mariadb() Ensure that file->external_ref points to TABLE object. storage/myisam/mi_extra.c: Dummy test-if-killed for standalone storage/myisam/mi_key.c: If ma_check_index_cond() fails, set my_errno and info->lastpos storage/myisam/mi_rkey.c: Ensure that info->lastpos= HA_OFFSET_ERROR in case of error Release/re-acquire locks to give queries that wait on them a chance to make progress Check if the query was killed, and return immediately if it was Reorder code to do less things in case of error. Added missing fast_mi_writeinfo() storage/myisam/mi_rnext.c: Check if the query was killed, and return immediately if it was Simplify old ICP code Added missing fast_ma_writeinfo(info) storage/myisam/mi_rnext_same.c: Check if the query was killed, and return immediately if it was Added missing fast_mi_writeinfo(info) storage/myisam/mi_rprev.c: Check if the query was killed, and return immediately if it was Simplify error handling of ICP Added missing fast_mi_writeinfo(info) storage/myisam/mi_search.c: Give error message if we find a wrong key storage/myisam/mi_static.c: Added pointer to test-if-killed function storage/myisam/myisamdef.h: New prototypes storage/xtradb/handler/ha_innodb.cc: Added DB_SEARCH_ABORTED_BY_USER and ha_innobase::is_thd_killed() Check if the query was killed, and return immediately if it was storage/xtradb/handler/ha_innodb.h: Added prototype storage/xtradb/include/db0err.h: Added DB_SEARCH_ABORTED_BY_USER storage/xtradb/include/row0mysql.h: Added possible ICP errors storage/xtradb/row/row0sel.c: Use ICP errors instead of constants. Detect if killed and return B_SEARCH_ABORTED_BY_USER --- mysys/my_handler_errors.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'mysys') diff --git a/mysys/my_handler_errors.h b/mysys/my_handler_errors.h index 4f0fd52e767..31445cd8671 100644 --- a/mysys/my_handler_errors.h +++ b/mysys/my_handler_errors.h @@ -64,6 +64,6 @@ static const char *handler_error_messages[]= "File too short; Expected more data in file", "Read page with wrong checksum", "Too many active concurrent transactions", - "Row is not visible by the current transaction" + "Row is not visible by the current transaction", + "Operation was interrupted by end user (probably kill command?)" }; - -- cgit v1.2.1 From 39616eb9ef974c69e73bcb80cd7e3c40228910fd Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Wed, 23 Feb 2011 02:06:58 +0200 Subject: Fixed compiler warnings and some test failures found by buildbot dbug/dbug.c: Fixed compiler warnings on windows mysql-test/r/index_intersect_innodb.result: Don't print number of rows as this varies mysql-test/suite/funcs_1/r/processlist_val_no_prot.result: Update to use new State mysql-test/suite/handler/heap.result: Update results mysql-test/suite/handler/heap.test: Changed key used in test as the key 'b' will not guarantee order of the two last elements (as the key value is the same) mysql-test/suite/pbxt/r/lock_multi.result: Update to use new State mysql-test/t/index_intersect.test: Don't print number of rows as this varies mysql-test/valgrind.supp: Added suppression for linux mysys/hash.c: Fixed compiler warnings on windows plugin/handler_socket/handlersocket/database.cpp: Fixed compiler warnings sql-common/client_plugin.c: Changed variable to define to avoid compiler warnings when variable is not used sql-common/my_time.c: Initialize all my_time elements to avoid compiler warnings storage/oqgraph/ha_oqgraph.cc: Use right function (to avoid compiler warning) storage/pbxt/src/table_xt.cc: Initialize variables (to avoid compiler warnings) --- mysys/hash.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'mysys') diff --git a/mysys/hash.c b/mysys/hash.c index 0e22ddcf215..452f410c896 100644 --- a/mysys/hash.c +++ b/mysys/hash.c @@ -181,8 +181,9 @@ my_hash_key(const HASH *hash, const uchar *record, size_t *length, static uint my_hash_mask(size_t hashnr, size_t buffmax, size_t maxlength) { - if ((hashnr & (buffmax-1)) < maxlength) return (hashnr & (buffmax-1)); - return (hashnr & ((buffmax >> 1) -1)); + if ((hashnr & (buffmax-1)) < maxlength) + return (uint) (hashnr & (buffmax-1)); + return (uint) (hashnr & ((buffmax >> 1) -1)); } static uint my_hash_rec_mask(const HASH *hash, HASH_LINK *pos, @@ -483,7 +484,8 @@ my_bool my_hash_insert(HASH *info, const uchar *record) my_bool my_hash_delete(HASH *hash, uchar *record) { - uint blength,pos2,pos_hashnr,lastpos_hashnr,idx,empty_index; + uint pos2,pos_hashnr,lastpos_hashnr,idx,empty_index; + size_t blength; HASH_LINK *data,*lastpos,*gpos,*pos,*pos3,*empty; DBUG_ENTER("my_hash_delete"); if (!hash->records) @@ -572,8 +574,8 @@ exit: my_bool my_hash_update(HASH *hash, uchar *record, uchar *old_key, size_t old_key_length) { - uint new_index,new_pos_index,blength,records; - size_t idx,empty; + uint new_index,new_pos_index,records; + size_t idx, empty, blength; HASH_LINK org_link,*data,*previous,*pos; DBUG_ENTER("my_hash_update"); @@ -655,7 +657,7 @@ my_bool my_hash_update(HASH *hash, uchar *record, uchar *old_key, if (new_index != new_pos_index) { /* Other record in wrong position */ data[empty] = *pos; - movelink(data,new_index,new_pos_index,empty); + movelink(data,new_index,new_pos_index, (uint) empty); org_link.next=NO_RECORD; data[new_index]= org_link; } @@ -663,7 +665,7 @@ my_bool my_hash_update(HASH *hash, uchar *record, uchar *old_key, { /* Link in chain at right position */ org_link.next=data[new_index].next; data[empty]=org_link; - data[new_index].next=empty; + data[new_index].next= (uint) empty; } DBUG_RETURN(0); } @@ -727,7 +729,8 @@ my_bool my_hash_check(HASH *hash) { int error; uint i,rec_link,found,max_links,seek,links,idx; - uint records,blength; + uint records; + size_t blength; HASH_LINK *data,*hash_info; records=hash->records; blength=hash->blength; -- cgit v1.2.1 From e6c45f5e1fc91e0ec0cffc97d5bcd1f77cec4de2 Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Thu, 24 Feb 2011 02:36:32 +0200 Subject: - Reduced memory requirements for mysqltest to 1/4.th This also gave a speedup for 5x for some tests. - Reduced memory usage from safe_mutex. - Fixed problem with failing tests that could not restart mysqld becasue the port was reserved - More DBUG information - Fixed bug where bitmap_set_prefix() wrote over buffer area. - Initialize n_pages_flushed in xtradb which was used uninitialized. client/mysqltest.cc: Reduced memory usage (400K -> 80 for simple test; 400M -> 70M for some other tests) - Smaller dynamic arrays at start - Made 'st_connection' significantly smaller by allocation 'mysql' on demand in mysql_init() and storing require_file in a mem_root. - Fixed that when one does --debug we get information from safemalloc in the trace (Most of changes are changing &connect->mysql to connect->mysql libmysql/libmysql.c: Don't call mysql_thread_end() if my_init() was called outside of mysql_server_init() This is needed to get information from my_end() into the DBUG log mysql-test/README: Fixed wrong comment mysql-test/mysql-test-run.pl: Reserv 20 ports / mysql-test-run thread. (Needed as some tests uses 9 mysqld servers) mysys/hash.c: More DBUG information mysys/my_bitmap.c: Fixed bug where bitmap_set_prefix() wrote over buffer area. mysys/safemalloc.c: More DBUG information mysys/thr_mutex.c: Initialize smaller arrays be default. sql-common/client.c: More DBUG_PRINT storage/xtradb/srv/srv0srv.c: Initialize n_pages_flushed which was used uninitialized. --- mysys/hash.c | 3 ++- mysys/my_bitmap.c | 3 +++ mysys/safemalloc.c | 4 +++- mysys/thr_mutex.c | 20 ++++++++++---------- 4 files changed, 18 insertions(+), 12 deletions(-) (limited to 'mysys') diff --git a/mysys/hash.c b/mysys/hash.c index 0e22ddcf215..924f0ef418d 100644 --- a/mysys/hash.c +++ b/mysys/hash.c @@ -130,7 +130,8 @@ static inline void my_hash_free_elements(HASH *hash) void my_hash_free(HASH *hash) { DBUG_ENTER("my_hash_free"); - DBUG_PRINT("enter",("hash: 0x%lx", (long) hash)); + DBUG_PRINT("enter",("hash: 0x%lx elements: %ld", + (long) hash, hash->records)); my_hash_free_elements(hash); hash->free= 0; diff --git a/mysys/my_bitmap.c b/mysys/my_bitmap.c index 0c3f45be374..c89aec67da8 100644 --- a/mysys/my_bitmap.c +++ b/mysys/my_bitmap.c @@ -274,7 +274,10 @@ void bitmap_set_prefix(MY_BITMAP *map, uint prefix_size) memset(m, 0xff, prefix_bytes); m+= prefix_bytes; if ((prefix_bits= prefix_size & 7)) + { *m++= (1 << prefix_bits)-1; + prefix_bytes++; + } if ((d= no_bytes_in_map(map)-prefix_bytes)) bzero(m, d); } diff --git a/mysys/safemalloc.c b/mysys/safemalloc.c index 16489ccc022..7067ebbe843 100644 --- a/mysys/safemalloc.c +++ b/mysys/safemalloc.c @@ -124,7 +124,8 @@ void *_mymalloc(size_t size, const char *filename, uint lineno, myf MyFlags) struct st_irem *irem; uchar *data; DBUG_ENTER("_mymalloc"); - DBUG_PRINT("enter",("Size: %lu", (ulong) size)); + DBUG_PRINT("enter",("Size: %lu Total alloc: %lu", (ulong) size, + sf_malloc_cur_memory)); if (!sf_malloc_quick) (void) _sanity (filename, lineno); @@ -316,6 +317,7 @@ void _myfree(void *ptr, const char *filename, uint lineno, myf myflags) sf_malloc_cur_memory-= irem->datasize; sf_malloc_count--; pthread_mutex_unlock(&THR_LOCK_malloc); + DBUG_PRINT("info", ("bytes freed: %ld", irem->datasize)); #ifndef HAVE_valgrind /* Mark this data as free'ed */ diff --git a/mysys/thr_mutex.c b/mysys/thr_mutex.c index 77f2286b3d1..9421a3b2d98 100644 --- a/mysys/thr_mutex.c +++ b/mysys/thr_mutex.c @@ -170,16 +170,16 @@ static int safe_mutex_lazy_init_deadlock_detection(safe_mutex_t *mp) pthread_mutex_lock(&THR_LOCK_mutex); mp->id= ++safe_mutex_id; pthread_mutex_unlock(&THR_LOCK_mutex); - hash_init(mp->locked_mutex, &my_charset_bin, - 1000, - offsetof(safe_mutex_deadlock_t, id), - sizeof(mp->id), - 0, 0, HASH_UNIQUE); - hash_init(mp->used_mutex, &my_charset_bin, - 1000, - offsetof(safe_mutex_t, id), - sizeof(mp->id), - 0, 0, HASH_UNIQUE); + hash_init2(mp->locked_mutex, 64, &my_charset_bin, + 128, + offsetof(safe_mutex_deadlock_t, id), + sizeof(mp->id), + 0, 0, HASH_UNIQUE); + hash_init2(mp->used_mutex, 64, &my_charset_bin, + 128, + offsetof(safe_mutex_t, id), + sizeof(mp->id), + 0, 0, HASH_UNIQUE); return 0; } -- cgit v1.2.1 From 07b5d554ee62543cea0dc5bb36883446b9dcaa4a Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Fri, 25 Feb 2011 14:55:40 +0200 Subject: - maria/ma_test_all.sh can now be run with --tmpdir=/dev/shm for faster testing - Fixed mysql-test-run failures on window - Fixed compiler warnings from my last push (sorry about that) - Fixed that maria_chk --repair --extended works again - Fixed compiler warnings about using not unitialized data mysql-test/mysql-test-run.pl: Better output mysql-test/suite/parts/inc/partition_check_drop.inc: Use remove_files_wildcard instead of rm mysys/safemalloc.c: Fixed argument to printf storage/maria/ma_cache.c: Don't give errors when running maria_chk storage/maria/ma_dynrec.c: Don't give errors when running maria_chk storage/maria/ma_rt_test.c: Added option --datadir for where to put logs and test data storage/maria/ma_test1.c: Added option --datadir for where to put logs and test data storage/maria/ma_test2.c: Added option --datadir for where to put logs and test data storage/maria/maria_chk.c: If --datadir is used but --logdir is not, set --logdir from --datadir (this reflects how --help said how things should work) storage/maria/maria_read_log.c: Changed short option for 'maria-log-dir-path' from -l to -h to be same as mysqld, maria_chk, ma_test1 etc.. storage/maria/unittest/ma_test_all-t: Allow one to specify --tmpdir for where to store logs and data storage/xtradb/buf/buf0buf.c: Fixed compiler warnings about using not unitialized data storage/xtradb/row/row0upd.c: Fixed compiler warnings about using not unitialized data storage/xtradb/srv/srv0srv.c: Fixed compiler warnings about using not unitialized data --- mysys/safemalloc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'mysys') diff --git a/mysys/safemalloc.c b/mysys/safemalloc.c index 7067ebbe843..2a61b8edede 100644 --- a/mysys/safemalloc.c +++ b/mysys/safemalloc.c @@ -125,7 +125,7 @@ void *_mymalloc(size_t size, const char *filename, uint lineno, myf MyFlags) uchar *data; DBUG_ENTER("_mymalloc"); DBUG_PRINT("enter",("Size: %lu Total alloc: %lu", (ulong) size, - sf_malloc_cur_memory)); + (ulong) sf_malloc_cur_memory)); if (!sf_malloc_quick) (void) _sanity (filename, lineno); @@ -317,7 +317,7 @@ void _myfree(void *ptr, const char *filename, uint lineno, myf myflags) sf_malloc_cur_memory-= irem->datasize; sf_malloc_count--; pthread_mutex_unlock(&THR_LOCK_malloc); - DBUG_PRINT("info", ("bytes freed: %ld", irem->datasize)); + DBUG_PRINT("info", ("bytes freed: %ld", (ulong) irem->datasize)); #ifndef HAVE_valgrind /* Mark this data as free'ed */ -- cgit v1.2.1 From a8a757c6bb32bbf291afdf33df861127489889ab Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 1 Mar 2011 13:24:36 +0100 Subject: wl#173 - temporal types with sub-second resolution and collateral changes. * introduce my_hrtime_t, my_timediff_t, and conversion macros * inroduce TIME_RESULT, but it can only be returned from Item::cmp_type(), never from Item::result_type() * pack_time/unpack_time function for "packed" representation of MYSQL_TIME in a longlong that can be compared * ADDTIME()/SUBTIME()/+- INTERVAL now work with TIME values * numbers aren't quoted in EXPLAIN EXTENDED * new column I_S.COLUMNS.DATETIME_PRECISION * date/time values are compares to anything as date/time, not as strings or numbers. * old timestamp(X) is no longer supported * MYSQL_TIME to string conversion functions take precision as an argument * unified the warnings from Field_timestamp/datetime/time/date/newdate store methods * Field_timestamp_hires, Field_datetime_hires, Field_time_hires * Field_temporal * Lazy_string class to pass a value (string, number, time) polymorphically down the stack * make_truncated_value_warning and Field::set_datetime_warning use Lazy_string as an argument, removed char*/int/double variants * removed Field::can_be_compared_as_longlong(). Use Field::cmp_type() == INT_RESULT instead * introduced Item::cmp_result() instead of Item::is_datetime() and Item::result_as_longlong() * in many cases date/time types are treated like other types, not as special cases * greatly simplified Arg_comparator (regarding date/time/year code) * SEC_TO_TIME is real function, not integer. * microsecond precision in NOW, CURTIME, etc * Item_temporal. All items derived from it only provide get_date, but no val* methods * replication of NOW(6) * Protocol::store(time) now takes the precision as an argument * @@TIMESTAMP is a double client/mysqlbinlog.cc: remove unneded casts include/my_sys.h: introduce my_hrtime_t, my_timediff_t, and conversion macros include/my_time.h: pack_time/unpack_time, etc. convenience functions to work with MYSQL_TIME::second_part libmysql/libmysql.c: str_to_time() is gone. str_to_datetime() does it now. my_TIME_to_str() takes the precision as an argument mysql-test/include/ps_conv.inc: time is not equal to datetime anymore mysql-test/r/distinct.result: a test for an old MySQL bug mysql-test/r/explain.result: numbers aren't quoted in EXPLAIN EXTENDED mysql-test/r/func_default.result: numbers aren't quoted in EXPLAIN EXTENDED mysql-test/r/func_sapdb.result: when decimals=NOT_FIXED_DEC it means "not fixed" indeed mysql-test/r/func_test.result: numbers aren't quoted in EXPLAIN EXTENDED mysql-test/r/func_time.result: ADDTIME()/SUBTIME()/+- INTERVAL now work with TIME values mysql-test/r/having.result: numbers aren't quoted in EXPLAIN EXTENDED mysql-test/r/information_schema.result: new column I_S.COLUMNS.DATETIME_PRECISION mysql-test/r/join_outer.result: numbers aren't quoted in EXPLAIN EXTENDED mysql-test/r/metadata.result: TIMESTAMP no longer has zerofill flag mysql-test/r/range.result: invalid datetime is not compared with as a string mysql-test/r/select.result: NO_ZERO_IN_DATE, etc only affect storage - according to the manual numbers aren't quoted in EXPLAIN EXTENDED mysql-test/r/subselect.result: numbers aren't quoted in EXPLAIN EXTENDED mysql-test/r/sysdate_is_now.result: when decimals=NOT_FIXED_DEC it means "not fixed" indeed mysql-test/r/type_blob.result: TIMESTAMP(N) is not deprecated mysql-test/r/type_timestamp.result: old TIMESTAMP(X) semantics is not supported anymore mysql-test/r/union.result: numbers aren't quoted in EXPLAIN EXTENDED mysql-test/r/varbinary.result: numbers aren't quoted in EXPLAIN EXTENDED mysql-test/t/distinct.test: test for an old MySQL bug mysql-test/t/func_time.test: +- INTERVAL now works with TIME values mysql-test/t/select.test: typo mysql-test/t/subselect.test: only one error per statement, please mysql-test/t/system_mysql_db_fix40123.test: old timestamp(X) is no longer supported mysql-test/t/system_mysql_db_fix50030.test: old timestamp(X) is no longer supported mysql-test/t/system_mysql_db_fix50117.test: old timestamp(X) is no longer supported mysql-test/t/type_blob.test: old timestamp(X) is no longer supported mysql-test/t/type_timestamp.test: old timestamp(X) is no longer supported mysys/my_getsystime.c: functions to get the time with microsecond precision mysys/my_init.c: move the my_getsystime.c initialization code to my_getsystime.c mysys/my_static.c: no need to make these variables extern mysys/my_static.h: no need to make these variables extern scripts/mysql_system_tables.sql: old timestamp(X) is no longer supported scripts/mysql_system_tables_fix.sql: old timestamp(X) is no longer supported scripts/mysqlhotcopy.sh: old timestamp(X) is no longer supported sql-common/my_time.c: * call str_to_time from str_to_datetime, as appropriate * date/time to string conversions take precision as an argument * number_to_time() * TIME_to_double() * pack_time() and unpack_time() sql/event_data_objects.cc: cast is not needed my_datetime_to_str() takes precision as an argument sql/event_db_repository.cc: avoid dangerous downcast (because the pointer is not always Field_timestamp, see events_1.test) sql/event_queue.cc: avoid silly double-work for cond_wait (having an endpoint of wait, subtract the current time to get the timeout, and use set_timespec() macro to fill in struct timespec, by adding the current time to the timeout) sql/field.cc: * remove virtual Field::get_time(), everyone should use only Field::get_date() * remove lots of #ifdef WORDS_BIGENDIAN * unified the warnings from Field_timestamp/datetime/time/date/newdate store methods * Field_timestamp_hires, Field_datetime_hires, Field_time_hires * Field_temporal * make_truncated_value_warning and Field::set_datetime_warning use Lazy_string as an argument, removed char*/int/double variants sql/field.h: * remove virtual Field::get_time(), everyone should use only Field::get_date() * remove lots of #ifdef WORDS_BIGENDIAN * unified the warnings from Field_timestamp/datetime/time/date/newdate store methods * Field_timestamp_hires, Field_datetime_hires, Field_time_hires * Field_temporal * make_truncated_value_warning and Field::set_datetime_warning use Lazy_string as an argument, removed char*/int/double variants * removed Field::can_be_compared_as_longlong(). Use Field::cmp_type() == INT_RESULT instead sql/filesort.cc: TIME_RESULT, cmp_time() sql/item.cc: * numbers aren't quoted in EXPLAIN EXTENDED * Item::cmp_result() instead of Item::is_datetime() and Item::result_as_longlong() * virtual Item::get_time() is gone * Item_param::field_type() is set correctly * Item_datetime, for a datetime constant * time to anything is compared as a time * Item_cache::print() prints the value is available * bug fixed in Item_cache_int::val_str() sql/item.h: * Item::print_value(), to be used from Item_xxx::print() when needed * Item::cmp_result() instead of Item::is_datetime() and Item::result_as_longlong() * virtual Item::get_time() is gone * Item_datetime, for a datetime constant * better default for cast_to_int_type() * Item_cache objects now *always* have the field_type() set sql/item_cmpfunc.cc: * get_year_value, get_time_value are gone. get_datetime_value does it all * get_value_a_func, get_value_b_func are gone * can_compare_as_dates() is gone too, TIME_RESULT is used instead * cmp_type() instead or result_type() when doing a comparison * compare_datetime and compate_e_datetime in the comparator_matrix, is_nulls_eq is gone * Item::cmp_result() instead of Item::is_datetime() and Item::result_as_longlong() sql/item_cmpfunc.h: greatly simplified Arg_comparator sql/item_create.cc: * fix a bug in error messages in CAST sql/item_func.cc: Item::cmp_result() instead of Item::is_datetime() and Item::result_as_longlong() mention all possibitiles in switch over Item_result values, or use default: sql/item_row.h: overwrite the default cmp_type() for Item_row, as no MYSQL_TYPE_xxx value corresponds to ROW_RESULT sql/item_timefunc.cc: rewrite make_datetime to support precision argument SEC_TO_TIME is real function, not integer. many functions that returned temporal values had duplicate code in val_* methods, some of them did not have get_date() which resulted in unnecessary date->str->date conversions. Now they all are derived from Item_temporal_func and *only* provide get_date, not val* methods. many fixes to set decimals (datetime precision) correctly. sql/item_timefunc.h: SEC_TO_TIME is real function, not integer. many functions that returned temporal values had duplicate code in val_* methods, some of them did not have get_date() which resulted in unnecessary date->str->date conversions. Now they all are derived from Item_temporal_func and *only* provide get_date, not val* methods. many fixes to set decimals (datetime precision) correctly. sql/log_event.cc: replication of NOW(6) sql/log_event.h: replication of NOW(6) sql/mysql_priv.h: Lazy_string class to pass a value (string, number, time) polymorphically down the stack. make_truncated_value_warning() that uses it. sql/mysqld.cc: datetime in Arg_comparator::comparator_matrix sql/opt_range.cc: cleanup: don't disable warnings before calling save_in_field_no_warnings() sql/protocol.cc: Protocol::store(time) now takes the precision as an argument sql/protocol.h: Protocol::store(time) now takes the precision as an argument sql/rpl_rli.cc: small cleanup sql/set_var.cc: SET TIMESTAMP=double sql/set_var.h: @@TIMESTAMP is a double sql/share/errmsg.txt: precision and scale are unsigned sql/slave.cc: replication of NOW(6) sql/sp_head.cc: cleanup sql/sql_class.cc: support for NOW(6) sql/sql_class.h: support for NOW(6) sql/sql_insert.cc: support for NOW(6) sql/sql_select.cc: use item->cmp_type(). move a comment where it belongs sql/sql_show.cc: new column I_S.COLUMNS.DATETIME_PRECISION sql/sql_yacc.yy: TIME(X), DATETIME(X), cast, NOW(X), CURTIME(X), etc sql/time.cc: fix date_add_interval() to support MYSQL_TIMESTAMP_TIME argument storage/myisam/ha_myisam.cc: TIMESTAMP no longer carries ZEROFIELD flag, still we keep MYI file compatible. strings/my_vsnprintf.c: warnings tests/mysql_client_test.c: old timestamp(X) does not work anymore datetime is no longer equal to time --- mysys/my_getsystime.c | 205 +++++++++++++++----------------------------------- mysys/my_init.c | 50 +----------- mysys/my_static.c | 5 -- mysys/my_static.h | 2 - 4 files changed, 62 insertions(+), 200 deletions(-) (limited to 'mysys') diff --git a/mysys/my_getsystime.c b/mysys/my_getsystime.c index b692b18bfc7..b467c49c6e6 100644 --- a/mysys/my_getsystime.c +++ b/mysys/my_getsystime.c @@ -13,9 +13,6 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ -/* get time since epoc in 100 nanosec units */ -/* thus to get the current time we should use the system function - with the highest possible resolution */ /* TODO: in functions my_micro_time() and my_micro_time_and_time() there @@ -27,14 +24,33 @@ #ifdef __NETWARE__ #include +#elif defined(__WIN__) +static ulonglong query_performance_frequency, query_performance_offset; +#elif defined(HAVE_GETHRTIME) +static ulonglong gethrtime_offset; #endif +/* + get time since epoc in 100 nanosec units + + NOTE: + Thus to get the current time we should use the system function + with the highest possible resolution + + The value is not subject to resetting or drifting by way of adjtime() or + settimeofday(), and thus it is *NOT* appropriate for getting the current + timestamp. It can be used for calculating time intervals, though. + And it's good enough for UUID. +*/ + ulonglong my_getsystime() { #ifdef HAVE_CLOCK_GETTIME struct timespec tp; clock_gettime(CLOCK_REALTIME, &tp); return (ulonglong)tp.tv_sec*10000000+(ulonglong)tp.tv_nsec/100; +#elif defined(HAVE_GETHRTIME) + return gethrtime()/100-gethrtime_offset; #elif defined(__WIN__) LARGE_INTEGER t_cnt; if (query_performance_frequency) @@ -57,169 +73,70 @@ ulonglong my_getsystime() #endif } +/* Return current time in microseconds since epoch */ -/* - Return current time - - SYNOPSIS - my_time() - flags If MY_WME is set, write error if time call fails - -*/ - -time_t my_time(myf flags __attribute__((unused))) -{ - time_t t; -#ifdef HAVE_GETHRTIME - (void) my_micro_time_and_time(&t); - return t; -#else - /* The following loop is here beacuse time() may fail on some systems */ - while ((t= time(0)) == (time_t) -1) - { - if (flags & MY_WME) - fprintf(stderr, "%s: Warning: time() call failed\n", my_progname); - } - return t; -#endif -} - - -/* - Return time in micro seconds - - SYNOPSIS - my_micro_time() - - NOTES - This function is to be used to measure performance in micro seconds. - As it's not defined whats the start time for the clock, this function - us only useful to measure time between two moments. - - For windows platforms we need the frequency value of the CUP. This is - initalized in my_init.c through QueryPerformanceFrequency(). - - If Windows platform doesn't support QueryPerformanceFrequency() we will - obtain the time via GetClockCount, which only supports milliseconds. - - RETURN - Value in microseconds from some undefined point in time -*/ - -ulonglong my_micro_time() +my_hrtime_t my_hrtime() { + my_hrtime_t hrtime; #if defined(__WIN__) ulonglong newtime; GetSystemTimeAsFileTime((FILETIME*)&newtime); - return (newtime/10); + hrtime.val= newtime/10; #elif defined(HAVE_GETHRTIME) - return gethrtime()/1000; -#else - ulonglong newtime; struct timeval t; /* The following loop is here because gettimeofday may fail on some systems */ while (gettimeofday(&t, NULL) != 0) {} - newtime= (ulonglong)t.tv_sec * 1000000 + t.tv_usec; - return newtime; -#endif /* defined(__WIN__) */ + hrtime.val= t.tv_sec*1000000 + t.tv_usec; +#else + hrtime.val= my_getsystime()/10; +#endif + return hrtime; } - /* - Return time in seconds and timer in microseconds (not different start!) - - SYNOPSIS - my_micro_time_and_time() - time_arg Will be set to seconds since epoch (00:00:00 UTC, - January 1, 1970) + This function is basically equivalent to - NOTES - This function is to be useful when we need both the time and microtime. - For example in MySQL this is used to get the query time start of a query - and to measure the time of a query (for the slow query log) + *interval= my_getsystime()/10; + *timestamp= my_time(); - IMPLEMENTATION - Value of time is as in time() call. - Value of microtime is same as my_micro_time(), which may be totally - unrealated to time() - - RETURN - Value in microseconds from some undefined point in time + but it avoids calling OS time functions twice, if possible. */ - -#define DELTA_FOR_SECONDS LL(500000000) /* Half a second */ - -ulonglong my_micro_time_and_time(time_t *time_arg) +void my_diff_and_hrtime(my_timediff_t *interval, my_hrtime_t *timestamp) { -#if defined(__WIN__) - ulonglong newtime; - GetSystemTimeAsFileTime((FILETIME*)&newtime); - *time_arg= (time_t) ((newtime - OFFSET_TO_EPOCH) / 10000000); - return (newtime/10); -#elif defined(HAVE_GETHRTIME) - /* - Solaris has a very slow time() call. We optimize this by using the very - fast gethrtime() call and only calling time() every 1/2 second - */ - static hrtime_t prev_gethrtime= 0; - static time_t cur_time= 0; - hrtime_t cur_gethrtime; - - pthread_mutex_lock(&THR_LOCK_time); - cur_gethrtime= gethrtime(); - if ((cur_gethrtime - prev_gethrtime) > DELTA_FOR_SECONDS) - { - cur_time= time(0); - prev_gethrtime= cur_gethrtime; - } - *time_arg= cur_time; - pthread_mutex_unlock(&THR_LOCK_time); - return cur_gethrtime/1000; + interval->val= my_getsystime() / 10; +#if defined(__WIN__) || defined(HAVE_GETHRTIME) + timestamp->val= my_hrtime(); #else - ulonglong newtime; - struct timeval t; - /* - The following loop is here because gettimeofday may fail on some systems - */ - while (gettimeofday(&t, NULL) != 0) - {} - *time_arg= t.tv_sec; - newtime= (ulonglong)t.tv_sec * 1000000 + t.tv_usec; - return newtime; -#endif /* defined(__WIN__) */ + timestamp->val= interval->val; +#endif } - -/* - Returns current time - - SYNOPSIS - my_time_possible_from_micro() - microtime Value from very recent my_micro_time() - - NOTES - This function returns the current time. The microtime argument is only used - if my_micro_time() uses a function that can safely be converted to the - current time. - - RETURN - current time -*/ - -time_t my_time_possible_from_micro(ulonglong microtime __attribute__((unused))) +void my_time_init() { -#if defined(__WIN__) - time_t t; - while ((t= time(0)) == (time_t) -1) - {} - return t; +#ifdef __WIN__ +#define OFFSET_TO_EPOC ((__int64) 134774 * 24 * 60 * 60 * 1000 * 1000 * 10) + FILETIME ft; + LARGE_INTEGER li, t_cnt; + DBUG_ASSERT(sizeof(LARGE_INTEGER) == sizeof(query_performance_frequency)); + if (QueryPerformanceFrequency((LARGE_INTEGER *)&query_performance_frequency) == 0) + query_performance_frequency= 0; + else + { + GetSystemTimeAsFileTime(&ft); + li.LowPart= ft.dwLowDateTime; + li.HighPart= ft.dwHighDateTime; + query_performance_offset= li.QuadPart-OFFSET_TO_EPOC; + QueryPerformanceCounter(&t_cnt); + query_performance_offset-= (t_cnt.QuadPart / + query_performance_frequency * 10000000 + + t_cnt.QuadPart % + query_performance_frequency * 10000000 / + query_performance_frequency); + } #elif defined(HAVE_GETHRTIME) - return my_time(0); /* Cached time */ -#else - return (time_t) (microtime / 1000000); -#endif /* defined(__WIN__) */ + gethrtime_offset= gethrtime(); +#endif } - diff --git a/mysys/my_init.c b/mysys/my_init.c index a60927be693..524d9062d02 100644 --- a/mysys/my_init.c +++ b/mysys/my_init.c @@ -101,6 +101,7 @@ my_bool my_init(void) DBUG_PROCESS((char*) (my_progname ? my_progname : "unknown")); if (!home_dir) { /* Don't initialize twice */ + my_time_init(); my_win_init(); if ((home_dir=getenv("HOME")) != 0) home_dir=intern_filename(home_dir_buff,home_dir); @@ -242,7 +243,6 @@ Voluntary context switches %ld, Involuntary context switches %ld\n", #ifdef __WIN__ - /* my_parameter_handler @@ -316,54 +316,6 @@ static void my_win_init(void) _tzset(); - - - - - - - - - - - - - - - - - - - - - - - - - /* The following is used by time functions */ -#define OFFSET_TO_EPOC ((__int64) 134774 * 24 * 60 * 60 * 1000 * 1000 * 10) -#define MS 10000000 - { - FILETIME ft; - LARGE_INTEGER li, t_cnt; - DBUG_ASSERT(sizeof(LARGE_INTEGER) == sizeof(query_performance_frequency)); - if (QueryPerformanceFrequency((LARGE_INTEGER *)&query_performance_frequency) == 0) - query_performance_frequency= 0; - else - { - GetSystemTimeAsFileTime(&ft); - li.LowPart= ft.dwLowDateTime; - li.HighPart= ft.dwHighDateTime; - query_performance_offset= li.QuadPart-OFFSET_TO_EPOC; - QueryPerformanceCounter(&t_cnt); - query_performance_offset-= (t_cnt.QuadPart / - query_performance_frequency * MS + - t_cnt.QuadPart % - query_performance_frequency * MS / - query_performance_frequency); - } - } - { /* Open HKEY_LOCAL_MACHINE\SOFTWARE\MySQL and set any strings found diff --git a/mysys/my_static.c b/mysys/my_static.c index a21a3d11104..c25a669dbe0 100644 --- a/mysys/my_static.c +++ b/mysys/my_static.c @@ -100,11 +100,6 @@ int (*fatal_error_handler_hook)(uint error,const char *str,myf MyFlags)= void (*debug_sync_C_callback_ptr)(const char *, size_t); #endif /* defined(ENABLED_DEBUG_SYNC) */ -#ifdef __WIN__ -/* from my_getsystime.c */ -ulonglong query_performance_frequency, query_performance_offset; -#endif - /* How to disable options */ my_bool NEAR my_disable_locking=0; my_bool NEAR my_disable_async_io=0; diff --git a/mysys/my_static.h b/mysys/my_static.h index 90168b099a8..5e853c2a559 100644 --- a/mysys/my_static.h +++ b/mysys/my_static.h @@ -66,8 +66,6 @@ extern struct st_irem *sf_malloc_root; extern struct st_my_file_info my_file_info_default[MY_NFILE]; -extern ulonglong query_performance_frequency, query_performance_offset; - #if defined(THREAD) && !defined(__WIN__) extern sigset_t my_signals; /* signals blocked by mf_brkhant */ #endif -- cgit v1.2.1 From 8250ceced6aab95347e00e2ff1c02730da0be4fe Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 24 Mar 2011 12:30:03 +0100 Subject: Fix compilation on Windows: - Fixes for type-conversion (time_t is not interchangeable with my_time_t on Windows as time_t s 64 bit while my_time_t is long) - BIGENDIAN-> ARCH_BIGENDIAN . BIGENDIAN constant is defined in winsock2.h (as 0) - added explicit cast for longlong->double conversion in sql/item.h (fixed many warnings) Also, HAVE_SNPRINTF is now defined and snprintf is defined to _snprintf in config-win.h --- mysys/mf_getdate.c | 2 +- mysys/my_getsystime.c | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'mysys') diff --git a/mysys/mf_getdate.c b/mysys/mf_getdate.c index 9475bebd107..af86322a856 100644 --- a/mysys/mf_getdate.c +++ b/mysys/mf_getdate.c @@ -17,7 +17,7 @@ #include "mysys_priv.h" #include - +#include /* get date as string diff --git a/mysys/my_getsystime.c b/mysys/my_getsystime.c index b467c49c6e6..827c65aef2b 100644 --- a/mysys/my_getsystime.c +++ b/mysys/my_getsystime.c @@ -108,7 +108,10 @@ void my_diff_and_hrtime(my_timediff_t *interval, my_hrtime_t *timestamp) { interval->val= my_getsystime() / 10; #if defined(__WIN__) || defined(HAVE_GETHRTIME) - timestamp->val= my_hrtime(); + { + my_hrtime_t t= my_hrtime(); + timestamp->val= t.val; + } #else timestamp->val= interval->val; #endif -- cgit v1.2.1 From e016a2f5f00774f8126974fa26c7550bf0e60c84 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 26 Mar 2011 11:59:34 +0100 Subject: lp:705210 Compiling with BUILD/compile-pentium64-debug fails --- mysys/mf_getdate.c | 2 +- mysys/my_getsystime.c | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) (limited to 'mysys') diff --git a/mysys/mf_getdate.c b/mysys/mf_getdate.c index af86322a856..9475bebd107 100644 --- a/mysys/mf_getdate.c +++ b/mysys/mf_getdate.c @@ -17,7 +17,7 @@ #include "mysys_priv.h" #include -#include + /* get date as string diff --git a/mysys/my_getsystime.c b/mysys/my_getsystime.c index 827c65aef2b..2b91c9d6b8c 100644 --- a/mysys/my_getsystime.c +++ b/mysys/my_getsystime.c @@ -108,10 +108,7 @@ void my_diff_and_hrtime(my_timediff_t *interval, my_hrtime_t *timestamp) { interval->val= my_getsystime() / 10; #if defined(__WIN__) || defined(HAVE_GETHRTIME) - { - my_hrtime_t t= my_hrtime(); - timestamp->val= t.val; - } + *timestamp= my_hrtime(); #else timestamp->val= interval->val; #endif -- cgit v1.2.1 From b1cd09600787e26665be7125c140a67474b654f1 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 3 Apr 2011 16:30:29 +0200 Subject: fix compilation warning --- mysys/mf_getdate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mysys') diff --git a/mysys/mf_getdate.c b/mysys/mf_getdate.c index 9475bebd107..70278e64003 100644 --- a/mysys/mf_getdate.c +++ b/mysys/mf_getdate.c @@ -42,7 +42,7 @@ void get_date(register char * to, int flag, time_t date) struct tm tm_tmp; #endif - skr=date ? (time_t) date : my_time(0); + skr=date ? date : (time_t) my_time(0); #if defined(HAVE_LOCALTIME_R) && defined(_REENTRANT) if (flag & GETDATE_GMT) gmtime_r(&skr,&tm_tmp); -- cgit v1.2.1 From 5346cb8d2745acd660b301092458e231c9f53319 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 19 Apr 2011 19:28:23 +0200 Subject: Fix my_hrtime() on Windows to return microseconds since Unix epoch. Prior to this patch it returned microseconds since 1 Jan 1601 (Windows epoch) --- mysys/my_getsystime.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'mysys') diff --git a/mysys/my_getsystime.c b/mysys/my_getsystime.c index 2b91c9d6b8c..125d389f3d5 100644 --- a/mysys/my_getsystime.c +++ b/mysys/my_getsystime.c @@ -26,6 +26,7 @@ #include #elif defined(__WIN__) static ulonglong query_performance_frequency, query_performance_offset; +#define OFFSET_TO_EPOC 116444736000000000LL #elif defined(HAVE_GETHRTIME) static ulonglong gethrtime_offset; #endif @@ -81,6 +82,7 @@ my_hrtime_t my_hrtime() #if defined(__WIN__) ulonglong newtime; GetSystemTimeAsFileTime((FILETIME*)&newtime); + newtime -= OFFSET_TO_EPOC; hrtime.val= newtime/10; #elif defined(HAVE_GETHRTIME) struct timeval t; @@ -117,7 +119,6 @@ void my_diff_and_hrtime(my_timediff_t *interval, my_hrtime_t *timestamp) void my_time_init() { #ifdef __WIN__ -#define OFFSET_TO_EPOC ((__int64) 134774 * 24 * 60 * 60 * 1000 * 1000 * 10) FILETIME ft; LARGE_INTEGER li, t_cnt; DBUG_ASSERT(sizeof(LARGE_INTEGER) == sizeof(query_performance_frequency)); -- cgit v1.2.1 From 96c1a87455f20db9ce651723628b4ac9a44d54a2 Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Tue, 3 May 2011 18:17:13 +0300 Subject: Fixed after-merge failures found by buildbot mysql-test/suite/funcs_1/r/is_columns_is.result: Updated test results mysql-test/suite/pbxt/r/client_xml.result: Updated test results mysys/Makefile.am: Added back my_port.c storage/xtradb/buf/buf0buddy.c: Removed not used variables storage/xtradb/fil/fil0fil.c: Added casts to fix compiler warnings --- mysys/Makefile.am | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'mysys') diff --git a/mysys/Makefile.am b/mysys/Makefile.am index 2c42534b93b..7915ec89d17 100644 --- a/mysys/Makefile.am +++ b/mysys/Makefile.am @@ -43,7 +43,8 @@ libmysys_la_SOURCES = my_init.c my_getwd.c mf_getdate.c my_mmap.c \ mf_wcomp.c mf_wfile.c my_gethwaddr.c \ mf_qsort.c mf_qsort2.c mf_sort.c \ ptr_cmp.c mf_radix.c queues.c my_getncpus.c \ - tree.c trie.c list.c hash.c array.c string.c typelib.c \ + tree.c trie.c list.c hash.c array.c string.c \ + typelib.c \ my_copy.c my_append.c my_lib.c \ my_delete.c my_rename.c my_redel.c \ my_chsize.c my_clock.c \ @@ -51,7 +52,7 @@ libmysys_la_SOURCES = my_init.c my_getwd.c mf_getdate.c my_mmap.c \ my_sync.c my_getopt.c my_mkdir.c \ default_modify.c default.c \ my_compress.c checksum.c \ - my_net.c my_sleep.c \ + my_net.c my_port.c my_sleep.c \ charset.c charset-def.c my_bitmap.c my_bit.c md5.c \ rijndael.c my_aes.c sha1.c \ my_compare.c my_netware.c my_largepage.c \ -- cgit v1.2.1 From 5ab92b1f850c62718907d166b47553440502043c Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Sun, 8 May 2011 13:24:06 +0300 Subject: Adding support for Dynamic columns (WL#34): - COLUMN_CREATE(column_nr, value, [column_nr,value]...) - COLUMN_ADD(blob,column_nr, value, column_nr,value]...) - COLUMN_DELETE(blob, column_nr, column_nr...) - COLUMN_EXISTS(blob, column_nr) - COLUMN_LIST(blob, column_nr) - COLUMN_GET(string, column_nr AS type) Added cast(X as DOUBLE) and cast(x as INT) Better warning and error messages for wrong cast's Created some sub functions to simplify and reuse code. Added a lot of conversation functions with error/warnings for what went wrong. Fixed some issues when casting time to datetime. Added functions to dynamic strings and Strings to allow one to move a string buffer from dynamic strings to String (to save malloc+ copy) Added dynamic columns library to libmysqlclient include/Makefile.am: Added ma_dyncol.h include/decimal.h: Added 'const' to arguments for some functions. include/my_sys.h: Added dynstr_reassociate() include/my_time.h: Added TIME_SUBSECOND_RANGE Added double_to_datetime() Added flag argument to str_to_time() libmysql/CMakeLists.txt: Added mysys/ma_dyncol.c libmysql/Makefile.shared: Added ma_dyncol libmysql/libmysql.c: Added argument to str_to_time() mysql-test/r/bigint.result: Better error messages mysql-test/r/cast.result: Better warning and error messages A lot of new cast() tests mysql-test/r/func_math.result: Better warning messages mysql-test/r/func_str.result: Better warning messages mysql-test/r/func_time.result: Better warning messages mysql-test/r/sp-vars.result: Better warning messages mysql-test/r/strict.result: Better warning messages New test result mysql-test/r/type_newdecimal.result: Better warning messages mysql-test/r/warnings.result: Better warning messages mysql-test/suite/funcs_1/r/innodb_func_view.result: Updated results after better cast warnings mysql-test/suite/funcs_1/r/memory_func_view.result: Updated results after better cast warnings mysql-test/suite/funcs_1/r/myisam_func_view.result: Updated results after better cast warnings mysql-test/suite/optimizer_unfixed_bugs/t/bug43448.test: Added begin...commit to speed up test. mysql-test/suite/parts/inc/part_supported_sql_funcs_delete.inc: Added begin...commit to speed up test. mysql-test/suite/parts/inc/partition_supported_sql_funcs.inc: Added begin...commit to speed up test. mysql-test/suite/parts/r/part_supported_sql_func_innodb.result: Added begin...commit to speed up test. mysql-test/suite/parts/r/part_supported_sql_func_myisam.result: Added begin...commit to speed up test. mysql-test/suite/parts/r/rpl_partition.result: Added begin...commit to speed up test. mysql-test/suite/parts/t/part_supported_sql_func_innodb.test: Removed duplicated --big_test mysql-test/suite/parts/t/rpl_partition.test: Added begin...commit to speed up test. mysql-test/suite/pbxt/r/cast.result: Updated results after better cast warnings mysql-test/suite/pbxt/r/func_str.result: Updated results after better cast warnings mysql-test/suite/pbxt/r/type_newdecimal.result: Updated results after better cast warnings mysql-test/suite/rpl/r/rpl_innodb_bug28430.result: Added begin...commit to speed up test. mysql-test/suite/rpl/t/rpl_innodb_bug28430.test: Added begin...commit to speed up test. mysql-test/suite/vcol/r/vcol_supported_sql_funcs_innodb.result: More warnings mysql-test/suite/vcol/r/vcol_supported_sql_funcs_myisam.result: More warnings mysql-test/t/cast.test: A lot of new cast() tests mysql-test/t/strict.test: Added new test mysys/CMakeLists.txt: Added ma_dyncol.c mysys/Makefile.am: Added ma_dyncol.c mysys/string.c: Added dynstr_reassociate() to move a buffer from dynamic_strings to some other allocator sql-common/my_time.c: Added 'fuzzydate' flag to str_to_time() Added support for microseconds to my_time_to_str() and my_datetime_to_str() Reset second_parts in number_to_datetime() Added double_to_datetime() sql/field.cc: Added double_to_longlong() and truncate_double() to simplify and reuse code sql/field.h: New prototypes sql/item.cc: Changed Item::get_date(MYSQL_TIME *ltime,uint fuzzydate) to be aware of type of argument. (Needed to make it microsecond safe and get better warnings). Updated call to str_to_time_with_warn() sql/item.h: Added struct st_dyncall_create_def used by dynamic columns Added virtual bool dynamic_result() to tell if type of argument may change over calls. sql/item_cmpfunc.cc: Added Item_func_dyncol_exists() sql/item_cmpfunc.h: Added class Item_func_dyncol_exists sql/item_create.cc: Added get_length_and_scale() to simplify other functions Simplified and extended create_func_cast() Added support for cast(X as double(X,Y)) Added functions to create dynamic column functions. sql/item_create.h: Added prototypes sql/item_func.cc: Extended cast functions Item_func_signed() and Item_func_unsigned() to work with dynamic types Added Item_double_typecast() sql/item_func.h: Added class Item_double_typecast() sql/item_strfunc.cc: Added functions for COLUMN_CREATE(), COLUMN_ADD(), COLUMN_GET() and COLUMN_LIST() sql/item_strfunc.h: Added classes for COLUMN_CREATE(), COLUMN_ADD(), COLUMN_GET() and COLUMN_LIST() sql/item_timefunc.cc: Added flag argument to str_to_time_with_warn() Updated Item_char_typecast() to handle result type that may change between calls (for dynamic columns) Added Item_time_typecast::get_date() to ensure that we cast a datetime to time properly. sql/item_timefunc.h: Added get_date() to Item_time_typecast() to allow proper results for casting time to datetime sql/lex.h: Added new SQL function names sql/my_decimal.cc: Added 'const' to some arguments. Better error message in case of errors (we now print out the wrong value) Added my_decimal2int() sql/my_decimal.h: Moved some constants to my_decimal_limits.h Updated prototypes. Made my_decimal2int() a function as it's rather long (no reason to have it inline) Added decimal2my_decimal() function. sql/mysql_priv.h: Prototypes for new functions sql/share/errmsg.txt: New error messages for wrong casts and dynamic columns sql/sql_acl.cc: Fixed indentation sql/sql_base.cc: Added dynamic_column_error_message() sql/sql_string.h: Added reassociate() to move a buffer to be owned by String object. sql/sql_yacc.yy: Added syntax for COLUMN_ functions. sql/time.cc: Updated str_to_datetime_with_warn() flag argument to same type as other functions Added conversion flag to str_to_time_with_warn() (Similar to all datetime functions) Added conversion functions with warnings: double_to_datetime_with_warn() and decimal_to_datetime_with_warn() strings/decimal.c: Added 'const' to arguments for some functions. unittest/mysys/Makefile.am: Added test for dynamic columns code --- mysys/CMakeLists.txt | 2 +- mysys/Makefile.am | 2 +- mysys/ma_dyncol.c | 2044 ++++++++++++++++++++++++++++++++++++++++++++++++++ mysys/string.c | 12 + 4 files changed, 2058 insertions(+), 2 deletions(-) create mode 100644 mysys/ma_dyncol.c (limited to 'mysys') diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index 8068318e4c2..9a89a371dcd 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -40,7 +40,7 @@ SET(MYSYS_SOURCES array.c charset-def.c charset.c checksum.c default.c default_ my_static.c my_symlink.c my_symlink2.c my_sync.c my_thr_init.c my_wincond.c my_windac.c my_winthread.c my_write.c ptr_cmp.c queues.c stacktrace.c rijndael.c safemalloc.c sha1.c string.c thr_alarm.c thr_lock.c thr_mutex.c - thr_rwlock.c tree.c typelib.c my_vle.c base64.c my_memmem.c my_getpagesize.c + thr_rwlock.c tree.c typelib.c my_vle.c base64.c my_memmem.c my_getpagesize.c ma_dyncol.c lf_alloc-pin.c lf_dynarray.c lf_hash.c my_atomic.c my_getncpus.c my_rnd.c my_uuid.c wqueue.c waiting_threads.c diff --git a/mysys/Makefile.am b/mysys/Makefile.am index 515791be7f1..756645449fa 100644 --- a/mysys/Makefile.am +++ b/mysys/Makefile.am @@ -27,7 +27,7 @@ libmysys_la_SOURCES = my_init.c my_getwd.c mf_getdate.c my_mmap.c \ mf_path.c mf_loadpath.c my_file.c \ my_open.c my_create.c my_dup.c my_seek.c my_read.c \ my_pread.c my_write.c my_getpagesize.c \ - my_crc32.c \ + my_crc32.c ma_dyncol.c \ mf_iocache.c mf_iocache2.c mf_cache.c mf_tempfile.c \ mf_tempdir.c my_lock.c mf_brkhant.c my_alarm.c \ my_malloc.c my_realloc.c my_once.c mulalloc.c \ diff --git a/mysys/ma_dyncol.c b/mysys/ma_dyncol.c new file mode 100644 index 00000000000..71eacc6b319 --- /dev/null +++ b/mysys/ma_dyncol.c @@ -0,0 +1,2044 @@ +/* Copyright (c) 2011, Monty Program Ab + Copyright (c) 2011, Oleksandr Byelkin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY ``AS IS'' AND ANY + EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. +*/ + +#include +#include +#include + +/* + Flag byte bits + + 2 bits which determinate size of offset in the header -1 +*/ +/* mask to get above bits */ +#define DYNCOL_FLG_OFFSET 3 +/* All known flags mask */ +#define DYNCOL_FLG_KNOWN 3 + +/* dynamic column size reserve */ +#define DYNCOL_SYZERESERVE 80 + +/* length of fixed string header 1 byte - flags, 2 bytes - columns counter */ +#define FIXED_HEADER_SIZE 3 + +#define COLUMN_NUMBER_SIZE 2 + +#define MAX_OFFSET_LENGTH 5 + +static enum enum_dyncol_func_result +dynamic_column_time_store(DYNAMIC_COLUMN *str, + MYSQL_TIME *value); +static enum enum_dyncol_func_result +dynamic_column_date_store(DYNAMIC_COLUMN *str, + MYSQL_TIME *value); +static enum enum_dyncol_func_result +dynamic_column_time_read_internal(DYNAMIC_COLUMN_VALUE *store_it_here, + uchar *data, size_t length); +static enum enum_dyncol_func_result +dynamic_column_date_read_internal(DYNAMIC_COLUMN_VALUE *store_it_here, + uchar *data, size_t length); + +/** + Initialize dynamic column string with (make it empty but correct format) + + @param str The string to initialize + @param size Amount of preallocated memory for the string. + + @retval FALSE OK + @retval TRUE error +*/ + +static my_bool dynamic_column_init_str(DYNAMIC_COLUMN *str, size_t size) +{ + DBUG_ASSERT(size != 0); + + /* + Make string with no fields (empty header) + - First \0 is flags + - other 2 \0 is number of fields + */ + if (init_dynamic_string(str, NULL, + size + FIXED_HEADER_SIZE, DYNCOL_SYZERESERVE)) + return TRUE; + bzero(str->str, FIXED_HEADER_SIZE); + str->length= FIXED_HEADER_SIZE; + return FALSE; +} + + +/** + Calculate how many bytes needed to store val as variable length integer + where first bit indicate continuation of the sequence. + + @param val The value for which we are calculating length + + @return number of bytes +*/ + +static size_t dynamic_column_var_uint_bytes(ulonglong val) +{ + size_t len= 0; + do + { + len++; + val>>= 7; + } while (val); + return len; +} + + +/** + Stores variable length unsigned integer value to a string + + @param str The string where to append the value + @param val The value to put in the string + + @return ER_DYNCOL_* return code + + @notes + This is used to store a number together with other data in the same + object. (Like decimals, length of string etc) + (As we don't know the length of this object, we can't store 0 in 0 bytes) +*/ + +static enum enum_dyncol_func_result +dynamic_column_var_uint_store(DYNAMIC_COLUMN *str, ulonglong val) +{ + if (dynstr_realloc(str, 10)) /* max what we can use */ + return ER_DYNCOL_RESOURCE; + + do + { + ulonglong rest= val >> 7; + str->str[str->length++]= ((val & 0x7f) | (rest ? 0x80 : 0x00)); + val= rest; + } while (val); + return ER_DYNCOL_OK; +} + + +/** + Reads variable length unsigned integer value from a string + + @param data The string from which the int should be read + @param data_length Max length of data + @param len Where to put length of the string read in bytes + + @return value of the unsigned integer read from the string + + In case of error, *len is set to 0 +*/ + +static ulonglong +dynamic_column_var_uint_get(uchar *data, size_t data_length, + size_t *len) +{ + ulonglong val= 0; + uint length; + uchar *end= data + data_length; + + for (length=0; data < end ; data++) + { + val+= (((ulonglong)((*data) & 0x7f)) << (length * 7)); + length++; + if (!((*data) & 0x80)) + { + /* End of data */ + *len= length; + return val; + } + } + /* Something was wrong with data */ + *len= 0; /* Mark error */ + return 0; +} + + +/** + Calculate how many bytes needed to store val as unsigned. + + @param val The value for which we are calculating length + + @return number of bytes (0-8) +*/ + +static size_t dynamic_column_uint_bytes(ulonglong val) +{ + size_t len; + + for (len= 0; val ; val>>= 8, len++) + ; + return len; +} + + +/** + Append the string with given unsigned int value. + + @param str The string where to put the value + @param val The value to put in the string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_uint_store(DYNAMIC_COLUMN *str, ulonglong val) +{ + if (dynstr_realloc(str, 8)) /* max what we can use */ + return ER_DYNCOL_RESOURCE; + + for (; val; val>>= 8) + str->str[str->length++]= (char) (val & 0xff); + return ER_DYNCOL_OK; +} + + +/** + Read unsigned int value of given length from the string + + @param store_it_here The structure to store the value + @param data The string which should be read + @param length The length (in bytes) of the value in nthe string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_uint_read(DYNAMIC_COLUMN_VALUE *store_it_here, + uchar *data, size_t length) +{ + ulonglong value= 0; + size_t i; + + for (i= 0; i < length; i++) + value+= ((ulonglong)data[i]) << (i*8); + + store_it_here->ulong_value= value; + return ER_DYNCOL_OK; +} + +/** + Calculate how many bytes needed to store val as signed in following encoding: + 0 -> 0 + -1 -> 1 + 1 -> 2 + -2 -> 3 + 2 -> 4 + ... + + @param val The value for which we are calculating length + + @return number of bytes +*/ + +static size_t dynamic_column_sint_bytes(longlong val) +{ + return dynamic_column_uint_bytes((val << 1) ^ + (val < 0 ? ULL(0xffffffffffffffff) : 0)); +} + + +/** + Append the string with given signed int value. + + @param str the string where to put the value + @param val the value to put in the string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_sint_store(DYNAMIC_COLUMN *str, longlong val) +{ + return dynamic_column_uint_store(str, + (val << 1) ^ + (val < 0 ? ULL(0xffffffffffffffff) : 0)); +} + + +/** + Read signed int value of given length from the string + + @param store_it_here The structure to store the value + @param data The string which should be read + @param length The length (in bytes) of the value in the string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_sint_read(DYNAMIC_COLUMN_VALUE *store_it_here, + uchar *data, size_t length) +{ + ulonglong val; + dynamic_column_uint_read(store_it_here, data, length); + val= store_it_here->ulong_value; + if (val & 1) + val= (val >> 1) ^ ULL(0xffffffffffffffff); + else + val>>= 1; + store_it_here->long_value= (longlong) val; + return ER_DYNCOL_OK; +} + + +/** + Calculate how many bytes needed to store the value. + + @param value The value for which we are calculating length + + @return number of bytes +*/ + +static size_t +dynamic_column_value_len(DYNAMIC_COLUMN_VALUE *value) +{ + switch (value->type) { + case DYN_COL_NULL: + return 0; + case DYN_COL_INT: + return dynamic_column_sint_bytes(value->long_value); + case DYN_COL_UINT: + return dynamic_column_uint_bytes(value->ulong_value); + case DYN_COL_DOUBLE: + return 8; + case DYN_COL_STRING: + return (dynamic_column_var_uint_bytes(value->charset->number) + + value->string_value.length); + case DYN_COL_DECIMAL: + return (dynamic_column_var_uint_bytes(value->decimal_value.intg) + + dynamic_column_var_uint_bytes(value->decimal_value.frac) + + decimal_bin_size(value->decimal_value.intg + + value->decimal_value.frac, + value->decimal_value.frac)); + case DYN_COL_DATETIME: + /* date+time in bits: 14 + 4 + 5 + 10 + 6 + 6 + 20 + 1 66bits ~= 9 bytes */ + return 9; + case DYN_COL_DATE: + /* date in dits: 14 + 4 + 5 = 23bits ~= 3bytes*/ + return 3; + case DYN_COL_TIME: + /* time in bits: 10 + 6 + 6 + 20 + 1 = 43bits ~= 6bytes*/ + return 6; + } + DBUG_ASSERT(0); + return 0; +} + + +/** + Append double value to a string + + @param str the string where to put the value + @param val the value to put in the string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_double_store(DYNAMIC_COLUMN *str, double val) +{ + if (dynstr_realloc(str, 8)) + return ER_DYNCOL_RESOURCE; + float8store(str->str + str->length, val); + str->length+= 8; + return ER_DYNCOL_OK; +} + + +/** + Read double value of given length from the string + + @param store_it_here The structure to store the value + @param data The string which should be read + @param length The length (in bytes) of the value in nthe string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_double_read(DYNAMIC_COLUMN_VALUE *store_it_here, + uchar *data, size_t length) +{ + if (length != 8) + return ER_DYNCOL_FORMAT; + float8get(store_it_here->double_value, data); + return ER_DYNCOL_OK; +} + + +/** + Append the string with given string value. + + @param str the string where to put the value + @param val the value to put in the string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_string_store(DYNAMIC_COLUMN *str, LEX_STRING *string, + CHARSET_INFO *charset) +{ + enum enum_dyncol_func_result rc; + if ((rc= dynamic_column_var_uint_store(str, charset->number))) + return rc; + if (dynstr_append_mem(str, string->str, string->length)) + return ER_DYNCOL_RESOURCE; + return ER_DYNCOL_OK; +} + + +/** + Read string value of given length from the packed string + + @param store_it_here The structure to store the value + @param data The packed string which should be read + @param length The length (in bytes) of the value in nthe string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_string_read(DYNAMIC_COLUMN_VALUE *store_it_here, + uchar *data, size_t length) +{ + size_t len; + uint charset_nr= dynamic_column_var_uint_get(data, length, &len); + if (len == 0) /* Wrong packed number */ + return ER_DYNCOL_FORMAT; + store_it_here->charset= get_charset(charset_nr, MYF(MY_WME)); + if (store_it_here->charset == NULL) + return ER_DYNCOL_UNKNOWN_CHARSET; + data+= len; + store_it_here->string_value.length= (length-= len); + store_it_here->string_value.str= (char*) data; + return ER_DYNCOL_OK; +} + + +/** + Append the string with given decimal value. + + @param str the string where to put the value + @param val the value to put in the string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_decimal_store(DYNAMIC_COLUMN *str, + decimal_t *value) +{ + uint bin_size= decimal_bin_size(value->intg + value->frac, value->frac); + if (dynstr_realloc(str, bin_size + 20)) + return ER_DYNCOL_RESOURCE; + + /* The following can't fail as memory is already allocated */ + (void) dynamic_column_var_uint_store(str, value->intg); + (void) dynamic_column_var_uint_store(str, value->frac); + + decimal2bin(value, (uchar *) str->str + str->length, + value->intg + value->frac, + value->frac); + str->length+= bin_size; + return ER_DYNCOL_OK; +} + + +/** + Prepare the value to be used as decimal. + + @param value The value structure which sould be setup. +*/ + +void dynamic_column_prepare_decimal(DYNAMIC_COLUMN_VALUE *value) +{ + value->decimal_value.buf= value->decimal_buffer; + value->decimal_value.len= DECIMAL_BUFF_LENGTH; + /* just to be safe */ + value->type= DYN_COL_DECIMAL; + decimal_make_zero(&value->decimal_value); +} + + +/** + Read decimal value of given length from the string + + @param store_it_here The structure to store the value + @param data The string which should be read + @param length The length (in bytes) of the value in nthe string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_decimal_read(DYNAMIC_COLUMN_VALUE *store_it_here, + uchar *data, size_t length) +{ + size_t intg_len, frac_len; + uint intg, frac; + + dynamic_column_prepare_decimal(store_it_here); + intg= dynamic_column_var_uint_get(data, length, &intg_len); + data+= intg_len; + frac= dynamic_column_var_uint_get(data, length - intg_len, &frac_len); + data+= frac_len; + + /* Check the size of data is correct */ + if (decimal_bin_size(intg + frac, frac) != + (int) (length - intg_len - frac_len)) + return ER_DYNCOL_FORMAT; + + if (bin2decimal(data, &store_it_here->decimal_value, intg + frac, frac) != + E_DEC_OK) + return ER_DYNCOL_FORMAT; + return ER_DYNCOL_OK; +} + + +/** + Append the string with given datetime value. + + @param str the string where to put the value + @param value the value to put in the string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_date_time_store(DYNAMIC_COLUMN *str, MYSQL_TIME *value) +{ + enum enum_dyncol_func_result rc; + /* + 0<----year---->00000!<-hours--><---microseconds---> + 12345678901234123412345 1123456789012345612345612345678901234567890 + <123456><123456><123456><123456><123456><123456><123456><123456><123456> + */ + if ((rc= dynamic_column_date_store(str, value)) || + (rc= dynamic_column_time_store(str, value))) + return rc; + return ER_DYNCOL_OK; +} + + +/** + Read datetime value of given length from the packed string + + @param store_it_here The structure to store the value + @param data The packed string which should be read + @param length The length (in bytes) of the value in nthe string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_date_time_read(DYNAMIC_COLUMN_VALUE *store_it_here, + uchar *data, size_t length) +{ + enum enum_dyncol_func_result rc= ER_DYNCOL_FORMAT; + /* + 0<----year---->00000!<-hours--><---microseconds---> + 12345678901234123412345 1123456789012345612345612345678901234567890 + <123456><123456><123456><123456><123456><123456><123456><123456><123456> + */ + if (length != 9) + goto err; + store_it_here->time_value.time_type= MYSQL_TIMESTAMP_DATETIME; + if ((rc= dynamic_column_date_read_internal(store_it_here, data, 3)) || + (rc= dynamic_column_time_read_internal(store_it_here, data + 3, 6))) + goto err; + return ER_DYNCOL_OK; + +err: + store_it_here->time_value.time_type= MYSQL_TIMESTAMP_ERROR; + return rc; +} + + +/** + Append the string with given time value. + + @param str the string where to put the value + @param value the value to put in the string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_time_store(DYNAMIC_COLUMN *str, MYSQL_TIME *value) +{ + uchar *buf; + if (dynstr_realloc(str, 6)) + return ER_DYNCOL_RESOURCE; + + buf= ((uchar *)str->str) + str->length; + + if (value->time_type == MYSQL_TIMESTAMP_NONE || + value->time_type == MYSQL_TIMESTAMP_ERROR || + value->time_type == MYSQL_TIMESTAMP_DATE) + value->neg= value->second_part= value->hour= + value->minute= value->second= 0; + DBUG_ASSERT(value->hour <= 838); + DBUG_ASSERT(value->minute <= 59); + DBUG_ASSERT(value->second <= 59); + DBUG_ASSERT(value->second_part <= 999999); + /* + 00000!<-hours--><---microseconds---> + 1123456789012345612345612345678901234567890 + <123456><123456><123456><123456><123456><123456> + */ + buf[0]= (value->second_part & 0xff); + buf[1]= ((value->second_part & 0xff00) >> 8); + buf[2]= (((value->second & 0xf) << 4) | + ((value->second_part & 0xf0000) >> 16)); + buf[3]= ((value->minute << 2) | ((value->second & 0x30) >> 4)); + buf[4]= (value->hour & 0xff); + buf[5]= ((value->neg ? 0x4 : 0) | (value->hour >> 8)); + str->length+= 6; + return ER_DYNCOL_OK; +} + + +/** + Read time value of given length from the packed string + + @param store_it_here The structure to store the value + @param data The packed string which should be read + @param length The length (in bytes) of the value in nthe string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_time_read(DYNAMIC_COLUMN_VALUE *store_it_here, + uchar *data, size_t length) +{ + store_it_here->time_value.year= store_it_here->time_value.month= + store_it_here->time_value.day= 0; + store_it_here->time_value.time_type= MYSQL_TIMESTAMP_TIME; + return dynamic_column_time_read_internal(store_it_here, data, length); +} + +/** + Internal function for reading time part from the string. + + @param store_it_here The structure to store the value + @param data The packed string which should be read + @param length The length (in bytes) of the value in nthe string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_time_read_internal(DYNAMIC_COLUMN_VALUE *store_it_here, + uchar *data, size_t length) +{ + if (length != 6) + goto err; + /* + 00000!<-hours--><---microseconds---> + 1123456789012345612345612345678901234567890 + <123456><123456><123456><123456><123456><123456> + */ + store_it_here->time_value.second_part= (data[0] | + (data[1] << 8) | + ((data[2] & 0xf) << 16)); + store_it_here->time_value.second= ((data[2] >> 4) | + ((data[3] & 0x3) << 4)); + store_it_here->time_value.minute= (data[3] >> 2); + store_it_here->time_value.hour= (((((uint)data[5]) & 0x3 ) << 8) | data[4]); + store_it_here->time_value.neg= ((data[5] & 0x4) ? 1 : 0); + if (store_it_here->time_value.second > 59 || + store_it_here->time_value.minute > 59 || + store_it_here->time_value.hour > 838 || + store_it_here->time_value.second_part > 999999) + goto err; + return ER_DYNCOL_OK; + +err: + store_it_here->time_value.time_type= MYSQL_TIMESTAMP_ERROR; + return ER_DYNCOL_FORMAT; +} + + +/** + Append the string with given date value. + + @param str the string where to put the value + @param value the value to put in the string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_date_store(DYNAMIC_COLUMN *str, MYSQL_TIME *value) +{ + uchar *buf; + if (dynstr_realloc(str, 3)) + return ER_DYNCOL_RESOURCE; + + buf= ((uchar *)str->str) + str->length; + if (value->time_type == MYSQL_TIMESTAMP_NONE || + value->time_type == MYSQL_TIMESTAMP_ERROR || + value->time_type == MYSQL_TIMESTAMP_TIME) + value->year= value->month= value->day = 0; + DBUG_ASSERT(value->year <= 9999); + DBUG_ASSERT(value->month <= 12); + DBUG_ASSERT(value->day <= 31); + /* + 0<----year----> + 012345678901234123412345 + <123456><123456><123456> + */ + buf[0]= (value->day | + ((value->month & 0x7) << 5)); + buf[1]= ((value->month >> 3) | ((value->year & 0x7F) << 1)); + buf[2]= (value->year >> 7); + str->length+= 3; + return ER_DYNCOL_OK; +} + + + +/** + Read date value of given length from the packed string + + @param store_it_here The structure to store the value + @param data The packed string which should be read + @param length The length (in bytes) of the value in nthe string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_date_read(DYNAMIC_COLUMN_VALUE *store_it_here, + uchar *data, size_t length) +{ + store_it_here->time_value.neg= store_it_here->time_value.second_part= + store_it_here->time_value.hour= store_it_here->time_value.minute= + store_it_here->time_value.second= 0; + store_it_here->time_value.time_type= MYSQL_TIMESTAMP_DATE; + return dynamic_column_date_read_internal(store_it_here, data, length); +} + +/** + Internal function for reading date part from the string. + + @param store_it_here The structure to store the value + @param data The packed string which should be read + @param length The length (in bytes) of the value in nthe string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_date_read_internal(DYNAMIC_COLUMN_VALUE *store_it_here, + uchar *data, + size_t length) +{ + if (length != 3) + goto err; + /* + 0<----year----> + 12345678901234123412345 + <123456><123456><123456> + */ + store_it_here->time_value.day= (data[0] & 0x1f); + store_it_here->time_value.month= (((data[1] & 0x1) << 3) | + (data[0] >> 5)); + store_it_here->time_value.year= ((((uint)data[2]) << 7) | + (data[1] >> 1)); + if (store_it_here->time_value.day > 31 || + store_it_here->time_value.month > 12 || + store_it_here->time_value.year > 9999) + goto err; + return ER_DYNCOL_OK; + +err: + store_it_here->time_value.time_type= MYSQL_TIMESTAMP_ERROR; + return ER_DYNCOL_FORMAT; +} + + +/** + Append the string with given value. + + @param str the string where to put the value + @param value the value to put in the string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +data_store(DYNAMIC_COLUMN *str, DYNAMIC_COLUMN_VALUE *value) +{ + switch (value->type) { + case DYN_COL_INT: + return dynamic_column_sint_store(str, value->long_value); + case DYN_COL_UINT: + return dynamic_column_uint_store(str, value->ulong_value); + case DYN_COL_DOUBLE: + return dynamic_column_double_store(str, value->double_value); + case DYN_COL_STRING: + return dynamic_column_string_store(str, &value->string_value, + value->charset); + case DYN_COL_DECIMAL: + return dynamic_column_decimal_store(str, &value->decimal_value); + case DYN_COL_DATETIME: + /* date+time in bits: 14 + 4 + 5 + 5 + 6 + 6 40bits = 5 bytes */ + return dynamic_column_date_time_store(str, &value->time_value); + case DYN_COL_DATE: + /* date in dits: 14 + 4 + 5 = 23bits ~= 3bytes*/ + return dynamic_column_date_store(str, &value->time_value); + case DYN_COL_TIME: + /* time in bits: 5 + 6 + 6 = 17bits ~= 3bytes*/ + return dynamic_column_time_store(str, &value->time_value); + case DYN_COL_NULL: + break; /* Impossible */ + } + DBUG_ASSERT(0); + return ER_DYNCOL_OK; /* Impossible */ +} + + +/** + Calculate length of offset field for given data length + + @param data_length Length of the data segment + + @return number of bytes +*/ + +static size_t dynamic_column_offset_bytes(size_t data_length) +{ + if (data_length < 0x1f) /* all 1 value is reserved */ + return 1; + if (data_length < 0x1fff) /* all 1 value is reserved */ + return 2; + if (data_length < 0x1fffff) /* all 1 value is reserved */ + return 3; + if (data_length < 0x1fffffff) /* all 1 value is reserved */ + return 4; + return MAX_OFFSET_LENGTH; /* For future */ +} + +/** + Store offset and type information in the given place + + @param place Beginning of the index entry + @param offset_size Size of offset field in bytes + @param type Type to be written + @param offset Offset to be written +*/ + +static void type_and_offset_store(uchar *place, size_t offset_size, + DYNAMIC_COLUMN_TYPE type, + size_t offset) +{ + ulong val = (((ulong) offset) << 3) | (type - 1); + DBUG_ASSERT(type != DYN_COL_NULL); + DBUG_ASSERT(((type - 1) & (~7)) == 0); /* fit in 3 bits */ + + /* Index entry starts with column number; Jump over it */ + place+= COLUMN_NUMBER_SIZE; + + switch (offset_size) { + case 1: + DBUG_ASSERT(offset < 0x1f); /* all 1 value is reserved */ + place[0]= (uchar)val; + break; + case 2: + DBUG_ASSERT(offset < 0x1fff); /* all 1 value is reserved */ + int2store(place, val); + break; + case 3: + DBUG_ASSERT(offset < 0x1fffff); /* all 1 value is reserved */ + int3store(place, val); + break; + case 4: + DBUG_ASSERT(offset < 0x1fffffff); /* all 1 value is reserved */ + int4store(place, val); + break; + default: + DBUG_ASSERT(0); /* impossible */ + } +} + + +/** + Read offset and type information from index entry + + @param type Where to put type info + @param offset Where to put offset info + @param place Beginning of the index entry + @param offset_size Size of offset field in bytes +*/ + +static void type_and_offset_read(DYNAMIC_COLUMN_TYPE *type, + size_t *offset, + uchar *place, size_t offset_size) +{ + ulong val; + + place+= COLUMN_NUMBER_SIZE; /* skip column number */ + switch (offset_size) { + case 1: + val= (ulong)place[0]; + break; + case 2: + val= uint2korr(place); + break; + case 3: + val= uint3korr(place); + break; + case 4: + val= uint4korr(place); + break; + default: + DBUG_ASSERT(0); /* impossible */ + } + *type= (val & 0x7) + 1; + *offset= val >> 3; +} + + +/** + Comparator function for references on column numbers for qsort +*/ + +static int column_sort(const void *a, const void *b) +{ + return **((uint **)a) - **((uint **)b); +} + + +/** + Write information to the fixed header + + @param str String where to write the header + @param offset_size Size of offset field in bytes + @param column_count Number of columns +*/ + +static void set_fixed_header(DYNAMIC_COLUMN *str, + uint offset_size, + uint column_count) +{ + DBUG_ASSERT(column_count <= 0xffff); + DBUG_ASSERT(offset_size <= 4); + str->str[0]= ((str->str[0] & ~DYNCOL_FLG_OFFSET) | + (offset_size - 1)); /* size of offset */ + int2store(str->str + 1, column_count); /* columns number */ + DBUG_ASSERT((str->str[0] & (~DYNCOL_FLG_KNOWN)) == 0); +} + +/* + Calculate entry size (E) and header size (H) by offset size (O) and column + count (C). +*/ + +#define calc_param(E,H,O,C) do { \ + (*(E))= (O) + COLUMN_NUMBER_SIZE; \ + (*(H))= (*(E)) * (C); \ +}while(0); + + +/** + Adds columns into the empty string + + @param str String where to write the data + @param header_size Size of the header without fixed part + @param offset_size Size of offset field in bytes + @param column_count Number of columns in the arrays + @parem not_null_count Number of non-null columns in the arrays + @param data_size Size of the data segment + @param column_numbers Array of columns numbers + @param values Array of columns values + @param new_str True if we need to allocate new string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_new_column_store(DYNAMIC_COLUMN *str, + size_t header_size, + size_t offset_size, + uint column_count, + uint not_null_count, + size_t data_size, + uint *column_numbers, + DYNAMIC_COLUMN_VALUE *values, + my_bool new_str) +{ + uchar *header_end; + uint **columns_order; + uint i; + uint entry_size= COLUMN_NUMBER_SIZE + offset_size; + enum enum_dyncol_func_result rc= ER_DYNCOL_RESOURCE; + + if (!(columns_order= malloc(sizeof(uint*)*column_count))) + return ER_DYNCOL_RESOURCE; + if (new_str) + { + if (dynamic_column_init_str(str, + data_size + header_size + DYNCOL_SYZERESERVE)) + goto err; + } + else + { + str->length= 0; + if (dynstr_realloc(str, data_size + header_size + DYNCOL_SYZERESERVE)) + goto err; + bzero(str->str, FIXED_HEADER_SIZE); + str->length= FIXED_HEADER_SIZE; + } + + /* sort columns for the header */ + for (i= 0; i < column_count; i++) + columns_order[i]= column_numbers + i; + qsort(columns_order, (size_t)column_count, sizeof(uint*), &column_sort); + + /* + For now we don't allow creating two columns with the same number + at the time of create. This can be fixed later to just use the later + by comparing the pointers. + */ + for (i= 0; i < column_count - 1; i++) + { + if (columns_order[i][0] > UINT_MAX16 || + columns_order[i][0] == columns_order[i + 1][0]) + { + rc= ER_DYNCOL_DATA; + goto err; + } + } + if (columns_order[i][0] > UINT_MAX16) + { + rc= ER_DYNCOL_DATA; + goto err; + } + + DBUG_ASSERT(str->max_length >= str->length + header_size); + set_fixed_header(str, offset_size, not_null_count); + str->length+= header_size; /* reserve place for header */ + header_end= (uchar *)str->str + FIXED_HEADER_SIZE; + for (i= 0; i < column_count; i++) + { + uint ord= columns_order[i] - column_numbers; + if (values[ord].type != DYN_COL_NULL) + { + /* Store header first in the str */ + int2store(header_end, column_numbers[ord]); + type_and_offset_store(header_end, offset_size, + values[ord].type, + str->length - header_size - FIXED_HEADER_SIZE); + + /* Store value in 'str + str->length' and increase str->length */ + if ((rc= data_store(str, values + ord))) + goto err; + header_end+= entry_size; + } + } + rc= ER_DYNCOL_OK; +err: + free(columns_order); + return rc; +} + +/** + Create packed string which contains given columns (internal) + + @param str String where to write the data + @param column_count Number of columns in the arrays + @param column_numbers Array of columns numbers + @param values Array of columns values + @param new_str True if we need allocate new string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_create_many_internal(DYNAMIC_COLUMN *str, + uint column_count, + uint *column_numbers, + DYNAMIC_COLUMN_VALUE *values, + my_bool new_str) +{ + size_t data_size= 0; + size_t header_size, offset_size; + uint i; + int not_null_column_count= 0; + + if (new_str) + { + /* to make dynstr_free() working in case of errors */ + bzero(str, sizeof(DYNAMIC_COLUMN)); + } + + for (i= 0; i < column_count; i++) + { + if (values[i].type != DYN_COL_NULL) + { + not_null_column_count++; + data_size+= dynamic_column_value_len(values + i); + } + } + + /* We can handle data up to 1fffffff = 536870911 bytes now */ + if ((offset_size= dynamic_column_offset_bytes(data_size)) >= + MAX_OFFSET_LENGTH) + return ER_DYNCOL_LIMIT; + + /* header entry is column number + offset & type */ + header_size= not_null_column_count * (offset_size + 2); + + return dynamic_new_column_store(str, + header_size, offset_size, + column_count, + not_null_column_count, + data_size, + column_numbers, values, + new_str); +} + + +/** + Create packed string which contains given columns + + @param str String where to write the data + @param column_count Number of columns in the arrays + @param column_numbers Array of columns numbers + @param values Array of columns values + + @return ER_DYNCOL_* return code +*/ + +enum enum_dyncol_func_result +dynamic_column_create_many(DYNAMIC_COLUMN *str, + uint column_count, + uint *column_numbers, + DYNAMIC_COLUMN_VALUE *values) +{ + DBUG_ENTER("dynamic_column_create_many"); + DBUG_RETURN(dynamic_column_create_many_internal(str, column_count, + column_numbers, values, + TRUE)); +} + + +/** + Create packed string which contains given column + + @param str String where to write the data + @param column_number Column number + @param value The columns value + + @return ER_DYNCOL_* return code +*/ + +enum enum_dyncol_func_result +dynamic_column_create(DYNAMIC_COLUMN *str, uint column_nr, + DYNAMIC_COLUMN_VALUE *value) +{ + DBUG_ENTER("dynamic_column_create"); + DBUG_RETURN(dynamic_column_create_many(str, 1, &column_nr, value)); +} + + +/** + Calculate length of data between given two header entries + + @param entry Pointer to the first entry + @param entry_next Pointer to the last entry + @param header_end Pointer to the header end + @param offset_size Size of offset field in bytes + @param last_offset Size of the data segment + + @return number of bytes +*/ + +static size_t get_length_interval(uchar *entry, uchar *entry_next, + uchar *header_end, size_t offset_size, + size_t last_offset) +{ + size_t offset, offset_next; + DYNAMIC_COLUMN_TYPE type, type_next; + DBUG_ASSERT(entry < entry_next); + + type_and_offset_read(&type, &offset, entry, offset_size); + if (entry_next >= header_end) + return (last_offset - offset); + type_and_offset_read(&type_next, &offset_next, entry_next, offset_size); + return (offset_next - offset); +} + +/* + Calculate length of data of one column + + + @param entry Pointer to the first entry + @param header_end Pointer to the header end + @param offset_size Size of offset field in bytes + @param last_offset Size of the data segment + + @return number of bytes +*/ + +static size_t get_length(uchar *entry, uchar *header_end, + size_t offset_size, + size_t last_offset) +{ + return get_length_interval(entry, + entry + offset_size + COLUMN_NUMBER_SIZE, + header_end, offset_size, last_offset); +} + + +/** + Comparator function for references to header entries for qsort +*/ + +static int header_compar(const void *a, const void *b) +{ + uint va= uint2korr((uchar*)a), vb= uint2korr((uchar*)b); + return (va > vb ? 1 : (va < vb ? -1 : 0)); +} + + +/** + Find column and fill information about it + + @param type Returns type of the column + @param data Returns a pointer to the data + @param length Returns length of the data + @param offset_size Size of offset field in bytes + @param column_count Number of column in the packed string + @param data_end Pointer to the data end + @param num Number of the column we want to fetch + @param entry_pos NULL or place where to put reference to the entry + + @return 0 ok + @return 1 error in data +*/ + +static my_bool +find_column(DYNAMIC_COLUMN_TYPE *type, uchar **data, size_t *length, + uchar *header, size_t offset_size, uint column_count, + uchar *data_end, uint num, uchar **entry_pos) +{ + uchar *entry; + size_t offset, total_data, header_size, entry_size; + uchar key[2+4]; + + if (!entry_pos) + entry_pos= &entry; + + calc_param(&entry_size, &header_size, offset_size, column_count); + + if (header + header_size > data_end) + return 1; + + int2store(key, num); + entry= bsearch(key, header, (size_t)column_count, entry_size, + &header_compar); + if (!entry) + { + /* Column not found */ + *type= DYN_COL_NULL; + *entry_pos= NULL; + return 0; + } + type_and_offset_read(type, &offset, entry, offset_size); + total_data= data_end - (header + header_size); + if (offset > total_data) + return 1; + *data= header + header_size + offset; + *length= get_length(entry, header + header_size, offset_size, + total_data); + /* + Check that the found data is withing the ranges. This can happen if + we get data with wrong offsets. + */ + if ((long) *length < 0 || offset + *length > total_data) + return 1; + + *entry_pos= entry; + return 0; +} + + +/** + Read and check the header of the dynamic string + + @param str Dynamic string + + @retval FALSE OK + @retval TRUE error + + Note + We don't check for str->length == 0 as all code that calls this + already have handled this case. +*/ + +static inline my_bool read_fixed_header(DYNAMIC_COLUMN *str, + size_t *offset_size, + uint *column_count) +{ + DBUG_ASSERT(str != NULL && str->length != 0); + if ((str->length < FIXED_HEADER_SIZE) || + (str->str[0] & (~DYNCOL_FLG_KNOWN))) + return 1; /* Wrong header */ + *offset_size= (str->str[0] & DYNCOL_FLG_OFFSET) + 1; + *column_count= uint2korr(str->str + 1); + return 0; +} + + +/** + Get dynamic column value + + @param str The packed string to extract the column + @param column_nr Number of column to fetch + @param store_it_here Where to store the extracted value + + @return ER_DYNCOL_* return code +*/ + +int dynamic_column_get(DYNAMIC_COLUMN *str, uint column_nr, + DYNAMIC_COLUMN_VALUE *store_it_here) +{ + uchar *data; + size_t offset_size, length; + uint column_count; + enum enum_dyncol_func_result rc= ER_DYNCOL_FORMAT; + + if (str->length == 0) + goto null; + + if (read_fixed_header(str, &offset_size, &column_count)) + goto err; + + if (column_count == 0) + goto null; + + if (find_column(&store_it_here->type, &data, &length, + (uchar*)str->str + FIXED_HEADER_SIZE, + offset_size, column_count, (uchar*)str->str + str->length, + column_nr, NULL)) + goto err; + + switch (store_it_here->type) { + case DYN_COL_INT: + rc= dynamic_column_sint_read(store_it_here, data, length); + break; + case DYN_COL_UINT: + rc= dynamic_column_uint_read(store_it_here, data, length); + break; + case DYN_COL_DOUBLE: + rc= dynamic_column_double_read(store_it_here, data, length); + break; + case DYN_COL_STRING: + rc= dynamic_column_string_read(store_it_here, data, length); + break; + case DYN_COL_DECIMAL: + rc= dynamic_column_decimal_read(store_it_here, data, length); + break; + case DYN_COL_DATETIME: + rc= dynamic_column_date_time_read(store_it_here, data, length); + break; + case DYN_COL_DATE: + rc= dynamic_column_date_read(store_it_here, data, length); + break; + case DYN_COL_TIME: + rc= dynamic_column_time_read(store_it_here, data, length); + break; + case DYN_COL_NULL: + rc= ER_DYNCOL_OK; + break; + default: + goto err; + } + return rc; + +null: + rc= ER_DYNCOL_OK; +err: + store_it_here->type= DYN_COL_NULL; + return rc; +} + +/** + Delete column with given number from the packed string + + @param str The packed string to delete the column + @param column_nr Number of column to delete + + @return ER_DYNCOL_* return code +*/ + +int dynamic_column_delete(DYNAMIC_COLUMN *str, uint column_nr) +{ + uchar *data, *header_entry, *read, *write; + size_t offset_size, new_offset_size, length, entry_size, new_entry_size, + header_size, new_header_size, data_size, new_data_size, + deleted_entry_offset; + uint column_count, i; + DYNAMIC_COLUMN_TYPE type; + + if (str->length == 0) + return ER_DYNCOL_OK; /* no columns */ + + if (read_fixed_header(str, &offset_size, &column_count)) + return ER_DYNCOL_FORMAT; + + if (column_count == 0) + { + str->length= 0; + return ER_DYNCOL_OK; /* no columns */ + } + + if (find_column(&type, &data, &length, (uchar*)str->str + FIXED_HEADER_SIZE, + offset_size, column_count, (uchar*)str->str + str->length, + column_nr, &header_entry)) + return ER_DYNCOL_FORMAT; + + if (type == DYN_COL_NULL) + return ER_DYNCOL_OK; /* no such column */ + + if (column_count == 1) + { + /* delete the only column; Return empty string */ + str->length= 0; + return ER_DYNCOL_OK; + } + + /* Calculate entry_size and header_size */ + calc_param(&entry_size, &header_size, offset_size, column_count); + data_size= str->length - FIXED_HEADER_SIZE - header_size; + + new_data_size= data_size - length; + if ((new_offset_size= dynamic_column_offset_bytes(new_data_size)) >= + MAX_OFFSET_LENGTH) + return ER_DYNCOL_LIMIT; + DBUG_ASSERT(new_offset_size <= offset_size); + + calc_param(&new_entry_size, &new_header_size, + new_offset_size, column_count - 1); + + deleted_entry_offset= ((data - (uchar*) str->str) - + header_size - FIXED_HEADER_SIZE); + + /* rewrite header*/ + set_fixed_header(str, new_offset_size, column_count - 1); + for (i= 0, write= read= (uchar *)str->str + FIXED_HEADER_SIZE; + i < column_count; + i++, read+= entry_size, write+= new_entry_size) + { + size_t offs; + uint nm; + DYNAMIC_COLUMN_TYPE tp; + if (read == header_entry) + { +#ifndef DBUG_OFF + nm= uint2korr(read); + type_and_offset_read(&tp, &offs, read, + offset_size); + DBUG_ASSERT(nm == column_nr); + DBUG_ASSERT(offs == deleted_entry_offset); +#endif + write-= new_entry_size; /* do not move writer */ + continue; /* skip removed field */ + } + + nm= uint2korr(read), + type_and_offset_read(&tp, &offs, read, + offset_size); + + if (offs > deleted_entry_offset) + offs-= length; /* data stored after removed data */ + + int2store(write, nm); + type_and_offset_store(write, new_offset_size, tp, offs); + } + + /* move data */ + { + size_t first_chunk_len= ((data - (uchar *)str->str) - + FIXED_HEADER_SIZE - header_size); + size_t second_chunk_len= new_data_size - first_chunk_len; + if (first_chunk_len) + memmove(str->str + FIXED_HEADER_SIZE + new_header_size, + str->str + FIXED_HEADER_SIZE + header_size, + first_chunk_len); + if (second_chunk_len) + memmove(str->str + + FIXED_HEADER_SIZE + new_header_size + first_chunk_len, + str->str + + FIXED_HEADER_SIZE + header_size + first_chunk_len + length, + second_chunk_len); + } + + /* fix str length */ + DBUG_ASSERT(str->length >= + FIXED_HEADER_SIZE + new_header_size + new_data_size); + str->length= FIXED_HEADER_SIZE + new_header_size + new_data_size; + + return ER_DYNCOL_OK; +} + + +/** + Check existence of the column in the packed string + + @param str The packed string to check the column + @param column_nr Number of column to check + + @return ER_DYNCOL_* return code +*/ + +enum enum_dyncol_func_result +dynamic_column_exists(DYNAMIC_COLUMN *str, uint column_nr) +{ + uchar *data; + size_t offset_size, length; + uint column_count; + DYNAMIC_COLUMN_TYPE type; + + if (str->length == 0) + return ER_DYNCOL_NO; /* no columns */ + + if (read_fixed_header(str, &offset_size, &column_count)) + return ER_DYNCOL_FORMAT; + + if (column_count == 0) + return ER_DYNCOL_NO; /* no columns */ + + if (find_column(&type, &data, &length, (uchar*)str->str + FIXED_HEADER_SIZE, + offset_size, column_count, (uchar*)str->str + str->length, + column_nr, NULL)) + return ER_DYNCOL_FORMAT; + + return (type != DYN_COL_NULL ? ER_DYNCOL_YES : ER_DYNCOL_NO); +} + + +/** + List not-null columns in the packed string + + @param str The packed string + @param array_of_uint Where to put reference on created array + + @return ER_DYNCOL_* return code +*/ + +enum enum_dyncol_func_result +dynamic_column_list(DYNAMIC_COLUMN *str, DYNAMIC_ARRAY *array_of_uint) +{ + uchar *read; + size_t offset_size, entry_size; + uint column_count, i; + + bzero(array_of_uint, sizeof(*array_of_uint)); /* In case of errors */ + if (str->length == 0) + return ER_DYNCOL_OK; /* no columns */ + + if (read_fixed_header(str, &offset_size, &column_count)) + return ER_DYNCOL_FORMAT; + + entry_size= COLUMN_NUMBER_SIZE + offset_size; + + if (entry_size * column_count + FIXED_HEADER_SIZE > str->length) + return ER_DYNCOL_FORMAT; + + if (init_dynamic_array(array_of_uint, sizeof(uint), + column_count, 0 CALLER_INFO)) + return ER_DYNCOL_RESOURCE; + + for (i= 0, read= (uchar *)str->str + FIXED_HEADER_SIZE; + i < column_count; + i++, read+= entry_size) + { + uint nm= uint2korr(read); + /* Insert can't never fail as it's pre-allocated above */ + (void) insert_dynamic(array_of_uint, (uchar *)&nm); + } + return ER_DYNCOL_OK; +} + + +/** + Find the place of the column in the header or place where it should be put + + @param num Number of the column + @param header Pointer to the header + @param entry_size Size of a header entry + @param column_count Number of columns in the packed string + @param entry Return pointer to the entry or next entry + + @retval TRUE found + @retval FALSE pointer set to the next row +*/ + +static my_bool +find_place(uint num, uchar *header, size_t entry_size, + uint column_count, uchar **entry) +{ + uint mid, start, end, val; + int flag; + + start= 0; + end= column_count -1; + mid= 1; + while (start != end) + { + uint val; + mid= (start + end) / 2; + val= uint2korr(header + mid * entry_size); + if ((flag= CMP_NUM(num, val)) <= 0) + end= mid; + else + start= mid + 1; + } + if (start != mid) + { + val= uint2korr(header + start * entry_size); + flag= CMP_NUM(num, val); + } + *entry= header + start * entry_size; + if (flag > 0) + *entry+= entry_size; /* Point at next bigger key */ + return flag == 0; +} + + +/* + Description of plan of adding/removing/updating a packed string +*/ + +typedef enum {PLAN_REPLACE, PLAN_ADD, PLAN_DELETE, PLAN_NOP} PLAN_ACT; + +struct st_plan { + DYNAMIC_COLUMN_VALUE *val; + uint *num; + uchar *place; + size_t length; + int hdelta, ddelta; + PLAN_ACT act; +}; +typedef struct st_plan PLAN; + + +static int plan_sort(const void *a, const void *b) +{ + return ((PLAN *)a)->num[0] - ((PLAN *)b)->num[0]; +} + +#define DELTA_CHECK(S, D, C) \ + if ((S) == 0) \ + (S)= (D); \ + else if (((S) > 0 && (D) < 0) || \ + ((S) < 0 && (D) > 0)) \ + { \ + (C)= TRUE; \ + break; \ + } \ + + +/** + Update the packed string with the given columns + + @param str String where to write the data + @param add_column_count Number of columns in the arrays + @param column_numbers Array of columns numbers + @param values Array of columns values + + @return ER_DYNCOL_* return code +*/ + +enum enum_dyncol_func_result +dynamic_column_update_many(DYNAMIC_COLUMN *str, + uint add_column_count, + uint *column_numbers, + DYNAMIC_COLUMN_VALUE *values) +{ + PLAN *plan; + uchar *header_end; + long data_delta= 0; + uint i, j, k; + uint new_column_count, column_count, not_null; + enum enum_dyncol_func_result rc; + int header_delta, header_delta_sign, data_delta_sign; + size_t offset_size, entry_size, header_size, data_size; + size_t new_offset_size, new_entry_size, new_header_size, new_data_size; + size_t max_offset; + my_bool copy; + + if (add_column_count == 0) + return ER_DYNCOL_OK; + + /* + Get columns in column order. As the data in 'str' is already + in column order this allows to replace all columns in one loop. + */ + + if (!(plan= my_malloc(sizeof(PLAN) * (add_column_count + 1), MYF(0)))) + return ER_DYNCOL_RESOURCE; + + not_null= add_column_count; + for (i= 0; i < add_column_count; i++) + { + if (column_numbers[i] > UINT_MAX16) + { + rc= ER_DYNCOL_DATA; + goto end; + } + + plan[i].val= values + i; + plan[i].num= column_numbers + i; + if (values[i].type == DYN_COL_NULL) + not_null--; + + } + + if (str->length == 0) + { + /* + Just add new columns. If there was no columns to add we return + an empty string. + */ + goto create_new_string; + } + + /* Check that header is ok */ + if (read_fixed_header(str, &offset_size, &column_count)) + { + rc= ER_DYNCOL_FORMAT; + goto end; + } + if (column_count == 0) + goto create_new_string; + + qsort(plan, (size_t)add_column_count, sizeof(PLAN), &plan_sort); + + new_column_count= column_count; + calc_param(&entry_size, &header_size, offset_size, column_count); + max_offset= str->length - (FIXED_HEADER_SIZE + header_size); + header_end= (uchar*) str->str + FIXED_HEADER_SIZE + header_size; + + if (header_size + FIXED_HEADER_SIZE > str->length) + { + rc= ER_DYNCOL_FORMAT; + goto end; + } + + /* + Calculate how many columns and data is added/deleted and make a 'plan' + for each of them. + */ + header_delta= 0; + for (i= 0; i < add_column_count; i++) + { + uchar *entry; + + /* + For now we don't allow creating two columns with the same number + at the time of create. This can be fixed later to just use the later + by comparing the pointers. + */ + if (i < add_column_count - 1 && plan[i].num[0] == plan[i + 1].num[0]) + { + rc= ER_DYNCOL_DATA; + goto end; + } + + /* Set common variables for all plans */ + plan[i].ddelta= data_delta; + /* get header delta in entries */ + plan[i].hdelta= header_delta; + plan[i].length= 0; /* Length if NULL */ + + if (find_place(plan[i].num[0], + (uchar *)str->str + FIXED_HEADER_SIZE, + entry_size, column_count, &entry)) + { + size_t entry_data_size; + + /* Data existed; We have to replace or delete it */ + + entry_data_size= get_length(entry, header_end, + offset_size, max_offset); + if ((long) entry_data_size < 0) + { + rc= ER_DYNCOL_FORMAT; + goto end; + } + + if (plan[i].val->type == DYN_COL_NULL) + { + /* Inserting a NULL means delete the old data */ + + plan[i].act= PLAN_DELETE; /* Remove old value */ + header_delta--; /* One row less in header */ + data_delta-= entry_data_size; /* Less data to store */ + } + else + { + /* Replace the value */ + + plan[i].act= PLAN_REPLACE; + /* get data delta in bytes */ + plan[i].length= dynamic_column_value_len(plan[i].val); + data_delta+= plan[i].length - entry_data_size; + } + } + else + { + /* Data did not exists. Add if it it's not NULL */ + + if (plan[i].val->type == DYN_COL_NULL) + { + plan[i].act= PLAN_NOP; /* Mark entry to be skiped */ + } + else + { + /* Add new value */ + + plan[i].act= PLAN_ADD; + header_delta++; /* One more row in header */ + /* get data delta in bytes */ + plan[i].length= dynamic_column_value_len(plan[i].val); + data_delta+= plan[i].length; + } + } + plan[i].place= entry; + } + plan[add_column_count].hdelta= header_delta; + plan[add_column_count].ddelta= data_delta; + new_column_count= column_count + header_delta; + + /* + Check if it is only "increasing" or only "decreasing" plan for (header + and data separately). + */ + data_size= str->length - header_size - FIXED_HEADER_SIZE; + new_data_size= data_size + data_delta; + if ((new_offset_size= dynamic_column_offset_bytes(new_data_size)) >= + MAX_OFFSET_LENGTH) + { + rc= ER_DYNCOL_LIMIT; + goto end; + } + + /* if (new_offset_size != offset_size) then we have to rewrite header */ + header_delta_sign= new_offset_size - offset_size; + data_delta_sign= 0; + copy= FALSE; + for (i= 0; i < add_column_count; i++) + { + /* This is the check for increasing/decreasing */ + DELTA_CHECK(header_delta_sign, plan[i].hdelta, copy); + DELTA_CHECK(data_delta_sign, plan[i].ddelta, copy); + } + + calc_param(&new_entry_size, &new_header_size, + new_offset_size, new_column_count); + + /* + The following code always make a copy. In future we can do a more + optimized version when data is only increasing / decreasing. + */ + + /*if (copy) */ + { + DYNAMIC_COLUMN tmp; + uchar *header_base= (uchar *)str->str + FIXED_HEADER_SIZE, + *write; + if (dynamic_column_init_str(&tmp, + (FIXED_HEADER_SIZE + new_header_size + + new_data_size + DYNCOL_SYZERESERVE))) + { + rc= ER_DYNCOL_RESOURCE; + goto end; + } + write= (uchar *)tmp.str + FIXED_HEADER_SIZE; + /* Adjust tmp to contain whole the future header */ + tmp.length= FIXED_HEADER_SIZE + new_header_size; + set_fixed_header(&tmp, new_offset_size, new_column_count); + data_delta= 0; + + /* + Copy data to the new string + i= index in array of changes + j= index in packed string header index + */ + + for (i= 0, j= 0; i < add_column_count || j < column_count; i++) + { + size_t first_offset; + uint start= j, end; + LINT_INIT(first_offset); + + /* + Search in i and j for the next column to add from i and where to + add. + */ + + while (i < add_column_count && plan[i].act == PLAN_NOP) + i++; /* skip NOP */ + if (i == add_column_count) + j= end= column_count; + else + { + /* + old data portion. We don't need to check that j < column_count + as plan[i].place is guaranteed to have a pointer inside the + data. + */ + while (header_base + j * entry_size < plan[i].place) + j++; + end= j; + if ((plan[i].act == PLAN_REPLACE || plan[i].act == PLAN_DELETE)) + j++; /* data at 'j' will be removed */ + } + + if (plan[i].ddelta == 0 && offset_size == new_offset_size) + { + uchar *read= header_base + start * entry_size; + DYNAMIC_COLUMN_TYPE tp; + /* + It's safe to copy the header unchanged. This is usually the + case for the first header block before any changed data. + */ + if (start < end) /* Avoid memcpy with 0 */ + { + size_t length= entry_size * (end - start); + memcpy(write, read, length); + write+= length; + } + /* Read first_offset */ + type_and_offset_read(&tp, &first_offset, read, offset_size); + } + else + { + /* + Adjust all headers since last loop. + We have to do this as the offset for data has moved + */ + for (k= start; k < end; k++) + { + uchar *read= header_base + k * entry_size; + size_t offs; + uint nm; + DYNAMIC_COLUMN_TYPE tp; + + nm= uint2korr(read); /* Column nummber */ + type_and_offset_read(&tp, &offs, read, offset_size); + if (k == start) + first_offset= offs; + offs+= plan[i].ddelta; + int2store(write, nm); + /* write rest of data at write + COLUMN_NUMBER_SIZE */ + type_and_offset_store(write, new_offset_size, tp, offs); + write+= new_entry_size; + } + } + + /* copy first the data that was not replaced in original packed data */ + if (start < end) + { + /* Add old data last in 'tmp' */ + size_t data_size= + get_length_interval(header_base + start * entry_size, + header_base + end * entry_size, + header_end, offset_size, max_offset); + if ((long) data_size < 0) + { + dynamic_column_column_free(&tmp); + rc= ER_DYNCOL_FORMAT; + goto end; + } + + memcpy(tmp.str + tmp.length, (char *)header_end + first_offset, + data_size); + tmp.length+= data_size; + } + + /* new data adding */ + if (i < add_column_count) + { + if( plan[i].act == PLAN_ADD || plan[i].act == PLAN_REPLACE) + { + int2store(write, plan[i].num[0]); + type_and_offset_store(write, new_offset_size, + plan[i].val[0].type, + tmp.length - + (FIXED_HEADER_SIZE + new_header_size)); + write+= new_entry_size; + data_store(&tmp, plan[i].val); /* Append new data */ + } + data_delta= plan[i].ddelta; + } + } + dynamic_column_column_free(str); + *str= tmp; + } + + rc= ER_DYNCOL_OK; + +end: + my_free(plan, MYF(0)); + return rc; + +create_new_string: + /* There is no columns from before, so let's just add the new ones */ + rc= ER_DYNCOL_OK; + if (not_null != 0) + rc= dynamic_column_create_many_internal(str, add_column_count, + column_numbers, values, + str->str == NULL); + goto end; +} + + +/** + Update the packed string with the given column + + @param str String where to write the data + @param column_number Array of columns number + @param values Array of columns values + + @return ER_DYNCOL_* return code +*/ + + +int dynamic_column_update(DYNAMIC_COLUMN *str, uint column_nr, + DYNAMIC_COLUMN_VALUE *value) +{ + return dynamic_column_update_many(str, 1, &column_nr, value); +} diff --git a/mysys/string.c b/mysys/string.c index 10a72b8a295..030178c6667 100644 --- a/mysys/string.c +++ b/mysys/string.c @@ -183,3 +183,15 @@ void dynstr_free(DYNAMIC_STRING *str) str->str=0; } } + + +/* Give over the control of the dynamic string to caller */ + +void dynstr_reassociate(DYNAMIC_STRING *str, char **ptr, size_t *length, + size_t *alloc_length) +{ + *ptr= str->str; + *length= str->length; + *alloc_length= str->max_length; + str->str=0; +} -- cgit v1.2.1 From c735c1b700ea882115db8b99db04c118ec5b45a1 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 11 May 2011 02:28:32 +0200 Subject: Fix compile error by changing #include to #include "mysys_priv.h" Fix various warnings about conversion from bigger to smaller integer types in assignments --- mysys/ma_dyncol.c | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) (limited to 'mysys') diff --git a/mysys/ma_dyncol.c b/mysys/ma_dyncol.c index 71eacc6b319..11754d4cec2 100644 --- a/mysys/ma_dyncol.c +++ b/mysys/ma_dyncol.c @@ -26,7 +26,7 @@ SUCH DAMAGE. */ -#include +#include "mysys_priv.h" #include #include @@ -429,7 +429,7 @@ dynamic_column_string_read(DYNAMIC_COLUMN_VALUE *store_it_here, uchar *data, size_t length) { size_t len; - uint charset_nr= dynamic_column_var_uint_get(data, length, &len); + uint charset_nr= (uint)dynamic_column_var_uint_get(data, length, &len); if (len == 0) /* Wrong packed number */ return ER_DYNCOL_FORMAT; store_it_here->charset= get_charset(charset_nr, MYF(MY_WME)); @@ -502,12 +502,12 @@ dynamic_column_decimal_read(DYNAMIC_COLUMN_VALUE *store_it_here, uchar *data, size_t length) { size_t intg_len, frac_len; - uint intg, frac; + int intg, frac; dynamic_column_prepare_decimal(store_it_here); - intg= dynamic_column_var_uint_get(data, length, &intg_len); + intg= (int)dynamic_column_var_uint_get(data, length, &intg_len); data+= intg_len; - frac= dynamic_column_var_uint_get(data, length - intg_len, &frac_len); + frac= (int)dynamic_column_var_uint_get(data, length - intg_len, &frac_len); data+= frac_len; /* Check the size of data is correct */ @@ -602,8 +602,13 @@ dynamic_column_time_store(DYNAMIC_COLUMN *str, MYSQL_TIME *value) if (value->time_type == MYSQL_TIMESTAMP_NONE || value->time_type == MYSQL_TIMESTAMP_ERROR || value->time_type == MYSQL_TIMESTAMP_DATE) - value->neg= value->second_part= value->hour= - value->minute= value->second= 0; + { + value->neg= 0; + value->second_part= 0; + value->hour= 0; + value->minute= 0; + value->second= 0; + } DBUG_ASSERT(value->hour <= 838); DBUG_ASSERT(value->minute <= 59); DBUG_ASSERT(value->second <= 59); @@ -615,7 +620,7 @@ dynamic_column_time_store(DYNAMIC_COLUMN *str, MYSQL_TIME *value) */ buf[0]= (value->second_part & 0xff); buf[1]= ((value->second_part & 0xff00) >> 8); - buf[2]= (((value->second & 0xf) << 4) | + buf[2]= (uchar)(((value->second & 0xf) << 4) | ((value->second_part & 0xf0000) >> 16)); buf[3]= ((value->minute << 2) | ((value->second & 0x30) >> 4)); buf[4]= (value->hour & 0xff); @@ -740,9 +745,11 @@ static enum enum_dyncol_func_result dynamic_column_date_read(DYNAMIC_COLUMN_VALUE *store_it_here, uchar *data, size_t length) { - store_it_here->time_value.neg= store_it_here->time_value.second_part= - store_it_here->time_value.hour= store_it_here->time_value.minute= - store_it_here->time_value.second= 0; + store_it_here->time_value.neg= 0; + store_it_here->time_value.second_part= 0; + store_it_here->time_value.hour= 0; + store_it_here->time_value.minute= 0; + store_it_here->time_value.second= 0; store_it_here->time_value.time_type= MYSQL_TIMESTAMP_DATE; return dynamic_column_date_read_internal(store_it_here, data, length); } -- cgit v1.2.1 From 8d52c2cffe0ff75ea9a4313c776cc5441e70aef3 Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Wed, 11 May 2011 13:59:17 +0300 Subject: Fixed compiler warnings and test cases problems found by buildbot mysql-test/r/dyncol.result: Updated test results mysql-test/r/index_intersect.result: Updated results mysql-test/r/index_intersect_innodb.result: Updated results mysql-test/t/dyncol.test: Added replace_result for floating point results that are different on windows Added round() around a result to get same result on all platforms. mysql-test/t/index_intersect.test: Added replace_result to fix that index_merge may put key names in different order. mysys/ma_dyncol.c: Fixed compiler warnings on Solaris sql/key.cc: Fixed compiler warnings on Solaris sql/mysqld.cc: Fixed compiler warning on windows support-files/compiler_warnings.supp: Suppressed an unintersting warning on Solaris --- mysys/ma_dyncol.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'mysys') diff --git a/mysys/ma_dyncol.c b/mysys/ma_dyncol.c index 71eacc6b319..6a8e4d689f3 100644 --- a/mysys/ma_dyncol.c +++ b/mysys/ma_dyncol.c @@ -905,6 +905,7 @@ static void type_and_offset_read(DYNAMIC_COLUMN_TYPE *type, uchar *place, size_t offset_size) { ulong val; + LINT_INIT(val); place+= COLUMN_NUMBER_SIZE; /* skip column number */ switch (offset_size) { @@ -1611,6 +1612,7 @@ find_place(uint num, uchar *header, size_t entry_size, { uint mid, start, end, val; int flag; + LINT_INIT(flag); /* 100 % safe */ start= 0; end= column_count -1; -- cgit v1.2.1 From f940c2ca14516268e3ed75cb93b69c7876fdc09f Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Wed, 11 May 2011 15:59:42 +0300 Subject: Fixed a memory overrun in dynamic columns when sending in a mailformed (too short in this case) string. mysql-test/t/dyncol.test: Added test case for mailformed string usage mysys/ma_dyncol.c: Added test for wrong dynamic string data --- mysys/ma_dyncol.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'mysys') diff --git a/mysys/ma_dyncol.c b/mysys/ma_dyncol.c index 6a8e4d689f3..a9cb3c42655 100644 --- a/mysys/ma_dyncol.c +++ b/mysys/ma_dyncol.c @@ -1963,6 +1963,13 @@ dynamic_column_update_many(DYNAMIC_COLUMN *str, type_and_offset_read(&tp, &offs, read, offset_size); if (k == start) first_offset= offs; + else if (offs < first_offset) + { + dynamic_column_column_free(&tmp); + rc= ER_DYNCOL_FORMAT; + goto end; + } + offs+= plan[i].ddelta; int2store(write, nm); /* write rest of data at write + COLUMN_NUMBER_SIZE */ @@ -1979,7 +1986,8 @@ dynamic_column_update_many(DYNAMIC_COLUMN *str, get_length_interval(header_base + start * entry_size, header_base + end * entry_size, header_end, offset_size, max_offset); - if ((long) data_size < 0) + if ((long) data_size < 0 || + data_size > max_offset - first_offset) { dynamic_column_column_free(&tmp); rc= ER_DYNCOL_FORMAT; -- cgit v1.2.1 From 4c81cef75d7871e2c77d6723813ac328c34603b5 Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Thu, 12 May 2011 02:19:28 +0300 Subject: Fixed bug when accessing wrong decimal value in dynamic string (Fixed lp:781233) Store decimal 0.0 in zero bytes in dynamic strings. mysqltest: Don't ignore error from mysql_stmt_fetch; This could cause rows to be missing from log when running with --ps-protocol Fixed wrong result length for CAST(... as TIME) client/mysqltest.cc: Don't ignore error from mysql_stmt_fetch; This could cause rows to be missing from log when running with --ps-protocol libmysql/libmysql.c: The max length for a TIME column is 17, not 15. mysql-test/r/dyncol.result: More tests mysql-test/t/dyncol.test: More tests mysys/ma_dyncol.c: Check content of decimal value on read and store to not get assert in decimal_bin_size(). Store decimal 0.0 in zero bytes in dynamic strings. This also solves a problem where decimal 0 had different internal representations. sql-common/my_time.c: Fixed DBUG_PRINT sql/item_timefunc.h: Fixed wrong result length for CAST(... as TIME). This was the cause of failures in buildbot when doing cast(... as time); sql/protocol.cc: More DBUG_PRINT --- mysys/ma_dyncol.c | 77 +++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 64 insertions(+), 13 deletions(-) (limited to 'mysys') diff --git a/mysys/ma_dyncol.c b/mysys/ma_dyncol.c index fd060e867ed..dcb03d7f073 100644 --- a/mysys/ma_dyncol.c +++ b/mysys/ma_dyncol.c @@ -312,7 +312,9 @@ dynamic_column_sint_read(DYNAMIC_COLUMN_VALUE *store_it_here, @param value The value for which we are calculating length - @return number of bytes + @return + Error: (size_t) ~0 + ok number of bytes */ static size_t @@ -331,11 +333,32 @@ dynamic_column_value_len(DYNAMIC_COLUMN_VALUE *value) return (dynamic_column_var_uint_bytes(value->charset->number) + value->string_value.length); case DYN_COL_DECIMAL: + { + int precision= value->decimal_value.intg + value->decimal_value.frac; + int scale= value->decimal_value.frac; + + if (precision == 0 || decimal_is_zero(&value->decimal_value)) + { + /* This is here to simplify dynamic_column_decimal_store() */ + value->decimal_value.intg= value->decimal_value.frac= 0; + return 0; + } + /* + Check if legal decimal; This is needed to not get an assert in + decimal_bin_size(). However this should be impossible as all + decimals entered here should be valid and we have the special check + above to handle the unlikely but possible case that decimal_value.intg + and decimal.frac is 0. + */ + if (scale < 0 || precision <= 0) + { + DBUG_ASSERT(0); /* Impossible */ + return (size_t) ~0; + } return (dynamic_column_var_uint_bytes(value->decimal_value.intg) + dynamic_column_var_uint_bytes(value->decimal_value.frac) + - decimal_bin_size(value->decimal_value.intg + - value->decimal_value.frac, - value->decimal_value.frac)); + decimal_bin_size(precision, scale)); + } case DYN_COL_DATETIME: /* date+time in bits: 14 + 4 + 5 + 10 + 6 + 6 + 20 + 1 66bits ~= 9 bytes */ return 9; @@ -455,7 +478,14 @@ static enum enum_dyncol_func_result dynamic_column_decimal_store(DYNAMIC_COLUMN *str, decimal_t *value) { - uint bin_size= decimal_bin_size(value->intg + value->frac, value->frac); + uint bin_size; + int precision= value->intg + value->frac; + + /* Store decimal zero as empty string */ + if (precision == 0) + return ER_DYNCOL_OK; + + bin_size= decimal_bin_size(precision, value->frac); if (dynstr_realloc(str, bin_size + 20)) return ER_DYNCOL_RESOURCE; @@ -464,8 +494,7 @@ dynamic_column_decimal_store(DYNAMIC_COLUMN *str, (void) dynamic_column_var_uint_store(str, value->frac); decimal2bin(value, (uchar *) str->str + str->length, - value->intg + value->frac, - value->frac); + precision, value->frac); str->length+= bin_size; return ER_DYNCOL_OK; } @@ -502,20 +531,29 @@ dynamic_column_decimal_read(DYNAMIC_COLUMN_VALUE *store_it_here, uchar *data, size_t length) { size_t intg_len, frac_len; - int intg, frac; + int intg, frac, precision, scale; dynamic_column_prepare_decimal(store_it_here); + /* Decimals 0.0 is stored as a zero length string */ + if (length == 0) + return ER_DYNCOL_OK; /* value contains zero */ + intg= (int)dynamic_column_var_uint_get(data, length, &intg_len); data+= intg_len; frac= (int)dynamic_column_var_uint_get(data, length - intg_len, &frac_len); data+= frac_len; /* Check the size of data is correct */ - if (decimal_bin_size(intg + frac, frac) != + precision= intg + frac; + scale= frac; + if (scale < 0 || precision <= 0 || scale > precision || + (length - intg_len - frac_len) > + (size_t) (DECIMAL_BUFF_LENGTH*sizeof(decimal_digit_t)) || + decimal_bin_size(intg + frac, frac) != (int) (length - intg_len - frac_len)) return ER_DYNCOL_FORMAT; - if (bin2decimal(data, &store_it_here->decimal_value, intg + frac, frac) != + if (bin2decimal(data, &store_it_here->decimal_value, precision, scale) != E_DEC_OK) return ER_DYNCOL_FORMAT; return ER_DYNCOL_OK; @@ -1113,8 +1151,11 @@ dynamic_column_create_many_internal(DYNAMIC_COLUMN *str, { if (values[i].type != DYN_COL_NULL) { + size_t tmp; not_null_column_count++; - data_size+= dynamic_column_value_len(values + i); + data_size+= (tmp=dynamic_column_value_len(values + i)); + if (tmp == (size_t) ~0) + return ER_DYNCOL_DATA; } } @@ -1822,7 +1863,12 @@ dynamic_column_update_many(DYNAMIC_COLUMN *str, plan[i].act= PLAN_REPLACE; /* get data delta in bytes */ - plan[i].length= dynamic_column_value_len(plan[i].val); + if ((plan[i].length= dynamic_column_value_len(plan[i].val)) == + (size_t) ~0) + { + rc= ER_DYNCOL_DATA; + goto end; + } data_delta+= plan[i].length - entry_data_size; } } @@ -1841,7 +1887,12 @@ dynamic_column_update_many(DYNAMIC_COLUMN *str, plan[i].act= PLAN_ADD; header_delta++; /* One more row in header */ /* get data delta in bytes */ - plan[i].length= dynamic_column_value_len(plan[i].val); + if ((plan[i].length= dynamic_column_value_len(plan[i].val)) == + (size_t) ~0) + { + rc= ER_DYNCOL_DATA; + goto end; + } data_delta+= plan[i].length; } } -- cgit v1.2.1 From 5c70f813f3d12974bf67d6ff98c0ff4b9ca71d7d Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Wed, 18 May 2011 13:36:12 +0300 Subject: Added HA_ERR_DISK_FULL handler error Original code by Zardosht Kasheff sql/handler.cc: Added HA_ERR_DISK_FULL and ENOSPC (for handler that uses normal errno). This sets 'fatal_error' to ensure that the error is logged to err file (which hopefully is on another disk...) --- mysys/my_handler_errors.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'mysys') diff --git a/mysys/my_handler_errors.h b/mysys/my_handler_errors.h index 19de02a00a8..6213ab188cc 100644 --- a/mysys/my_handler_errors.h +++ b/mysys/my_handler_errors.h @@ -64,5 +64,6 @@ static const char *handler_error_messages[]= "Read page with wrong checksum", "Too many active concurrent transactions", "Row is not visible by the current transaction", - "Operation was interrupted by end user (probably kill command?)" + "Operation was interrupted by end user (probably kill command?)", + "Disk full" }; -- cgit v1.2.1 From 1177bea2f1cd0c9f887670b65c9a403a26f07d76 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 18 May 2011 16:27:19 +0300 Subject: Rewritten patch of percona - switching query cache on and off, removing comments. client/mysqltest.cc: Column names. mysql-test/r/grant_cache_no_prot.result: fix of text. mysql-test/r/grant_cache_ps_prot.result: Fix of test. mysql-test/r/query_cache.result: Switching on and off query cache. mysql-test/t/query_cache.test: Switching on and off query cache. mysys/charset.c: Fix of parser. sql/handler.cc: thd added to parameters. sql/log_event.cc: thd added to parameters. sql/log_event_old.cc: thd added to parameters. sql/mysql_priv.h: Fixed functions definitions. sql/mysqld.cc: Comments stripping. sql/set_var.cc: Switching on and off query cache. sql/set_var.h: Switching on and off query cache. sql/share/errmsg.txt: New errors. sql/sql_cache.cc: Switching query cache on and off, removing comments. sql/sql_cache.h: thd added to parameters. sql/sql_class.h: Comments stripping. sql/sql_db.cc: thd added to parameters. sql/sql_lex.cc: lex fixed. sql/sql_parse.cc: thd added to parameters. --- mysys/charset.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'mysys') diff --git a/mysys/charset.c b/mysys/charset.c index 7b72f1f5ae3..c7756583b19 100644 --- a/mysys/charset.c +++ b/mysys/charset.c @@ -97,6 +97,9 @@ static my_bool init_state_maps(struct charset_info_st *cs) state_map[(uchar)'@']= (uchar) MY_LEX_USER_END; state_map[(uchar) '`']= (uchar) MY_LEX_USER_VARIABLE_DELIMITER; state_map[(uchar)'"']= (uchar) MY_LEX_STRING_OR_DELIMITER; + state_map[(uchar)'-']= (uchar) MY_LEX_MINUS_OR_COMMENT; + state_map[(uchar)',']= (uchar) MY_LEX_COMMA; + state_map[(uchar)'?']= (uchar) MY_LEX_PLACEHOLDER; /* Create a second map to make it faster to find identifiers -- cgit v1.2.1 From 03b33425e5a3de3fad070aa21e7fc7baf8104c81 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 19 May 2011 19:05:35 +0200 Subject: many changes to my_getsystime.c: * my_getsystime() is only an interval timer. Its value can beused for calculating time intervals. * renamed my_getsystime() to my_interval_timer(), to make the semantics clearer and let the compiler catch wrong usages of my_getsystime() (also future ones, that may come in merges). * increased its granularity from 100ns to 1ns, old value was for UUID, but as UUID can no longer use it directly there is no need to downgrade the OS provided value * fixed the UUID code to anchor the my_interval_timer() on the epoch, as required by the UUID standard. That is, this was only needed by UUID, and now I've moved it to UUID code from my_getsystime(). * fixed other wrong usages of my_getsystime() - e.g. in calculating times for pthread_cond_timedwait. It was buggy and could've caused long waits if OS clock would be changed. --- mysys/my_getsystime.c | 101 ++++++++++++++++---------------------------------- mysys/queues.c | 6 +-- 2 files changed, 35 insertions(+), 72 deletions(-) (limited to 'mysys') diff --git a/mysys/my_getsystime.c b/mysys/my_getsystime.c index 125d389f3d5..1e713ea26c3 100644 --- a/mysys/my_getsystime.c +++ b/mysys/my_getsystime.c @@ -14,67 +14,66 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ -/* - TODO: in functions my_micro_time() and my_micro_time_and_time() there - exists some common code that should be merged into a function. -*/ - #include "mysys_priv.h" #include "my_static.h" #ifdef __NETWARE__ #include #elif defined(__WIN__) -static ulonglong query_performance_frequency, query_performance_offset; #define OFFSET_TO_EPOC 116444736000000000LL -#elif defined(HAVE_GETHRTIME) -static ulonglong gethrtime_offset; +static ulonglong query_performance_frequency; #endif /* - get time since epoc in 100 nanosec units + return number of nanoseconds since unspecified (but always the same) + point in the past NOTE: Thus to get the current time we should use the system function with the highest possible resolution - The value is not subject to resetting or drifting by way of adjtime() or - settimeofday(), and thus it is *NOT* appropriate for getting the current - timestamp. It can be used for calculating time intervals, though. - And it's good enough for UUID. + The value is not not anchored to any specific point in time (e.g. epoch) nor + is it subject to resetting or drifting by way of adjtime() or settimeofday(), + and thus it is *NOT* appropriate for getting the current timestamp. It can be + used for calculating time intervals, though. */ -ulonglong my_getsystime() +ulonglong my_interval_timer() { #ifdef HAVE_CLOCK_GETTIME struct timespec tp; - clock_gettime(CLOCK_REALTIME, &tp); - return (ulonglong)tp.tv_sec*10000000+(ulonglong)tp.tv_nsec/100; + clock_gettime(CLOCK_MONOTONIC, &tp); + return tp.tv_sec*1000000000ULL+tp.tv_nsec; #elif defined(HAVE_GETHRTIME) - return gethrtime()/100-gethrtime_offset; + return gethrtime(); #elif defined(__WIN__) LARGE_INTEGER t_cnt; if (query_performance_frequency) { QueryPerformanceCounter(&t_cnt); - return ((t_cnt.QuadPart / query_performance_frequency * 10000000) + - ((t_cnt.QuadPart % query_performance_frequency) * 10000000 / - query_performance_frequency) + query_performance_offset); + return (t_cnt.QuadPart / query_performance_frequency * 1000000000ULL) + + ((t_cnt.QuadPart % query_performance_frequency) * 1000000000ULL / + query_performance_frequency); + } + else + { + ulonglong newtime; + GetSystemTimeAsFileTime((FILETIME*)&newtime); + return newtime*100ULL; } - return 0; #elif defined(__NETWARE__) NXTime_t tm; NXGetTime(NX_SINCE_1970, NX_NSECONDS, &tm); - return (ulonglong)tm/100; + return (ulonglong)tm; #else /* TODO: check for other possibilities for hi-res timestamping */ struct timeval tv; gettimeofday(&tv,NULL); - return (ulonglong)tv.tv_sec*10000000+(ulonglong)tv.tv_usec*10; + return tv.tv_sec*1000000000ULL+tv.tv_usec*1000ULL; #endif } -/* Return current time in microseconds since epoch */ +/* Return current time in HRTIME_RESOLUTION (microseconds) since epoch */ my_hrtime_t my_hrtime() { @@ -84,60 +83,24 @@ my_hrtime_t my_hrtime() GetSystemTimeAsFileTime((FILETIME*)&newtime); newtime -= OFFSET_TO_EPOC; hrtime.val= newtime/10; -#elif defined(HAVE_GETHRTIME) - struct timeval t; - /* - The following loop is here because gettimeofday may fail on some systems - */ - while (gettimeofday(&t, NULL) != 0) - {} - hrtime.val= t.tv_sec*1000000 + t.tv_usec; +#elif defined(HAVE_CLOCK_GETTIME) + struct timespec tp; + clock_gettime(CLOCK_REALTIME, &tp); + return tp.tv_sec*1000000ULL+tp.tv_nsec/1000ULL; #else - hrtime.val= my_getsystime()/10; + struct timeval t; + /* The following loop is here because gettimeofday may fail on some systems */ + while (gettimeofday(&t, NULL) != 0) {} + hrtime.val= t.tv_sec*1000000ULL + t.tv_usec; #endif return hrtime; } -/* - This function is basically equivalent to - - *interval= my_getsystime()/10; - *timestamp= my_time(); - - but it avoids calling OS time functions twice, if possible. -*/ -void my_diff_and_hrtime(my_timediff_t *interval, my_hrtime_t *timestamp) -{ - interval->val= my_getsystime() / 10; -#if defined(__WIN__) || defined(HAVE_GETHRTIME) - *timestamp= my_hrtime(); -#else - timestamp->val= interval->val; -#endif -} - void my_time_init() { #ifdef __WIN__ - FILETIME ft; - LARGE_INTEGER li, t_cnt; - DBUG_ASSERT(sizeof(LARGE_INTEGER) == sizeof(query_performance_frequency)); + compile_time_assert(sizeof(LARGE_INTEGER) == sizeof(query_performance_frequency)); if (QueryPerformanceFrequency((LARGE_INTEGER *)&query_performance_frequency) == 0) query_performance_frequency= 0; - else - { - GetSystemTimeAsFileTime(&ft); - li.LowPart= ft.dwLowDateTime; - li.HighPart= ft.dwHighDateTime; - query_performance_offset= li.QuadPart-OFFSET_TO_EPOC; - QueryPerformanceCounter(&t_cnt); - query_performance_offset-= (t_cnt.QuadPart / - query_performance_frequency * 10000000 + - t_cnt.QuadPart % - query_performance_frequency * 10000000 / - query_performance_frequency); - } -#elif defined(HAVE_GETHRTIME) - gethrtime_offset= gethrtime(); #endif } diff --git a/mysys/queues.c b/mysys/queues.c index 9c85e493141..ff7fc1bc308 100644 --- a/mysys/queues.c +++ b/mysys/queues.c @@ -596,15 +596,15 @@ bool do_test(uint no_parts, uint l_max_ind, bool l_fix_used) static void start_measurement() { - start_time= my_getsystime(); + start_time= my_interval_timer(); } static void stop_measurement() { - ulonglong stop_time= my_getsystime(); + ulonglong stop_time= my_interval_timer(); uint time_in_micros; stop_time-= start_time; - stop_time/= 10; /* Convert to microseconds */ + stop_time/= 1000; /* Convert to microseconds */ time_in_micros= (uint)stop_time; printf("Time expired is %u microseconds \n", time_in_micros); } -- cgit v1.2.1 From b519f2b626ebd1f3243a21dc883cefa6a26460f9 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 28 May 2011 16:57:58 +0200 Subject: Fix compile errors and warnings and test errors introduced by microseconds push. Also, change windows timespec definition to be Unix-ish - simplifies handling a lot. --- mysys/my_wincond.c | 37 ++++++++++++++----------------------- 1 file changed, 14 insertions(+), 23 deletions(-) (limited to 'mysys') diff --git a/mysys/my_wincond.c b/mysys/my_wincond.c index b869b22bdea..e181b24cdef 100644 --- a/mysys/my_wincond.c +++ b/mysys/my_wincond.c @@ -77,31 +77,22 @@ int pthread_cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex, struct timespec *abstime) { int result; - long timeout; - union ft64 now; - + DWORD timeout; + long long timeout_us; + my_hrtime_t now; + my_hrtime_t then; if( abstime != NULL ) { - GetSystemTimeAsFileTime(&now.ft); - - /* - Calculate time left to abstime - - subtract start time from current time(values are in 100ns units) - - convert to millisec by dividing with 10000 - */ - timeout= (long)((abstime->tv.i64 - now.i64) / 10000); - - /* Don't allow the timeout to be negative */ - if (timeout < 0) - timeout= 0L; - - /* - Make sure the calucated timeout does not exceed original timeout - value which could cause "wait for ever" if system time changes - */ - if (timeout > abstime->max_timeout_msec) - timeout= abstime->max_timeout_msec; - + now= my_hrtime(); + then.val= 1000000ULL*abstime->tv_sec + abstime->tv_nsec/1000; + timeout_us= then.val - now.val; + + if (timeout_us < 0) + timeout= 0; + else if (timeout_us > 1000ULL*INFINITE) + timeout= INFINITE; + else + timeout= (DWORD)(timeout_us/1000); } else { -- cgit v1.2.1 From 6ae42b75b894b5c8a338536861601f2d6d3db686 Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Mon, 6 Jun 2011 16:39:06 +0300 Subject: Fixed lock sorting and lock check issues with thr_lock that caused warnings when running test suite. Safety check that could cause core dump when doing create table with virtual column. mysql-test/mysql-test-run.pl: Show also warnings from thr_lock (which starts with just Warning, not Warning:) mysql-test/r/lock.result: Added test that showed not relevant warning when using table locks. mysql-test/t/lock.test: Added test that showed not relevant warning when using table locks. mysys/thr_lock.c: Fixed sorting of locks. (Old sort code didn't handle case where TL_WRITE_CONCURRENT_INSERT must be sorted before TL_WRITE) Added more information to check_locks warning output. Fixed wrong testing of multiple different write locks for same table. sql/item_cmpfunc.cc: Safety check that could cause core dump when doing create table with virtual column. --- mysys/thr_lock.c | 112 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 70 insertions(+), 42 deletions(-) (limited to 'mysys') diff --git a/mysys/thr_lock.c b/mysys/thr_lock.c index f7908ab57bc..32593ca3f4f 100644 --- a/mysys/thr_lock.c +++ b/mysys/thr_lock.c @@ -112,25 +112,37 @@ static inline pthread_cond_t *get_cond(void) /* + Sort locks in priority order + + LOCK_CMP() + A First lock + B Second lock + + Return: + 0 if A >= B + 1 if A < B + Priority for locks (decides in which order locks are locked) We want all write locks to be first, followed by read locks. Locks from MERGE tables has a little lower priority than other locks, to allow one to release merge tables without having to unlock and re-lock other locks. The lower the number, the higher the priority for the lock. - Read locks should have 4, write locks should have 0. - UNLOCK is 8, to force these last in thr_merge_locks. For MERGE tables we add 2 (THR_LOCK_MERGE_PRIV) to the lock priority. THR_LOCK_LATE_PRIV (1) is used when one locks other tables to be merged with existing locks. This way we prioritize the original locks over the new locks. */ -static uint lock_priority[(uint)TL_WRITE_ONLY+1] = -{ 8, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0}; - -#define LOCK_CMP(A,B) ((uchar*) ((A)->lock) + lock_priority[(uint) (A)->type] + (A)->priority < (uchar*) ((B)->lock) + lock_priority[(uint) (B)->type] + (B)->priority) +inline int LOCK_CMP(THR_LOCK_DATA *A, THR_LOCK_DATA *B) +{ + if (A->lock != B->lock) + return A->lock < B->lock; + if (A->type != B->type) + return A->type > B->type; /* Prioritize read */ + return A->priority < B->priority; +} /* For the future (now the thread specific cond is alloced by my_pthread.c) @@ -215,6 +227,7 @@ static int check_lock(struct st_lock_list *list, const char* lock_type, static void check_locks(THR_LOCK *lock, const char *where, + enum thr_lock_type type, my_bool allow_no_locks) { uint old_found_errors=found_errors; @@ -279,6 +292,7 @@ static void check_locks(THR_LOCK *lock, const char *where, found_errors++; fprintf(stderr, "Warning at '%s': Write lock %d waiting while no exclusive read locks\n",where,(int) lock->write_wait.data->type); + DBUG_PRINT("warning", ("Warning at '%s': Write lock %d waiting while no exclusive read locks\n",where,(int) lock->write_wait.data->type)); } } } @@ -293,8 +307,10 @@ static void check_locks(THR_LOCK *lock, const char *where, if (data->type != TL_WRITE_CONCURRENT_INSERT) { fprintf(stderr, - "Warning at '%s': Found TL_WRITE_CONCURRENT_INSERT lock mixed with other write locks\n", - where); + "Warning at '%s': Found TL_WRITE_CONCURRENT_INSERT lock mixed with other write lock: %d\n", + where, data->type); + DBUG_PRINT("warning", ("Warning at '%s': Found TL_WRITE_CONCURRENT_INSERT lock mixed with other write lock: %d\n", + where, data->type)); break; } } @@ -309,26 +325,34 @@ static void check_locks(THR_LOCK *lock, const char *where, fprintf(stderr, "Warning at '%s': Found WRITE_ALLOW_WRITE lock waiting for WRITE_ALLOW_WRITE lock\n", where); + DBUG_PRINT("warning", ("Warning at '%s': Found WRITE_ALLOW_WRITE lock waiting for WRITE_ALLOW_WRITE lock\n", + where)); + } } if (lock->read.data) { - if (!thr_lock_owner_equal(lock->write.data->owner, - lock->read.data->owner) && + THR_LOCK_DATA *data; + for (data=lock->read.data ; data ; data=data->next) + { + if (!thr_lock_owner_equal(lock->write.data->owner, + data->owner) && ((lock->write.data->type > TL_WRITE_DELAYED && lock->write.data->type != TL_WRITE_ONLY) || ((lock->write.data->type == TL_WRITE_CONCURRENT_INSERT || lock->write.data->type == TL_WRITE_ALLOW_WRITE) && - lock->read_no_write_count))) - { - found_errors++; - fprintf(stderr, - "Warning at '%s': Found lock of type %d that is write and read locked\n", - where, lock->write.data->type); - DBUG_PRINT("warning",("At '%s': Found lock of type %d that is write and read locked\n", - where, lock->write.data->type)); - - } + data->type == TL_READ_NO_INSERT))) + { + found_errors++; + fprintf(stderr, + "Warning at '%s' for lock: %d: Found lock of type %d that is write and read locked. Read_no_write_count: %d\n", + where, (int) type, lock->write.data->type, + lock->read_no_write_count); + DBUG_PRINT("warning",("At '%s' for lock %d: Found lock of type %d that is write and read locked\n", + where, (int) type, + lock->write.data->type)); + } + } } if (lock->read_wait.data) { @@ -354,7 +378,7 @@ static void check_locks(THR_LOCK *lock, const char *where, } #else /* EXTRA_DEBUG */ -#define check_locks(A,B,C) +#define check_locks(A,B,C,D) #endif @@ -531,13 +555,14 @@ wait_for_lock(struct st_lock_list *wait, THR_LOCK_DATA *data, else wait->last=data->prev; data->type= TL_UNLOCK; /* No lock */ - check_locks(data->lock, "killed or timed out wait_for_lock", 1); + check_locks(data->lock, "killed or timed out wait_for_lock", data->type, + 1); wake_up_waiters(data->lock); } else { DBUG_PRINT("thr_lock", ("lock aborted")); - check_locks(data->lock, "aborted wait_for_lock", 0); + check_locks(data->lock, "aborted wait_for_lock", data->type, 0); } } else @@ -546,7 +571,7 @@ wait_for_lock(struct st_lock_list *wait, THR_LOCK_DATA *data, if (data->lock->get_status) (*data->lock->get_status)(data->status_param, data->type == TL_WRITE_CONCURRENT_INSERT); - check_locks(data->lock,"got wait_for_lock",0); + check_locks(data->lock,"got wait_for_lock", data->type, 0); } pthread_mutex_unlock(&data->lock->mutex); @@ -579,7 +604,7 @@ thr_lock(THR_LOCK_DATA *data, THR_LOCK_OWNER *owner, (long) data, data->owner->info->thread_id, (long) lock, (int) lock_type)); check_locks(lock,(uint) lock_type <= (uint) TL_READ_NO_INSERT ? - "enter read_lock" : "enter write_lock",0); + "enter read_lock" : "enter write_lock", lock_type, 0); if ((int) lock_type <= (int) TL_READ_NO_INSERT) { /* Request for READ lock */ @@ -608,7 +633,7 @@ thr_lock(THR_LOCK_DATA *data, THR_LOCK_OWNER *owner, lock->read.last= &data->next; if (lock_type == TL_READ_NO_INSERT) lock->read_no_write_count++; - check_locks(lock,"read lock with old write lock",0); + check_locks(lock,"read lock with old write lock", lock_type, 0); if (lock->get_status) (*lock->get_status)(data->status_param, 0); statistic_increment(locks_immediate,&THR_LOCK_lock); @@ -632,7 +657,7 @@ thr_lock(THR_LOCK_DATA *data, THR_LOCK_OWNER *owner, lock->read.last= &data->next; if (lock_type == TL_READ_NO_INSERT) lock->read_no_write_count++; - check_locks(lock,"read lock with no write locks",0); + check_locks(lock,"read lock with no write locks", lock_type, 0); if (lock->get_status) (*lock->get_status)(data->status_param, 0); statistic_increment(locks_immediate,&THR_LOCK_lock); @@ -717,7 +742,7 @@ thr_lock(THR_LOCK_DATA *data, THR_LOCK_OWNER *owner, (*lock->write.last)=data; /* Add to running fifo */ data->prev=lock->write.last; lock->write.last= &data->next; - check_locks(lock,"second write lock",0); + check_locks(lock,"second write lock", lock_type, 0); if (lock->get_status) (*lock->get_status)(data->status_param, lock_type == TL_WRITE_CONCURRENT_INSERT); @@ -755,7 +780,7 @@ thr_lock(THR_LOCK_DATA *data, THR_LOCK_OWNER *owner, lock->write.last= &data->next; if (lock->get_status) (*lock->get_status)(data->status_param, concurrent_insert); - check_locks(lock,"only write lock",0); + check_locks(lock,"only write lock", lock_type, 0); statistic_increment(locks_immediate,&THR_LOCK_lock); goto end; } @@ -791,7 +816,7 @@ static inline void free_all_read_locks(THR_LOCK *lock, { THR_LOCK_DATA *data=lock->read_wait.data; - check_locks(lock,"before freeing read locks",1); + check_locks(lock,"before freeing read locks", TL_UNLOCK, 1); /* move all locks from read_wait list to read list */ (*lock->read.last)=data; @@ -833,7 +858,7 @@ static inline void free_all_read_locks(THR_LOCK *lock, *lock->read_wait.last=0; if (!lock->read_wait.data) lock->write_lock_count=0; - check_locks(lock,"after giving read locks",0); + check_locks(lock,"after giving read locks", TL_UNLOCK, 0); } /* Unlock lock and free next thread on same lock */ @@ -846,7 +871,7 @@ void thr_unlock(THR_LOCK_DATA *data, uint unlock_flags) DBUG_PRINT("lock",("data: 0x%lx thread: 0x%lx lock: 0x%lx", (long) data, data->owner->info->thread_id, (long) lock)); pthread_mutex_lock(&lock->mutex); - check_locks(lock,"start of release lock",0); + check_locks(lock,"start of release lock", lock_type, 0); if (((*data->prev)=data->next)) /* remove from lock-list */ data->next->prev= data->prev; @@ -880,9 +905,9 @@ void thr_unlock(THR_LOCK_DATA *data, uint unlock_flags) if (lock_type == TL_READ_NO_INSERT) lock->read_no_write_count--; data->type=TL_UNLOCK; /* Mark unlocked */ - check_locks(lock,"after releasing lock",1); + check_locks(lock,"after releasing lock", lock_type, 1); wake_up_waiters(lock); - check_locks(lock,"end of thr_unlock",1); + check_locks(lock,"end of thr_unlock", lock_type, 1); pthread_mutex_unlock(&lock->mutex); DBUG_VOID_RETURN; } @@ -1007,7 +1032,7 @@ static void wake_up_waiters(THR_LOCK *lock) free_all_read_locks(lock,0); } end: - check_locks(lock, "after waking up waiters", 0); + check_locks(lock, "after waking up waiters", TL_UNLOCK, 0); DBUG_VOID_RETURN; } @@ -1317,7 +1342,7 @@ void thr_downgrade_write_lock(THR_LOCK_DATA *in_data, DBUG_ASSERT(old_lock_type == TL_WRITE_ONLY); DBUG_ASSERT(old_lock_type > new_lock_type); in_data->type= new_lock_type; - check_locks(lock,"after downgrading lock",0); + check_locks(lock,"after downgrading lock", old_lock_type, 0); #if TO_BE_REMOVED switch (old_lock_type) @@ -1417,7 +1442,8 @@ void thr_downgrade_write_lock(THR_LOCK_DATA *in_data, data->prev= lock->write.last; lock->write.last= &data->next; data->next= 0; - check_locks(lock, "Started write lock after downgrade",0); + check_locks(lock, "Started write lock after downgrade", new_lock_type, + 0); data->cond= 0; pthread_cond_signal(cond); } @@ -1470,13 +1496,15 @@ void thr_downgrade_write_lock(THR_LOCK_DATA *in_data, if (data->type == TL_READ_NO_INSERT) lock->read_no_write_count++; - check_locks(lock, "Started read lock after downgrade",0); + check_locks(lock, "Started read lock after downgrade", new_lock_type, + 0); data->cond= 0; pthread_cond_signal(cond); } } } - check_locks(lock,"after starting waiters after downgrading lock",0); + check_locks(lock,"after starting waiters after downgrading lock", + new_lock_type, 0); #endif pthread_mutex_unlock(&lock->mutex); DBUG_VOID_RETURN; @@ -1497,7 +1525,7 @@ my_bool thr_upgrade_write_delay_lock(THR_LOCK_DATA *data, pthread_mutex_unlock(&lock->mutex); DBUG_RETURN(data->type == TL_UNLOCK); /* Test if Aborted */ } - check_locks(lock,"before upgrading lock",0); + check_locks(lock,"before upgrading lock", data->type, 0); /* TODO: Upgrade to TL_WRITE_CONCURRENT_INSERT in some cases */ data->type= new_lock_type; /* Upgrade lock */ @@ -1525,11 +1553,11 @@ my_bool thr_upgrade_write_delay_lock(THR_LOCK_DATA *data, lock->write_wait.last= &data->next; data->prev= &lock->write_wait.data; lock->write_wait.data=data; - check_locks(lock,"upgrading lock",0); + check_locks(lock,"upgrading lock", new_lock_type, 0); } else { - check_locks(lock,"waiting for lock",0); + check_locks(lock,"waiting for lock", new_lock_type, 0); } res= wait_for_lock(&lock->write_wait,data,1); if (res == THR_LOCK_SUCCESS && lock->start_trans) -- cgit v1.2.1 From 4171483b539555f50336d4304d931ef743cf7011 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sun, 12 Jun 2011 15:52:07 +0200 Subject: Backport Fix for Bug#24509 - 2048 file descriptor limit on windows needs increasing. The patch replaces the use of the POSIX I/O interfaces in mysys on Windows with the Win32 API calls (CreateFile, WriteFile, etc). The Windows HANDLE for the open file is stored in the my_file_info struct, along with a flag for append mode (because the Windows API does not support opening files in append mode in all cases) The default max open files has been increased to 16384 and can be increased further by setting --max-open-files= during the server start. Noteworthy benefit of this patch is that it removes limits from the table_cache size - allowing for more simultaneus users --- mysys/CMakeLists.txt | 2 +- mysys/default_modify.c | 8 +- mysys/mf_iocache.c | 3 - mysys/my_chsize.c | 19 +- mysys/my_create.c | 15 +- mysys/my_dup.c | 6 +- mysys/my_file.c | 1 + mysys/my_fopen.c | 48 ++-- mysys/my_fstream.c | 17 +- mysys/my_init.c | 3 +- mysys/my_lib.c | 26 +- mysys/my_lock.c | 137 ++++++++-- mysys/my_mmap.c | 6 +- mysys/my_open.c | 227 +---------------- mysys/my_pread.c | 81 +++--- mysys/my_quick.c | 22 +- mysys/my_read.c | 7 +- mysys/my_seek.c | 17 +- mysys/my_static.c | 2 +- mysys/my_sync.c | 2 +- mysys/my_winerr.c | 123 +++++++++ mysys/my_winfile.c | 681 +++++++++++++++++++++++++++++++++++++++++++++++++ mysys/my_write.c | 36 ++- mysys/mysys_priv.h | 24 ++ 24 files changed, 1122 insertions(+), 391 deletions(-) create mode 100644 mysys/my_winerr.c create mode 100644 mysys/my_winfile.c (limited to 'mysys') diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index 9ab19222caf..0cf10e6e993 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -38,7 +38,7 @@ SET(MYSYS_SOURCES array.c charset-def.c charset.c checksum.c default.c default_ my_mkdir.c my_mmap.c my_net.c my_once.c my_open.c my_pread.c my_pthread.c my_quick.c my_read.c my_realloc.c my_redel.c my_rename.c my_seek.c my_sleep.c my_static.c my_symlink.c my_symlink2.c my_sync.c my_thr_init.c my_wincond.c - my_windac.c my_winthread.c my_write.c ptr_cmp.c queues.c stacktrace.c + my_winerr.c my_winfile.c my_windac.c my_winthread.c my_write.c ptr_cmp.c queues.c stacktrace.c rijndael.c safemalloc.c sha1.c string.c thr_alarm.c thr_lock.c thr_mutex.c thr_rwlock.c tree.c typelib.c my_vle.c base64.c my_memmem.c my_getpagesize.c lf_alloc-pin.c lf_dynarray.c lf_hash.c diff --git a/mysys/default_modify.c b/mysys/default_modify.c index 88df0122da2..ccbf47176a6 100644 --- a/mysys/default_modify.c +++ b/mysys/default_modify.c @@ -21,7 +21,7 @@ #define BUFF_SIZE 1024 #define RESERVE 1024 /* Extend buffer with this extent */ -#ifdef __WIN__ +#ifdef _WIN32 #define NEWLINE "\r\n" #define NEWLINE_LEN 2 #else @@ -78,7 +78,7 @@ int modify_defaults_file(const char *file_location, const char *option, DBUG_RETURN(2); /* my_fstat doesn't use the flag parameter */ - if (my_fstat(fileno(cnf_file), &file_stat, MYF(0))) + if (my_fstat(my_fileno(cnf_file), &file_stat, MYF(0))) goto malloc_err; if (option && option_value) @@ -96,7 +96,7 @@ int modify_defaults_file(const char *file_location, const char *option, NEWLINE_LEN + /* Space for newline */ RESERVE); /* Some additional space */ - buffer_size= (file_stat.st_size + + buffer_size= (uint)(file_stat.st_size + 1); /* The ending zero */ /* @@ -213,7 +213,7 @@ int modify_defaults_file(const char *file_location, const char *option, if (opt_applied) { /* Don't write the file if there are no changes to be made */ - if (my_chsize(fileno(cnf_file), (my_off_t) (dst_ptr - file_buffer), 0, + if (my_chsize(my_fileno(cnf_file), (my_off_t) (dst_ptr - file_buffer), 0, MYF(MY_WME)) || my_fseek(cnf_file, 0, MY_SEEK_SET, MYF(0)) || my_fwrite(cnf_file, (uchar*) file_buffer, (size_t) (dst_ptr - file_buffer), diff --git a/mysys/mf_iocache.c b/mysys/mf_iocache.c index 3824669365f..1905437cc0a 100644 --- a/mysys/mf_iocache.c +++ b/mysys/mf_iocache.c @@ -1659,9 +1659,6 @@ int my_block_write(register IO_CACHE *info, const uchar *Buffer, size_t Count, Buffer+=length; pos+= length; Count-= length; -#ifndef HAVE_PREAD - info->seek_not_done=1; -#endif } /* Check if we want to write inside the used part of the buffer.*/ diff --git a/mysys/my_chsize.c b/mysys/my_chsize.c index b1dbb22c687..b9013811b34 100644 --- a/mysys/my_chsize.c +++ b/mysys/my_chsize.c @@ -52,20 +52,13 @@ int my_chsize(File fd, my_off_t newlength, int filler, myf MyFlags) if (oldsize > newlength) { -#if defined(HAVE_SETFILEPOINTER) - /* This is for the moment only true on windows */ - long is_success; - HANDLE win_file= (HANDLE) _get_osfhandle(fd); - long length_low, length_high; - length_low= (long) (ulong) newlength; - length_high= (long) ((ulonglong) newlength >> 32); - is_success= SetFilePointer(win_file, length_low, &length_high, FILE_BEGIN); - if (is_success == -1 && (my_errno= GetLastError()) != NO_ERROR) +#ifdef _WIN32 + if (my_win_chsize(fd, newlength)) + { + my_errno= errno; goto err; - if (SetEndOfFile(win_file)) - DBUG_RETURN(0); - my_errno= GetLastError(); - goto err; + } + DBUG_RETURN(0); #elif defined(HAVE_FTRUNCATE) if (ftruncate(fd, (off_t) newlength)) { diff --git a/mysys/my_create.c b/mysys/my_create.c index 5c9a1e027d2..d0436276d03 100644 --- a/mysys/my_create.c +++ b/mysys/my_create.c @@ -18,7 +18,7 @@ #include "mysys_err.h" #include #include -#if defined(__WIN__) +#if defined(_WIN32) #include #endif @@ -41,16 +41,12 @@ File my_create(const char *FileName, int CreateFlags, int access_flags, FileName, CreateFlags, access_flags, MyFlags)); #if !defined(NO_OPEN_3) - fd = open((char *) FileName, access_flags | O_CREAT, + fd= open((char *) FileName, access_flags | O_CREAT, CreateFlags ? CreateFlags : my_umask); -#elif defined(VMS) - fd = open((char *) FileName, access_flags | O_CREAT, 0, - "ctx=stm","ctx=bin"); -#elif defined(__WIN__) - fd= my_sopen((char *) FileName, access_flags | O_CREAT | O_BINARY, - SH_DENYNO, MY_S_IREAD | MY_S_IWRITE); +#elif defined(_WIN32) + fd= my_win_open(FileName, access_flags | O_CREAT); #else - fd = open(FileName, access_flags); + fd= open(FileName, access_flags); #endif if ((MyFlags & MY_SYNC_DIR) && (fd >=0) && @@ -71,6 +67,7 @@ File my_create(const char *FileName, int CreateFlags, int access_flags, if (unlikely(fd >= 0 && rc < 0)) { int tmp= my_errno; + my_close(fd, MyFlags); my_delete(FileName, MyFlags); my_errno= tmp; } diff --git a/mysys/my_dup.c b/mysys/my_dup.c index 55f5e0c0099..5fdd6e9f364 100644 --- a/mysys/my_dup.c +++ b/mysys/my_dup.c @@ -29,7 +29,11 @@ File my_dup(File file, myf MyFlags) const char *filename; DBUG_ENTER("my_dup"); DBUG_PRINT("my",("file: %d MyFlags: %d", file, MyFlags)); - fd = dup(file); +#ifdef _WIN32 + fd= my_win_dup(file); +#else + fd= dup(file); +#endif filename= (((uint) file < my_file_limit) ? my_file_info[(int) file].name : "Unknown"); DBUG_RETURN(my_register_filename(fd, filename, FILE_BY_DUP, diff --git a/mysys/my_file.c b/mysys/my_file.c index 594f361437f..d8d51b91ab2 100644 --- a/mysys/my_file.c +++ b/mysys/my_file.c @@ -97,6 +97,7 @@ uint my_set_max_open_files(uint files) DBUG_ENTER("my_set_max_open_files"); DBUG_PRINT("enter",("files: %u my_file_limit: %u", files, my_file_limit)); + files+= MY_FILE_MIN; files= set_max_open_files(min(files, OS_FILE_LIMIT)); if (files <= MY_NFILE) DBUG_RETURN(files); diff --git a/mysys/my_fopen.c b/mysys/my_fopen.c index a4b0c9f895d..ae631a59353 100644 --- a/mysys/my_fopen.c +++ b/mysys/my_fopen.c @@ -46,24 +46,14 @@ FILE *my_fopen(const char *filename, int flags, myf MyFlags) DBUG_ENTER("my_fopen"); DBUG_PRINT("my",("Name: '%s' flags: %d MyFlags: %d", filename, flags, MyFlags)); - /* - if we are not creating, then we need to use my_access to make sure - the file exists since Windows doesn't handle files like "com1.sym" - very well - */ -#ifdef __WIN__ - if (check_if_legal_filename(filename)) - { - errno= EACCES; - fd= 0; - } - else + + make_ftype(type,flags); + +#ifdef _WIN32 + fd= my_win_fopen(filename, type); +#else + fd= fopen(filename, type); #endif - { - make_ftype(type,flags); - fd = fopen(filename, type); - } - if (fd != 0) { /* @@ -71,18 +61,20 @@ FILE *my_fopen(const char *filename, int flags, myf MyFlags) on some OS (SUNOS). Actually the filename save isn't that important so we can ignore if this doesn't work. */ - if ((uint) fileno(fd) >= my_file_limit) + + int filedesc= my_fileno(fd); + if ((uint)filedesc >= my_file_limit) { thread_safe_increment(my_stream_opened,&THR_LOCK_open); DBUG_RETURN(fd); /* safeguard */ } pthread_mutex_lock(&THR_LOCK_open); - if ((my_file_info[fileno(fd)].name = (char*) + if ((my_file_info[filedesc].name= (char*) my_strdup(filename,MyFlags))) { my_stream_opened++; my_file_total_opened++; - my_file_info[fileno(fd)].type = STREAM_BY_FOPEN; + my_file_info[filedesc].type= STREAM_BY_FOPEN; pthread_mutex_unlock(&THR_LOCK_open); DBUG_PRINT("exit",("stream: 0x%lx", (long) fd)); DBUG_RETURN(fd); @@ -240,8 +232,13 @@ int my_fclose(FILE *fd, myf MyFlags) DBUG_PRINT("my",("stream: 0x%lx MyFlags: %d", (long) fd, MyFlags)); pthread_mutex_lock(&THR_LOCK_open); - file=fileno(fd); - if ((err = fclose(fd)) < 0) + file= my_fileno(fd); +#ifndef _WIN32 + err= fclose(fd); +#else + err= my_win_fclose(fd); +#endif + if(err < 0) { my_errno=errno; if (MyFlags & (MY_FAE | MY_WME)) @@ -272,7 +269,12 @@ FILE *my_fdopen(File Filedes, const char *name, int Flags, myf MyFlags) Filedes, Flags, MyFlags)); make_ftype(type,Flags); - if ((fd = fdopen(Filedes, type)) == 0) +#ifdef _WIN32 + fd= my_win_fdopen(Filedes, type); +#else + fd= fdopen(Filedes, type); +#endif + if (!fd) { my_errno=errno; if (MyFlags & (MY_FAE | MY_WME)) diff --git a/mysys/my_fstream.c b/mysys/my_fstream.c index f3b5418b906..2059e1a9f18 100644 --- a/mysys/my_fstream.c +++ b/mysys/my_fstream.c @@ -56,11 +56,11 @@ size_t my_fread(FILE *stream, uchar *Buffer, size_t Count, myf MyFlags) { if (ferror(stream)) my_error(EE_READ, MYF(ME_BELL+ME_WAITTANG), - my_filename(fileno(stream)),errno); + my_filename(my_fileno(stream)),errno); else if (MyFlags & (MY_NABP | MY_FNABP)) my_error(EE_EOFERR, MYF(ME_BELL+ME_WAITTANG), - my_filename(fileno(stream)),errno); + my_filename(my_fileno(stream)),errno); } my_errno=errno ? errno : -1; if (ferror(stream) || MyFlags & (MY_NABP | MY_FNABP)) @@ -142,7 +142,7 @@ size_t my_fwrite(FILE *stream, const uchar *Buffer, size_t Count, myf MyFlags) if (MyFlags & (MY_WME | MY_FAE | MY_FNABP)) { my_error(EE_WRITE, MYF(ME_BELL+ME_WAITTANG), - my_filename(fileno(stream)),errno); + my_filename(my_fileno(stream)),errno); } writtenbytes= (size_t) -1; /* Return that we got error */ break; @@ -182,3 +182,14 @@ my_off_t my_ftell(FILE *stream, myf MyFlags __attribute__((unused))) DBUG_PRINT("exit",("ftell: %lu",(ulong) pos)); DBUG_RETURN((my_off_t) pos); } /* my_ftell */ + + +/* Get a File corresponding to the stream*/ +int my_fileno(FILE *f) +{ +#ifdef _WIN32 + return my_win_fileno(f); +#else + return fileno(f); +#endif +} diff --git a/mysys/my_init.c b/mysys/my_init.c index e7ab9ba7a1f..7f174cea1f0 100644 --- a/mysys/my_init.c +++ b/mysys/my_init.c @@ -274,6 +274,7 @@ void my_parameter_handler(const wchar_t * expression, const wchar_t * function, { DBUG_PRINT("my",("Expression: %s function: %s file: %s, line: %d", expression, function, file, line)); + __debugbreak(); } @@ -298,7 +299,7 @@ int handle_rtc_failure(int err_type, const char *file, int line, fprintf(stderr, " At %s:%d\n", file, line); va_end(args); (void) fflush(stderr); - + __debugbreak(); return 0; /* Error is handled */ } #pragma runtime_checks("", restore) diff --git a/mysys/my_lib.c b/mysys/my_lib.c index c18d14fb549..033f8789b49 100644 --- a/mysys/my_lib.c +++ b/mysys/my_lib.c @@ -35,8 +35,7 @@ # if defined(HAVE_NDIR_H) # include # endif -# if defined(__WIN__) -# include +# if defined(_WIN32) # ifdef __BORLANDC__ # include # endif @@ -92,7 +91,7 @@ static int comp_names(struct fileinfo *a, struct fileinfo *b) } /* comp_names */ -#if !defined(__WIN__) +#if !defined(_WIN32) MY_DIR *my_dir(const char *path, myf MyFlags) { @@ -507,19 +506,24 @@ error: DBUG_RETURN((MY_DIR *) NULL); } /* my_dir */ -#endif /* __WIN__ */ +#endif /* _WIN32 */ /**************************************************************************** ** File status ** Note that MY_STAT is assumed to be same as struct stat ****************************************************************************/ -int my_fstat(int Filedes, MY_STAT *stat_area, + +int my_fstat(File Filedes, MY_STAT *stat_area, myf MyFlags __attribute__((unused))) { DBUG_ENTER("my_fstat"); DBUG_PRINT("my",("fd: %d MyFlags: %d", Filedes, MyFlags)); +#ifdef _WIN32 + DBUG_RETURN(my_win_fstat(Filedes, stat_area)); +#else DBUG_RETURN(fstat(Filedes, (struct stat *) stat_area)); +#endif } @@ -531,11 +535,15 @@ MY_STAT *my_stat(const char *path, MY_STAT *stat_area, myf my_flags) (long) stat_area, my_flags)); if ((m_used= (stat_area == NULL))) - if (!(stat_area = (MY_STAT *) my_malloc(sizeof(MY_STAT), my_flags))) + if (!(stat_area= (MY_STAT *) my_malloc(sizeof(MY_STAT), my_flags))) goto error; - if (! stat((char *) path, (struct stat *) stat_area) ) - DBUG_RETURN(stat_area); - +#ifndef _WIN32 + if (! stat((char *) path, (struct stat *) stat_area) ) + DBUG_RETURN(stat_area); +#else + if (! my_win_stat(path, stat_area) ) + DBUG_RETURN(stat_area); +#endif DBUG_PRINT("error",("Got errno: %d from stat", errno)); my_errno= errno; if (m_used) /* Free if new area */ diff --git a/mysys/my_lock.c b/mysys/my_lock.c index 8450fcfc30a..6bbb177e4b6 100644 --- a/mysys/my_lock.c +++ b/mysys/my_lock.c @@ -22,13 +22,113 @@ #undef NO_ALARM_LOOP #endif #include -#ifdef __WIN__ -#include -#endif #ifdef __NETWARE__ #include #endif +#ifdef _WIN32 +#define WIN_LOCK_INFINITE -1 +#define WIN_LOCK_SLEEP_MILLIS 100 + +static int win_lock(File fd, int locktype, my_off_t start, my_off_t length, + int timeout_sec) +{ + LARGE_INTEGER liOffset,liLength; + DWORD dwFlags; + OVERLAPPED ov= {0}; + HANDLE hFile= (HANDLE)my_get_osfhandle(fd); + DWORD lastError= 0; + int i; + int timeout_millis= timeout_sec * 1000; + + DBUG_ENTER("win_lock"); + + liOffset.QuadPart= start; + liLength.QuadPart= length; + + ov.Offset= liOffset.LowPart; + ov.OffsetHigh= liOffset.HighPart; + + if (locktype == F_UNLCK) + { + if (UnlockFileEx(hFile, 0, liLength.LowPart, liLength.HighPart, &ov)) + DBUG_RETURN(0); + /* + For compatibility with fcntl implementation, ignore error, + if region was not locked + */ + if (GetLastError() == ERROR_NOT_LOCKED) + { + SetLastError(0); + DBUG_RETURN(0); + } + goto error; + } + else if (locktype == F_RDLCK) + /* read lock is mapped to a shared lock. */ + dwFlags= 0; + else + /* write lock is mapped to an exclusive lock. */ + dwFlags= LOCKFILE_EXCLUSIVE_LOCK; + + /* + Drop old lock first to avoid double locking. + During analyze of Bug#38133 (Myisamlog test fails on Windows) + I met the situation that the program myisamlog locked the file + exclusively, then additionally shared, then did one unlock, and + then blocked on an attempt to lock it exclusively again. + Unlocking before every lock fixed the problem. + Note that this introduces a race condition. When the application + wants to convert an exclusive lock into a shared one, it will now + first unlock the file and then lock it shared. A waiting exclusive + lock could step in here. For reasons described in Bug#38133 and + Bug#41124 (Server hangs on Windows with --external-locking after + INSERT...SELECT) and in the review thread at + http://lists.mysql.com/commits/60721 it seems to be the better + option than not to unlock here. + If one day someone notices a way how to do file lock type changes + on Windows without unlocking before taking the new lock, please + change this code accordingly to fix the race condition. + */ + if (!UnlockFileEx(hFile, 0, liLength.LowPart, liLength.HighPart, &ov) && + (GetLastError() != ERROR_NOT_LOCKED)) + goto error; + + if (timeout_sec == WIN_LOCK_INFINITE) + { + if (LockFileEx(hFile, dwFlags, 0, liLength.LowPart, liLength.HighPart, &ov)) + DBUG_RETURN(0); + goto error; + } + + dwFlags|= LOCKFILE_FAIL_IMMEDIATELY; + timeout_millis= timeout_sec * 1000; + /* Try lock in a loop, until the lock is acquired or timeout happens */ + for(i= 0; ;i+= WIN_LOCK_SLEEP_MILLIS) + { + if (LockFileEx(hFile, dwFlags, 0, liLength.LowPart, liLength.HighPart, &ov)) + DBUG_RETURN(0); + + if (GetLastError() != ERROR_LOCK_VIOLATION) + goto error; + + if (i >= timeout_millis) + break; + Sleep(WIN_LOCK_SLEEP_MILLIS); + } + + /* timeout */ + errno= EAGAIN; + DBUG_RETURN(-1); + +error: + my_osmaperr(GetLastError()); + DBUG_RETURN(-1); +} +#endif + + + /* Lock a part of a file @@ -97,29 +197,16 @@ int my_lock(File fd, int locktype, my_off_t start, my_off_t length, DBUG_RETURN(0); } } -#elif defined(HAVE_LOCKING) - /* Windows */ +#elif defined(_WIN32) { - my_bool error= FALSE; - pthread_mutex_lock(&my_file_info[fd].mutex); - if (MyFlags & MY_SEEK_NOT_DONE) - { - if( my_seek(fd,start,MY_SEEK_SET,MYF(MyFlags & ~MY_SEEK_NOT_DONE)) - == MY_FILEPOS_ERROR ) - { - /* - If my_seek fails my_errno will already contain an error code; - just unlock and return error code. - */ - DBUG_PRINT("error",("my_errno: %d (%d)",my_errno,errno)); - pthread_mutex_unlock(&my_file_info[fd].mutex); - DBUG_RETURN(-1); - } - } - error= locking(fd,locktype,(ulong) length) && errno != EINVAL; - pthread_mutex_unlock(&my_file_info[fd].mutex); - if (!error) - DBUG_RETURN(0); + int timeout_sec; + if (MyFlags & MY_NO_WAIT) + timeout_sec= 0; + else + timeout_sec= WIN_LOCK_INFINITE; + + if(win_lock(fd, locktype, start, length, timeout_sec) == 0) + DBUG_RETURN(0); } #else #if defined(HAVE_FCNTL) diff --git a/mysys/my_mmap.c b/mysys/my_mmap.c index 023a06fd896..303d8efaf30 100644 --- a/mysys/my_mmap.c +++ b/mysys/my_mmap.c @@ -27,17 +27,17 @@ int my_msync(int fd, void *addr, size_t len, int flags) return my_sync(fd, MYF(0)); } -#elif defined(__WIN__) +#elif defined(_WIN32) static SECURITY_ATTRIBUTES mmap_security_attributes= {sizeof(SECURITY_ATTRIBUTES), 0, TRUE}; void *my_mmap(void *addr, size_t len, int prot, - int flags, int fd, my_off_t offset) + int flags, File fd, my_off_t offset) { HANDLE hFileMap; LPVOID ptr; - HANDLE hFile= (HANDLE)_get_osfhandle(fd); + HANDLE hFile= (HANDLE)my_get_osfhandle(fd); if (hFile == INVALID_HANDLE_VALUE) return MAP_FAILED; diff --git a/mysys/my_open.c b/mysys/my_open.c index fe7f65c450b..b1788506832 100644 --- a/mysys/my_open.c +++ b/mysys/my_open.c @@ -17,9 +17,7 @@ #include "mysys_err.h" #include #include -#if defined(__WIN__) -#include -#endif + /* Open a file @@ -43,29 +41,8 @@ File my_open(const char *FileName, int Flags, myf MyFlags) DBUG_ENTER("my_open"); DBUG_PRINT("my",("Name: '%s' Flags: %d MyFlags: %d", FileName, Flags, MyFlags)); -#if defined(__WIN__) - /* - Check that we don't try to open or create a file name that may - cause problems for us in the future (like PRN) - */ - if (check_if_legal_filename(FileName)) - { - errno= EACCES; - DBUG_RETURN(my_register_filename(-1, FileName, FILE_BY_OPEN, - EE_FILENOTFOUND, MyFlags)); - } -#ifndef __WIN__ - if (Flags & O_SHARE) - fd = sopen((char *) FileName, (Flags & ~O_SHARE) | O_BINARY, SH_DENYNO, - MY_S_IREAD | MY_S_IWRITE); - else - fd = open((char *) FileName, Flags | O_BINARY, - MY_S_IREAD | MY_S_IWRITE); -#else - fd= my_sopen((char *) FileName, (Flags & ~O_SHARE) | O_BINARY, SH_DENYNO, - MY_S_IREAD | MY_S_IWRITE); -#endif - +#if defined(_WIN32) + fd= my_win_open(FileName, Flags); #elif !defined(NO_OPEN_3) fd = open(FileName, Flags, my_umask); /* Normal unix */ #else @@ -94,11 +71,14 @@ int my_close(File fd, myf MyFlags) DBUG_PRINT("my",("fd: %d MyFlags: %d",fd, MyFlags)); pthread_mutex_lock(&THR_LOCK_open); +#ifndef _WIN32 do { err= close(fd); } while (err == -1 && errno == EINTR); - +#else + err= my_win_close(fd); +#endif if (err) { DBUG_PRINT("error",("Got error %d on close",err)); @@ -109,9 +89,6 @@ int my_close(File fd, myf MyFlags) if ((uint) fd < my_file_limit && my_file_info[fd].type != UNOPEN) { my_free(my_file_info[fd].name, MYF(0)); -#if defined(THREAD) && !defined(HAVE_PREAD) - pthread_mutex_destroy(&my_file_info[fd].mutex); -#endif my_file_info[fd].type = UNOPEN; } my_file_opened--; @@ -141,16 +118,12 @@ File my_register_filename(File fd, const char *FileName, enum file_type type_of_file, uint error_message_number, myf MyFlags) { DBUG_ENTER("my_register_filename"); - if ((int) fd >= 0) + if ((int) fd >= MY_FILE_MIN) { if ((uint) fd >= my_file_limit) { -#if defined(THREAD) && !defined(HAVE_PREAD) - my_errno= EMFILE; -#else thread_safe_increment(my_file_opened,&THR_LOCK_open); DBUG_RETURN(fd); /* safeguard */ -#endif } else { @@ -160,9 +133,6 @@ File my_register_filename(File fd, const char *FileName, enum file_type my_file_opened++; my_file_total_opened++; my_file_info[fd].type = type_of_file; -#if defined(THREAD) && !defined(HAVE_PREAD) - pthread_mutex_init(&my_file_info[fd].mutex,MY_MUTEX_INIT_FAST); -#endif pthread_mutex_unlock(&THR_LOCK_open); DBUG_PRINT("exit",("fd: %d",fd)); DBUG_RETURN(fd); @@ -187,188 +157,7 @@ File my_register_filename(File fd, const char *FileName, enum file_type DBUG_RETURN(-1); } -#ifdef __WIN__ - -extern void __cdecl _dosmaperr(unsigned long); - -/* - Open a file with sharing. Similar to _sopen() from libc, but allows managing - share delete on win32 - - SYNOPSIS - my_sopen() - path fully qualified file name - oflag operation flags - shflag share flag - pmode permission flags - - RETURN VALUE - File descriptor of opened file if success - -1 and sets errno if fails. -*/ - -File my_sopen(const char *path, int oflag, int shflag, int pmode) -{ - int fh; /* handle of opened file */ - int mask; - HANDLE osfh; /* OS handle of opened file */ - DWORD fileaccess; /* OS file access (requested) */ - DWORD fileshare; /* OS file sharing mode */ - DWORD filecreate; /* OS method of opening/creating */ - DWORD fileattrib; /* OS file attribute flags */ - SECURITY_ATTRIBUTES SecurityAttributes; - - SecurityAttributes.nLength= sizeof(SecurityAttributes); - SecurityAttributes.lpSecurityDescriptor= NULL; - SecurityAttributes.bInheritHandle= !(oflag & _O_NOINHERIT); - /* - * decode the access flags - */ - switch (oflag & (_O_RDONLY | _O_WRONLY | _O_RDWR)) { - case _O_RDONLY: /* read access */ - fileaccess= GENERIC_READ; - break; - case _O_WRONLY: /* write access */ - fileaccess= GENERIC_WRITE; - break; - case _O_RDWR: /* read and write access */ - fileaccess= GENERIC_READ | GENERIC_WRITE; - break; - default: /* error, bad oflag */ - errno= EINVAL; - _doserrno= 0L; /* not an OS error */ - return -1; - } - - /* - * decode sharing flags - */ - switch (shflag) { - case _SH_DENYRW: /* exclusive access except delete */ - fileshare= FILE_SHARE_DELETE; - break; - case _SH_DENYWR: /* share read and delete access */ - fileshare= FILE_SHARE_READ | FILE_SHARE_DELETE; - break; - case _SH_DENYRD: /* share write and delete access */ - fileshare= FILE_SHARE_WRITE | FILE_SHARE_DELETE; - break; - case _SH_DENYNO: /* share read, write and delete access */ - fileshare= FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE; - break; - case _SH_DENYRWD: /* exclusive access */ - fileshare= 0L; - break; - case _SH_DENYWRD: /* share read access */ - fileshare= FILE_SHARE_READ; - break; - case _SH_DENYRDD: /* share write access */ - fileshare= FILE_SHARE_WRITE; - break; - case _SH_DENYDEL: /* share read and write access */ - fileshare= FILE_SHARE_READ | FILE_SHARE_WRITE; - break; - default: /* error, bad shflag */ - errno= EINVAL; - _doserrno= 0L; /* not an OS error */ - return -1; - } - - /* - * decode open/create method flags - */ - switch (oflag & (_O_CREAT | _O_EXCL | _O_TRUNC)) { - case 0: - case _O_EXCL: /* ignore EXCL w/o CREAT */ - filecreate= OPEN_EXISTING; - break; - - case _O_CREAT: - filecreate= OPEN_ALWAYS; - break; - - case _O_CREAT | _O_EXCL: - case _O_CREAT | _O_TRUNC | _O_EXCL: - filecreate= CREATE_NEW; - break; - - case _O_TRUNC: - case _O_TRUNC | _O_EXCL: /* ignore EXCL w/o CREAT */ - filecreate= TRUNCATE_EXISTING; - break; - - case _O_CREAT | _O_TRUNC: - filecreate= CREATE_ALWAYS; - break; - - default: - /* this can't happen ... all cases are covered */ - errno= EINVAL; - _doserrno= 0L; - return -1; - } - - /* - * decode file attribute flags if _O_CREAT was specified - */ - fileattrib= FILE_ATTRIBUTE_NORMAL; /* default */ - if (oflag & _O_CREAT) - { - _umask((mask= _umask(0))); - - if (!((pmode & ~mask) & _S_IWRITE)) - fileattrib= FILE_ATTRIBUTE_READONLY; - } - - /* - * Set temporary file (delete-on-close) attribute if requested. - */ - if (oflag & _O_TEMPORARY) - { - fileattrib|= FILE_FLAG_DELETE_ON_CLOSE; - fileaccess|= DELETE; - } - - /* - * Set temporary file (delay-flush-to-disk) attribute if requested. - */ - if (oflag & _O_SHORT_LIVED) - fileattrib|= FILE_ATTRIBUTE_TEMPORARY; - - /* - * Set sequential or random access attribute if requested. - */ - if (oflag & _O_SEQUENTIAL) - fileattrib|= FILE_FLAG_SEQUENTIAL_SCAN; - else if (oflag & _O_RANDOM) - fileattrib|= FILE_FLAG_RANDOM_ACCESS; - - /* - * try to open/create the file - */ - if ((osfh= CreateFile(path, fileaccess, fileshare, &SecurityAttributes, - filecreate, fileattrib, NULL)) == INVALID_HANDLE_VALUE) - { - /* - * OS call to open/create file failed! map the error, release - * the lock, and return -1. note that it's not necessary to - * call _free_osfhnd (it hasn't been used yet). - */ - _dosmaperr(GetLastError()); /* map error */ - return -1; /* return error to caller */ - } - - if ((fh= _open_osfhandle((intptr_t)osfh, - oflag & (_O_APPEND | _O_RDONLY | _O_TEXT))) == -1) - { - _dosmaperr(GetLastError()); /* map error */ - CloseHandle(osfh); - } - - return fh; /* return handle */ -} -#endif /* __WIN__ */ #ifdef EXTRA_DEBUG diff --git a/mysys/my_pread.c b/mysys/my_pread.c index 836f5a92963..6421a2da601 100644 --- a/mysys/my_pread.c +++ b/mysys/my_pread.c @@ -18,7 +18,7 @@ #include "my_base.h" #include #include -#ifdef HAVE_PREAD +#ifndef _WIN32 #include #endif @@ -48,9 +48,7 @@ size_t my_pread(File Filedes, uchar *Buffer, size_t Count, my_off_t offset, { size_t readbytes; int error= 0; -#ifndef HAVE_PREAD - int save_errno; -#endif + #ifndef DBUG_OFF char llbuf[22]; DBUG_ENTER("my_pread"); @@ -61,20 +59,15 @@ size_t my_pread(File Filedes, uchar *Buffer, size_t Count, my_off_t offset, for (;;) { errno= 0; /* Linux, Windows don't reset this on EOF/success */ -#ifndef HAVE_PREAD - pthread_mutex_lock(&my_file_info[Filedes].mutex); - readbytes= (uint) -1; - error= (lseek(Filedes, offset, MY_SEEK_SET) == (my_off_t) -1 || - (readbytes= read(Filedes, Buffer, (uint) Count)) != Count); - save_errno= errno; - pthread_mutex_unlock(&my_file_info[Filedes].mutex); - if (error) - { - errno= save_errno; +#ifdef _WIN32 + readbytes= my_win_pread(Filedes, Buffer, Count, offset); #else - if ((error= ((readbytes= pread(Filedes, Buffer, Count, offset)) != Count))) - { + readbytes= pread(Filedes, Buffer, Count, offset); #endif + error = (readbytes != Count); + + if (error) + { my_errno= errno ? errno : -1; if (errno == 0 || (readbytes != (size_t) -1 && (MyFlags & (MY_NABP | MY_FNABP)))) @@ -91,19 +84,19 @@ size_t my_pread(File Filedes, uchar *Buffer, size_t Count, my_off_t offset, #endif if (MyFlags & (MY_WME | MY_FAE | MY_FNABP)) { - if (readbytes == (size_t) -1) - my_error(EE_READ, MYF(ME_BELL+ME_WAITTANG), - my_filename(Filedes),my_errno); - else if (MyFlags & (MY_NABP | MY_FNABP)) - my_error(EE_EOFERR, MYF(ME_BELL+ME_WAITTANG), - my_filename(Filedes),my_errno); + if (readbytes == (size_t) -1) + my_error(EE_READ, MYF(ME_BELL+ME_WAITTANG), + my_filename(Filedes),my_errno); + else if (MyFlags & (MY_NABP | MY_FNABP)) + my_error(EE_EOFERR, MYF(ME_BELL+ME_WAITTANG), + my_filename(Filedes),my_errno); } if (readbytes == (size_t) -1 || (MyFlags & (MY_FNABP | MY_NABP))) - DBUG_RETURN(MY_FILE_ERROR); /* Return with error */ + DBUG_RETURN(MY_FILE_ERROR); /* Return with error */ } if (MyFlags & (MY_NABP | MY_FNABP)) - DBUG_RETURN(0); /* Read went ok; Return 0 */ - DBUG_RETURN(readbytes); /* purecov: inspected */ + DBUG_RETURN(0); /* Read went ok; Return 0 */ + DBUG_RETURN(readbytes); /* purecov: inspected */ } } /* my_pread */ @@ -132,7 +125,7 @@ size_t my_pread(File Filedes, uchar *Buffer, size_t Count, my_off_t offset, size_t my_pwrite(int Filedes, const uchar *Buffer, size_t Count, my_off_t offset, myf MyFlags) { - size_t writenbytes, written; + size_t writtenbytes, written; uint errors; #ifndef DBUG_OFF char llbuf[22]; @@ -146,28 +139,22 @@ size_t my_pwrite(int Filedes, const uchar *Buffer, size_t Count, for (;;) { -#ifndef HAVE_PREAD - int error; - writenbytes= (size_t) -1; - pthread_mutex_lock(&my_file_info[Filedes].mutex); - error= (lseek(Filedes, offset, MY_SEEK_SET) != (my_off_t) -1 && - (writenbytes = write(Filedes, Buffer, (uint) Count)) == Count); - pthread_mutex_unlock(&my_file_info[Filedes].mutex); - if (error) - break; +#ifdef _WIN32 + writtenbytes= my_win_pwrite(Filedes, Buffer, Count,offset); #else - if ((writenbytes= pwrite(Filedes, Buffer, Count,offset)) == Count) + writtenbytes= pwrite(Filedes, Buffer, Count, offset); +#endif + if (writtenbytes == Count) break; my_errno= errno; -#endif - if (writenbytes != (size_t) -1) + if (writtenbytes != (size_t) -1) { /* Safegueard */ - written+=writenbytes; - Buffer+=writenbytes; - Count-=writenbytes; - offset+=writenbytes; + written+=writtenbytes; + Buffer+=writtenbytes; + Count-=writtenbytes; + offset+=writtenbytes; } - DBUG_PRINT("error",("Write only %u bytes", (uint) writenbytes)); + DBUG_PRINT("error",("Write only %u bytes", (uint) writtenbytes)); #ifndef NO_BACKGROUND #ifdef THREAD if (my_thread_var->abort) @@ -180,15 +167,15 @@ size_t my_pwrite(int Filedes, const uchar *Buffer, size_t Count, errors++; continue; } - if ((writenbytes && writenbytes != (size_t) -1) || my_errno == EINTR) + if ((writtenbytes && writtenbytes != (size_t) -1) || my_errno == EINTR) continue; /* Retry */ #endif if (MyFlags & (MY_NABP | MY_FNABP)) { if (MyFlags & (MY_WME | MY_FAE | MY_FNABP)) { - my_error(EE_WRITE, MYF(ME_BELL | ME_WAITTANG), - my_filename(Filedes),my_errno); + my_error(EE_WRITE, MYF(ME_BELL | ME_WAITTANG), + my_filename(Filedes),my_errno); } DBUG_RETURN(MY_FILE_ERROR); /* Error on read */ } @@ -198,5 +185,5 @@ size_t my_pwrite(int Filedes, const uchar *Buffer, size_t Count, DBUG_EXECUTE_IF("check", my_seek(Filedes, -1, SEEK_SET, MYF(0));); if (MyFlags & (MY_NABP | MY_FNABP)) DBUG_RETURN(0); /* Want only errors */ - DBUG_RETURN(writenbytes+written); /* purecov: inspected */ + DBUG_RETURN(writtenbytes+written); /* purecov: inspected */ } /* my_pwrite */ diff --git a/mysys/my_quick.c b/mysys/my_quick.c index 0ba20a5bdee..b93e7e17224 100644 --- a/mysys/my_quick.c +++ b/mysys/my_quick.c @@ -19,11 +19,19 @@ #include "my_nosys.h" +#ifdef _WIN32 +extern size_t my_win_read(File Filedes,uchar *Buffer,size_t Count); +#endif + size_t my_quick_read(File Filedes,uchar *Buffer,size_t Count,myf MyFlags) { size_t readbytes; - - if ((readbytes = read(Filedes, Buffer, (uint) Count)) != Count) +#ifdef _WIN32 + readbytes= my_win_read(Filedes, Buffer, Count); +#else + readbytes= read(Filedes, Buffer, Count); +#endif + if(readbytes != Count) { #ifndef DBUG_OFF if ((readbytes == 0 || readbytes == (size_t) -1) && errno == EINTR) @@ -40,8 +48,13 @@ size_t my_quick_read(File Filedes,uchar *Buffer,size_t Count,myf MyFlags) } -size_t my_quick_write(File Filedes,const uchar *Buffer,size_t Count) + +size_t my_quick_write(File Filedes, const uchar *Buffer, size_t Count) { +#ifdef _WIN32 + return my_win_write(Filedes, Buffer, Count); +#else + #ifndef DBUG_OFF size_t writtenbytes; #endif @@ -50,7 +63,7 @@ size_t my_quick_write(File Filedes,const uchar *Buffer,size_t Count) #ifndef DBUG_OFF writtenbytes = #endif - (size_t) write(Filedes,Buffer, (uint) Count)) != Count) + (size_t) write(Filedes,Buffer,Count)) != Count) { #ifndef DBUG_OFF if ((writtenbytes == 0 || writtenbytes == (size_t) -1) && errno == EINTR) @@ -64,4 +77,5 @@ size_t my_quick_write(File Filedes,const uchar *Buffer,size_t Count) return (size_t) -1; } return 0; +#endif } diff --git a/mysys/my_read.c b/mysys/my_read.c index 25ffe73d813..9c76193aa63 100644 --- a/mysys/my_read.c +++ b/mysys/my_read.c @@ -44,7 +44,12 @@ size_t my_read(File Filedes, uchar *Buffer, size_t Count, myf MyFlags) for (;;) { errno= 0; /* Linux, Windows don't reset this on EOF/success */ - if ((readbytes= read(Filedes, Buffer, (uint) Count)) != Count) +#ifdef _WIN32 + readbytes= my_win_read(Filedes, Buffer, Count); +#else + readbytes= my_read(Fildes, Buffer, Count); +#endif + if (readbytes != Count) { my_errno= errno; if (errno == 0 || (readbytes != (size_t) -1 && diff --git a/mysys/my_seek.c b/mysys/my_seek.c index d186d56869a..24941517487 100644 --- a/mysys/my_seek.c +++ b/mysys/my_seek.c @@ -56,16 +56,11 @@ my_off_t my_seek(File fd, my_off_t pos, int whence, myf MyFlags) Make sure we are using a valid file descriptor! */ DBUG_ASSERT(fd != -1); -#if defined(THREAD) && !defined(HAVE_PREAD) - if (MyFlags & MY_THREADSAFE) - { - pthread_mutex_lock(&my_file_info[fd].mutex); - newpos= lseek(fd, pos, whence); - pthread_mutex_unlock(&my_file_info[fd].mutex); - } - else +#ifdef _WIN32 + newpos= my_win_lseek(fd, pos, whence); +#else + newpos= lseek(fd, pos, whence); #endif - newpos= lseek(fd, pos, whence); if (newpos == (os_off_t) -1) { my_errno= errno; @@ -91,7 +86,9 @@ my_off_t my_tell(File fd, myf MyFlags) DBUG_ENTER("my_tell"); DBUG_PRINT("my",("fd: %d MyFlags: %d",fd, MyFlags)); DBUG_ASSERT(fd >= 0); -#ifdef HAVE_TELL +#ifdef _WIN32 + pos= my_seek(fd, 0, MY_SEEK_CUR,0); +#elif defined(HAVE_TELL) pos=tell(fd); #else pos=lseek(fd, 0L, MY_SEEK_CUR); diff --git a/mysys/my_static.c b/mysys/my_static.c index 08653d03d21..629ee26efb9 100644 --- a/mysys/my_static.c +++ b/mysys/my_static.c @@ -35,7 +35,7 @@ int NEAR my_umask=0664, NEAR my_umask_dir=0777; #ifndef THREAD int NEAR my_errno=0; #endif -struct st_my_file_info my_file_info_default[MY_NFILE]= {{0,UNOPEN}}; +struct st_my_file_info my_file_info_default[MY_NFILE]; uint my_file_limit= MY_NFILE; struct st_my_file_info *my_file_info= my_file_info_default; diff --git a/mysys/my_sync.c b/mysys/my_sync.c index dae20b0163e..9c5fbce7ab7 100644 --- a/mysys/my_sync.c +++ b/mysys/my_sync.c @@ -71,7 +71,7 @@ int my_sync(File fd, myf my_flags) if (res == -1 && errno == ENOLCK) res= 0; /* Result Bug in Old FreeBSD */ #elif defined(__WIN__) - res= _commit(fd); + res= my_win_fsync(fd); #else #error Cannot find a way to sync a file, durability in danger res= 0; /* No sync (strange OS) */ diff --git a/mysys/my_winerr.c b/mysys/my_winerr.c new file mode 100644 index 00000000000..534078b6737 --- /dev/null +++ b/mysys/my_winerr.c @@ -0,0 +1,123 @@ +/* Copyright (C) 2008 MySQL AB + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* + Convert Windows API error (GetLastError() to Posix equivalent (errno) + The exported function my_osmaperr() is modelled after and borrows + heavily from undocumented _dosmaperr()(found of the static Microsoft C runtime). +*/ + +#include +#include + + +struct errentry +{ + unsigned long oscode; /* OS return value */ + int sysv_errno; /* System V error code */ +}; + +static struct errentry errtable[]= { + { ERROR_INVALID_FUNCTION, EINVAL }, /* 1 */ + { ERROR_FILE_NOT_FOUND, ENOENT }, /* 2 */ + { ERROR_PATH_NOT_FOUND, ENOENT }, /* 3 */ + { ERROR_TOO_MANY_OPEN_FILES, EMFILE }, /* 4 */ + { ERROR_ACCESS_DENIED, EACCES }, /* 5 */ + { ERROR_INVALID_HANDLE, EBADF }, /* 6 */ + { ERROR_ARENA_TRASHED, ENOMEM }, /* 7 */ + { ERROR_NOT_ENOUGH_MEMORY, ENOMEM }, /* 8 */ + { ERROR_INVALID_BLOCK, ENOMEM }, /* 9 */ + { ERROR_BAD_ENVIRONMENT, E2BIG }, /* 10 */ + { ERROR_BAD_FORMAT, ENOEXEC }, /* 11 */ + { ERROR_INVALID_ACCESS, EINVAL }, /* 12 */ + { ERROR_INVALID_DATA, EINVAL }, /* 13 */ + { ERROR_INVALID_DRIVE, ENOENT }, /* 15 */ + { ERROR_CURRENT_DIRECTORY, EACCES }, /* 16 */ + { ERROR_NOT_SAME_DEVICE, EXDEV }, /* 17 */ + { ERROR_NO_MORE_FILES, ENOENT }, /* 18 */ + { ERROR_LOCK_VIOLATION, EACCES }, /* 33 */ + { ERROR_BAD_NETPATH, ENOENT }, /* 53 */ + { ERROR_NETWORK_ACCESS_DENIED, EACCES }, /* 65 */ + { ERROR_BAD_NET_NAME, ENOENT }, /* 67 */ + { ERROR_FILE_EXISTS, EEXIST }, /* 80 */ + { ERROR_CANNOT_MAKE, EACCES }, /* 82 */ + { ERROR_FAIL_I24, EACCES }, /* 83 */ + { ERROR_INVALID_PARAMETER, EINVAL }, /* 87 */ + { ERROR_NO_PROC_SLOTS, EAGAIN }, /* 89 */ + { ERROR_DRIVE_LOCKED, EACCES }, /* 108 */ + { ERROR_BROKEN_PIPE, EPIPE }, /* 109 */ + { ERROR_DISK_FULL, ENOSPC }, /* 112 */ + { ERROR_INVALID_TARGET_HANDLE, EBADF }, /* 114 */ + { ERROR_INVALID_HANDLE, EINVAL }, /* 124 */ + { ERROR_WAIT_NO_CHILDREN, ECHILD }, /* 128 */ + { ERROR_CHILD_NOT_COMPLETE, ECHILD }, /* 129 */ + { ERROR_DIRECT_ACCESS_HANDLE, EBADF }, /* 130 */ + { ERROR_NEGATIVE_SEEK, EINVAL }, /* 131 */ + { ERROR_SEEK_ON_DEVICE, EACCES }, /* 132 */ + { ERROR_DIR_NOT_EMPTY, ENOTEMPTY }, /* 145 */ + { ERROR_NOT_LOCKED, EACCES }, /* 158 */ + { ERROR_BAD_PATHNAME, ENOENT }, /* 161 */ + { ERROR_MAX_THRDS_REACHED, EAGAIN }, /* 164 */ + { ERROR_LOCK_FAILED, EACCES }, /* 167 */ + { ERROR_ALREADY_EXISTS, EEXIST }, /* 183 */ + { ERROR_FILENAME_EXCED_RANGE, ENOENT }, /* 206 */ + { ERROR_NESTING_NOT_ALLOWED, EAGAIN }, /* 215 */ + { ERROR_NOT_ENOUGH_QUOTA, ENOMEM } /* 1816 */ +}; + +/* size of the table */ +#define ERRTABLESIZE (sizeof(errtable)/sizeof(errtable[0])) + +/* The following two constants must be the minimum and maximum +values in the (contiguous) range of Exec Failure errors. */ +#define MIN_EXEC_ERROR ERROR_INVALID_STARTING_CODESEG +#define MAX_EXEC_ERROR ERROR_INFLOOP_IN_RELOC_CHAIN + +/* These are the low and high value in the range of errors that are +access violations */ +#define MIN_EACCES_RANGE ERROR_WRITE_PROTECT +#define MAX_EACCES_RANGE ERROR_SHARING_BUFFER_EXCEEDED + + +static int get_errno_from_oserr(unsigned long oserrno) +{ + int i; + + /* check the table for the OS error code */ + for (i= 0; i < ERRTABLESIZE; ++i) + { + if (oserrno == errtable[i].oscode) + { + return errtable[i].sysv_errno; + } + } + + /* The error code wasn't in the table. We check for a range of */ + /* EACCES errors or exec failure errors (ENOEXEC). Otherwise */ + /* EINVAL is returned. */ + + if (oserrno >= MIN_EACCES_RANGE && oserrno <= MAX_EACCES_RANGE) + return EACCES; + else if (oserrno >= MIN_EXEC_ERROR && oserrno <= MAX_EXEC_ERROR) + return ENOEXEC; + else + return EINVAL; +} + +/* Set errno corresponsing to GetLastError() value */ +void my_osmaperr ( unsigned long oserrno) +{ + errno= get_errno_from_oserr(oserrno); +} diff --git a/mysys/my_winfile.c b/mysys/my_winfile.c new file mode 100644 index 00000000000..f63c35ba47b --- /dev/null +++ b/mysys/my_winfile.c @@ -0,0 +1,681 @@ +/* Copyright (C) 2008 MySQL AB, 2008-2009 Sun Microsystems, Inc + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* + The purpose of this file is to provide implementation of file IO routines on + Windows that can be thought as drop-in replacement for corresponding C runtime + functionality. + + Compared to Windows CRT, this one + - does not have the same file descriptor + limitation (default is 16384 and can be increased further, whereas CRT poses + a hard limit of 2048 file descriptors) + - the file operations are not serialized + - positional IO pread/pwrite is ported here. + - no text mode for files, all IO is "binary" + + Naming convention: + All routines are prefixed with my_win_, e.g Posix open() is implemented with + my_win_open() + + Implemented are + - POSIX routines(e.g open, read, lseek ...) + - Some ANSI C stream routines (fopen, fdopen, fileno, fclose) + - Windows CRT equvalients (my_get_osfhandle, open_osfhandle) + + Worth to note: + - File descriptors used here are located in a range that is not compatible + with CRT on purpose. Attempt to use a file descriptor from Windows CRT library + range in my_win_* function will be punished with DBUG_ASSERT() + + - File streams (FILE *) are actually from the C runtime. The routines provided + here are useful only in scernarios that use low-level IO with my_win_fileno() +*/ + +#ifdef _WIN32 + +#include "mysys_priv.h" +#include +#include + +/* Associates a file descriptor with an existing operating-system file handle.*/ +File my_open_osfhandle(HANDLE handle, int oflag) +{ + int offset= -1; + uint i; + DBUG_ENTER("my_open_osfhandle"); + + pthread_mutex_lock(&THR_LOCK_open); + for(i= MY_FILE_MIN; i < my_file_limit;i++) + { + if(my_file_info[i].fhandle == 0) + { + struct st_my_file_info *finfo= &(my_file_info[i]); + finfo->type= FILE_BY_OPEN; + finfo->fhandle= handle; + finfo->oflag= oflag; + offset= i; + break; + } + } + pthread_mutex_unlock(&THR_LOCK_open); + if(offset == -1) + errno= EMFILE; /* to many file handles open */ + DBUG_RETURN(offset); +} + + +static void invalidate_fd(File fd) +{ + DBUG_ENTER("invalidate_fd"); + DBUG_ASSERT(fd >= MY_FILE_MIN && fd < (int)my_file_limit); + my_file_info[fd].fhandle= 0; + DBUG_VOID_RETURN; +} + + +/* Get Windows handle for a file descriptor */ +HANDLE my_get_osfhandle(File fd) +{ + DBUG_ENTER("my_get_osfhandle"); + DBUG_ASSERT(fd >= MY_FILE_MIN && fd < (int)my_file_limit); + DBUG_RETURN(my_file_info[fd].fhandle); +} + + +static int my_get_open_flags(File fd) +{ + DBUG_ENTER("my_get_open_flags"); + DBUG_ASSERT(fd >= MY_FILE_MIN && fd < (int)my_file_limit); + DBUG_RETURN(my_file_info[fd].oflag); +} + + +/* + Open a file with sharing. Similar to _sopen() from libc, but allows managing + share delete on win32 + + SYNOPSIS + my_win_sopen() + path file name + oflag operation flags + shflag share flag + pmode permission flags + + RETURN VALUE + File descriptor of opened file if success + -1 and sets errno if fails. +*/ + +File my_win_sopen(const char *path, int oflag, int shflag, int pmode) +{ + int fh; /* handle of opened file */ + int mask; + HANDLE osfh; /* OS handle of opened file */ + DWORD fileaccess; /* OS file access (requested) */ + DWORD fileshare; /* OS file sharing mode */ + DWORD filecreate; /* OS method of opening/creating */ + DWORD fileattrib; /* OS file attribute flags */ + SECURITY_ATTRIBUTES SecurityAttributes; + + DBUG_ENTER("my_win_sopen"); + + if (check_if_legal_filename(path)) + { + errno= EACCES; + DBUG_RETURN(-1); + } + SecurityAttributes.nLength= sizeof(SecurityAttributes); + SecurityAttributes.lpSecurityDescriptor= NULL; + SecurityAttributes.bInheritHandle= !(oflag & _O_NOINHERIT); + + /* decode the access flags */ + switch (oflag & (_O_RDONLY | _O_WRONLY | _O_RDWR)) { + case _O_RDONLY: /* read access */ + fileaccess= GENERIC_READ; + break; + case _O_WRONLY: /* write access */ + fileaccess= GENERIC_WRITE; + break; + case _O_RDWR: /* read and write access */ + fileaccess= GENERIC_READ | GENERIC_WRITE; + break; + default: /* error, bad oflag */ + errno= EINVAL; + DBUG_RETURN(-1); + } + + /* decode sharing flags */ + switch (shflag) { + case _SH_DENYRW: /* exclusive access except delete */ + fileshare= FILE_SHARE_DELETE; + break; + case _SH_DENYWR: /* share read and delete access */ + fileshare= FILE_SHARE_READ | FILE_SHARE_DELETE; + break; + case _SH_DENYRD: /* share write and delete access */ + fileshare= FILE_SHARE_WRITE | FILE_SHARE_DELETE; + break; + case _SH_DENYNO: /* share read, write and delete access */ + fileshare= FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE; + break; + case _SH_DENYRWD: /* exclusive access */ + fileshare= 0L; + break; + case _SH_DENYWRD: /* share read access */ + fileshare= FILE_SHARE_READ; + break; + case _SH_DENYRDD: /* share write access */ + fileshare= FILE_SHARE_WRITE; + break; + case _SH_DENYDEL: /* share read and write access */ + fileshare= FILE_SHARE_READ | FILE_SHARE_WRITE; + break; + default: /* error, bad shflag */ + errno= EINVAL; + DBUG_RETURN(-1); + } + + /* decode open/create method flags */ + switch (oflag & (_O_CREAT | _O_EXCL | _O_TRUNC)) { + case 0: + case _O_EXCL: /* ignore EXCL w/o CREAT */ + filecreate= OPEN_EXISTING; + break; + + case _O_CREAT: + filecreate= OPEN_ALWAYS; + break; + + case _O_CREAT | _O_EXCL: + case _O_CREAT | _O_TRUNC | _O_EXCL: + filecreate= CREATE_NEW; + break; + + case _O_TRUNC: + case _O_TRUNC | _O_EXCL: /* ignore EXCL w/o CREAT */ + filecreate= TRUNCATE_EXISTING; + break; + + case _O_CREAT | _O_TRUNC: + filecreate= CREATE_ALWAYS; + break; + + default: + /* this can't happen ... all cases are covered */ + errno= EINVAL; + DBUG_RETURN(-1); + } + + /* decode file attribute flags if _O_CREAT was specified */ + fileattrib= FILE_ATTRIBUTE_NORMAL; /* default */ + if (oflag & _O_CREAT) + { + _umask((mask= _umask(0))); + + if (!((pmode & ~mask) & _S_IWRITE)) + fileattrib= FILE_ATTRIBUTE_READONLY; + } + + /* Set temporary file (delete-on-close) attribute if requested. */ + if (oflag & _O_TEMPORARY) + { + fileattrib|= FILE_FLAG_DELETE_ON_CLOSE; + fileaccess|= DELETE; + } + + /* Set temporary file (delay-flush-to-disk) attribute if requested.*/ + if (oflag & _O_SHORT_LIVED) + fileattrib|= FILE_ATTRIBUTE_TEMPORARY; + + /* Set sequential or random access attribute if requested. */ + if (oflag & _O_SEQUENTIAL) + fileattrib|= FILE_FLAG_SEQUENTIAL_SCAN; + else if (oflag & _O_RANDOM) + fileattrib|= FILE_FLAG_RANDOM_ACCESS; + + /* try to open/create the file */ + if ((osfh= CreateFile(path, fileaccess, fileshare, &SecurityAttributes, + filecreate, fileattrib, NULL)) == INVALID_HANDLE_VALUE) + { + /* + OS call to open/create file failed! map the error, release + the lock, and return -1. note that it's not necessary to + call _free_osfhnd (it hasn't been used yet). + */ + my_osmaperr(GetLastError()); /* map error */ + DBUG_RETURN(-1); /* return error to caller */ + } + + if ((fh= my_open_osfhandle(osfh, + oflag & (_O_APPEND | _O_RDONLY | _O_TEXT))) == -1) + { + CloseHandle(osfh); + } + + DBUG_RETURN(fh); /* return handle */ +} + + +File my_win_open(const char *path, int flags) +{ + DBUG_ENTER("my_win_open"); + DBUG_RETURN(my_win_sopen((char *) path, flags | _O_BINARY, _SH_DENYNO, + _S_IREAD | S_IWRITE)); +} + + +int my_win_close(File fd) +{ + DBUG_ENTER("my_win_close"); + if(CloseHandle(my_get_osfhandle(fd))) + { + invalidate_fd(fd); + DBUG_RETURN(0); + } + my_osmaperr(GetLastError()); + DBUG_RETURN(-1); +} + + +size_t my_win_pread(File Filedes, uchar *Buffer, size_t Count, my_off_t offset) +{ + DWORD nBytesRead; + HANDLE hFile; + OVERLAPPED ov= {0}; + LARGE_INTEGER li; + + DBUG_ENTER("my_win_pread"); + + if(!Count) + DBUG_RETURN(0); +#ifdef _WIN64 + if(Count > UINT_MAX) + Count= UINT_MAX; +#endif + + hFile= (HANDLE)my_get_osfhandle(Filedes); + li.QuadPart= offset; + ov.Offset= li.LowPart; + ov.OffsetHigh= li.HighPart; + + if(!ReadFile(hFile, Buffer, (DWORD)Count, &nBytesRead, &ov)) + { + DWORD lastError= GetLastError(); + /* + ERROR_BROKEN_PIPE is returned when no more data coming + through e.g. a command pipe in windows : see MSDN on ReadFile. + */ + if(lastError == ERROR_HANDLE_EOF || lastError == ERROR_BROKEN_PIPE) + DBUG_RETURN(0); /*return 0 at EOF*/ + my_osmaperr(lastError); + DBUG_RETURN((size_t)-1); + } + DBUG_RETURN(nBytesRead); +} + + +size_t my_win_read(File Filedes, uchar *Buffer, size_t Count) +{ + DWORD nBytesRead; + HANDLE hFile; + + DBUG_ENTER("my_win_read"); + if(!Count) + DBUG_RETURN(0); +#ifdef _WIN64 + if(Count > UINT_MAX) + Count= UINT_MAX; +#endif + + hFile= (HANDLE)my_get_osfhandle(Filedes); + + if(!ReadFile(hFile, Buffer, (DWORD)Count, &nBytesRead, NULL)) + { + DWORD lastError= GetLastError(); + /* + ERROR_BROKEN_PIPE is returned when no more data coming + through e.g. a command pipe in windows : see MSDN on ReadFile. + */ + if(lastError == ERROR_HANDLE_EOF || lastError == ERROR_BROKEN_PIPE) + DBUG_RETURN(0); /*return 0 at EOF*/ + my_osmaperr(lastError); + DBUG_RETURN((size_t)-1); + } + DBUG_RETURN(nBytesRead); +} + + +size_t my_win_pwrite(File Filedes, const uchar *Buffer, size_t Count, + my_off_t offset) +{ + DWORD nBytesWritten; + HANDLE hFile; + OVERLAPPED ov= {0}; + LARGE_INTEGER li; + + DBUG_ENTER("my_win_pwrite"); + DBUG_PRINT("my",("Filedes: %d, Buffer: %p, Count: %llu, offset: %llu", + Filedes, Buffer, (ulonglong)Count, (ulonglong)offset)); + + if(!Count) + DBUG_RETURN(0); + +#ifdef _WIN64 + if(Count > UINT_MAX) + Count= UINT_MAX; +#endif + + hFile= (HANDLE)my_get_osfhandle(Filedes); + li.QuadPart= offset; + ov.Offset= li.LowPart; + ov.OffsetHigh= li.HighPart; + + if(!WriteFile(hFile, Buffer, (DWORD)Count, &nBytesWritten, &ov)) + { + my_osmaperr(GetLastError()); + DBUG_RETURN((size_t)-1); + } + else + DBUG_RETURN(nBytesWritten); +} + + +my_off_t my_win_lseek(File fd, my_off_t pos, int whence) +{ + LARGE_INTEGER offset; + LARGE_INTEGER newpos; + + DBUG_ENTER("my_win_lseek"); + + /* Check compatibility of Windows and Posix seek constants */ + compile_time_assert(FILE_BEGIN == SEEK_SET && FILE_CURRENT == SEEK_CUR + && FILE_END == SEEK_END); + + offset.QuadPart= pos; + if(!SetFilePointerEx(my_get_osfhandle(fd), offset, &newpos, whence)) + { + my_osmaperr(GetLastError()); + newpos.QuadPart= -1; + } + DBUG_RETURN(newpos.QuadPart); +} + + +#ifndef FILE_WRITE_TO_END_OF_FILE +#define FILE_WRITE_TO_END_OF_FILE 0xffffffff +#endif +size_t my_win_write(File fd, const uchar *Buffer, size_t Count) +{ + DWORD nWritten; + OVERLAPPED ov; + OVERLAPPED *pov= NULL; + HANDLE hFile; + + DBUG_ENTER("my_win_write"); + DBUG_PRINT("my",("Filedes: %d, Buffer: %p, Count %llu", fd, Buffer, + (ulonglong)Count)); + + if(!Count) + DBUG_RETURN(0); + +#ifdef _WIN64 + if(Count > UINT_MAX) + Count= UINT_MAX; +#endif + + if(my_get_open_flags(fd) & _O_APPEND) + { + /* + Atomic append to the end of file is is done by special initialization of + the OVERLAPPED structure. See MSDN WriteFile documentation for more info. + */ + memset(&ov, 0, sizeof(ov)); + ov.Offset= FILE_WRITE_TO_END_OF_FILE; + ov.OffsetHigh= -1; + pov= &ov; + } + + hFile= my_get_osfhandle(fd); + if(!WriteFile(hFile, Buffer, (DWORD)Count, &nWritten, pov)) + { + my_osmaperr(GetLastError()); + DBUG_RETURN((size_t)-1); + } + DBUG_RETURN(nWritten); +} + + +int my_win_chsize(File fd, my_off_t newlength) +{ + HANDLE hFile; + LARGE_INTEGER length; + DBUG_ENTER("my_win_chsize"); + + hFile= (HANDLE) my_get_osfhandle(fd); + length.QuadPart= newlength; + if (!SetFilePointerEx(hFile, length , NULL , FILE_BEGIN)) + goto err; + if (!SetEndOfFile(hFile)) + goto err; + DBUG_RETURN(0); +err: + my_osmaperr(GetLastError()); + my_errno= errno; + DBUG_RETURN(-1); +} + + +/* Get the file descriptor for stdin,stdout or stderr */ +static File my_get_stdfile_descriptor(FILE *stream) +{ + HANDLE hFile; + DWORD nStdHandle; + DBUG_ENTER("my_get_stdfile_descriptor"); + + if(stream == stdin) + nStdHandle= STD_INPUT_HANDLE; + else if(stream == stdout) + nStdHandle= STD_OUTPUT_HANDLE; + else if(stream == stderr) + nStdHandle= STD_ERROR_HANDLE; + else + DBUG_RETURN(-1); + + hFile= GetStdHandle(nStdHandle); + if(hFile != INVALID_HANDLE_VALUE) + DBUG_RETURN(my_open_osfhandle(hFile, 0)); + DBUG_RETURN(-1); +} + + +File my_win_fileno(FILE *file) +{ + HANDLE hFile= (HANDLE)_get_osfhandle(fileno(file)); + int retval= -1; + uint i; + + DBUG_ENTER("my_win_fileno"); + + for(i= MY_FILE_MIN; i < my_file_limit; i++) + { + if(my_file_info[i].fhandle == hFile) + { + retval= i; + break; + } + } + if(retval == -1) + /* try std stream */ + DBUG_RETURN(my_get_stdfile_descriptor(file)); + DBUG_RETURN(retval); +} + + +FILE *my_win_fopen(const char *filename, const char *type) +{ + FILE *file; + int flags= 0; + DBUG_ENTER("my_win_open"); + + /* + If we are not creating, then we need to use my_access to make sure + the file exists since Windows doesn't handle files like "com1.sym" + very well + */ + if (check_if_legal_filename(filename)) + { + errno= EACCES; + DBUG_RETURN(NULL); + } + + file= fopen(filename, type); + if(!file) + DBUG_RETURN(NULL); + + if(strchr(type,'a') != NULL) + flags= O_APPEND; + + /* + Register file handle in my_table_info. + Necessary for my_fileno() + */ + if(my_open_osfhandle((HANDLE)_get_osfhandle(fileno(file)), flags) < 0) + { + fclose(file); + DBUG_RETURN(NULL); + } + DBUG_RETURN(file); +} + + +FILE * my_win_fdopen(File fd, const char *type) +{ + FILE *file; + int crt_fd; + int flags= 0; + + DBUG_ENTER("my_win_fdopen"); + + if(strchr(type,'a') != NULL) + flags= O_APPEND; + /* Convert OS file handle to CRT file descriptor and then call fdopen*/ + crt_fd= _open_osfhandle((intptr_t)my_get_osfhandle(fd), flags); + if(crt_fd < 0) + file= NULL; + else + file= fdopen(crt_fd, type); + DBUG_RETURN(file); +} + + +int my_win_fclose(FILE *file) +{ + File fd; + + DBUG_ENTER("my_win_close"); + fd= my_fileno(file); + if(fd < 0) + DBUG_RETURN(-1); + if(fclose(file) < 0) + DBUG_RETURN(-1); + invalidate_fd(fd); + DBUG_RETURN(0); +} + + + +/* + Quick and dirty my_fstat() implementation for Windows. + Use CRT fstat on temporarily allocated file descriptor. + Patch file size, because size that fstat returns is not + reliable (may be outdated) +*/ +int my_win_fstat(File fd, struct _stati64 *buf) +{ + int crt_fd; + int retval; + HANDLE hFile, hDup; + + DBUG_ENTER("my_win_fstat"); + + hFile= my_get_osfhandle(fd); + if(!DuplicateHandle( GetCurrentProcess(), hFile, GetCurrentProcess(), + &hDup ,0,FALSE,DUPLICATE_SAME_ACCESS)) + { + my_osmaperr(GetLastError()); + DBUG_RETURN(-1); + } + if ((crt_fd= _open_osfhandle((intptr_t)hDup,0)) < 0) + DBUG_RETURN(-1); + + retval= _fstati64(crt_fd, buf); + if(retval == 0) + { + /* File size returned by stat is not accurate (may be outdated), fix it*/ + GetFileSizeEx(hDup, (PLARGE_INTEGER) (&(buf->st_size))); + } + _close(crt_fd); + DBUG_RETURN(retval); +} + + + +int my_win_stat( const char *path, struct _stati64 *buf) +{ + DBUG_ENTER("my_win_stat"); + if(_stati64( path, buf) == 0) + { + /* File size returned by stat is not accurate (may be outdated), fix it*/ + WIN32_FILE_ATTRIBUTE_DATA data; + if (GetFileAttributesEx(path, GetFileExInfoStandard, &data)) + { + LARGE_INTEGER li; + li.LowPart= data.nFileSizeLow; + li.HighPart= data.nFileSizeHigh; + buf->st_size= li.QuadPart; + } + DBUG_RETURN(0); + } + DBUG_RETURN(-1); +} + + + +int my_win_fsync(File fd) +{ + DBUG_ENTER("my_win_fsync"); + if(FlushFileBuffers(my_get_osfhandle(fd))) + DBUG_RETURN(0); + my_osmaperr(GetLastError()); + DBUG_RETURN(-1); +} + + + +int my_win_dup(File fd) +{ + HANDLE hDup; + DBUG_ENTER("my_win_dup"); + if (DuplicateHandle(GetCurrentProcess(), my_get_osfhandle(fd), + GetCurrentProcess(), &hDup, 0, FALSE, DUPLICATE_SAME_ACCESS)) + { + DBUG_RETURN(my_open_osfhandle(hDup, my_get_open_flags(fd))); + } + my_osmaperr(GetLastError()); + DBUG_RETURN(-1); +} + +#endif /*_WIN32*/ diff --git a/mysys/my_write.c b/mysys/my_write.c index 52127545888..5b916d6cb65 100644 --- a/mysys/my_write.c +++ b/mysys/my_write.c @@ -22,7 +22,7 @@ size_t my_write(int Filedes, const uchar *Buffer, size_t Count, myf MyFlags) { - size_t writenbytes, written; + size_t writtenbytes, written; uint errors; DBUG_ENTER("my_write"); DBUG_PRINT("my",("fd: %d Buffer: 0x%lx Count: %lu MyFlags: %d", @@ -35,17 +35,27 @@ size_t my_write(int Filedes, const uchar *Buffer, size_t Count, myf MyFlags) for (;;) { - if ((writenbytes= write(Filedes, Buffer, Count)) == Count) +#ifdef _WIN32 + if(Filedes < 0) + { + errno = EINVAL; + return -1; + } + writtenbytes= my_win_write(Filedes, Buffer, Count); +#else + writtenbytes= write(Filedes, Buffer, Count); +#endif + if (writtenbytes == Count) break; - if (writenbytes != (size_t) -1) + if (writtenbytes != (size_t) -1) { /* Safeguard */ - written+=writenbytes; - Buffer+=writenbytes; - Count-=writenbytes; + written+= writtenbytes; + Buffer+= writtenbytes; + Count-= writtenbytes; } - my_errno=errno; + my_errno= errno; DBUG_PRINT("error",("Write only %ld bytes, error: %d", - (long) writenbytes, my_errno)); + (long) writtenbytes, my_errno)); #ifndef NO_BACKGROUND #ifdef THREAD if (my_thread_var->abort) @@ -59,19 +69,19 @@ size_t my_write(int Filedes, const uchar *Buffer, size_t Count, myf MyFlags) continue; } - if ((writenbytes == 0 || writenbytes == (size_t) -1)) + if ((writtenbytes == 0 || writtenbytes == (size_t) -1)) { if (my_errno == EINTR) { DBUG_PRINT("debug", ("my_write() was interrupted and returned %ld", - (long) writenbytes)); + (long) writtenbytes)); continue; /* Interrupted */ } - if (!writenbytes && !errors++) /* Retry once */ + if (!writtenbytes && !errors++) /* Retry once */ { /* We may come here if the file quota is exeeded */ - errno=EFBIG; /* Assume this is the error */ + errno= EFBIG; /* Assume this is the error */ continue; } } @@ -92,5 +102,5 @@ size_t my_write(int Filedes, const uchar *Buffer, size_t Count, myf MyFlags) } if (MyFlags & (MY_NABP | MY_FNABP)) DBUG_RETURN(0); /* Want only errors */ - DBUG_RETURN(writenbytes+written); + DBUG_RETURN(writtenbytes+written); } /* my_write */ diff --git a/mysys/mysys_priv.h b/mysys/mysys_priv.h index 113b64005f2..6d39999aa86 100644 --- a/mysys/mysys_priv.h +++ b/mysys/mysys_priv.h @@ -45,3 +45,27 @@ extern pthread_mutex_t THR_LOCK_charset, THR_LOCK_time; void my_error_unregister_all(void); void my_thread_destroy_mutex(void); my_bool my_wait_for_other_threads_to_die(uint number_of_threads); + +#ifdef _WIN32 +/* my_winfile.c exports, should not be used outside mysys */ +extern File my_win_open(const char *path, int oflag); +extern int my_win_close(File fd); +extern size_t my_win_read(File fd, uchar *buffer, size_t count); +extern size_t my_win_write(File fd, const uchar *buffer, size_t count); +extern size_t my_win_pread(File fd, uchar *buffer, size_t count, + my_off_t offset); +extern size_t my_win_pwrite(File fd, const uchar *buffer, size_t count, + my_off_t offset); +extern my_off_t my_win_lseek(File fd, my_off_t pos, int whence); +extern int my_win_chsize(File fd, my_off_t newlength); +extern FILE* my_win_fopen(const char *filename, const char *type); +extern File my_win_fclose(FILE *file); +extern File my_win_fileno(FILE *file); +extern FILE* my_win_fdopen(File Filedes, const char *type); +extern int my_win_stat(const char *path, struct _stat64 *buf); +extern int my_win_fstat(File fd, struct _stat64 *buf); +extern int my_win_fsync(File fd); +extern File my_win_dup(File fd); +extern File my_win_sopen(const char *path, int oflag, int shflag, int perm); +extern File my_open_osfhandle(HANDLE handle, int oflag); +#endif -- cgit v1.2.1 From fe054adfcaecbd891da802517826a951bb4ca377 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sun, 12 Jun 2011 16:09:28 +0200 Subject: Backport fix for MySQL bug #56405 : use native windows condition variables and rwlocks in mysys, if Windows supports it. --- mysys/my_wincond.c | 232 +++++++++++++++++++++++++++++++++++++++++---------- mysys/my_winthread.c | 15 +++- mysys/thr_rwlock.c | 160 +++++++++++++++++++++++++++++++++++ 3 files changed, 363 insertions(+), 44 deletions(-) (limited to 'mysys') diff --git a/mysys/my_wincond.c b/mysys/my_wincond.c index b869b22bdea..13bf0b78766 100644 --- a/mysys/my_wincond.c +++ b/mysys/my_wincond.c @@ -26,7 +26,108 @@ #include #include -int pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr) + +/* + Windows native condition variables. We use runtime loading / function + pointers, because they are not available on XP +*/ + +/* Prototypes and function pointers for condition variable functions */ +typedef VOID (WINAPI * InitializeConditionVariableProc) + (PCONDITION_VARIABLE ConditionVariable); + +typedef BOOL (WINAPI * SleepConditionVariableCSProc) + (PCONDITION_VARIABLE ConditionVariable, + PCRITICAL_SECTION CriticalSection, + DWORD dwMilliseconds); + +typedef VOID (WINAPI * WakeAllConditionVariableProc) + (PCONDITION_VARIABLE ConditionVariable); + +typedef VOID (WINAPI * WakeConditionVariableProc) + (PCONDITION_VARIABLE ConditionVariable); + +static InitializeConditionVariableProc my_InitializeConditionVariable; +static SleepConditionVariableCSProc my_SleepConditionVariableCS; +static WakeAllConditionVariableProc my_WakeAllConditionVariable; +static WakeConditionVariableProc my_WakeConditionVariable; + + +/** + Indicates if we have native condition variables, + initialized first time pthread_cond_init is called. +*/ + +static BOOL have_native_conditions= FALSE; + + +/** + Check if native conditions can be used, load function pointers +*/ + +static void check_native_cond_availability(void) +{ + HMODULE module= GetModuleHandle("kernel32"); + + my_InitializeConditionVariable= (InitializeConditionVariableProc) + GetProcAddress(module, "InitializeConditionVariable"); + my_SleepConditionVariableCS= (SleepConditionVariableCSProc) + GetProcAddress(module, "SleepConditionVariableCS"); + my_WakeAllConditionVariable= (WakeAllConditionVariableProc) + GetProcAddress(module, "WakeAllConditionVariable"); + my_WakeConditionVariable= (WakeConditionVariableProc) + GetProcAddress(module, "WakeConditionVariable"); + + if (my_InitializeConditionVariable) + have_native_conditions= TRUE; +} + + + +/** + Convert abstime to milliseconds +*/ + +static DWORD get_milliseconds(const struct timespec *abstime) +{ + long long millis; + union ft64 now; + + if (abstime == NULL) + return INFINITE; + + GetSystemTimeAsFileTime(&now.ft); + + /* + Calculate time left to abstime + - subtract start time from current time(values are in 100ns units) + - convert to millisec by dividing with 10000 + */ + millis= (abstime->tv.i64 - now.i64) / 10000; + + /* Don't allow the timeout to be negative */ + if (millis < 0) + return 0; + + /* + Make sure the calculated timeout does not exceed original timeout + value which could cause "wait for ever" if system time changes + */ + if (millis > abstime->max_timeout_msec) + millis= abstime->max_timeout_msec; + + if (millis > UINT_MAX) + millis= UINT_MAX; + + return (DWORD)millis; +} + + +/* + Old (pre-vista) implementation using events +*/ + +static int legacy_cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr) { cond->waiting= 0; InitializeCriticalSection(&cond->lock_waiting); @@ -55,7 +156,8 @@ int pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr) return 0; } -int pthread_cond_destroy(pthread_cond_t *cond) + +static int legacy_cond_destroy(pthread_cond_t *cond) { DeleteCriticalSection(&cond->lock_waiting); @@ -67,48 +169,13 @@ int pthread_cond_destroy(pthread_cond_t *cond) } -int pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex) -{ - return pthread_cond_timedwait(cond,mutex,NULL); -} - - -int pthread_cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex, +static int legacy_cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex, struct timespec *abstime) { int result; - long timeout; - union ft64 now; - - if( abstime != NULL ) - { - GetSystemTimeAsFileTime(&now.ft); - - /* - Calculate time left to abstime - - subtract start time from current time(values are in 100ns units) - - convert to millisec by dividing with 10000 - */ - timeout= (long)((abstime->tv.i64 - now.i64) / 10000); - - /* Don't allow the timeout to be negative */ - if (timeout < 0) - timeout= 0L; - - /* - Make sure the calucated timeout does not exceed original timeout - value which could cause "wait for ever" if system time changes - */ - if (timeout > abstime->max_timeout_msec) - timeout= abstime->max_timeout_msec; - - } - else - { - /* No time specified; don't expire */ - timeout= INFINITE; - } + DWORD timeout; + timeout= get_milliseconds(abstime); /* Block access if previous broadcast hasn't finished. This is just for safety and should normally not @@ -144,7 +211,7 @@ int pthread_cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex, return result == WAIT_TIMEOUT ? ETIMEDOUT : 0; } -int pthread_cond_signal(pthread_cond_t *cond) +static int legacy_cond_signal(pthread_cond_t *cond) { EnterCriticalSection(&cond->lock_waiting); @@ -157,7 +224,7 @@ int pthread_cond_signal(pthread_cond_t *cond) } -int pthread_cond_broadcast(pthread_cond_t *cond) +static int legacy_cond_broadcast(pthread_cond_t *cond) { EnterCriticalSection(&cond->lock_waiting); /* @@ -179,6 +246,87 @@ int pthread_cond_broadcast(pthread_cond_t *cond) } +/* + Posix API functions. Just choose between native and legacy implementation. +*/ + +int pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr) +{ + /* + Once initialization is used here rather than in my_init(), to + 1) avoid my_init() pitfalls- undefined order in which initialization should + run + 2) be potentially useful C++ (in static constructors that run before main()) + 3) just to simplify the API. + Also, the overhead of my_pthread_once is very small. + */ + static my_pthread_once_t once_control= MY_PTHREAD_ONCE_INIT; + my_pthread_once(&once_control, check_native_cond_availability); + + if (have_native_conditions) + { + my_InitializeConditionVariable(&cond->native_cond); + return 0; + } + else + return legacy_cond_init(cond, attr); +} + + +int pthread_cond_destroy(pthread_cond_t *cond) +{ + if (have_native_conditions) + return 0; /* no destroy function */ + else + return legacy_cond_destroy(cond); +} + + +int pthread_cond_broadcast(pthread_cond_t *cond) +{ + if (have_native_conditions) + { + my_WakeAllConditionVariable(&cond->native_cond); + return 0; + } + else + return legacy_cond_broadcast(cond); +} + + +int pthread_cond_signal(pthread_cond_t *cond) +{ + if (have_native_conditions) + { + my_WakeConditionVariable(&cond->native_cond); + return 0; + } + else + return legacy_cond_signal(cond); +} + + +int pthread_cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex, + struct timespec *abstime) +{ + if (have_native_conditions) + { + DWORD timeout= get_milliseconds(abstime); + if (!my_SleepConditionVariableCS(&cond->native_cond, mutex, timeout)) + return ETIMEDOUT; + return 0; + } + else + return legacy_cond_timedwait(cond, mutex, abstime); +} + + +int pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex) +{ + return pthread_cond_timedwait(cond, mutex, NULL); +} + + int pthread_attr_init(pthread_attr_t *connect_att) { connect_att->dwStackSize = 0; diff --git a/mysys/my_winthread.c b/mysys/my_winthread.c index 6adf24ef543..5fbad39597c 100644 --- a/mysys/my_winthread.c +++ b/mysys/my_winthread.c @@ -156,8 +156,19 @@ int win_pthread_setspecific(void *a,void *b,uint length) int my_pthread_once(my_pthread_once_t *once_control, void (*init_routine)(void)) { - LONG state= InterlockedCompareExchange(once_control, MY_PTHREAD_ONCE_INPROGRESS, - MY_PTHREAD_ONCE_INIT); + LONG state; + + /* + Do "dirty" read to find out if initialization is already done, to + save an interlocked operation in common case. Memory barriers are ensured by + Visual C++ volatile implementation. + */ + if (*once_control == MY_PTHREAD_ONCE_DONE) + return 0; + + state= InterlockedCompareExchange(once_control, MY_PTHREAD_ONCE_INPROGRESS, + MY_PTHREAD_ONCE_INIT); + switch(state) { case MY_PTHREAD_ONCE_INIT: diff --git a/mysys/thr_rwlock.c b/mysys/thr_rwlock.c index ea98a854a4d..2978d91090d 100644 --- a/mysys/thr_rwlock.c +++ b/mysys/thr_rwlock.c @@ -19,6 +19,119 @@ #if defined(THREAD) && !defined(HAVE_PTHREAD_RWLOCK_RDLOCK) && !defined(HAVE_RWLOCK_INIT) #include +#ifdef _WIN32 + +static BOOL have_srwlock= FALSE; +/* Prototypes and function pointers for windows functions */ +typedef VOID (WINAPI* srw_func) (PSRWLOCK SRWLock); +typedef BOOL (WINAPI* srw_bool_func) (PSRWLOCK SRWLock); + +static srw_func my_InitializeSRWLock; +static srw_func my_AcquireSRWLockExclusive; +static srw_func my_ReleaseSRWLockExclusive; +static srw_func my_AcquireSRWLockShared; +static srw_func my_ReleaseSRWLockShared; + +static srw_bool_func my_TryAcquireSRWLockExclusive; +static srw_bool_func my_TryAcquireSRWLockShared; + +/** + Check for presence of Windows slim reader writer lock function. + Load function pointers. +*/ + +static void check_srwlock_availability(void) +{ + HMODULE module= GetModuleHandle("kernel32"); + + my_InitializeSRWLock= (srw_func) GetProcAddress(module, + "InitializeSRWLock"); + my_AcquireSRWLockExclusive= (srw_func) GetProcAddress(module, + "AcquireSRWLockExclusive"); + my_AcquireSRWLockShared= (srw_func) GetProcAddress(module, + "AcquireSRWLockShared"); + my_ReleaseSRWLockExclusive= (srw_func) GetProcAddress(module, + "ReleaseSRWLockExclusive"); + my_ReleaseSRWLockShared= (srw_func) GetProcAddress(module, + "ReleaseSRWLockShared"); + my_TryAcquireSRWLockExclusive= (srw_bool_func) GetProcAddress(module, + "TryAcquireSRWLockExclusive"); + my_TryAcquireSRWLockShared= (srw_bool_func) GetProcAddress(module, + "TryAcquireSRWLockShared"); + + /* + We currently require TryAcquireSRWLockExclusive. This API is missing on + Vista, this means SRWLock are only used starting with Win7. + + If "trylock" usage for rwlocks is eliminated from server codebase (it is used + in a single place currently, in query cache), then SRWLock can be enabled on + Vista too. In this case condition below needs to be changed to e.g check + for my_InitializeSRWLock. + */ + + if (my_TryAcquireSRWLockExclusive) + have_srwlock= TRUE; + +} + + +static int srw_init(my_rw_lock_t *rwp) +{ + my_InitializeSRWLock(&rwp->srwlock); + rwp->have_exclusive_srwlock = FALSE; + return 0; +} + + +static int srw_rdlock(my_rw_lock_t *rwp) +{ + my_AcquireSRWLockShared(&rwp->srwlock); + return 0; +} + + +static int srw_tryrdlock(my_rw_lock_t *rwp) +{ + + if (!my_TryAcquireSRWLockShared(&rwp->srwlock)) + return EBUSY; + return 0; +} + + +static int srw_wrlock(my_rw_lock_t *rwp) +{ + my_AcquireSRWLockExclusive(&rwp->srwlock); + rwp->have_exclusive_srwlock= TRUE; + return 0; +} + + +static int srw_trywrlock(my_rw_lock_t *rwp) +{ + if (!my_TryAcquireSRWLockExclusive(&rwp->srwlock)) + return EBUSY; + rwp->have_exclusive_srwlock= TRUE; + return 0; +} + + +static int srw_unlock(my_rw_lock_t *rwp) +{ + if (rwp->have_exclusive_srwlock) + { + rwp->have_exclusive_srwlock= FALSE; + my_ReleaseSRWLockExclusive(&rwp->srwlock); + } + else + { + my_ReleaseSRWLockShared(&rwp->srwlock); + } + return 0; +} + +#endif /*_WIN32 */ + /* Source base from Sun Microsystems SPILT, simplified for MySQL use -- Joshua Chamas @@ -62,6 +175,22 @@ int my_rwlock_init(rw_lock_t *rwp, void *arg __attribute__((unused))) { pthread_condattr_t cond_attr; +#ifdef _WIN32 + /* + Once initialization is used here rather than in my_init(), in order to + - avoid my_init() pitfalls- (undefined order in which initialization should + run) + - be potentially useful C++ (static constructors) + - just to simplify the API. + Also, the overhead is of my_pthread_once is very small. + */ + static my_pthread_once_t once_control= MY_PTHREAD_ONCE_INIT; + my_pthread_once(&once_control, check_srwlock_availability); + + if (have_srwlock) + return srw_init(rwp); +#endif + pthread_mutex_init( &rwp->lock, MY_MUTEX_INIT_FAST); pthread_condattr_init( &cond_attr ); pthread_cond_init( &rwp->readers, &cond_attr ); @@ -77,6 +206,10 @@ int my_rwlock_init(rw_lock_t *rwp, void *arg __attribute__((unused))) int my_rwlock_destroy(rw_lock_t *rwp) { +#ifdef _WIN32 + if (have_srwlock) + return 0; /* no destroy function */ +#endif pthread_mutex_destroy( &rwp->lock ); pthread_cond_destroy( &rwp->readers ); pthread_cond_destroy( &rwp->writers ); @@ -86,6 +219,11 @@ int my_rwlock_destroy(rw_lock_t *rwp) int my_rw_rdlock(rw_lock_t *rwp) { +#ifdef _WIN32 + if (have_srwlock) + return srw_rdlock(rwp); +#endif + pthread_mutex_lock(&rwp->lock); /* active or queued writers */ @@ -100,6 +238,12 @@ int my_rw_rdlock(rw_lock_t *rwp) int my_rw_tryrdlock(rw_lock_t *rwp) { int res; + +#ifdef _WIN32 + if (have_srwlock) + return srw_tryrdlock(rwp); +#endif + pthread_mutex_lock(&rwp->lock); if ((rwp->state < 0 ) || rwp->waiters) res= EBUSY; /* Can't get lock */ @@ -115,6 +259,11 @@ int my_rw_tryrdlock(rw_lock_t *rwp) int my_rw_wrlock(rw_lock_t *rwp) { +#ifdef _WIN32 + if (have_srwlock) + return srw_wrlock(rwp); +#endif + pthread_mutex_lock(&rwp->lock); rwp->waiters++; /* another writer queued */ @@ -130,6 +279,12 @@ int my_rw_wrlock(rw_lock_t *rwp) int my_rw_trywrlock(rw_lock_t *rwp) { int res; + +#ifdef _WIN32 + if (have_srwlock) + return srw_trywrlock(rwp); +#endif + pthread_mutex_lock(&rwp->lock); if (rwp->state) res= EBUSY; /* Can't get lock */ @@ -145,6 +300,11 @@ int my_rw_trywrlock(rw_lock_t *rwp) int my_rw_unlock(rw_lock_t *rwp) { +#ifdef _WIN32 + if (have_srwlock) + return srw_unlock(rwp); +#endif + DBUG_PRINT("rw_unlock", ("state: %d waiters: %d", rwp->state, rwp->waiters)); pthread_mutex_lock(&rwp->lock); -- cgit v1.2.1 From 6cf9e92e74089dcc700d6f0577ef9e7208027442 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sun, 19 Jun 2011 00:29:49 +0200 Subject: fix compile error on *nix --- mysys/my_read.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'mysys') diff --git a/mysys/my_read.c b/mysys/my_read.c index 9c76193aa63..5fef4455f62 100644 --- a/mysys/my_read.c +++ b/mysys/my_read.c @@ -45,9 +45,9 @@ size_t my_read(File Filedes, uchar *Buffer, size_t Count, myf MyFlags) { errno= 0; /* Linux, Windows don't reset this on EOF/success */ #ifdef _WIN32 - readbytes= my_win_read(Filedes, Buffer, Count); + readbytes= my_win_read(Filedes, Buffer, Count); #else - readbytes= my_read(Fildes, Buffer, Count); + readbytes= read(Filedes, Buffer, Count); #endif if (readbytes != Count) { -- cgit v1.2.1 From d1d19a68830c71d5aa6dfd71ecdfab5170904783 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sun, 19 Jun 2011 00:51:41 +0200 Subject: add missing DBUG_RETURN --- mysys/my_write.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mysys') diff --git a/mysys/my_write.c b/mysys/my_write.c index 5b916d6cb65..1d59c62a005 100644 --- a/mysys/my_write.c +++ b/mysys/my_write.c @@ -39,7 +39,7 @@ size_t my_write(int Filedes, const uchar *Buffer, size_t Count, myf MyFlags) if(Filedes < 0) { errno = EINVAL; - return -1; + DBUG_RETURN(-1); } writtenbytes= my_win_write(Filedes, Buffer, Count); #else -- cgit v1.2.1 From 4243aee728dd038e23f05a69d60208e9cebd8635 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sun, 19 Jun 2011 17:19:22 +0200 Subject: Fix "make dist" : add my_winfile.c and my_winerr.c to EXTRA_DIST list --- mysys/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mysys') diff --git a/mysys/Makefile.am b/mysys/Makefile.am index 1dd4cc0f780..edeba73a5ee 100644 --- a/mysys/Makefile.am +++ b/mysys/Makefile.am @@ -74,7 +74,7 @@ endif EXTRA_DIST = thr_alarm.c thr_lock.c my_pthread.c my_thr_init.c \ thr_mutex.c thr_rwlock.c waiting_threads.c \ CMakeLists.txt mf_soundex.c \ - my_conio.c my_wincond.c my_winthread.c + my_conio.c my_wincond.c my_winthread.c my_winfile.c my_winerr.c # test_dir_DEPENDENCIES= $(LIBRARIES) # testhash_DEPENDENCIES= $(LIBRARIES) # test_charset_DEPENDENCIES= $(LIBRARIES) -- cgit v1.2.1 From b48dc8306f2b729bef09f9cdf30d7897726b873e Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sun, 26 Jun 2011 01:07:39 +0200 Subject: set errno to EBADF, if file descriptor < 0 in my_write() --- mysys/my_write.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'mysys') diff --git a/mysys/my_write.c b/mysys/my_write.c index 1d59c62a005..be493049d5c 100644 --- a/mysys/my_write.c +++ b/mysys/my_write.c @@ -38,8 +38,8 @@ size_t my_write(int Filedes, const uchar *Buffer, size_t Count, myf MyFlags) #ifdef _WIN32 if(Filedes < 0) { - errno = EINVAL; - DBUG_RETURN(-1); + my_errno= errno= EBADF; + DBUG_RETURN((size_t)-1); } writtenbytes= my_win_write(Filedes, Buffer, Count); #else -- cgit v1.2.1