Merge branch '10.2' of github.com:MariaDB/server into 10.2-mariarocks

and a few trivial test result updates

19 files changed, 537 insertions, 141 deletions

diff --git a/include/CMakeLists.txt b/include/CMakeLists.txt
index 8ebd72b9c58..e6de8515fdc 100644
--- a/include/CMakeLists.txt
+++ b/include/CMakeLists.txt
@@ -61,6 +61,7 @@ SET(HEADERS
   my_compiler.h
   handler_state.h
   handler_ername.h
+  json_lib.h
 )
 
 INSTALL(FILES ${HEADERS} DESTINATION ${INSTALL_INCLUDEDIR} COMPONENT Development)
diff --git a/include/atomic/nolock.h b/include/atomic/nolock.h
deleted file mode 100644
index 2137445a075..00000000000
--- a/include/atomic/nolock.h
+++ /dev/null
@@ -1,54 +0,0 @@
-#ifndef ATOMIC_NOLOCK_INCLUDED
-#define ATOMIC_NOLOCK_INCLUDED
-
-/* Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
-
-   This program is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published by
-   the Free Software Foundation; version 2 of the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
-
-#if defined(__i386__) || defined(_MSC_VER) || defined(__x86_64__)   \
-    || defined(HAVE_GCC_ATOMIC_BUILTINS) \
-    || defined(HAVE_SOLARIS_ATOMIC)
-
-#  ifdef MY_ATOMIC_MODE_DUMMY
-#    define LOCK_prefix ""
-#  else
-#    define LOCK_prefix "lock"
-#  endif
-/*
-  We choose implementation as follows:
-  ------------------------------------
-  On Windows using Visual C++ the native implementation should be
-  preferrable. When using gcc we prefer the Solaris implementation
-  before the gcc because of stability preference, we choose gcc
-  builtins if available, otherwise we choose the somewhat broken
-  native x86 implementation. If neither Visual C++ or gcc we still
-  choose the Solaris implementation on Solaris (mainly for SunStudio
-  compilers).
-*/
-#  if defined(_MSC_VER)
-#    include "generic-msvc.h"
-#  elif __GNUC__
-#    if defined(HAVE_SOLARIS_ATOMIC)
-#      include "solaris.h"
-#    elif defined(HAVE_GCC_ATOMIC_BUILTINS)
-#      include "gcc_builtins.h"
-#    elif defined(__i386__) || defined(__x86_64__)
-#      include "x86-gcc.h"
-#    endif
-#  elif defined(HAVE_SOLARIS_ATOMIC)
-#    include "solaris.h"
-#  endif
-#endif
-
-#endif /* ATOMIC_NOLOCK_INCLUDED */
diff --git a/include/atomic/x86-gcc.h b/include/atomic/x86-gcc.h
index 173e32e790c..3a081d9bbc5 100644
--- a/include/atomic/x86-gcc.h
+++ b/include/atomic/x86-gcc.h
@@ -28,6 +28,12 @@
 */
 #undef MY_ATOMIC_HAS_8_AND_16
 
+#ifdef MY_ATOMIC_MODE_DUMMY
+#define LOCK_prefix ""
+#else
+#define LOCK_prefix "lock"
+#endif
+
 #ifdef __x86_64__
 #  ifdef MY_ATOMIC_NO_XADD
 #    define MY_ATOMIC_MODE "gcc-amd64" LOCK_prefix "-no-xadd"
diff --git a/include/byte_order_generic_x86.h b/include/byte_order_generic_x86.h
index 0a71a17829b..a97dd0f43a3 100644
--- a/include/byte_order_generic_x86.h
+++ b/include/byte_order_generic_x86.h
@@ -27,19 +27,9 @@
 				  ((uint32) (uchar) (A)[0])))
 #define sint4korr(A)	(*((const long *) (A)))
 #define uint2korr(A)	(*((const uint16 *) (A)))
-
-/*
-  Attention: Please, note, uint3korr reads 4 bytes (not 3)!
-  It means, that you have to provide enough allocated space.
-*/
-#if defined(HAVE_valgrind) && !defined(_WIN32)
 #define uint3korr(A)	(uint32) (((uint32) ((uchar) (A)[0])) +\
 				  (((uint32) ((uchar) (A)[1])) << 8) +\
 				  (((uint32) ((uchar) (A)[2])) << 16))
-#else
-#define uint3korr(A)	(long) (*((const unsigned int *) (A)) & 0xFFFFFF)
-#endif
-
 #define uint4korr(A)	(*((const uint32 *) (A)))
 #define uint5korr(A)	((ulonglong)(((uint32) ((uchar) (A)[0])) +\
 				    (((uint32) ((uchar) (A)[1])) << 8) +\
diff --git a/include/byte_order_generic_x86_64.h b/include/byte_order_generic_x86_64.h
index 05c144f83cb..fbc6e1f536b 100644
--- a/include/byte_order_generic_x86_64.h
+++ b/include/byte_order_generic_x86_64.h
@@ -28,17 +28,9 @@
 				  ((uint32) (uchar) (A)[0])))
 #define sint4korr(A)	(int32)  (*((int32 *) (A)))
 #define uint2korr(A)	(uint16) (*((uint16 *) (A)))
-/*
-  Attention: Please, note, uint3korr reads 4 bytes (not 3)!
-  It means, that you have to provide enough allocated space.
-*/
-#if defined(HAVE_valgrind) && !defined(_WIN32)
 #define uint3korr(A)	(uint32) (((uint32) ((uchar) (A)[0])) +\
 				  (((uint32) ((uchar) (A)[1])) << 8) +\
 				  (((uint32) ((uchar) (A)[2])) << 16))
-#else
-#define uint3korr(A)	(uint32) (*((unsigned int *) (A)) & 0xFFFFFF)
-#endif
 #define uint4korr(A)	(uint32) (*((uint32 *) (A)))
 
 
diff --git a/include/json_lib.h b/include/json_lib.h
new file mode 100644
index 00000000000..ce7f27317bc
--- /dev/null
+++ b/include/json_lib.h
@@ -0,0 +1,399 @@
+#ifndef JSON_LIB_INCLUDED
+#define JSON_LIB_INCLUDED
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define JSON_DEPTH_LIMIT 32
+
+/*
+  When error happens, the c_next of the JSON engine contains the
+  character that caused the error, and the c_str is the position
+  in string where the error occurs.
+*/
+enum json_errors {
+  JE_BAD_CHR= -1,      /* Invalid character, charset handler cannot read it. */
+
+  JE_NOT_JSON_CHR= -2, /* Character met not used in JSON. */
+                       /* ASCII 00-08 for instance.       */
+
+  JE_EOS= -3,          /* Unexpected end of string. */
+
+  JE_SYN= -4,          /* The next character breaks the JSON syntax. */
+
+  JE_STRING_CONST= -5, /* Character disallowed in string constant. */
+
+  JE_ESCAPING= -6,     /* Error in the escaping. */
+
+  JE_DEPTH= -7,        /* The limit on the JSON depth was overrun. */
+};
+
+
+typedef struct st_json_string_t
+{
+  const uchar *c_str;    /* Current position in JSON string */
+  const uchar *str_end;  /* The end on the string. */
+  my_wc_t c_next;        /* UNICODE of the last read character */
+  int error;             /* error code. */
+
+  CHARSET_INFO *cs;      /* Character set of the JSON string. */
+
+  my_charset_conv_mb_wc wc; /* UNICODE conversion function. */
+                            /* It's taken out of the cs just to speed calls. */
+} json_string_t;
+
+
+void json_string_set_cs(json_string_t *s, CHARSET_INFO *i_cs);
+void json_string_set_str(json_string_t *s,
+                         const uchar *str, const uchar *end);
+#define json_next_char(j) \
+  (j)->wc((j)->cs, &(j)->c_next, (j)->c_str, (j)->str_end)
+#define json_eos(j) ((j)->c_str >= (j)->str_end)
+/*
+  read_string_const_chr() reads the next character of the string constant
+  and saves it to the js->c_next.
+  It takes into account possible escapings, so if for instance
+  the string is '\b', the read_string_const_chr() sets 8.
+*/
+int json_read_string_const_chr(json_string_t *js);
+
+
+/*
+  Various JSON-related operations expect JSON path as a parameter.
+  The path is a string like this "$.keyA[2].*"
+  The path itself is a number of steps specifying either a key or a position
+  in an array. Some of them can be wildcards.
+  So the representation of the JSON path is the json_path_t class
+  containing an array of json_path_step_t objects.
+*/
+
+
+/* Path step types - actually bitmasks to let '&' or '|' operations. */
+enum json_path_step_types
+{
+  JSON_PATH_KEY_NULL=0,
+  JSON_PATH_KEY=1,   /* Must be equal to JSON_VALUE_OBJECT. */
+  JSON_PATH_ARRAY=2, /* Must be equal to JSON_VALUE_ARRAY. */
+  JSON_PATH_KEY_OR_ARRAY=3,
+  JSON_PATH_WILD=4, /* Step like .* or [*] */
+  JSON_PATH_DOUBLE_WILD=8, /* Step like **.k or **[1] */
+  JSON_PATH_KEY_WILD= 1+4,
+  JSON_PATH_KEY_DOUBLEWILD= 1+8,
+  JSON_PATH_ARRAY_WILD= 2+4,
+  JSON_PATH_ARRAY_DOUBLEWILD= 2+8
+};
+
+
+typedef struct st_json_path_step_t
+{
+  enum json_path_step_types type;  /* The type of the step -   */
+                                   /* see json_path_step_types */
+  const uchar *key; /* Pointer to the beginning of the key. */
+  const uchar *key_end;  /* Pointer to the end of the key. */
+  uint n_item;      /* Item number in an array. No meaning for the key step. */
+} json_path_step_t;
+
+
+typedef struct st_json_path_t
+{
+  json_string_t s;  /* The string to be parsed. */
+  json_path_step_t steps[JSON_DEPTH_LIMIT]; /* Steps of the path. */
+  json_path_step_t *last_step; /* Points to the last step. */
+
+  int mode_strict; /* TRUE if the path specified as 'strict' */
+  enum json_path_step_types types_used; /* The '|' of all step's 'type'-s */
+} json_path_t;
+
+
+int json_path_setup(json_path_t *p,
+                    CHARSET_INFO *i_cs, const uchar *str, const uchar *end);
+
+
+/*
+  The set of functions and structures below provides interface
+  to the JSON text parser.
+  Running the parser normally goes like this:
+
+    json_engine_t j_eng;   // structure keeps parser's data
+    json_scan_start(j_eng) // begin the parsing
+
+    do
+    {
+      // The parser has read next piece of JSON
+      // and set fields of j_eng structure accordingly.
+      // So let's see what we have:
+      switch (j_eng.state)
+      {
+        case JST_KEY:
+           // Handle key name. See the json_read_keyname_chr()
+           // Probably compare it with the keyname we're looking for
+        case JST_VALUE:
+           // Handle value. It is either value of the key or an array item.
+           // see the json_read_value()
+        case JST_OBJ_START:
+          // parser found an object (the '{' in JSON)
+        case JST_OBJ_END:
+          // parser found the end of the object (the '}' in JSON)
+        case JST_ARRAY_START:
+          // parser found an array (the '[' in JSON)
+        case JST_ARRAY_END:
+          // parser found the end of the array (the ']' in JSON)
+
+      };
+    } while (json_scan_next() == 0);  // parse next structure
+
+    
+    if (j_eng.s.error)  // we need to check why the loop ended.
+                        // Did we get to the end of JSON, or came upon error.
+    {
+       signal_error_in_JSON()
+    }
+
+
+  Parts of JSON can be quickly skipped. If we are not interested
+  in a particular key, we can just skip it with json_skip_key() call.
+  Similarly json_skip_level() goes right to the end of an object
+  or an array.
+*/
+
+
+/* These are JSON parser states that user can expect and handle.  */
+enum json_states {
+  JST_VALUE,       /* value found      */
+  JST_KEY,         /* key found        */
+  JST_OBJ_START,   /* object           */
+  JST_OBJ_END,     /* object ended     */
+  JST_ARRAY_START, /* array            */
+  JST_ARRAY_END,   /* array ended      */
+  NR_JSON_USER_STATES
+};
+
+
+enum json_value_types
+{
+  JSON_VALUE_OBJECT=1,
+  JSON_VALUE_ARRAY=2,
+  JSON_VALUE_STRING,
+  JSON_VALUE_NUMBER,
+  JSON_VALUE_TRUE,
+  JSON_VALUE_FALSE,
+  JSON_VALUE_NULL
+};
+
+
+enum json_num_flags
+{
+  JSON_NUM_NEG=1,        /* Number is negative. */
+  JSON_NUM_FRAC_PART=2,  /* The fractional part is not empty. */
+  JSON_NUM_EXP=4,        /* The number has the 'e' part. */
+};
+
+
+typedef struct st_json_engine_t
+{
+  json_string_t s;  /* String to parse. */
+  int sav_c_len;    /* Length of the current character.
+                       Can be more than 1 for multibyte charsets */
+
+  int state; /* The state of the parser. One of 'enum json_states'.
+                It tells us what construction of JSON we've just read. */
+
+  /* These values are only set after the json_read_value() call. */
+  enum json_value_types value_type; /* type of the value.*/
+  const uchar *value;      /* Points to the value. */
+  const uchar *value_begin;/* Points to where the value starts in the JSON. */
+  uint num_flags;  /* the details of the JSON_VALUE_NUMBER, is it negative,
+                      or if it has the fractional part.
+                      See the enum json_num_flags. */
+
+  /*
+    In most cases the 'value' and 'value_begin' are equal.
+    They only differ if the value is a string constants. Then 'value_begin'
+    points to the starting quotation mark, while the 'value' - to
+    the first character of the string.
+  */
+
+  const uchar *value_end; /* Points to the next character after the value. */
+  int value_len; /* The length of the value. Does not count quotations for */
+                 /* string constants. */
+
+  int stack[JSON_DEPTH_LIMIT]; /* Keeps the stack of nested JSON structures. */
+  int *stack_p;                /* The 'stack' pointer. */
+} json_engine_t;
+
+
+int json_scan_start(json_engine_t *je,
+                        CHARSET_INFO *i_cs, const uchar *str, const uchar *end);
+int json_scan_next(json_engine_t *j);
+
+
+/*
+  json_read_keyname_chr() function assists parsing the name of an JSON key.
+  It only can be called when the json_engine is in JST_KEY.
+  The json_read_keyname_chr() reads one character of the name of the key,
+  and puts it in j_eng.s.next_c.
+  Typical usage is like this:
+
+  if (j_eng.state == JST_KEY)
+  {
+    while (json_read_keyname_chr(&j) == 0)
+    {
+      //handle next character i.e. match it against the pattern
+    }
+  }
+*/
+
+int json_read_keyname_chr(json_engine_t *j);
+
+
+/*
+  Check if the name of the current JSON key matches
+  the step of the path.
+*/
+int json_key_matches(json_engine_t *je, json_string_t *k);
+
+
+/*
+  json_read_value() function parses the JSON value syntax,
+  so that we can handle the value of a key or an array item.
+  It only returns meaningful result when the engine is in
+  the JST_VALUE state.
+
+  Typical usage is like this:
+
+  if (j_eng.state ==  JST_VALUE)
+  {
+    json_read_value(&j_eng);
+    switch(j_eng.value_type)
+    {
+      case JSON_VALUE_STRING:
+        // get the string
+        str= j_eng.value;
+        str_length= j_eng.value_len;
+      case JSON_VALUE_NUMBER:
+        // get the number
+      ... etc
+    }
+*/
+int json_read_value(json_engine_t *j);
+
+
+/*
+  json_skip_key() makes parser skip the content of the current
+  JSON key quickly.
+  It can be called only when the json_engine state is JST_KEY.
+  Typical usage is:
+
+  if (j_eng.state == JST_KEY)
+  {
+    if (key_does_not_match(j_eng))
+      json_skip_key(j_eng);
+  }
+*/
+
+int json_skip_key(json_engine_t *j);
+
+
+typedef const int *json_level_t;
+
+/*
+  json_skip_to_level() makes parser quickly get out of nested
+  loops and arrays. It is used when we're not interested in what is
+  there in the rest of these structures.
+  The 'level' should be remembered in advance.
+        json_level_t level= json_get_level(j);
+        .... // getting into the nested JSON structures
+        json_skip_to_level(j, level);
+*/
+#define json_get_level(j) (j->stack_p)
+
+int json_skip_to_level(json_engine_t *j, json_level_t level);
+
+/*
+  json_skip_level() works as above with just current structre.
+  So it gets to the end of the current JSON array or object.
+*/
+#define json_skip_level(json_engine) \
+  json_skip_to_level((json_engine), (json_engine)->stack_p)
+
+
+#define json_skip_array_item json_skip_key
+
+/*
+  Checks if the current value is of scalar type -
+  not an OBJECT nor ARRAY.
+*/
+#define json_value_scalar(je)  ((je)->value_type > JSON_VALUE_ARRAY)
+
+
+/*
+  Look for the JSON PATH in the json string.
+  Function can be called several times with same JSON/PATH to
+  find multiple matches.
+  On the first call, the json_engine_t parameter should be
+  initialized with the JSON string, and the json_path_t with the JSON path
+  appropriately. The 'p_cur_step' should point at the first
+  step of the path.
+  The 'array_counters' is the array of JSON_DEPTH_LIMIT size.
+  It stores the array counters of the parsed JSON.
+  If function returns 0, it means it found the match. The position of
+  the match is je->s.c_str. Then we can call the json_find_path()
+  with same engine/path/p_cur_step to get the next match.
+  Non-zero return means no matches found.
+  Check je->s.error to see if there was an error in JSON.
+*/
+int json_find_path(json_engine_t *je,
+                   json_path_t *p, json_path_step_t **p_cur_step,
+                   uint *array_counters);
+
+
+typedef struct st_json_find_paths_t
+{
+  uint n_paths;
+  json_path_t *paths;
+  uint cur_depth;
+  uint *path_depths;
+  uint array_counters[JSON_DEPTH_LIMIT];
+} json_find_paths_t;
+
+
+int json_find_paths_first(json_engine_t *je, json_find_paths_t *state,
+                          uint n_paths, json_path_t *paths, uint *path_depths);
+int json_find_paths_next(json_engine_t *je, json_find_paths_t *state);
+
+
+/*
+  Converst JSON string constant into ordinary string constant
+  which can involve unpacking json escapes and changing character set.
+  Returns negative integer in the case of an error,
+  the length of the result otherwise.
+*/
+int json_unescape(CHARSET_INFO *json_cs,
+                  const uchar *json_str, const uchar *json_end,
+                  CHARSET_INFO *res_cs,
+                  uchar *res, uchar *res_end);
+
+/*
+  Converst ordinary string constant into JSON string constant.
+  which can involve appropriate escaping and changing character set.
+  Returns negative integer in the case of an error,
+  the length of the result otherwise.
+*/
+int json_escape(CHARSET_INFO *str_cs, const uchar *str, const uchar *str_end,
+                CHARSET_INFO *json_cs, uchar *json, uchar *json_end);
+
+
+/*
+  Appends the ASCII string to the json with the charset conversion.
+*/
+int json_append_ascii(CHARSET_INFO *json_cs,
+                      uchar *json, uchar *json_end,
+                      const uchar *ascii, const uchar *ascii_end);
+
+#ifdef  __cplusplus
+}
+#endif
+
+#endif /* JSON_LIB_INCLUDED */
+
diff --git a/include/m_ctype.h b/include/m_ctype.h
index c50763ed570..04a82953f0a 100644
--- a/include/m_ctype.h
+++ b/include/m_ctype.h
@@ -184,7 +184,6 @@ extern MY_UNI_CTYPE my_uni_ctype[256];
 #define MY_CS_MBMAXLEN  6     /* Maximum supported mbmaxlen */
 #define MY_CS_IS_TOOSMALL(rc) ((rc) >= MY_CS_TOOSMALL6 && (rc) <= MY_CS_TOOSMALL)
 
-
 #define MY_SEQ_INTTAIL	1
 #define MY_SEQ_SPACES	2
 #define MY_SEQ_NONSPACES 3 /* Skip non-space characters, including bad bytes */
@@ -397,7 +396,6 @@ typedef struct
 */
 typedef struct
 {
-  MY_STRCOPY_STATUS m_native_copy_status;
   const char *m_cannot_convert_error_pos;
 } MY_STRCONV_STATUS;
 
@@ -410,9 +408,6 @@ struct my_charset_handler_st
   size_t  (*numchars)(CHARSET_INFO *, const char *b, const char *e);
   size_t  (*charpos)(CHARSET_INFO *, const char *b, const char *e,
                      size_t pos);
-  size_t  (*well_formed_len)(CHARSET_INFO *,
-                             const char *b,const char *e,
-                             size_t nchars, int *error);
   size_t  (*lengthsp)(CHARSET_INFO *, const char *ptr, size_t length);
   size_t  (*numcells)(CHARSET_INFO *, const char *b, const char *e);
   
@@ -817,8 +812,6 @@ int my_wildcmp_bin(CHARSET_INFO *,
 size_t my_numchars_8bit(CHARSET_INFO *, const char *b, const char *e);
 size_t my_numcells_8bit(CHARSET_INFO *, const char *b, const char *e);
 size_t my_charpos_8bit(CHARSET_INFO *, const char *b, const char *e, size_t pos);
-size_t my_well_formed_len_8bit(CHARSET_INFO *, const char *b, const char *e,
-                             size_t pos, int *error);
 size_t my_well_formed_char_length_8bit(CHARSET_INFO *cs,
                                        const char *b, const char *e,
                                        size_t nchars,
@@ -850,8 +843,6 @@ int my_wildcmp_mb(CHARSET_INFO *,
 size_t my_numchars_mb(CHARSET_INFO *, const char *b, const char *e);
 size_t my_numcells_mb(CHARSET_INFO *, const char *b, const char *e);
 size_t my_charpos_mb(CHARSET_INFO *, const char *b, const char *e, size_t pos);
-size_t my_well_formed_len_mb(CHARSET_INFO *, const char *b, const char *e,
-                             size_t pos, int *error);
 uint my_instr_mb(CHARSET_INFO *,
                  const char *b, size_t b_length,
                  const char *s, size_t s_length,
@@ -994,7 +985,9 @@ uint32 my_convert_using_func(char *to, uint32 to_length, CHARSET_INFO *to_cs,
 */
 size_t my_convert_fix(CHARSET_INFO *dstcs, char *dst, size_t dst_length,
                       CHARSET_INFO *srccs, const char *src, size_t src_length,
-                      size_t nchars, MY_STRCONV_STATUS *status);
+                      size_t nchars,
+                      MY_STRCOPY_STATUS *copy_status,
+                      MY_STRCONV_STATUS *conv_status);
 
 #define	_MY_U	01	/* Upper case */
 #define	_MY_L	02	/* Lower case */
@@ -1089,6 +1082,22 @@ uint my_charlen_fix(CHARSET_INFO *cs, const char *str, const char *end)
 }
 
 
+/*
+  A compatibility replacement pure C function for the former
+    cs->cset->well_formed_len().
+  In C++ code please use Well_formed_prefix::length() instead.
+*/
+static inline size_t
+my_well_formed_length(CHARSET_INFO *cs, const char *b, const char *e,
+                      size_t nchars, int *error)
+{
+  MY_STRCOPY_STATUS status;
+  (void) cs->cset->well_formed_char_length(cs, b, e, nchars, &status);
+  *error= status.m_well_formed_error_pos == NULL ? 0 : 1;
+  return status.m_source_end_pos - b;
+}
+
+
 #define my_caseup_str(s, a)           ((s)->cset->caseup_str((s), (a)))
 #define my_casedn_str(s, a)           ((s)->cset->casedn_str((s), (a)))
 #define my_strntol(s, a, b, c, d, e)  ((s)->cset->strntol((s),(a),(b),(c),(d),(e)))
diff --git a/include/maria.h b/include/maria.h
index bdd53f3d183..1305b6444c0 100644
--- a/include/maria.h
+++ b/include/maria.h
@@ -149,7 +149,7 @@ typedef struct st_maria_create_info
   uint null_bytes;
   uint old_options;
   enum data_file_type org_data_file_type;
-  uint8 language;
+  uint16 language;
   my_bool with_auto_increment, transactional;
 } MARIA_CREATE_INFO;
 
diff --git a/include/my_atomic.h b/include/my_atomic.h
index 2034bf48987..8f13a0ab89b 100644
--- a/include/my_atomic.h
+++ b/include/my_atomic.h
@@ -112,9 +112,26 @@
 #undef MY_ATOMIC_HAS_8_16
 
 /*
- * Attempt to do atomic ops without locks
- */
-#include "atomic/nolock.h"
+  We choose implementation as follows:
+  ------------------------------------
+  On Windows using Visual C++ the native implementation should be
+  preferrable. When using gcc we prefer the Solaris implementation
+  before the gcc because of stability preference, we choose gcc
+  builtins if available, otherwise we choose the somewhat broken
+  native x86 implementation. If neither Visual C++ or gcc we still
+  choose the Solaris implementation on Solaris (mainly for SunStudio
+  compilers).
+*/
+#if defined(_MSC_VER)
+#include "atomic/generic-msvc.h"
+#elif defined(HAVE_SOLARIS_ATOMIC)
+#include "atomic/solaris.h"
+#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
+#include "atomic/gcc_builtins.h"
+#elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
+#include "atomic/x86-gcc.h"
+#endif
+
 
 #ifndef make_atomic_cas_body
 /* nolock.h was not able to generate even a CAS function, fall back */
@@ -282,8 +299,16 @@ make_atomic_store(ptr)
 
 #if SIZEOF_LONG == 4
 #define my_atomic_addlong(A,B) my_atomic_add32((int32*) (A), (B))
+#define my_atomic_loadlong(A) my_atomic_load32((int32*) (A))
+#define my_atomic_storelong(A,B) my_atomic_store32((int32*) (A), (B))
+#define my_atomic_faslong(A,B) my_atomic_fas32((int32*) (A), (B))
+#define my_atomic_caslong(A,B,C) my_atomic_cas32((int32*) (A), (int32*) (B), (C))
 #else
 #define my_atomic_addlong(A,B) my_atomic_add64((int64*) (A), (B))
+#define my_atomic_loadlong(A) my_atomic_load64((int64*) (A))
+#define my_atomic_storelong(A,B) my_atomic_store64((int64*) (A), (B))
+#define my_atomic_faslong(A,B) my_atomic_fas64((int64*) (A), (B))
+#define my_atomic_caslong(A,B,C) my_atomic_cas64((int64*) (A), (int64*) (B), (C))
 #endif
 
 #ifdef _atomic_h_cleanup_
diff --git a/include/my_base.h b/include/my_base.h
index f5842685f9d..84b2e28d340 100644
--- a/include/my_base.h
+++ b/include/my_base.h
@@ -566,7 +566,7 @@ typedef ulong key_part_map;
 #define HA_STATE_KEY_CHANGED	128
 #define HA_STATE_WRITE_AT_END	256	/* set in _ps_find_writepos */
 #define HA_STATE_BUFF_SAVED	512	/* If current keybuff is info->buff */
-#define HA_STATE_ROW_CHANGED	1024	/* To invalide ROW cache */
+#define HA_STATE_ROW_CHANGED	1024	/* To invalidate ROW cache */
 #define HA_STATE_EXTEND_BLOCK	2048
 #define HA_STATE_RNEXT_SAME	4096	/* rnext_same occupied lastkey2 */
 
diff --git a/include/my_bitmap.h b/include/my_bitmap.h
index 9c9550e3141..3e242280be4 100644
--- a/include/my_bitmap.h
+++ b/include/my_bitmap.h
@@ -58,6 +58,7 @@ extern my_bool bitmap_is_overlapping(const MY_BITMAP *map1,
 extern my_bool bitmap_test_and_set(MY_BITMAP *map, uint bitmap_bit);
 extern my_bool bitmap_test_and_clear(MY_BITMAP *map, uint bitmap_bit);
 extern my_bool bitmap_fast_test_and_set(MY_BITMAP *map, uint bitmap_bit);
+extern my_bool bitmap_fast_test_and_clear(MY_BITMAP *map, uint bitmap_bit);
 extern my_bool bitmap_union_is_set_all(const MY_BITMAP *map1,
                                        const MY_BITMAP *map2);
 extern my_bool bitmap_exists_intersection(const MY_BITMAP **bitmap_array,
diff --git a/include/my_compare.h b/include/my_compare.h
index 6b76483074b..12f9971d49b 100644
--- a/include/my_compare.h
+++ b/include/my_compare.h
@@ -57,7 +57,7 @@ typedef struct st_HA_KEYSEG		/* Key-portion */
   uint16 language;
   uint8  type;				/* Type of key (for sort) */
   uint8  null_bit;			/* bitmask to test for NULL */
-  uint8  bit_start,bit_end;		/* if bit field */
+  uint8  bit_start;
   uint8  bit_length;                    /* Length of bit part */
 } HA_KEYSEG;
 
diff --git a/include/my_global.h b/include/my_global.h
index 0f24570c2f7..4f40d3e0615 100644
--- a/include/my_global.h
+++ b/include/my_global.h
@@ -879,8 +879,7 @@ typedef long long	my_ptrdiff_t;
   and related routines are refactored.
 */
 
-#define my_offsetof(TYPE, MEMBER) \
-        ((size_t)((char *)&(((TYPE *)0x10)->MEMBER) - (char*)0x10))
+#define my_offsetof(TYPE, MEMBER) PTR_BYTE_DIFF(&((TYPE *)0x10)->MEMBER, 0x10)
 
 #define NullS		(char *) 0
 
diff --git a/include/my_sys.h b/include/my_sys.h
index 528950f4e22..c8f3e1bf3a4 100644
--- a/include/my_sys.h
+++ b/include/my_sys.h
@@ -1,5 +1,5 @@
 /* Copyright (c) 2000, 2013, Oracle and/or its affiliates.
-   Copyright (c) 2010, 2013, Monty Program Ab.
+   Copyright (c) 2010, 2016, Monty Program Ab.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -279,7 +279,7 @@ extern my_bool my_use_symdir;
 extern ulong	my_default_record_cache_size;
 extern my_bool  my_disable_locking, my_disable_async_io,
                 my_disable_flush_key_blocks, my_disable_symlinks;
-extern my_bool my_disable_sync;
+extern my_bool my_disable_sync, my_disable_copystat_in_redel;
 extern char	wild_many,wild_one,wild_prefix;
 extern const char *charsets_dir;
 extern my_bool timed_mutexes;
@@ -698,7 +698,7 @@ extern FILE *my_fdopen(File Filedes,const char *name, int Flags,myf MyFlags);
 extern FILE *my_freopen(const char *path, const char *mode, FILE *stream);
 extern int my_fclose(FILE *fd,myf MyFlags);
 extern int my_vfprintf(FILE *stream, const char* format, va_list args);
-extern void my_strerror(char *buf, size_t len, int nr);
+extern const char* my_strerror(char *buf, size_t len, int nr);
 extern int my_fprintf(FILE *stream, const char* format, ...);
 extern File my_fileno(FILE *fd);
 extern int my_chsize(File fd,my_off_t newlength, int filler, myf MyFlags);
diff --git a/include/myisamchk.h b/include/myisamchk.h
index 64724de1789..643241d84e5 100644
--- a/include/myisamchk.h
+++ b/include/myisamchk.h
@@ -16,14 +16,6 @@
 
 /* Definitions needed for myisamchk/mariachk.c */
 
-/*
-  Entries marked as "QQ to be removed" are NOT used to
-  pass check/repair options to xxx_check.c. They are used
-  internally by xxxchk.c or/and ha_xxxx.cc and should NOT
-  be stored together with other flags. They should be removed
-  from the following list to make addition of new flags possible.
-*/
-
 #ifndef _myisamchk_h
 #define _myisamchk_h
 
@@ -66,6 +58,7 @@ typedef enum
   MI_STATS_METHOD_IGNORE_NULLS
 } enum_handler_stats_method;
 
+struct st_myisam_info;
 
 typedef struct st_handler_check_param
 {
@@ -122,6 +115,8 @@ typedef struct st_handler_check_param
   uint progress_counter;             /* How often to call _report_progress() */
   ulonglong progress, max_progress;
 
+  int (*fix_record)(struct st_myisam_info *info, uchar *record, int keynum);
+
   mysql_mutex_t print_msg_mutex;
   my_bool need_print_msg_lock;
   myf malloc_flags;
diff --git a/include/mysql.h b/include/mysql.h
index eab859270f7..80e75c264e8 100644
--- a/include/mysql.h
+++ b/include/mysql.h
@@ -138,6 +138,12 @@ typedef unsigned long long my_ulonglong;
 #define ER_WARN_DATA_TRUNCATED WARN_DATA_TRUNCATED
 #define WARN_PLUGIN_DELETE_BUILTIN ER_PLUGIN_DELETE_BUILTIN
 #define ER_FK_DUP_NAME ER_DUP_CONSTRAINT_NAME
+#define ER_VIRTUAL_COLUMN_FUNCTION_IS_NOT_ALLOWED ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED
+#define ER_PRIMARY_KEY_BASED_ON_VIRTUAL_COLUMN ER_PRIMARY_KEY_BASED_ON_GENERATED_COLUMN
+#define ER_WRONG_FK_OPTION_FOR_VIRTUAL_COLUMN ER_WRONG_FK_OPTION_FOR_GENERATED_COLUMN
+#define ER_WARNING_NON_DEFAULT_VALUE_FOR_VIRTUAL_COLUMN ER_WARNING_NON_DEFAULT_VALUE_FOR_GENERATED_COLUMN
+#define ER_UNSUPPORTED_ACTION_ON_VIRTUAL_COLUMN ER_UNSUPPORTED_ACTION_ON_GENERATED_COLUMN
+#define ER_UNSUPPORTED_ENGINE_FOR_VIRTUAL_COLUMNS ER_UNSUPPORTED_ENGINE_FOR_GENERATED_COLUMNS
 
 typedef struct st_mysql_rows {
   struct st_mysql_rows *next;		/* list of rows */
diff --git a/include/mysql.h.pp b/include/mysql.h.pp
index 857f5b99a90..9d93b9ad325 100644
--- a/include/mysql.h.pp
+++ b/include/mysql.h.pp
@@ -11,11 +11,18 @@ enum enum_server_command
   COM_STMT_RESET, COM_SET_OPTION, COM_STMT_FETCH, COM_DAEMON,
   COM_MDB_GAP_BEG,
   COM_MDB_GAP_END=250,
-  COM_SLAVE_WORKER,
-  COM_SLAVE_IO,
-  COM_SLAVE_SQL,
-  COM_MULTI,
-  COM_END
+  COM_SLAVE_WORKER=251,
+  COM_SLAVE_IO=252,
+  COM_SLAVE_SQL=253,
+  COM_MULTI=254,
+  COM_END=255
+};
+enum enum_indicator_type
+{
+  STMT_INDICATOR_NONE= 0,
+  STMT_INDICATOR_NULL,
+  STMT_INDICATOR_DEFAULT,
+  STMT_INDICATOR_IGNORE
 };
 struct st_vio;
 typedef struct st_vio Vio;
@@ -109,6 +116,8 @@ my_bool net_write_command(NET *net,unsigned char command,
      const unsigned char *packet, size_t len);
 int net_real_write(NET *net,const unsigned char *packet, size_t len);
 unsigned long my_net_read_packet(NET *net, my_bool read_from_server);
+unsigned long my_net_read_packet_reallen(NET *net, my_bool read_from_server,
+                                         unsigned long* reallen);
 struct sockaddr;
 int my_connect(my_socket s, const struct sockaddr *name, unsigned int namelen,
         unsigned int timeout);
diff --git a/include/mysql_com.h b/include/mysql_com.h
index 461800f3ce7..c399520022d 100644
--- a/include/mysql_com.h
+++ b/include/mysql_com.h
@@ -114,12 +114,24 @@ enum enum_server_command
   /* don't forget to update const char *command_name[] in sql_parse.cc */
   COM_MDB_GAP_BEG,
   COM_MDB_GAP_END=250,
-  COM_SLAVE_WORKER,
-  COM_SLAVE_IO,
-  COM_SLAVE_SQL,
-  COM_MULTI,
+  COM_SLAVE_WORKER=251,
+  COM_SLAVE_IO=252,
+  COM_SLAVE_SQL=253,
+  COM_MULTI=254,
   /* Must be last */
-  COM_END
+  COM_END=255
+};
+
+
+/*
+  Bulk PS protocol indicator value:
+*/
+enum enum_indicator_type
+{
+  STMT_INDICATOR_NONE= 0,
+  STMT_INDICATOR_NULL,
+  STMT_INDICATOR_DEFAULT,
+  STMT_INDICATOR_IGNORE
 };
 
 /* sql type stored in .frm files for virtual fields */
@@ -202,43 +214,43 @@ enum enum_server_command
 #define REFRESH_FAST            (1ULL << 31) /* Intern flag */
 
 #define CLIENT_LONG_PASSWORD	0	/* obsolete flag */
-#define CLIENT_MYSQL            1       /* mysql/old mariadb server/client */
-#define CLIENT_FOUND_ROWS	2	/* Found instead of affected rows */
-#define CLIENT_LONG_FLAG	4	/* Get all column flags */
-#define CLIENT_CONNECT_WITH_DB	8	/* One can specify db on connect */
-#define CLIENT_NO_SCHEMA	16	/* Don't allow database.table.column */
-#define CLIENT_COMPRESS		32	/* Can use compression protocol */
-#define CLIENT_ODBC		64	/* Odbc client */
-#define CLIENT_LOCAL_FILES	128	/* Can use LOAD DATA LOCAL */
-#define CLIENT_IGNORE_SPACE	256	/* Ignore spaces before '(' */
-#define CLIENT_PROTOCOL_41	512	/* New 4.1 protocol */
-#define CLIENT_INTERACTIVE	1024	/* This is an interactive client */
-#define CLIENT_SSL              2048	/* Switch to SSL after handshake */
-#define CLIENT_IGNORE_SIGPIPE   4096    /* IGNORE sigpipes */
-#define CLIENT_TRANSACTIONS	8192	/* Client knows about transactions */
-#define CLIENT_RESERVED         16384   /* Old flag for 4.1 protocol  */
-#define CLIENT_SECURE_CONNECTION 32768  /* New 4.1 authentication */
-#define CLIENT_MULTI_STATEMENTS (1UL << 16) /* Enable/disable multi-stmt support */
-#define CLIENT_MULTI_RESULTS    (1UL << 17) /* Enable/disable multi-results */
-#define CLIENT_PS_MULTI_RESULTS (1UL << 18) /* Multi-results in PS-protocol */
-
-#define CLIENT_PLUGIN_AUTH  (1UL << 19) /* Client supports plugin authentication */
-#define CLIENT_CONNECT_ATTRS (1UL << 20) /* Client supports connection attributes */
+#define CLIENT_MYSQL            1ULL       /* mysql/old mariadb server/client */
+#define CLIENT_FOUND_ROWS	2ULL	/* Found instead of affected rows */
+#define CLIENT_LONG_FLAG	4ULL	/* Get all column flags */
+#define CLIENT_CONNECT_WITH_DB	8ULL	/* One can specify db on connect */
+#define CLIENT_NO_SCHEMA	16ULL	/* Don't allow database.table.column */
+#define CLIENT_COMPRESS		32ULL	/* Can use compression protocol */
+#define CLIENT_ODBC		64ULL	/* Odbc client */
+#define CLIENT_LOCAL_FILES	128ULL	/* Can use LOAD DATA LOCAL */
+#define CLIENT_IGNORE_SPACE	256ULL	/* Ignore spaces before '(' */
+#define CLIENT_PROTOCOL_41	512ULL	/* New 4.1 protocol */
+#define CLIENT_INTERACTIVE	1024ULL	/* This is an interactive client */
+#define CLIENT_SSL              2048ULL	/* Switch to SSL after handshake */
+#define CLIENT_IGNORE_SIGPIPE   4096ULL    /* IGNORE sigpipes */
+#define CLIENT_TRANSACTIONS	8192ULL	/* Client knows about transactions */
+#define CLIENT_RESERVED         16384ULL   /* Old flag for 4.1 protocol  */
+#define CLIENT_SECURE_CONNECTION 32768ULL  /* New 4.1 authentication */
+#define CLIENT_MULTI_STATEMENTS (1ULL << 16) /* Enable/disable multi-stmt support */
+#define CLIENT_MULTI_RESULTS    (1ULL << 17) /* Enable/disable multi-results */
+#define CLIENT_PS_MULTI_RESULTS (1ULL << 18) /* Multi-results in PS-protocol */
+
+#define CLIENT_PLUGIN_AUTH  (1ULL << 19) /* Client supports plugin authentication */
+#define CLIENT_CONNECT_ATTRS (1ULL << 20) /* Client supports connection attributes */
 /* Enable authentication response packet to be larger than 255 bytes. */
-#define CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA (1UL << 21)
+#define CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA (1ULL << 21)
 /* Don't close the connection for a connection with expired password. */
-#define CLIENT_CAN_HANDLE_EXPIRED_PASSWORDS (1UL << 22)
+#define CLIENT_CAN_HANDLE_EXPIRED_PASSWORDS (1ULL << 22)
 
 /**
   Capable of handling server state change information. Its a hint to the
   server to include the state change information in Ok packet.
 */
-#define CLIENT_SESSION_TRACK (1UL << 23)
+#define CLIENT_SESSION_TRACK (1ULL << 23)
 /* Client no longer needs EOF packet */
-#define CLIENT_DEPRECATE_EOF (1UL << 24)
+#define CLIENT_DEPRECATE_EOF (1ULL << 24)
 
-#define CLIENT_PROGRESS_OBSOLETE  (1UL << 29)
-#define CLIENT_SSL_VERIFY_SERVER_CERT (1UL << 30)
+#define CLIENT_PROGRESS_OBSOLETE  (1ULL << 29)
+#define CLIENT_SSL_VERIFY_SERVER_CERT (1ULL << 30)
 /*
   It used to be that if mysql_real_connect() failed, it would delete any
   options set by the client, unless the CLIENT_REMEMBER_OPTIONS flag was
@@ -248,7 +260,7 @@ enum enum_server_command
   always preserve any options set in case of failed connect, and this
   option is effectively always set.
 */
-#define CLIENT_REMEMBER_OPTIONS (1UL << 31)
+#define CLIENT_REMEMBER_OPTIONS (1ULL << 31)
 
 /* MariaDB extended capability flags */
 #define MARIADB_CLIENT_FLAGS_MASK 0xffffffff00000000ULL
@@ -256,6 +268,8 @@ enum enum_server_command
 #define MARIADB_CLIENT_PROGRESS (1ULL << 32)
 /* support COM_MULTI */
 #define MARIADB_CLIENT_COM_MULTI (1ULL << 33)
+/* support of array binding */
+#define MARIADB_CLIENT_STMT_BULK_OPERATIONS (1ULL << 34)
 
 #ifdef HAVE_COMPRESS
 #define CAN_CLIENT_COMPRESS CLIENT_COMPRESS
@@ -295,7 +309,8 @@ enum enum_server_command
                            CLIENT_SESSION_TRACK |\
                            CLIENT_DEPRECATE_EOF |\
                            CLIENT_CONNECT_ATTRS |\
-                           MARIADB_CLIENT_COM_MULTI)
+                           MARIADB_CLIENT_COM_MULTI |\
+                           MARIADB_CLIENT_STMT_BULK_OPERATIONS)
 
 /*
   To be added later:
@@ -585,6 +600,8 @@ my_bool	net_write_command(NET *net,unsigned char command,
 			  const unsigned char *packet, size_t len);
 int	net_real_write(NET *net,const unsigned char *packet, size_t len);
 unsigned long my_net_read_packet(NET *net, my_bool read_from_server);
+unsigned long my_net_read_packet_reallen(NET *net, my_bool read_from_server,
+                                         unsigned long* reallen);
 #define my_net_read(A) my_net_read_packet((A), 0)
 
 #ifdef MY_GLOBAL_INCLUDED
diff --git a/include/sql_common.h b/include/sql_common.h
index 49616f6d56c..bbf459e1e55 100644
--- a/include/sql_common.h
+++ b/include/sql_common.h
@@ -104,6 +104,7 @@ cli_advanced_command(MYSQL *mysql, enum enum_server_command command,
 		     const unsigned char *arg, ulong arg_length,
                      my_bool skip_check, MYSQL_STMT *stmt);
 unsigned long cli_safe_read(MYSQL *mysql);
+unsigned long cli_safe_read_reallen(MYSQL *mysql, ulong* reallen);
 void net_clear_error(NET *net);
 void set_stmt_errmsg(MYSQL_STMT *stmt, NET *net);
 void set_stmt_error(MYSQL_STMT *stmt, int errcode, const char *sqlstate,